diff --git a/aaparser/aaparser.go b/aaparser/aaparser.go deleted file mode 100644 index 507298f..0000000 --- a/aaparser/aaparser.go +++ /dev/null @@ -1,92 +0,0 @@ -// Package aaparser is a convenience package interacting with `apparmor_parser`. -package aaparser - -import ( - "fmt" - "os/exec" - "path/filepath" - "strconv" - "strings" -) - -const ( - binary = "apparmor_parser" -) - -// GetVersion returns the major and minor version of apparmor_parser. -func GetVersion() (int, error) { - output, err := cmd("", "--version") - if err != nil { - return -1, err - } - - return parseVersion(output) -} - -// LoadProfile runs `apparmor_parser -r -W` on a specified apparmor profile to -// replace and write it to disk. -func LoadProfile(profilePath string) error { - _, err := cmd(filepath.Dir(profilePath), "-r", "-W", filepath.Base(profilePath)) - if err != nil { - return err - } - return nil -} - -// cmd runs `apparmor_parser` with the passed arguments. -func cmd(dir string, arg ...string) (string, error) { - c := exec.Command(binary, arg...) - c.Dir = dir - - output, err := c.CombinedOutput() - if err != nil { - return "", fmt.Errorf("running `%s %s` failed with output: %s\nerror: %v", c.Path, strings.Join(c.Args, " "), string(output), err) - } - - return string(output), nil -} - -// parseVersion takes the output from `apparmor_parser --version` and returns -// a representation of the {major, minor, patch} version as a single number of -// the form MMmmPPP {major, minor, patch}. -func parseVersion(output string) (int, error) { - // output is in the form of the following: - // AppArmor parser version 2.9.1 - // Copyright (C) 1999-2008 Novell Inc. - // Copyright 2009-2012 Canonical Ltd. - - lines := strings.SplitN(output, "\n", 2) - words := strings.Split(lines[0], " ") - version := words[len(words)-1] - - // split by major minor version - v := strings.Split(version, ".") - if len(v) == 0 || len(v) > 3 { - return -1, fmt.Errorf("parsing version failed for output: `%s`", output) - } - - // Default the versions to 0. - var majorVersion, minorVersion, patchLevel int - - majorVersion, err := strconv.Atoi(v[0]) - if err != nil { - return -1, err - } - - if len(v) > 1 { - minorVersion, err = strconv.Atoi(v[1]) - if err != nil { - return -1, err - } - } - if len(v) > 2 { - patchLevel, err = strconv.Atoi(v[2]) - if err != nil { - return -1, err - } - } - - // major*10^5 + minor*10^3 + patch*10^0 - numericVersion := majorVersion*1e5 + minorVersion*1e3 + patchLevel - return numericVersion, nil -} diff --git a/aaparser/aaparser_test.go b/aaparser/aaparser_test.go deleted file mode 100644 index 69bc8d2..0000000 --- a/aaparser/aaparser_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package aaparser - -import ( - "testing" -) - -type versionExpected struct { - output string - version int -} - -func TestParseVersion(t *testing.T) { - versions := []versionExpected{ - { - output: `AppArmor parser version 2.10 -Copyright (C) 1999-2008 Novell Inc. -Copyright 2009-2012 Canonical Ltd. - -`, - version: 210000, - }, - { - output: `AppArmor parser version 2.8 -Copyright (C) 1999-2008 Novell Inc. -Copyright 2009-2012 Canonical Ltd. - -`, - version: 208000, - }, - { - output: `AppArmor parser version 2.20 -Copyright (C) 1999-2008 Novell Inc. -Copyright 2009-2012 Canonical Ltd. - -`, - version: 220000, - }, - { - output: `AppArmor parser version 2.05 -Copyright (C) 1999-2008 Novell Inc. -Copyright 2009-2012 Canonical Ltd. - -`, - version: 205000, - }, - { - output: `AppArmor parser version 2.9.95 -Copyright (C) 1999-2008 Novell Inc. -Copyright 2009-2012 Canonical Ltd. - -`, - version: 209095, - }, - { - output: `AppArmor parser version 3.14.159 -Copyright (C) 1999-2008 Novell Inc. -Copyright 2009-2012 Canonical Ltd. - -`, - version: 314159, - }, - } - - for _, v := range versions { - version, err := parseVersion(v.output) - if err != nil { - t.Fatalf("expected error to be nil for %#v, got: %v", v, err) - } - if version != v.version { - t.Fatalf("expected version to be %d, was %d, for: %#v\n", v.version, version, v) - } - } -} diff --git a/authorization/api.go b/authorization/api.go deleted file mode 100644 index fc82c46..0000000 --- a/authorization/api.go +++ /dev/null @@ -1,54 +0,0 @@ -package authorization - -const ( - // AuthZApiRequest is the url for daemon request authorization - AuthZApiRequest = "AuthZPlugin.AuthZReq" - - // AuthZApiResponse is the url for daemon response authorization - AuthZApiResponse = "AuthZPlugin.AuthZRes" - - // AuthZApiImplements is the name of the interface all AuthZ plugins implement - AuthZApiImplements = "authz" -) - -// Request holds data required for authZ plugins -type Request struct { - // User holds the user extracted by AuthN mechanism - User string `json:"User,omitempty"` - - // UserAuthNMethod holds the mechanism used to extract user details (e.g., krb) - UserAuthNMethod string `json:"UserAuthNMethod,omitempty"` - - // RequestMethod holds the HTTP method (GET/POST/PUT) - RequestMethod string `json:"RequestMethod,omitempty"` - - // RequestUri holds the full HTTP uri (e.g., /v1.21/version) - RequestURI string `json:"RequestUri,omitempty"` - - // RequestBody stores the raw request body sent to the docker daemon - RequestBody []byte `json:"RequestBody,omitempty"` - - // RequestHeaders stores the raw request headers sent to the docker daemon - RequestHeaders map[string]string `json:"RequestHeaders,omitempty"` - - // ResponseStatusCode stores the status code returned from docker daemon - ResponseStatusCode int `json:"ResponseStatusCode,omitempty"` - - // ResponseBody stores the raw response body sent from docker daemon - ResponseBody []byte `json:"ResponseBody,omitempty"` - - // ResponseHeaders stores the response headers sent to the docker daemon - ResponseHeaders map[string]string `json:"ResponseHeaders,omitempty"` -} - -// Response represents authZ plugin response -type Response struct { - // Allow indicating whether the user is allowed or not - Allow bool `json:"Allow"` - - // Msg stores the authorization message - Msg string `json:"Msg,omitempty"` - - // Err stores a message in case there's an error - Err string `json:"Err,omitempty"` -} diff --git a/authorization/authz.go b/authorization/authz.go deleted file mode 100644 index ebbc380..0000000 --- a/authorization/authz.go +++ /dev/null @@ -1,179 +0,0 @@ -package authorization - -import ( - "bufio" - "bytes" - "fmt" - "io" - "net/http" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/ioutils" -) - -const maxBodySize = 1048576 // 1MB - -// NewCtx creates new authZ context, it is used to store authorization information related to a specific docker -// REST http session -// A context provides two method: -// Authenticate Request: -// Call authZ plugins with current REST request and AuthN response -// Request contains full HTTP packet sent to the docker daemon -// https://docs.docker.com/reference/api/docker_remote_api/ -// -// Authenticate Response: -// Call authZ plugins with full info about current REST request, REST response and AuthN response -// The response from this method may contains content that overrides the daemon response -// This allows authZ plugins to filter privileged content -// -// If multiple authZ plugins are specified, the block/allow decision is based on ANDing all plugin results -// For response manipulation, the response from each plugin is piped between plugins. Plugin execution order -// is determined according to daemon parameters -func NewCtx(authZPlugins []Plugin, user, userAuthNMethod, requestMethod, requestURI string) *Ctx { - return &Ctx{ - plugins: authZPlugins, - user: user, - userAuthNMethod: userAuthNMethod, - requestMethod: requestMethod, - requestURI: requestURI, - } -} - -// Ctx stores a single request-response interaction context -type Ctx struct { - user string - userAuthNMethod string - requestMethod string - requestURI string - plugins []Plugin - // authReq stores the cached request object for the current transaction - authReq *Request -} - -// AuthZRequest authorized the request to the docker daemon using authZ plugins -func (ctx *Ctx) AuthZRequest(w http.ResponseWriter, r *http.Request) error { - var body []byte - if sendBody(ctx.requestURI, r.Header) && r.ContentLength > 0 && r.ContentLength < maxBodySize { - var err error - body, r.Body, err = drainBody(r.Body) - if err != nil { - return err - } - } - - var h bytes.Buffer - if err := r.Header.Write(&h); err != nil { - return err - } - - ctx.authReq = &Request{ - User: ctx.user, - UserAuthNMethod: ctx.userAuthNMethod, - RequestMethod: ctx.requestMethod, - RequestURI: ctx.requestURI, - RequestBody: body, - RequestHeaders: headers(r.Header), - } - - for _, plugin := range ctx.plugins { - logrus.Debugf("AuthZ request using plugin %s", plugin.Name()) - - authRes, err := plugin.AuthZRequest(ctx.authReq) - if err != nil { - return fmt.Errorf("plugin %s failed with error: %s", plugin.Name(), err) - } - - if !authRes.Allow { - return newAuthorizationError(plugin.Name(), authRes.Msg) - } - } - - return nil -} - -// AuthZResponse authorized and manipulates the response from docker daemon using authZ plugins -func (ctx *Ctx) AuthZResponse(rm ResponseModifier, r *http.Request) error { - ctx.authReq.ResponseStatusCode = rm.StatusCode() - ctx.authReq.ResponseHeaders = headers(rm.Header()) - - if sendBody(ctx.requestURI, rm.Header()) { - ctx.authReq.ResponseBody = rm.RawBody() - } - - for _, plugin := range ctx.plugins { - logrus.Debugf("AuthZ response using plugin %s", plugin.Name()) - - authRes, err := plugin.AuthZResponse(ctx.authReq) - if err != nil { - return fmt.Errorf("plugin %s failed with error: %s", plugin.Name(), err) - } - - if !authRes.Allow { - return newAuthorizationError(plugin.Name(), authRes.Msg) - } - } - - rm.FlushAll() - - return nil -} - -// drainBody dump the body (if it's length is less than 1MB) without modifying the request state -func drainBody(body io.ReadCloser) ([]byte, io.ReadCloser, error) { - bufReader := bufio.NewReaderSize(body, maxBodySize) - newBody := ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() }) - - data, err := bufReader.Peek(maxBodySize) - // Body size exceeds max body size - if err == nil { - logrus.Warnf("Request body is larger than: '%d' skipping body", maxBodySize) - return nil, newBody, nil - } - // Body size is less than maximum size - if err == io.EOF { - return data, newBody, nil - } - // Unknown error - return nil, newBody, err -} - -// sendBody returns true when request/response body should be sent to AuthZPlugin -func sendBody(url string, header http.Header) bool { - // Skip body for auth endpoint - if strings.HasSuffix(url, "/auth") { - return false - } - - // body is sent only for text or json messages - return header.Get("Content-Type") == "application/json" -} - -// headers returns flatten version of the http headers excluding authorization -func headers(header http.Header) map[string]string { - v := make(map[string]string, 0) - for k, values := range header { - // Skip authorization headers - if strings.EqualFold(k, "Authorization") || strings.EqualFold(k, "X-Registry-Config") || strings.EqualFold(k, "X-Registry-Auth") { - continue - } - for _, val := range values { - v[k] = val - } - } - return v -} - -// authorizationError represents an authorization deny error -type authorizationError struct { - error -} - -// HTTPErrorStatusCode returns the authorization error status code (forbidden) -func (e authorizationError) HTTPErrorStatusCode() int { - return http.StatusForbidden -} - -func newAuthorizationError(plugin, msg string) authorizationError { - return authorizationError{error: fmt.Errorf("authorization denied by plugin %s: %s", plugin, msg)} -} diff --git a/authorization/authz_unix_test.go b/authorization/authz_unix_test.go deleted file mode 100644 index e13303f..0000000 --- a/authorization/authz_unix_test.go +++ /dev/null @@ -1,282 +0,0 @@ -// +build !windows - -// TODO Windows: This uses a Unix socket for testing. This might be possible -// to port to Windows using a named pipe instead. - -package authorization - -import ( - "bytes" - "encoding/json" - "io/ioutil" - "net" - "net/http" - "net/http/httptest" - "os" - "path" - "reflect" - "strings" - "testing" - - "github.com/docker/docker/pkg/plugins" - "github.com/docker/go-connections/tlsconfig" - "github.com/gorilla/mux" -) - -const ( - pluginAddress = "authz-test-plugin.sock" -) - -func TestAuthZRequestPluginError(t *testing.T) { - server := authZPluginTestServer{t: t} - server.start() - defer server.stop() - - authZPlugin := createTestPlugin(t) - - request := Request{ - User: "user", - RequestBody: []byte("sample body"), - RequestURI: "www.authz.com/auth", - RequestMethod: "GET", - RequestHeaders: map[string]string{"header": "value"}, - } - server.replayResponse = Response{ - Err: "an error", - } - - actualResponse, err := authZPlugin.AuthZRequest(&request) - if err != nil { - t.Fatalf("Failed to authorize request %v", err) - } - - if !reflect.DeepEqual(server.replayResponse, *actualResponse) { - t.Fatal("Response must be equal") - } - if !reflect.DeepEqual(request, server.recordedRequest) { - t.Fatal("Requests must be equal") - } -} - -func TestAuthZRequestPlugin(t *testing.T) { - server := authZPluginTestServer{t: t} - server.start() - defer server.stop() - - authZPlugin := createTestPlugin(t) - - request := Request{ - User: "user", - RequestBody: []byte("sample body"), - RequestURI: "www.authz.com/auth", - RequestMethod: "GET", - RequestHeaders: map[string]string{"header": "value"}, - } - server.replayResponse = Response{ - Allow: true, - Msg: "Sample message", - } - - actualResponse, err := authZPlugin.AuthZRequest(&request) - if err != nil { - t.Fatalf("Failed to authorize request %v", err) - } - - if !reflect.DeepEqual(server.replayResponse, *actualResponse) { - t.Fatal("Response must be equal") - } - if !reflect.DeepEqual(request, server.recordedRequest) { - t.Fatal("Requests must be equal") - } -} - -func TestAuthZResponsePlugin(t *testing.T) { - server := authZPluginTestServer{t: t} - server.start() - defer server.stop() - - authZPlugin := createTestPlugin(t) - - request := Request{ - User: "user", - RequestURI: "someting.com/auth", - RequestBody: []byte("sample body"), - } - server.replayResponse = Response{ - Allow: true, - Msg: "Sample message", - } - - actualResponse, err := authZPlugin.AuthZResponse(&request) - if err != nil { - t.Fatalf("Failed to authorize request %v", err) - } - - if !reflect.DeepEqual(server.replayResponse, *actualResponse) { - t.Fatal("Response must be equal") - } - if !reflect.DeepEqual(request, server.recordedRequest) { - t.Fatal("Requests must be equal") - } -} - -func TestResponseModifier(t *testing.T) { - r := httptest.NewRecorder() - m := NewResponseModifier(r) - m.Header().Set("h1", "v1") - m.Write([]byte("body")) - m.WriteHeader(500) - - m.FlushAll() - if r.Header().Get("h1") != "v1" { - t.Fatalf("Header value must exists %s", r.Header().Get("h1")) - } - if !reflect.DeepEqual(r.Body.Bytes(), []byte("body")) { - t.Fatalf("Body value must exists %s", r.Body.Bytes()) - } - if r.Code != 500 { - t.Fatalf("Status code must be correct %d", r.Code) - } -} - -func TestDrainBody(t *testing.T) { - tests := []struct { - length int // length is the message length send to drainBody - expectedBodyLength int // expectedBodyLength is the expected body length after drainBody is called - }{ - {10, 10}, // Small message size - {maxBodySize - 1, maxBodySize - 1}, // Max message size - {maxBodySize * 2, 0}, // Large message size (skip copying body) - - } - - for _, test := range tests { - msg := strings.Repeat("a", test.length) - body, closer, err := drainBody(ioutil.NopCloser(bytes.NewReader([]byte(msg)))) - if err != nil { - t.Fatal(err) - } - if len(body) != test.expectedBodyLength { - t.Fatalf("Body must be copied, actual length: '%d'", len(body)) - } - if closer == nil { - t.Fatal("Closer must not be nil") - } - modified, err := ioutil.ReadAll(closer) - if err != nil { - t.Fatalf("Error must not be nil: '%v'", err) - } - if len(modified) != len(msg) { - t.Fatalf("Result should not be truncated. Original length: '%d', new length: '%d'", len(msg), len(modified)) - } - } -} - -func TestResponseModifierOverride(t *testing.T) { - r := httptest.NewRecorder() - m := NewResponseModifier(r) - m.Header().Set("h1", "v1") - m.Write([]byte("body")) - m.WriteHeader(500) - - overrideHeader := make(http.Header) - overrideHeader.Add("h1", "v2") - overrideHeaderBytes, err := json.Marshal(overrideHeader) - if err != nil { - t.Fatalf("override header failed %v", err) - } - - m.OverrideHeader(overrideHeaderBytes) - m.OverrideBody([]byte("override body")) - m.OverrideStatusCode(404) - m.FlushAll() - if r.Header().Get("h1") != "v2" { - t.Fatalf("Header value must exists %s", r.Header().Get("h1")) - } - if !reflect.DeepEqual(r.Body.Bytes(), []byte("override body")) { - t.Fatalf("Body value must exists %s", r.Body.Bytes()) - } - if r.Code != 404 { - t.Fatalf("Status code must be correct %d", r.Code) - } -} - -// createTestPlugin creates a new sample authorization plugin -func createTestPlugin(t *testing.T) *authorizationPlugin { - pwd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - - client, err := plugins.NewClient("unix:///"+path.Join(pwd, pluginAddress), &tlsconfig.Options{InsecureSkipVerify: true}) - if err != nil { - t.Fatalf("Failed to create client %v", err) - } - - return &authorizationPlugin{name: "plugin", plugin: client} -} - -// AuthZPluginTestServer is a simple server that implements the authZ plugin interface -type authZPluginTestServer struct { - listener net.Listener - t *testing.T - // request stores the request sent from the daemon to the plugin - recordedRequest Request - // response stores the response sent from the plugin to the daemon - replayResponse Response - server *httptest.Server -} - -// start starts the test server that implements the plugin -func (t *authZPluginTestServer) start() { - r := mux.NewRouter() - l, err := net.Listen("unix", pluginAddress) - if err != nil { - t.t.Fatal(err) - } - t.listener = l - r.HandleFunc("/Plugin.Activate", t.activate) - r.HandleFunc("/"+AuthZApiRequest, t.auth) - r.HandleFunc("/"+AuthZApiResponse, t.auth) - t.server = &httptest.Server{ - Listener: l, - Config: &http.Server{ - Handler: r, - Addr: pluginAddress, - }, - } - t.server.Start() -} - -// stop stops the test server that implements the plugin -func (t *authZPluginTestServer) stop() { - t.server.Close() - os.Remove(pluginAddress) - if t.listener != nil { - t.listener.Close() - } -} - -// auth is a used to record/replay the authentication api messages -func (t *authZPluginTestServer) auth(w http.ResponseWriter, r *http.Request) { - t.recordedRequest = Request{} - body, err := ioutil.ReadAll(r.Body) - if err != nil { - t.t.Fatal(err) - } - r.Body.Close() - json.Unmarshal(body, &t.recordedRequest) - b, err := json.Marshal(t.replayResponse) - if err != nil { - t.t.Fatal(err) - } - w.Write(b) -} - -func (t *authZPluginTestServer) activate(w http.ResponseWriter, r *http.Request) { - b, err := json.Marshal(plugins.Manifest{Implements: []string{AuthZApiImplements}}) - if err != nil { - t.t.Fatal(err) - } - w.Write(b) -} diff --git a/authorization/middleware.go b/authorization/middleware.go deleted file mode 100644 index 73511a8..0000000 --- a/authorization/middleware.go +++ /dev/null @@ -1,60 +0,0 @@ -package authorization - -import ( - "net/http" - - "github.com/Sirupsen/logrus" - "golang.org/x/net/context" -) - -// Middleware uses a list of plugins to -// handle authorization in the API requests. -type Middleware struct { - plugins []Plugin -} - -// NewMiddleware creates a new Middleware -// with a slice of plugins. -func NewMiddleware(p []Plugin) Middleware { - return Middleware{ - plugins: p, - } -} - -// WrapHandler returns a new handler function wrapping the previous one in the request chain. -func (m Middleware) WrapHandler(handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error) func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - - user := "" - userAuthNMethod := "" - - // Default authorization using existing TLS connection credentials - // FIXME: Non trivial authorization mechanisms (such as advanced certificate validations, kerberos support - // and ldap) will be extracted using AuthN feature, which is tracked under: - // https://github.com/docker/docker/pull/20883 - if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { - user = r.TLS.PeerCertificates[0].Subject.CommonName - userAuthNMethod = "TLS" - } - - authCtx := NewCtx(m.plugins, user, userAuthNMethod, r.Method, r.RequestURI) - - if err := authCtx.AuthZRequest(w, r); err != nil { - logrus.Errorf("AuthZRequest for %s %s returned error: %s", r.Method, r.RequestURI, err) - return err - } - - rw := NewResponseModifier(w) - - if err := handler(ctx, rw, r, vars); err != nil { - logrus.Errorf("Handler for %s %s returned error: %s", r.Method, r.RequestURI, err) - return err - } - - if err := authCtx.AuthZResponse(rw, r); err != nil { - logrus.Errorf("AuthZResponse for %s %s returned error: %s", r.Method, r.RequestURI, err) - return err - } - return nil - } -} diff --git a/authorization/plugin.go b/authorization/plugin.go deleted file mode 100644 index fc5c7ef..0000000 --- a/authorization/plugin.go +++ /dev/null @@ -1,92 +0,0 @@ -package authorization - -import ( - "sync" - - "github.com/docker/docker/pkg/plugins" -) - -// Plugin allows third party plugins to authorize requests and responses -// in the context of docker API -type Plugin interface { - // Name returns the registered plugin name - Name() string - - // AuthZRequest authorizes the request from the client to the daemon - AuthZRequest(*Request) (*Response, error) - - // AuthZResponse authorizes the response from the daemon to the client - AuthZResponse(*Request) (*Response, error) -} - -// NewPlugins constructs and initializes the authorization plugins based on plugin names -func NewPlugins(names []string) []Plugin { - plugins := []Plugin{} - pluginsMap := make(map[string]struct{}) - for _, name := range names { - if _, ok := pluginsMap[name]; ok { - continue - } - pluginsMap[name] = struct{}{} - plugins = append(plugins, newAuthorizationPlugin(name)) - } - return plugins -} - -// authorizationPlugin is an internal adapter to docker plugin system -type authorizationPlugin struct { - plugin *plugins.Client - name string - once sync.Once -} - -func newAuthorizationPlugin(name string) Plugin { - return &authorizationPlugin{name: name} -} - -func (a *authorizationPlugin) Name() string { - return a.name -} - -func (a *authorizationPlugin) AuthZRequest(authReq *Request) (*Response, error) { - if err := a.initPlugin(); err != nil { - return nil, err - } - - authRes := &Response{} - if err := a.plugin.Call(AuthZApiRequest, authReq, authRes); err != nil { - return nil, err - } - - return authRes, nil -} - -func (a *authorizationPlugin) AuthZResponse(authReq *Request) (*Response, error) { - if err := a.initPlugin(); err != nil { - return nil, err - } - - authRes := &Response{} - if err := a.plugin.Call(AuthZApiResponse, authReq, authRes); err != nil { - return nil, err - } - - return authRes, nil -} - -// initPlugin initializes the authorization plugin if needed -func (a *authorizationPlugin) initPlugin() error { - // Lazy loading of plugins - var err error - a.once.Do(func() { - if a.plugin == nil { - plugin, e := plugins.Get(a.name, AuthZApiImplements) - if e != nil { - err = e - return - } - a.plugin = plugin.Client() - } - }) - return err -} diff --git a/authorization/response.go b/authorization/response.go deleted file mode 100644 index 9eec82d..0000000 --- a/authorization/response.go +++ /dev/null @@ -1,201 +0,0 @@ -package authorization - -import ( - "bufio" - "bytes" - "encoding/json" - "fmt" - "net" - "net/http" - - "github.com/Sirupsen/logrus" -) - -// ResponseModifier allows authorization plugins to read and modify the content of the http.response -type ResponseModifier interface { - http.ResponseWriter - http.Flusher - http.CloseNotifier - - // RawBody returns the current http content - RawBody() []byte - - // RawHeaders returns the current content of the http headers - RawHeaders() ([]byte, error) - - // StatusCode returns the current status code - StatusCode() int - - // OverrideBody replaces the body of the HTTP reply - OverrideBody(b []byte) - - // OverrideHeader replaces the headers of the HTTP reply - OverrideHeader(b []byte) error - - // OverrideStatusCode replaces the status code of the HTTP reply - OverrideStatusCode(statusCode int) - - // Flush flushes all data to the HTTP response - FlushAll() error - - // Hijacked indicates the response has been hijacked by the Docker daemon - Hijacked() bool -} - -// NewResponseModifier creates a wrapper to an http.ResponseWriter to allow inspecting and modifying the content -func NewResponseModifier(rw http.ResponseWriter) ResponseModifier { - return &responseModifier{rw: rw, header: make(http.Header)} -} - -// responseModifier is used as an adapter to http.ResponseWriter in order to manipulate and explore -// the http request/response from docker daemon -type responseModifier struct { - // The original response writer - rw http.ResponseWriter - // body holds the response body - body []byte - // header holds the response header - header http.Header - // statusCode holds the response status code - statusCode int - // hijacked indicates the request has been hijacked - hijacked bool -} - -func (rm *responseModifier) Hijacked() bool { - return rm.hijacked -} - -// WriterHeader stores the http status code -func (rm *responseModifier) WriteHeader(s int) { - - // Use original request if hijacked - if rm.hijacked { - rm.rw.WriteHeader(s) - return - } - - rm.statusCode = s -} - -// Header returns the internal http header -func (rm *responseModifier) Header() http.Header { - - // Use original header if hijacked - if rm.hijacked { - return rm.rw.Header() - } - - return rm.header -} - -// StatusCode returns the http status code -func (rm *responseModifier) StatusCode() int { - return rm.statusCode -} - -// OverrideBody replaces the body of the HTTP response -func (rm *responseModifier) OverrideBody(b []byte) { - rm.body = b -} - -// OverrideStatusCode replaces the status code of the HTTP response -func (rm *responseModifier) OverrideStatusCode(statusCode int) { - rm.statusCode = statusCode -} - -// OverrideHeader replaces the headers of the HTTP response -func (rm *responseModifier) OverrideHeader(b []byte) error { - header := http.Header{} - if err := json.Unmarshal(b, &header); err != nil { - return err - } - rm.header = header - return nil -} - -// Write stores the byte array inside content -func (rm *responseModifier) Write(b []byte) (int, error) { - - if rm.hijacked { - return rm.rw.Write(b) - } - - rm.body = append(rm.body, b...) - return len(b), nil -} - -// Body returns the response body -func (rm *responseModifier) RawBody() []byte { - return rm.body -} - -func (rm *responseModifier) RawHeaders() ([]byte, error) { - var b bytes.Buffer - if err := rm.header.Write(&b); err != nil { - return nil, err - } - return b.Bytes(), nil -} - -// Hijack returns the internal connection of the wrapped http.ResponseWriter -func (rm *responseModifier) Hijack() (net.Conn, *bufio.ReadWriter, error) { - - rm.hijacked = true - rm.FlushAll() - - hijacker, ok := rm.rw.(http.Hijacker) - if !ok { - return nil, nil, fmt.Errorf("Internal response writer doesn't support the Hijacker interface") - } - return hijacker.Hijack() -} - -// CloseNotify uses the internal close notify API of the wrapped http.ResponseWriter -func (rm *responseModifier) CloseNotify() <-chan bool { - closeNotifier, ok := rm.rw.(http.CloseNotifier) - if !ok { - logrus.Error("Internal response writer doesn't support the CloseNotifier interface") - return nil - } - return closeNotifier.CloseNotify() -} - -// Flush uses the internal flush API of the wrapped http.ResponseWriter -func (rm *responseModifier) Flush() { - flusher, ok := rm.rw.(http.Flusher) - if !ok { - logrus.Error("Internal response writer doesn't support the Flusher interface") - return - } - - rm.FlushAll() - flusher.Flush() -} - -// FlushAll flushes all data to the HTTP response -func (rm *responseModifier) FlushAll() error { - // Copy the status code - if rm.statusCode > 0 { - rm.rw.WriteHeader(rm.statusCode) - } - - // Copy the header - for k, vv := range rm.header { - for _, v := range vv { - rm.rw.Header().Add(k, v) - } - } - - var err error - if len(rm.body) > 0 { - // Write body - _, err = rm.rw.Write(rm.body) - } - - // Clean previous data - rm.body = nil - rm.statusCode = 0 - rm.header = http.Header{} - return err -} diff --git a/broadcaster/unbuffered.go b/broadcaster/unbuffered.go deleted file mode 100644 index 784d65d..0000000 --- a/broadcaster/unbuffered.go +++ /dev/null @@ -1,49 +0,0 @@ -package broadcaster - -import ( - "io" - "sync" -) - -// Unbuffered accumulates multiple io.WriteCloser by stream. -type Unbuffered struct { - mu sync.Mutex - writers []io.WriteCloser -} - -// Add adds new io.WriteCloser. -func (w *Unbuffered) Add(writer io.WriteCloser) { - w.mu.Lock() - w.writers = append(w.writers, writer) - w.mu.Unlock() -} - -// Write writes bytes to all writers. Failed writers will be evicted during -// this call. -func (w *Unbuffered) Write(p []byte) (n int, err error) { - w.mu.Lock() - var evict []int - for i, sw := range w.writers { - if n, err := sw.Write(p); err != nil || n != len(p) { - // On error, evict the writer - evict = append(evict, i) - } - } - for n, i := range evict { - w.writers = append(w.writers[:i-n], w.writers[i-n+1:]...) - } - w.mu.Unlock() - return len(p), nil -} - -// Clean closes and removes all writers. Last non-eol-terminated part of data -// will be saved. -func (w *Unbuffered) Clean() error { - w.mu.Lock() - for _, sw := range w.writers { - sw.Close() - } - w.writers = nil - w.mu.Unlock() - return nil -} diff --git a/broadcaster/unbuffered_test.go b/broadcaster/unbuffered_test.go deleted file mode 100644 index 9f8e72b..0000000 --- a/broadcaster/unbuffered_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package broadcaster - -import ( - "bytes" - "errors" - "strings" - - "testing" -) - -type dummyWriter struct { - buffer bytes.Buffer - failOnWrite bool -} - -func (dw *dummyWriter) Write(p []byte) (n int, err error) { - if dw.failOnWrite { - return 0, errors.New("Fake fail") - } - return dw.buffer.Write(p) -} - -func (dw *dummyWriter) String() string { - return dw.buffer.String() -} - -func (dw *dummyWriter) Close() error { - return nil -} - -func TestUnbuffered(t *testing.T) { - writer := new(Unbuffered) - - // Test 1: Both bufferA and bufferB should contain "foo" - bufferA := &dummyWriter{} - writer.Add(bufferA) - bufferB := &dummyWriter{} - writer.Add(bufferB) - writer.Write([]byte("foo")) - - if bufferA.String() != "foo" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - - if bufferB.String() != "foo" { - t.Errorf("Buffer contains %v", bufferB.String()) - } - - // Test2: bufferA and bufferB should contain "foobar", - // while bufferC should only contain "bar" - bufferC := &dummyWriter{} - writer.Add(bufferC) - writer.Write([]byte("bar")) - - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - - if bufferB.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferB.String()) - } - - if bufferC.String() != "bar" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - - // Test3: Test eviction on failure - bufferA.failOnWrite = true - writer.Write([]byte("fail")) - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - if bufferC.String() != "barfail" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - // Even though we reset the flag, no more writes should go in there - bufferA.failOnWrite = false - writer.Write([]byte("test")) - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - if bufferC.String() != "barfailtest" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - - // Test4: Test eviction on multiple simultaneous failures - bufferB.failOnWrite = true - bufferC.failOnWrite = true - bufferD := &dummyWriter{} - writer.Add(bufferD) - writer.Write([]byte("yo")) - writer.Write([]byte("ink")) - if strings.Contains(bufferB.String(), "yoink") { - t.Errorf("bufferB received write. contents: %q", bufferB) - } - if strings.Contains(bufferC.String(), "yoink") { - t.Errorf("bufferC received write. contents: %q", bufferC) - } - if g, w := bufferD.String(), "yoink"; g != w { - t.Errorf("bufferD = %q, want %q", g, w) - } - - writer.Clean() -} - -type devNullCloser int - -func (d devNullCloser) Close() error { - return nil -} - -func (d devNullCloser) Write(buf []byte) (int, error) { - return len(buf), nil -} - -// This test checks for races. It is only useful when run with the race detector. -func TestRaceUnbuffered(t *testing.T) { - writer := new(Unbuffered) - c := make(chan bool) - go func() { - writer.Add(devNullCloser(0)) - c <- true - }() - writer.Write([]byte("hello")) - <-c -} - -func BenchmarkUnbuffered(b *testing.B) { - writer := new(Unbuffered) - setUpWriter := func() { - for i := 0; i < 100; i++ { - writer.Add(devNullCloser(0)) - writer.Add(devNullCloser(0)) - writer.Add(devNullCloser(0)) - } - } - testLine := "Line that thinks that it is log line from docker" - var buf bytes.Buffer - for i := 0; i < 100; i++ { - buf.Write([]byte(testLine + "\n")) - } - // line without eol - buf.Write([]byte(testLine)) - testText := buf.Bytes() - b.SetBytes(int64(5 * len(testText))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - setUpWriter() - b.StartTimer() - - for j := 0; j < 5; j++ { - if _, err := writer.Write(testText); err != nil { - b.Fatal(err) - } - } - - b.StopTimer() - writer.Clean() - b.StartTimer() - } -} diff --git a/discovery/README.md b/discovery/README.md deleted file mode 100644 index 39777c2..0000000 --- a/discovery/README.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -page_title: Docker discovery -page_description: discovery -page_keywords: docker, clustering, discovery ---- - -# Discovery - -Docker comes with multiple Discovery backends. - -## Backends - -### Using etcd - -Point your Docker Engine instances to a common etcd instance. You can specify -the address Docker uses to advertise the node using the `--cluster-advertise` -flag. - -```bash -$ docker daemon -H= --cluster-advertise= --cluster-store etcd://,/ -``` - -### Using consul - -Point your Docker Engine instances to a common Consul instance. You can specify -the address Docker uses to advertise the node using the `--cluster-advertise` -flag. - -```bash -$ docker daemon -H= --cluster-advertise= --cluster-store consul:/// -``` - -### Using zookeeper - -Point your Docker Engine instances to a common Zookeeper instance. You can specify -the address Docker uses to advertise the node using the `--cluster-advertise` -flag. - -```bash -$ docker daemon -H= --cluster-advertise= --cluster-store zk://,/ -``` diff --git a/discovery/backends.go b/discovery/backends.go deleted file mode 100644 index 65364c9..0000000 --- a/discovery/backends.go +++ /dev/null @@ -1,107 +0,0 @@ -package discovery - -import ( - "fmt" - "net" - "strings" - "time" - - log "github.com/Sirupsen/logrus" -) - -var ( - // Backends is a global map of discovery backends indexed by their - // associated scheme. - backends = make(map[string]Backend) -) - -// Register makes a discovery backend available by the provided scheme. -// If Register is called twice with the same scheme an error is returned. -func Register(scheme string, d Backend) error { - if _, exists := backends[scheme]; exists { - return fmt.Errorf("scheme already registered %s", scheme) - } - log.WithField("name", scheme).Debug("Registering discovery service") - backends[scheme] = d - return nil -} - -func parse(rawurl string) (string, string) { - parts := strings.SplitN(rawurl, "://", 2) - - // nodes:port,node2:port => nodes://node1:port,node2:port - if len(parts) == 1 { - return "nodes", parts[0] - } - return parts[0], parts[1] -} - -// ParseAdvertise parses the --cluster-advertise daemon config which accepts -// : or : -func ParseAdvertise(advertise string) (string, error) { - var ( - iface *net.Interface - addrs []net.Addr - err error - ) - - addr, port, err := net.SplitHostPort(advertise) - - if err != nil { - return "", fmt.Errorf("invalid --cluster-advertise configuration: %s: %v", advertise, err) - } - - ip := net.ParseIP(addr) - // If it is a valid ip-address, use it as is - if ip != nil { - return advertise, nil - } - - // If advertise is a valid interface name, get the valid ipv4 address and use it to advertise - ifaceName := addr - iface, err = net.InterfaceByName(ifaceName) - if err != nil { - return "", fmt.Errorf("invalid cluster advertise IP address or interface name (%s) : %v", advertise, err) - } - - addrs, err = iface.Addrs() - if err != nil { - return "", fmt.Errorf("unable to get advertise IP address from interface (%s) : %v", advertise, err) - } - - if addrs == nil || len(addrs) == 0 { - return "", fmt.Errorf("no available advertise IP address in interface (%s)", advertise) - } - - addr = "" - for _, a := range addrs { - ip, _, err := net.ParseCIDR(a.String()) - if err != nil { - return "", fmt.Errorf("error deriving advertise ip-address in interface (%s) : %v", advertise, err) - } - if ip.To4() == nil || ip.IsLoopback() { - continue - } - addr = ip.String() - break - } - if addr == "" { - return "", fmt.Errorf("couldnt find a valid ip-address in interface %s", advertise) - } - - addr = net.JoinHostPort(addr, port) - return addr, nil -} - -// New returns a new Discovery given a URL, heartbeat and ttl settings. -// Returns an error if the URL scheme is not supported. -func New(rawurl string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) (Backend, error) { - scheme, uri := parse(rawurl) - if backend, exists := backends[scheme]; exists { - log.WithFields(log.Fields{"name": scheme, "uri": uri}).Debug("Initializing discovery service") - err := backend.Initialize(uri, heartbeat, ttl, clusterOpts) - return backend, err - } - - return nil, ErrNotSupported -} diff --git a/discovery/discovery.go b/discovery/discovery.go deleted file mode 100644 index ca7f587..0000000 --- a/discovery/discovery.go +++ /dev/null @@ -1,35 +0,0 @@ -package discovery - -import ( - "errors" - "time" -) - -var ( - // ErrNotSupported is returned when a discovery service is not supported. - ErrNotSupported = errors.New("discovery service not supported") - - // ErrNotImplemented is returned when discovery feature is not implemented - // by discovery backend. - ErrNotImplemented = errors.New("not implemented in this discovery service") -) - -// Watcher provides watching over a cluster for nodes joining and leaving. -type Watcher interface { - // Watch the discovery for entry changes. - // Returns a channel that will receive changes or an error. - // Providing a non-nil stopCh can be used to stop watching. - Watch(stopCh <-chan struct{}) (<-chan Entries, <-chan error) -} - -// Backend is implemented by discovery backends which manage cluster entries. -type Backend interface { - // Watcher must be provided by every backend. - Watcher - - // Initialize the discovery with URIs, a heartbeat, a ttl and optional settings. - Initialize(string, time.Duration, time.Duration, map[string]string) error - - // Register to the discovery. - Register(string) error -} diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go deleted file mode 100644 index 6084f3e..0000000 --- a/discovery/discovery_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package discovery - -import ( - "testing" - - "github.com/go-check/check" -) - -// Hook up gocheck into the "go test" runner. -func Test(t *testing.T) { check.TestingT(t) } - -type DiscoverySuite struct{} - -var _ = check.Suite(&DiscoverySuite{}) - -func (s *DiscoverySuite) TestNewEntry(c *check.C) { - entry, err := NewEntry("127.0.0.1:2375") - c.Assert(err, check.IsNil) - c.Assert(entry.Equals(&Entry{Host: "127.0.0.1", Port: "2375"}), check.Equals, true) - c.Assert(entry.String(), check.Equals, "127.0.0.1:2375") - - entry, err = NewEntry("[2001:db8:0:f101::2]:2375") - c.Assert(err, check.IsNil) - c.Assert(entry.Equals(&Entry{Host: "2001:db8:0:f101::2", Port: "2375"}), check.Equals, true) - c.Assert(entry.String(), check.Equals, "[2001:db8:0:f101::2]:2375") - - _, err = NewEntry("127.0.0.1") - c.Assert(err, check.NotNil) -} - -func (s *DiscoverySuite) TestParse(c *check.C) { - scheme, uri := parse("127.0.0.1:2375") - c.Assert(scheme, check.Equals, "nodes") - c.Assert(uri, check.Equals, "127.0.0.1:2375") - - scheme, uri = parse("localhost:2375") - c.Assert(scheme, check.Equals, "nodes") - c.Assert(uri, check.Equals, "localhost:2375") - - scheme, uri = parse("scheme://127.0.0.1:2375") - c.Assert(scheme, check.Equals, "scheme") - c.Assert(uri, check.Equals, "127.0.0.1:2375") - - scheme, uri = parse("scheme://localhost:2375") - c.Assert(scheme, check.Equals, "scheme") - c.Assert(uri, check.Equals, "localhost:2375") - - scheme, uri = parse("") - c.Assert(scheme, check.Equals, "nodes") - c.Assert(uri, check.Equals, "") -} - -func (s *DiscoverySuite) TestCreateEntries(c *check.C) { - entries, err := CreateEntries(nil) - c.Assert(entries, check.DeepEquals, Entries{}) - c.Assert(err, check.IsNil) - - entries, err = CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", "[2001:db8:0:f101::2]:2375", ""}) - c.Assert(err, check.IsNil) - expected := Entries{ - &Entry{Host: "127.0.0.1", Port: "2375"}, - &Entry{Host: "127.0.0.2", Port: "2375"}, - &Entry{Host: "2001:db8:0:f101::2", Port: "2375"}, - } - c.Assert(entries.Equals(expected), check.Equals, true) - - _, err = CreateEntries([]string{"127.0.0.1", "127.0.0.2"}) - c.Assert(err, check.NotNil) -} - -func (s *DiscoverySuite) TestContainsEntry(c *check.C) { - entries, err := CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", ""}) - c.Assert(err, check.IsNil) - c.Assert(entries.Contains(&Entry{Host: "127.0.0.1", Port: "2375"}), check.Equals, true) - c.Assert(entries.Contains(&Entry{Host: "127.0.0.3", Port: "2375"}), check.Equals, false) -} - -func (s *DiscoverySuite) TestEntriesEquality(c *check.C) { - entries := Entries{ - &Entry{Host: "127.0.0.1", Port: "2375"}, - &Entry{Host: "127.0.0.2", Port: "2375"}, - } - - // Same - c.Assert(entries.Equals(Entries{ - &Entry{Host: "127.0.0.1", Port: "2375"}, - &Entry{Host: "127.0.0.2", Port: "2375"}, - }), check. - Equals, true) - - // Different size - c.Assert(entries.Equals(Entries{ - &Entry{Host: "127.0.0.1", Port: "2375"}, - &Entry{Host: "127.0.0.2", Port: "2375"}, - &Entry{Host: "127.0.0.3", Port: "2375"}, - }), check. - Equals, false) - - // Different content - c.Assert(entries.Equals(Entries{ - &Entry{Host: "127.0.0.1", Port: "2375"}, - &Entry{Host: "127.0.0.42", Port: "2375"}, - }), check. - Equals, false) - -} - -func (s *DiscoverySuite) TestEntriesDiff(c *check.C) { - entry1 := &Entry{Host: "1.1.1.1", Port: "1111"} - entry2 := &Entry{Host: "2.2.2.2", Port: "2222"} - entry3 := &Entry{Host: "3.3.3.3", Port: "3333"} - entries := Entries{entry1, entry2} - - // No diff - added, removed := entries.Diff(Entries{entry2, entry1}) - c.Assert(added, check.HasLen, 0) - c.Assert(removed, check.HasLen, 0) - - // Add - added, removed = entries.Diff(Entries{entry2, entry3, entry1}) - c.Assert(added, check.HasLen, 1) - c.Assert(added.Contains(entry3), check.Equals, true) - c.Assert(removed, check.HasLen, 0) - - // Remove - added, removed = entries.Diff(Entries{entry2}) - c.Assert(added, check.HasLen, 0) - c.Assert(removed, check.HasLen, 1) - c.Assert(removed.Contains(entry1), check.Equals, true) - - // Add and remove - added, removed = entries.Diff(Entries{entry1, entry3}) - c.Assert(added, check.HasLen, 1) - c.Assert(added.Contains(entry3), check.Equals, true) - c.Assert(removed, check.HasLen, 1) - c.Assert(removed.Contains(entry2), check.Equals, true) -} diff --git a/discovery/entry.go b/discovery/entry.go deleted file mode 100644 index ce23bbf..0000000 --- a/discovery/entry.go +++ /dev/null @@ -1,94 +0,0 @@ -package discovery - -import "net" - -// NewEntry creates a new entry. -func NewEntry(url string) (*Entry, error) { - host, port, err := net.SplitHostPort(url) - if err != nil { - return nil, err - } - return &Entry{host, port}, nil -} - -// An Entry represents a host. -type Entry struct { - Host string - Port string -} - -// Equals returns true if cmp contains the same data. -func (e *Entry) Equals(cmp *Entry) bool { - return e.Host == cmp.Host && e.Port == cmp.Port -} - -// String returns the string form of an entry. -func (e *Entry) String() string { - return net.JoinHostPort(e.Host, e.Port) -} - -// Entries is a list of *Entry with some helpers. -type Entries []*Entry - -// Equals returns true if cmp contains the same data. -func (e Entries) Equals(cmp Entries) bool { - // Check if the file has really changed. - if len(e) != len(cmp) { - return false - } - for i := range e { - if !e[i].Equals(cmp[i]) { - return false - } - } - return true -} - -// Contains returns true if the Entries contain a given Entry. -func (e Entries) Contains(entry *Entry) bool { - for _, curr := range e { - if curr.Equals(entry) { - return true - } - } - return false -} - -// Diff compares two entries and returns the added and removed entries. -func (e Entries) Diff(cmp Entries) (Entries, Entries) { - added := Entries{} - for _, entry := range cmp { - if !e.Contains(entry) { - added = append(added, entry) - } - } - - removed := Entries{} - for _, entry := range e { - if !cmp.Contains(entry) { - removed = append(removed, entry) - } - } - - return added, removed -} - -// CreateEntries returns an array of entries based on the given addresses. -func CreateEntries(addrs []string) (Entries, error) { - entries := Entries{} - if addrs == nil { - return entries, nil - } - - for _, addr := range addrs { - if len(addr) == 0 { - continue - } - entry, err := NewEntry(addr) - if err != nil { - return nil, err - } - entries = append(entries, entry) - } - return entries, nil -} diff --git a/discovery/file/file.go b/discovery/file/file.go deleted file mode 100644 index b4f870b..0000000 --- a/discovery/file/file.go +++ /dev/null @@ -1,109 +0,0 @@ -package file - -import ( - "fmt" - "io/ioutil" - "strings" - "time" - - "github.com/docker/docker/pkg/discovery" -) - -// Discovery is exported -type Discovery struct { - heartbeat time.Duration - path string -} - -func init() { - Init() -} - -// Init is exported -func Init() { - discovery.Register("file", &Discovery{}) -} - -// Initialize is exported -func (s *Discovery) Initialize(path string, heartbeat time.Duration, ttl time.Duration, _ map[string]string) error { - s.path = path - s.heartbeat = heartbeat - return nil -} - -func parseFileContent(content []byte) []string { - var result []string - for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") { - line = strings.TrimSpace(line) - // Ignoring line starts with # - if strings.HasPrefix(line, "#") { - continue - } - // Inlined # comment also ignored. - if strings.Contains(line, "#") { - line = line[0:strings.Index(line, "#")] - // Trim additional spaces caused by above stripping. - line = strings.TrimSpace(line) - } - for _, ip := range discovery.Generate(line) { - result = append(result, ip) - } - } - return result -} - -func (s *Discovery) fetch() (discovery.Entries, error) { - fileContent, err := ioutil.ReadFile(s.path) - if err != nil { - return nil, fmt.Errorf("failed to read '%s': %v", s.path, err) - } - return discovery.CreateEntries(parseFileContent(fileContent)) -} - -// Watch is exported -func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) { - ch := make(chan discovery.Entries) - errCh := make(chan error) - ticker := time.NewTicker(s.heartbeat) - - go func() { - defer close(errCh) - defer close(ch) - - // Send the initial entries if available. - currentEntries, err := s.fetch() - if err != nil { - errCh <- err - } else { - ch <- currentEntries - } - - // Periodically send updates. - for { - select { - case <-ticker.C: - newEntries, err := s.fetch() - if err != nil { - errCh <- err - continue - } - - // Check if the file has really changed. - if !newEntries.Equals(currentEntries) { - ch <- newEntries - } - currentEntries = newEntries - case <-stopCh: - ticker.Stop() - return - } - } - }() - - return ch, errCh -} - -// Register is exported -func (s *Discovery) Register(addr string) error { - return discovery.ErrNotImplemented -} diff --git a/discovery/file/file_test.go b/discovery/file/file_test.go deleted file mode 100644 index 667f00b..0000000 --- a/discovery/file/file_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package file - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/docker/docker/pkg/discovery" - - "github.com/go-check/check" -) - -// Hook up gocheck into the "go test" runner. -func Test(t *testing.T) { check.TestingT(t) } - -type DiscoverySuite struct{} - -var _ = check.Suite(&DiscoverySuite{}) - -func (s *DiscoverySuite) TestInitialize(c *check.C) { - d := &Discovery{} - d.Initialize("/path/to/file", 1000, 0, nil) - c.Assert(d.path, check.Equals, "/path/to/file") -} - -func (s *DiscoverySuite) TestNew(c *check.C) { - d, err := discovery.New("file:///path/to/file", 0, 0, nil) - c.Assert(err, check.IsNil) - c.Assert(d.(*Discovery).path, check.Equals, "/path/to/file") -} - -func (s *DiscoverySuite) TestContent(c *check.C) { - data := ` -1.1.1.[1:2]:1111 -2.2.2.[2:4]:2222 -` - ips := parseFileContent([]byte(data)) - c.Assert(ips, check.HasLen, 5) - c.Assert(ips[0], check.Equals, "1.1.1.1:1111") - c.Assert(ips[1], check.Equals, "1.1.1.2:1111") - c.Assert(ips[2], check.Equals, "2.2.2.2:2222") - c.Assert(ips[3], check.Equals, "2.2.2.3:2222") - c.Assert(ips[4], check.Equals, "2.2.2.4:2222") -} - -func (s *DiscoverySuite) TestRegister(c *check.C) { - discovery := &Discovery{path: "/path/to/file"} - c.Assert(discovery.Register("0.0.0.0"), check.NotNil) -} - -func (s *DiscoverySuite) TestParsingContentsWithComments(c *check.C) { - data := ` -### test ### -1.1.1.1:1111 # inline comment -# 2.2.2.2:2222 - ### empty line with comment - 3.3.3.3:3333 -### test ### -` - ips := parseFileContent([]byte(data)) - c.Assert(ips, check.HasLen, 2) - c.Assert("1.1.1.1:1111", check.Equals, ips[0]) - c.Assert("3.3.3.3:3333", check.Equals, ips[1]) -} - -func (s *DiscoverySuite) TestWatch(c *check.C) { - data := ` -1.1.1.1:1111 -2.2.2.2:2222 -` - expected := discovery.Entries{ - &discovery.Entry{Host: "1.1.1.1", Port: "1111"}, - &discovery.Entry{Host: "2.2.2.2", Port: "2222"}, - } - - // Create a temporary file and remove it. - tmp, err := ioutil.TempFile(os.TempDir(), "discovery-file-test") - c.Assert(err, check.IsNil) - c.Assert(tmp.Close(), check.IsNil) - c.Assert(os.Remove(tmp.Name()), check.IsNil) - - // Set up file discovery. - d := &Discovery{} - d.Initialize(tmp.Name(), 1000, 0, nil) - stopCh := make(chan struct{}) - ch, errCh := d.Watch(stopCh) - - // Make sure it fires errors since the file doesn't exist. - c.Assert(<-errCh, check.NotNil) - // We have to drain the error channel otherwise Watch will get stuck. - go func() { - for range errCh { - } - }() - - // Write the file and make sure we get the expected value back. - c.Assert(ioutil.WriteFile(tmp.Name(), []byte(data), 0600), check.IsNil) - c.Assert(<-ch, check.DeepEquals, expected) - - // Add a new entry and look it up. - expected = append(expected, &discovery.Entry{Host: "3.3.3.3", Port: "3333"}) - f, err := os.OpenFile(tmp.Name(), os.O_APPEND|os.O_WRONLY, 0600) - c.Assert(err, check.IsNil) - c.Assert(f, check.NotNil) - _, err = f.WriteString("\n3.3.3.3:3333\n") - c.Assert(err, check.IsNil) - f.Close() - c.Assert(<-ch, check.DeepEquals, expected) - - // Stop and make sure it closes all channels. - close(stopCh) - c.Assert(<-ch, check.IsNil) - c.Assert(<-errCh, check.IsNil) -} diff --git a/discovery/generator.go b/discovery/generator.go deleted file mode 100644 index d222982..0000000 --- a/discovery/generator.go +++ /dev/null @@ -1,35 +0,0 @@ -package discovery - -import ( - "fmt" - "regexp" - "strconv" -) - -// Generate takes care of IP generation -func Generate(pattern string) []string { - re, _ := regexp.Compile(`\[(.+):(.+)\]`) - submatch := re.FindStringSubmatch(pattern) - if submatch == nil { - return []string{pattern} - } - - from, err := strconv.Atoi(submatch[1]) - if err != nil { - return []string{pattern} - } - to, err := strconv.Atoi(submatch[2]) - if err != nil { - return []string{pattern} - } - - template := re.ReplaceAllString(pattern, "%d") - - var result []string - for val := from; val <= to; val++ { - entry := fmt.Sprintf(template, val) - result = append(result, entry) - } - - return result -} diff --git a/discovery/generator_test.go b/discovery/generator_test.go deleted file mode 100644 index 6281c46..0000000 --- a/discovery/generator_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package discovery - -import ( - "github.com/go-check/check" -) - -func (s *DiscoverySuite) TestGeneratorNotGenerate(c *check.C) { - ips := Generate("127.0.0.1") - c.Assert(len(ips), check.Equals, 1) - c.Assert(ips[0], check.Equals, "127.0.0.1") -} - -func (s *DiscoverySuite) TestGeneratorWithPortNotGenerate(c *check.C) { - ips := Generate("127.0.0.1:8080") - c.Assert(len(ips), check.Equals, 1) - c.Assert(ips[0], check.Equals, "127.0.0.1:8080") -} - -func (s *DiscoverySuite) TestGeneratorMatchFailedNotGenerate(c *check.C) { - ips := Generate("127.0.0.[1]") - c.Assert(len(ips), check.Equals, 1) - c.Assert(ips[0], check.Equals, "127.0.0.[1]") -} - -func (s *DiscoverySuite) TestGeneratorWithPort(c *check.C) { - ips := Generate("127.0.0.[1:11]:2375") - c.Assert(len(ips), check.Equals, 11) - c.Assert(ips[0], check.Equals, "127.0.0.1:2375") - c.Assert(ips[1], check.Equals, "127.0.0.2:2375") - c.Assert(ips[2], check.Equals, "127.0.0.3:2375") - c.Assert(ips[3], check.Equals, "127.0.0.4:2375") - c.Assert(ips[4], check.Equals, "127.0.0.5:2375") - c.Assert(ips[5], check.Equals, "127.0.0.6:2375") - c.Assert(ips[6], check.Equals, "127.0.0.7:2375") - c.Assert(ips[7], check.Equals, "127.0.0.8:2375") - c.Assert(ips[8], check.Equals, "127.0.0.9:2375") - c.Assert(ips[9], check.Equals, "127.0.0.10:2375") - c.Assert(ips[10], check.Equals, "127.0.0.11:2375") -} - -func (s *DiscoverySuite) TestGenerateWithMalformedInputAtRangeStart(c *check.C) { - malformedInput := "127.0.0.[x:11]:2375" - ips := Generate(malformedInput) - c.Assert(len(ips), check.Equals, 1) - c.Assert(ips[0], check.Equals, malformedInput) -} - -func (s *DiscoverySuite) TestGenerateWithMalformedInputAtRangeEnd(c *check.C) { - malformedInput := "127.0.0.[1:x]:2375" - ips := Generate(malformedInput) - c.Assert(len(ips), check.Equals, 1) - c.Assert(ips[0], check.Equals, malformedInput) -} diff --git a/discovery/kv/kv.go b/discovery/kv/kv.go deleted file mode 100644 index f371c0c..0000000 --- a/discovery/kv/kv.go +++ /dev/null @@ -1,192 +0,0 @@ -package kv - -import ( - "fmt" - "path" - "strings" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/discovery" - "github.com/docker/go-connections/tlsconfig" - "github.com/docker/libkv" - "github.com/docker/libkv/store" - "github.com/docker/libkv/store/consul" - "github.com/docker/libkv/store/etcd" - "github.com/docker/libkv/store/zookeeper" -) - -const ( - defaultDiscoveryPath = "docker/nodes" -) - -// Discovery is exported -type Discovery struct { - backend store.Backend - store store.Store - heartbeat time.Duration - ttl time.Duration - prefix string - path string -} - -func init() { - Init() -} - -// Init is exported -func Init() { - // Register to libkv - zookeeper.Register() - consul.Register() - etcd.Register() - - // Register to internal discovery service - discovery.Register("zk", &Discovery{backend: store.ZK}) - discovery.Register("consul", &Discovery{backend: store.CONSUL}) - discovery.Register("etcd", &Discovery{backend: store.ETCD}) -} - -// Initialize is exported -func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) error { - var ( - parts = strings.SplitN(uris, "/", 2) - addrs = strings.Split(parts[0], ",") - err error - ) - - // A custom prefix to the path can be optionally used. - if len(parts) == 2 { - s.prefix = parts[1] - } - - s.heartbeat = heartbeat - s.ttl = ttl - - // Use a custom path if specified in discovery options - dpath := defaultDiscoveryPath - if clusterOpts["kv.path"] != "" { - dpath = clusterOpts["kv.path"] - } - - s.path = path.Join(s.prefix, dpath) - - var config *store.Config - if clusterOpts["kv.cacertfile"] != "" && clusterOpts["kv.certfile"] != "" && clusterOpts["kv.keyfile"] != "" { - log.Info("Initializing discovery with TLS") - tlsConfig, err := tlsconfig.Client(tlsconfig.Options{ - CAFile: clusterOpts["kv.cacertfile"], - CertFile: clusterOpts["kv.certfile"], - KeyFile: clusterOpts["kv.keyfile"], - }) - if err != nil { - return err - } - config = &store.Config{ - // Set ClientTLS to trigger https (bug in libkv/etcd) - ClientTLS: &store.ClientTLSConfig{ - CACertFile: clusterOpts["kv.cacertfile"], - CertFile: clusterOpts["kv.certfile"], - KeyFile: clusterOpts["kv.keyfile"], - }, - // The actual TLS config that will be used - TLS: tlsConfig, - } - } else { - log.Info("Initializing discovery without TLS") - } - - // Creates a new store, will ignore options given - // if not supported by the chosen store - s.store, err = libkv.NewStore(s.backend, addrs, config) - return err -} - -// Watch the store until either there's a store error or we receive a stop request. -// Returns false if we shouldn't attempt watching the store anymore (stop request received). -func (s *Discovery) watchOnce(stopCh <-chan struct{}, watchCh <-chan []*store.KVPair, discoveryCh chan discovery.Entries, errCh chan error) bool { - for { - select { - case pairs := <-watchCh: - if pairs == nil { - return true - } - - log.WithField("discovery", s.backend).Debugf("Watch triggered with %d nodes", len(pairs)) - - // Convert `KVPair` into `discovery.Entry`. - addrs := make([]string, len(pairs)) - for _, pair := range pairs { - addrs = append(addrs, string(pair.Value)) - } - - entries, err := discovery.CreateEntries(addrs) - if err != nil { - errCh <- err - } else { - discoveryCh <- entries - } - case <-stopCh: - // We were requested to stop watching. - return false - } - } -} - -// Watch is exported -func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) { - ch := make(chan discovery.Entries) - errCh := make(chan error) - - go func() { - defer close(ch) - defer close(errCh) - - // Forever: Create a store watch, watch until we get an error and then try again. - // Will only stop if we receive a stopCh request. - for { - // Create the path to watch if it does not exist yet - exists, err := s.store.Exists(s.path) - if err != nil { - errCh <- err - } - if !exists { - if err := s.store.Put(s.path, []byte(""), &store.WriteOptions{IsDir: true}); err != nil { - errCh <- err - } - } - - // Set up a watch. - watchCh, err := s.store.WatchTree(s.path, stopCh) - if err != nil { - errCh <- err - } else { - if !s.watchOnce(stopCh, watchCh, ch, errCh) { - return - } - } - - // If we get here it means the store watch channel was closed. This - // is unexpected so let's retry later. - errCh <- fmt.Errorf("Unexpected watch error") - time.Sleep(s.heartbeat) - } - }() - return ch, errCh -} - -// Register is exported -func (s *Discovery) Register(addr string) error { - opts := &store.WriteOptions{TTL: s.ttl} - return s.store.Put(path.Join(s.path, addr), []byte(addr), opts) -} - -// Store returns the underlying store used by KV discovery. -func (s *Discovery) Store() store.Store { - return s.store -} - -// Prefix returns the store prefix -func (s *Discovery) Prefix() string { - return s.prefix -} diff --git a/discovery/kv/kv_test.go b/discovery/kv/kv_test.go deleted file mode 100644 index dab3939..0000000 --- a/discovery/kv/kv_test.go +++ /dev/null @@ -1,324 +0,0 @@ -package kv - -import ( - "errors" - "io/ioutil" - "os" - "path" - "testing" - "time" - - "github.com/docker/docker/pkg/discovery" - "github.com/docker/libkv" - "github.com/docker/libkv/store" - - "github.com/go-check/check" -) - -// Hook up gocheck into the "go test" runner. -func Test(t *testing.T) { check.TestingT(t) } - -type DiscoverySuite struct{} - -var _ = check.Suite(&DiscoverySuite{}) - -func (ds *DiscoverySuite) TestInitialize(c *check.C) { - storeMock := &FakeStore{ - Endpoints: []string{"127.0.0.1"}, - } - d := &Discovery{backend: store.CONSUL} - d.Initialize("127.0.0.1", 0, 0, nil) - d.store = storeMock - - s := d.store.(*FakeStore) - c.Assert(s.Endpoints, check.HasLen, 1) - c.Assert(s.Endpoints[0], check.Equals, "127.0.0.1") - c.Assert(d.path, check.Equals, defaultDiscoveryPath) - - storeMock = &FakeStore{ - Endpoints: []string{"127.0.0.1:1234"}, - } - d = &Discovery{backend: store.CONSUL} - d.Initialize("127.0.0.1:1234/path", 0, 0, nil) - d.store = storeMock - - s = d.store.(*FakeStore) - c.Assert(s.Endpoints, check.HasLen, 1) - c.Assert(s.Endpoints[0], check.Equals, "127.0.0.1:1234") - c.Assert(d.path, check.Equals, "path/"+defaultDiscoveryPath) - - storeMock = &FakeStore{ - Endpoints: []string{"127.0.0.1:1234", "127.0.0.2:1234", "127.0.0.3:1234"}, - } - d = &Discovery{backend: store.CONSUL} - d.Initialize("127.0.0.1:1234,127.0.0.2:1234,127.0.0.3:1234/path", 0, 0, nil) - d.store = storeMock - - s = d.store.(*FakeStore) - c.Assert(s.Endpoints, check.HasLen, 3) - c.Assert(s.Endpoints[0], check.Equals, "127.0.0.1:1234") - c.Assert(s.Endpoints[1], check.Equals, "127.0.0.2:1234") - c.Assert(s.Endpoints[2], check.Equals, "127.0.0.3:1234") - - c.Assert(d.path, check.Equals, "path/"+defaultDiscoveryPath) -} - -// Extremely limited mock store so we can test initialization -type Mock struct { - // Endpoints passed to InitializeMock - Endpoints []string - - // Options passed to InitializeMock - Options *store.Config -} - -func NewMock(endpoints []string, options *store.Config) (store.Store, error) { - s := &Mock{} - s.Endpoints = endpoints - s.Options = options - return s, nil -} -func (s *Mock) Put(key string, value []byte, opts *store.WriteOptions) error { - return errors.New("Put not supported") -} -func (s *Mock) Get(key string) (*store.KVPair, error) { - return nil, errors.New("Get not supported") -} -func (s *Mock) Delete(key string) error { - return errors.New("Delete not supported") -} - -// Exists mock -func (s *Mock) Exists(key string) (bool, error) { - return false, errors.New("Exists not supported") -} - -// Watch mock -func (s *Mock) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) { - return nil, errors.New("Watch not supported") -} - -// WatchTree mock -func (s *Mock) WatchTree(prefix string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) { - return nil, errors.New("WatchTree not supported") -} - -// NewLock mock -func (s *Mock) NewLock(key string, options *store.LockOptions) (store.Locker, error) { - return nil, errors.New("NewLock not supported") -} - -// List mock -func (s *Mock) List(prefix string) ([]*store.KVPair, error) { - return nil, errors.New("List not supported") -} - -// DeleteTree mock -func (s *Mock) DeleteTree(prefix string) error { - return errors.New("DeleteTree not supported") -} - -// AtomicPut mock -func (s *Mock) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) { - return false, nil, errors.New("AtomicPut not supported") -} - -// AtomicDelete mock -func (s *Mock) AtomicDelete(key string, previous *store.KVPair) (bool, error) { - return false, errors.New("AtomicDelete not supported") -} - -// Close mock -func (s *Mock) Close() { - return -} - -func (ds *DiscoverySuite) TestInitializeWithCerts(c *check.C) { - cert := `-----BEGIN CERTIFICATE----- -MIIDCDCCAfKgAwIBAgIICifG7YeiQOEwCwYJKoZIhvcNAQELMBIxEDAOBgNVBAMT -B1Rlc3QgQ0EwHhcNMTUxMDAxMjMwMDAwWhcNMjAwOTI5MjMwMDAwWjASMRAwDgYD -VQQDEwdUZXN0IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1wRC -O+flnLTK5ImjTurNRHwSejuqGbc4CAvpB0hS+z0QlSs4+zE9h80aC4hz+6caRpds -+J908Q+RvAittMHbpc7VjbZP72G6fiXk7yPPl6C10HhRSoSi3nY+B7F2E8cuz14q -V2e+ejhWhSrBb/keyXpcyjoW1BOAAJ2TIclRRkICSCZrpXUyXxAvzXfpFXo1RhSb -UywN11pfiCQzDUN7sPww9UzFHuAHZHoyfTr27XnJYVUerVYrCPq8vqfn//01qz55 -Xs0hvzGdlTFXhuabFtQnKFH5SNwo/fcznhB7rePOwHojxOpXTBepUCIJLbtNnWFT -V44t9gh5IqIWtoBReQIDAQABo2YwZDAOBgNVHQ8BAf8EBAMCAAYwEgYDVR0TAQH/ -BAgwBgEB/wIBAjAdBgNVHQ4EFgQUZKUI8IIjIww7X/6hvwggQK4bD24wHwYDVR0j -BBgwFoAUZKUI8IIjIww7X/6hvwggQK4bD24wCwYJKoZIhvcNAQELA4IBAQDES2cz -7sCQfDCxCIWH7X8kpi/JWExzUyQEJ0rBzN1m3/x8ySRxtXyGekimBqQwQdFqlwMI -xzAQKkh3ue8tNSzRbwqMSyH14N1KrSxYS9e9szJHfUasoTpQGPmDmGIoRJuq1h6M -ej5x1SCJ7GWCR6xEXKUIE9OftXm9TdFzWa7Ja3OHz/mXteii8VXDuZ5ACq6EE5bY -8sP4gcICfJ5fTrpTlk9FIqEWWQrCGa5wk95PGEj+GJpNogjXQ97wVoo/Y3p1brEn -t5zjN9PAq4H1fuCMdNNA+p1DHNwd+ELTxcMAnb2ajwHvV6lKPXutrTFc4umJToBX -FpTxDmJHEV4bzUzh ------END CERTIFICATE----- -` - key := `-----BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEA1wRCO+flnLTK5ImjTurNRHwSejuqGbc4CAvpB0hS+z0QlSs4 -+zE9h80aC4hz+6caRpds+J908Q+RvAittMHbpc7VjbZP72G6fiXk7yPPl6C10HhR -SoSi3nY+B7F2E8cuz14qV2e+ejhWhSrBb/keyXpcyjoW1BOAAJ2TIclRRkICSCZr -pXUyXxAvzXfpFXo1RhSbUywN11pfiCQzDUN7sPww9UzFHuAHZHoyfTr27XnJYVUe -rVYrCPq8vqfn//01qz55Xs0hvzGdlTFXhuabFtQnKFH5SNwo/fcznhB7rePOwHoj -xOpXTBepUCIJLbtNnWFTV44t9gh5IqIWtoBReQIDAQABAoIBAHSWipORGp/uKFXj -i/mut776x8ofsAxhnLBARQr93ID+i49W8H7EJGkOfaDjTICYC1dbpGrri61qk8sx -qX7p3v/5NzKwOIfEpirgwVIqSNYe/ncbxnhxkx6tXtUtFKmEx40JskvSpSYAhmmO -1XSx0E/PWaEN/nLgX/f1eWJIlxlQkk3QeqL+FGbCXI48DEtlJ9+MzMu4pAwZTpj5 -5qtXo5JJ0jRGfJVPAOznRsYqv864AhMdMIWguzk6EGnbaCWwPcfcn+h9a5LMdony -MDHfBS7bb5tkF3+AfnVY3IBMVx7YlsD9eAyajlgiKu4zLbwTRHjXgShy+4Oussz0 -ugNGnkECgYEA/hi+McrZC8C4gg6XqK8+9joD8tnyDZDz88BQB7CZqABUSwvjDqlP -L8hcwo/lzvjBNYGkqaFPUICGWKjeCtd8pPS2DCVXxDQX4aHF1vUur0uYNncJiV3N -XQz4Iemsa6wnKf6M67b5vMXICw7dw0HZCdIHD1hnhdtDz0uVpeevLZ8CgYEA2KCT -Y43lorjrbCgMqtlefkr3GJA9dey+hTzCiWEOOqn9RqGoEGUday0sKhiLofOgmN2B -LEukpKIey8s+Q/cb6lReajDVPDsMweX8i7hz3Wa4Ugp4Xa5BpHqu8qIAE2JUZ7bU -t88aQAYE58pUF+/Lq1QzAQdrjjzQBx6SrBxieecCgYEAvukoPZEC8mmiN1VvbTX+ -QFHmlZha3QaDxChB+QUe7bMRojEUL/fVnzkTOLuVFqSfxevaI/km9n0ac5KtAchV -xjp2bTnBb5EUQFqjopYktWA+xO07JRJtMfSEmjZPbbay1kKC7rdTfBm961EIHaRj -xZUf6M+rOE8964oGrdgdLlECgYEA046GQmx6fh7/82FtdZDRQp9tj3SWQUtSiQZc -qhO59Lq8mjUXz+MgBuJXxkiwXRpzlbaFB0Bca1fUoYw8o915SrDYf/Zu2OKGQ/qa -V81sgiVmDuEgycR7YOlbX6OsVUHrUlpwhY3hgfMe6UtkMvhBvHF/WhroBEIJm1pV -PXZ/CbMCgYEApNWVktFBjOaYfY6SNn4iSts1jgsQbbpglg3kT7PLKjCAhI6lNsbk -dyT7ut01PL6RaW4SeQWtrJIVQaM6vF3pprMKqlc5XihOGAmVqH7rQx9rtQB5TicL -BFrwkQE4HQtQBV60hYQUzzlSk44VFDz+jxIEtacRHaomDRh2FtOTz+I= ------END RSA PRIVATE KEY----- -` - certFile, err := ioutil.TempFile("", "cert") - c.Assert(err, check.IsNil) - defer os.Remove(certFile.Name()) - certFile.Write([]byte(cert)) - certFile.Close() - keyFile, err := ioutil.TempFile("", "key") - c.Assert(err, check.IsNil) - defer os.Remove(keyFile.Name()) - keyFile.Write([]byte(key)) - keyFile.Close() - - libkv.AddStore("mock", NewMock) - d := &Discovery{backend: "mock"} - err = d.Initialize("127.0.0.3:1234", 0, 0, map[string]string{ - "kv.cacertfile": certFile.Name(), - "kv.certfile": certFile.Name(), - "kv.keyfile": keyFile.Name(), - }) - c.Assert(err, check.IsNil) - s := d.store.(*Mock) - c.Assert(s.Options.TLS, check.NotNil) - c.Assert(s.Options.TLS.RootCAs, check.NotNil) - c.Assert(s.Options.TLS.Certificates, check.HasLen, 1) -} - -func (ds *DiscoverySuite) TestWatch(c *check.C) { - mockCh := make(chan []*store.KVPair) - - storeMock := &FakeStore{ - Endpoints: []string{"127.0.0.1:1234"}, - mockKVChan: mockCh, - } - - d := &Discovery{backend: store.CONSUL} - d.Initialize("127.0.0.1:1234/path", 0, 0, nil) - d.store = storeMock - - expected := discovery.Entries{ - &discovery.Entry{Host: "1.1.1.1", Port: "1111"}, - &discovery.Entry{Host: "2.2.2.2", Port: "2222"}, - } - kvs := []*store.KVPair{ - {Key: path.Join("path", defaultDiscoveryPath, "1.1.1.1"), Value: []byte("1.1.1.1:1111")}, - {Key: path.Join("path", defaultDiscoveryPath, "2.2.2.2"), Value: []byte("2.2.2.2:2222")}, - } - - stopCh := make(chan struct{}) - ch, errCh := d.Watch(stopCh) - - // It should fire an error since the first WatchTree call failed. - c.Assert(<-errCh, check.ErrorMatches, "test error") - // We have to drain the error channel otherwise Watch will get stuck. - go func() { - for range errCh { - } - }() - - // Push the entries into the store channel and make sure discovery emits. - mockCh <- kvs - c.Assert(<-ch, check.DeepEquals, expected) - - // Add a new entry. - expected = append(expected, &discovery.Entry{Host: "3.3.3.3", Port: "3333"}) - kvs = append(kvs, &store.KVPair{Key: path.Join("path", defaultDiscoveryPath, "3.3.3.3"), Value: []byte("3.3.3.3:3333")}) - mockCh <- kvs - c.Assert(<-ch, check.DeepEquals, expected) - - close(mockCh) - // Give it enough time to call WatchTree. - time.Sleep(3 * time.Second) - - // Stop and make sure it closes all channels. - close(stopCh) - c.Assert(<-ch, check.IsNil) - c.Assert(<-errCh, check.IsNil) -} - -// FakeStore implements store.Store methods. It mocks all store -// function in a simple, naive way. -type FakeStore struct { - Endpoints []string - Options *store.Config - mockKVChan <-chan []*store.KVPair - - watchTreeCallCount int -} - -func (s *FakeStore) Put(key string, value []byte, options *store.WriteOptions) error { - return nil -} - -func (s *FakeStore) Get(key string) (*store.KVPair, error) { - return nil, nil -} - -func (s *FakeStore) Delete(key string) error { - return nil -} - -func (s *FakeStore) Exists(key string) (bool, error) { - return true, nil -} - -func (s *FakeStore) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) { - return nil, nil -} - -// WatchTree will fail the first time, and return the mockKVchan afterwards. -// This is the behavior we need for testing.. If we need 'moar', should update this. -func (s *FakeStore) WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) { - if s.watchTreeCallCount == 0 { - s.watchTreeCallCount = 1 - return nil, errors.New("test error") - } - // First calls error - return s.mockKVChan, nil -} - -func (s *FakeStore) NewLock(key string, options *store.LockOptions) (store.Locker, error) { - return nil, nil -} - -func (s *FakeStore) List(directory string) ([]*store.KVPair, error) { - return []*store.KVPair{}, nil -} - -func (s *FakeStore) DeleteTree(directory string) error { - return nil -} - -func (s *FakeStore) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) { - return true, nil, nil -} - -func (s *FakeStore) AtomicDelete(key string, previous *store.KVPair) (bool, error) { - return true, nil -} - -func (s *FakeStore) Close() { -} diff --git a/discovery/memory/memory.go b/discovery/memory/memory.go deleted file mode 100644 index ba8b1f5..0000000 --- a/discovery/memory/memory.go +++ /dev/null @@ -1,93 +0,0 @@ -package memory - -import ( - "sync" - "time" - - "github.com/docker/docker/pkg/discovery" -) - -// Discovery implements a discovery backend that keeps -// data in memory. -type Discovery struct { - heartbeat time.Duration - values []string - mu sync.Mutex -} - -func init() { - Init() -} - -// Init registers the memory backend on demand. -func Init() { - discovery.Register("memory", &Discovery{}) -} - -// Initialize sets the heartbeat for the memory backend. -func (s *Discovery) Initialize(_ string, heartbeat time.Duration, _ time.Duration, _ map[string]string) error { - s.heartbeat = heartbeat - s.values = make([]string, 0) - return nil -} - -// Watch sends periodic discovery updates to a channel. -func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) { - ch := make(chan discovery.Entries) - errCh := make(chan error) - ticker := time.NewTicker(s.heartbeat) - - go func() { - defer close(errCh) - defer close(ch) - - // Send the initial entries if available. - var currentEntries discovery.Entries - var err error - - s.mu.Lock() - if len(s.values) > 0 { - currentEntries, err = discovery.CreateEntries(s.values) - } - s.mu.Unlock() - - if err != nil { - errCh <- err - } else if currentEntries != nil { - ch <- currentEntries - } - - // Periodically send updates. - for { - select { - case <-ticker.C: - s.mu.Lock() - newEntries, err := discovery.CreateEntries(s.values) - s.mu.Unlock() - if err != nil { - errCh <- err - continue - } - - // Check if the file has really changed. - if !newEntries.Equals(currentEntries) { - ch <- newEntries - } - currentEntries = newEntries - case <-stopCh: - ticker.Stop() - return - } - } - }() - - return ch, errCh -} - -// Register adds a new address to the discovery. -func (s *Discovery) Register(addr string) error { - s.mu.Lock() - s.values = append(s.values, addr) - s.mu.Unlock() - return nil -} diff --git a/discovery/memory/memory_test.go b/discovery/memory/memory_test.go deleted file mode 100644 index c2da0a0..0000000 --- a/discovery/memory/memory_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package memory - -import ( - "testing" - - "github.com/docker/docker/pkg/discovery" - "github.com/go-check/check" -) - -// Hook up gocheck into the "go test" runner. -func Test(t *testing.T) { check.TestingT(t) } - -type discoverySuite struct{} - -var _ = check.Suite(&discoverySuite{}) - -func (s *discoverySuite) TestWatch(c *check.C) { - d := &Discovery{} - d.Initialize("foo", 1000, 0, nil) - stopCh := make(chan struct{}) - ch, errCh := d.Watch(stopCh) - - // We have to drain the error channel otherwise Watch will get stuck. - go func() { - for range errCh { - } - }() - - expected := discovery.Entries{ - &discovery.Entry{Host: "1.1.1.1", Port: "1111"}, - } - - c.Assert(d.Register("1.1.1.1:1111"), check.IsNil) - c.Assert(<-ch, check.DeepEquals, expected) - - expected = discovery.Entries{ - &discovery.Entry{Host: "1.1.1.1", Port: "1111"}, - &discovery.Entry{Host: "2.2.2.2", Port: "2222"}, - } - - c.Assert(d.Register("2.2.2.2:2222"), check.IsNil) - c.Assert(<-ch, check.DeepEquals, expected) - - // Stop and make sure it closes all channels. - close(stopCh) - c.Assert(<-ch, check.IsNil) - c.Assert(<-errCh, check.IsNil) -} diff --git a/discovery/nodes/nodes.go b/discovery/nodes/nodes.go deleted file mode 100644 index c0e3c07..0000000 --- a/discovery/nodes/nodes.go +++ /dev/null @@ -1,54 +0,0 @@ -package nodes - -import ( - "fmt" - "strings" - "time" - - "github.com/docker/docker/pkg/discovery" -) - -// Discovery is exported -type Discovery struct { - entries discovery.Entries -} - -func init() { - Init() -} - -// Init is exported -func Init() { - discovery.Register("nodes", &Discovery{}) -} - -// Initialize is exported -func (s *Discovery) Initialize(uris string, _ time.Duration, _ time.Duration, _ map[string]string) error { - for _, input := range strings.Split(uris, ",") { - for _, ip := range discovery.Generate(input) { - entry, err := discovery.NewEntry(ip) - if err != nil { - return fmt.Errorf("%s, please check you are using the correct discovery (missing token:// ?)", err.Error()) - } - s.entries = append(s.entries, entry) - } - } - - return nil -} - -// Watch is exported -func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) { - ch := make(chan discovery.Entries) - go func() { - defer close(ch) - ch <- s.entries - <-stopCh - }() - return ch, nil -} - -// Register is exported -func (s *Discovery) Register(addr string) error { - return discovery.ErrNotImplemented -} diff --git a/discovery/nodes/nodes_test.go b/discovery/nodes/nodes_test.go deleted file mode 100644 index e26568c..0000000 --- a/discovery/nodes/nodes_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package nodes - -import ( - "testing" - - "github.com/docker/docker/pkg/discovery" - - "github.com/go-check/check" -) - -// Hook up gocheck into the "go test" runner. -func Test(t *testing.T) { check.TestingT(t) } - -type DiscoverySuite struct{} - -var _ = check.Suite(&DiscoverySuite{}) - -func (s *DiscoverySuite) TestInitialize(c *check.C) { - d := &Discovery{} - d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0, nil) - c.Assert(len(d.entries), check.Equals, 2) - c.Assert(d.entries[0].String(), check.Equals, "1.1.1.1:1111") - c.Assert(d.entries[1].String(), check.Equals, "2.2.2.2:2222") -} - -func (s *DiscoverySuite) TestInitializeWithPattern(c *check.C) { - d := &Discovery{} - d.Initialize("1.1.1.[1:2]:1111,2.2.2.[2:4]:2222", 0, 0, nil) - c.Assert(len(d.entries), check.Equals, 5) - c.Assert(d.entries[0].String(), check.Equals, "1.1.1.1:1111") - c.Assert(d.entries[1].String(), check.Equals, "1.1.1.2:1111") - c.Assert(d.entries[2].String(), check.Equals, "2.2.2.2:2222") - c.Assert(d.entries[3].String(), check.Equals, "2.2.2.3:2222") - c.Assert(d.entries[4].String(), check.Equals, "2.2.2.4:2222") -} - -func (s *DiscoverySuite) TestWatch(c *check.C) { - d := &Discovery{} - d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0, nil) - expected := discovery.Entries{ - &discovery.Entry{Host: "1.1.1.1", Port: "1111"}, - &discovery.Entry{Host: "2.2.2.2", Port: "2222"}, - } - ch, _ := d.Watch(nil) - c.Assert(expected.Equals(<-ch), check.Equals, true) -} - -func (s *DiscoverySuite) TestRegister(c *check.C) { - d := &Discovery{} - c.Assert(d.Register("0.0.0.0"), check.NotNil) -} diff --git a/filenotify/filenotify.go b/filenotify/filenotify.go deleted file mode 100644 index 23befae..0000000 --- a/filenotify/filenotify.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package filenotify provides a mechanism for watching file(s) for changes. -// Generally leans on fsnotify, but provides a poll-based notifier which fsnotify does not support. -// These are wrapped up in a common interface so that either can be used interchangeably in your code. -package filenotify - -import "gopkg.in/fsnotify.v1" - -// FileWatcher is an interface for implementing file notification watchers -type FileWatcher interface { - Events() <-chan fsnotify.Event - Errors() <-chan error - Add(name string) error - Remove(name string) error - Close() error -} - -// New tries to use an fs-event watcher, and falls back to the poller if there is an error -func New() (FileWatcher, error) { - if watcher, err := NewEventWatcher(); err == nil { - return watcher, nil - } - return NewPollingWatcher(), nil -} - -// NewPollingWatcher returns a poll-based file watcher -func NewPollingWatcher() FileWatcher { - return &filePoller{ - events: make(chan fsnotify.Event), - errors: make(chan error), - } -} - -// NewEventWatcher returns an fs-event based file watcher -func NewEventWatcher() (FileWatcher, error) { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - return &fsNotifyWatcher{watcher}, nil -} diff --git a/filenotify/fsnotify.go b/filenotify/fsnotify.go deleted file mode 100644 index 4203883..0000000 --- a/filenotify/fsnotify.go +++ /dev/null @@ -1,18 +0,0 @@ -package filenotify - -import "gopkg.in/fsnotify.v1" - -// fsNotify wraps the fsnotify package to satisfy the FileNotifer interface -type fsNotifyWatcher struct { - *fsnotify.Watcher -} - -// GetEvents returns the fsnotify event channel receiver -func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event { - return w.Watcher.Events -} - -// GetErrors returns the fsnotify error channel receiver -func (w *fsNotifyWatcher) Errors() <-chan error { - return w.Watcher.Errors -} diff --git a/filenotify/poller.go b/filenotify/poller.go deleted file mode 100644 index 5261085..0000000 --- a/filenotify/poller.go +++ /dev/null @@ -1,204 +0,0 @@ -package filenotify - -import ( - "errors" - "fmt" - "os" - "sync" - "time" - - "github.com/Sirupsen/logrus" - - "gopkg.in/fsnotify.v1" -) - -var ( - // errPollerClosed is returned when the poller is closed - errPollerClosed = errors.New("poller is closed") - // errNoSuchPoller is returned when trying to remove a watch that doesn't exist - errNoSuchWatch = errors.New("poller does not exist") -) - -// watchWaitTime is the time to wait between file poll loops -const watchWaitTime = 200 * time.Millisecond - -// filePoller is used to poll files for changes, especially in cases where fsnotify -// can't be run (e.g. when inotify handles are exhausted) -// filePoller satisfies the FileWatcher interface -type filePoller struct { - // watches is the list of files currently being polled, close the associated channel to stop the watch - watches map[string]chan struct{} - // events is the channel to listen to for watch events - events chan fsnotify.Event - // errors is the channel to listen to for watch errors - errors chan error - // mu locks the poller for modification - mu sync.Mutex - // closed is used to specify when the poller has already closed - closed bool -} - -// Add adds a filename to the list of watches -// once added the file is polled for changes in a separate goroutine -func (w *filePoller) Add(name string) error { - w.mu.Lock() - defer w.mu.Unlock() - - if w.closed == true { - return errPollerClosed - } - - f, err := os.Open(name) - if err != nil { - return err - } - fi, err := os.Stat(name) - if err != nil { - return err - } - - if w.watches == nil { - w.watches = make(map[string]chan struct{}) - } - if _, exists := w.watches[name]; exists { - return fmt.Errorf("watch exists") - } - chClose := make(chan struct{}) - w.watches[name] = chClose - - go w.watch(f, fi, chClose) - return nil -} - -// Remove stops and removes watch with the specified name -func (w *filePoller) Remove(name string) error { - w.mu.Lock() - defer w.mu.Unlock() - return w.remove(name) -} - -func (w *filePoller) remove(name string) error { - if w.closed == true { - return errPollerClosed - } - - chClose, exists := w.watches[name] - if !exists { - return errNoSuchWatch - } - close(chClose) - delete(w.watches, name) - return nil -} - -// Events returns the event channel -// This is used for notifications on events about watched files -func (w *filePoller) Events() <-chan fsnotify.Event { - return w.events -} - -// Errors returns the errors channel -// This is used for notifications about errors on watched files -func (w *filePoller) Errors() <-chan error { - return w.errors -} - -// Close closes the poller -// All watches are stopped, removed, and the poller cannot be added to -func (w *filePoller) Close() error { - w.mu.Lock() - defer w.mu.Unlock() - - if w.closed { - return nil - } - - w.closed = true - for name := range w.watches { - w.remove(name) - delete(w.watches, name) - } - return nil -} - -// sendEvent publishes the specified event to the events channel -func (w *filePoller) sendEvent(e fsnotify.Event, chClose <-chan struct{}) error { - select { - case w.events <- e: - case <-chClose: - return fmt.Errorf("closed") - } - return nil -} - -// sendErr publishes the specified error to the errors channel -func (w *filePoller) sendErr(e error, chClose <-chan struct{}) error { - select { - case w.errors <- e: - case <-chClose: - return fmt.Errorf("closed") - } - return nil -} - -// watch is responsible for polling the specified file for changes -// upon finding changes to a file or errors, sendEvent/sendErr is called -func (w *filePoller) watch(f *os.File, lastFi os.FileInfo, chClose chan struct{}) { - defer f.Close() - for { - time.Sleep(watchWaitTime) - select { - case <-chClose: - logrus.Debugf("watch for %s closed", f.Name()) - return - default: - } - - fi, err := os.Stat(f.Name()) - if err != nil { - // if we got an error here and lastFi is not set, we can presume that nothing has changed - // This should be safe since before `watch()` is called, a stat is performed, there is any error `watch` is not called - if lastFi == nil { - continue - } - // If it doesn't exist at this point, it must have been removed - // no need to send the error here since this is a valid operation - if os.IsNotExist(err) { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Remove, Name: f.Name()}, chClose); err != nil { - return - } - lastFi = nil - continue - } - // at this point, send the error - if err := w.sendErr(err, chClose); err != nil { - return - } - continue - } - - if lastFi == nil { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Create, Name: fi.Name()}, chClose); err != nil { - return - } - lastFi = fi - continue - } - - if fi.Mode() != lastFi.Mode() { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Chmod, Name: fi.Name()}, chClose); err != nil { - return - } - lastFi = fi - continue - } - - if fi.ModTime() != lastFi.ModTime() || fi.Size() != lastFi.Size() { - if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Write, Name: fi.Name()}, chClose); err != nil { - return - } - lastFi = fi - continue - } - } -} diff --git a/filenotify/poller_test.go b/filenotify/poller_test.go deleted file mode 100644 index 4f50262..0000000 --- a/filenotify/poller_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package filenotify - -import ( - "fmt" - "io/ioutil" - "os" - "runtime" - "testing" - "time" - - "gopkg.in/fsnotify.v1" -) - -func TestPollerAddRemove(t *testing.T) { - w := NewPollingWatcher() - - if err := w.Add("no-such-file"); err == nil { - t.Fatal("should have gotten error when adding a non-existent file") - } - if err := w.Remove("no-such-file"); err == nil { - t.Fatal("should have gotten error when removing non-existent watch") - } - - f, err := ioutil.TempFile("", "asdf") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(f.Name()) - - if err := w.Add(f.Name()); err != nil { - t.Fatal(err) - } - - if err := w.Remove(f.Name()); err != nil { - t.Fatal(err) - } -} - -func TestPollerEvent(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("No chmod on Windows") - } - w := NewPollingWatcher() - - f, err := ioutil.TempFile("", "test-poller") - if err != nil { - t.Fatal("error creating temp file") - } - defer os.RemoveAll(f.Name()) - f.Close() - - if err := w.Add(f.Name()); err != nil { - t.Fatal(err) - } - - select { - case <-w.Events(): - t.Fatal("got event before anything happened") - case <-w.Errors(): - t.Fatal("got error before anything happened") - default: - } - - if err := ioutil.WriteFile(f.Name(), []byte("hello"), 644); err != nil { - t.Fatal(err) - } - if err := assertEvent(w, fsnotify.Write); err != nil { - t.Fatal(err) - } - - if err := os.Chmod(f.Name(), 600); err != nil { - t.Fatal(err) - } - if err := assertEvent(w, fsnotify.Chmod); err != nil { - t.Fatal(err) - } - - if err := os.Remove(f.Name()); err != nil { - t.Fatal(err) - } - if err := assertEvent(w, fsnotify.Remove); err != nil { - t.Fatal(err) - } -} - -func TestPollerClose(t *testing.T) { - w := NewPollingWatcher() - if err := w.Close(); err != nil { - t.Fatal(err) - } - // test double-close - if err := w.Close(); err != nil { - t.Fatal(err) - } - - f, err := ioutil.TempFile("", "asdf") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(f.Name()) - if err := w.Add(f.Name()); err == nil { - t.Fatal("should have gotten error adding watch for closed watcher") - } -} - -func assertEvent(w FileWatcher, eType fsnotify.Op) error { - var err error - select { - case e := <-w.Events(): - if e.Op != eType { - err = fmt.Errorf("got wrong event type, expected %q: %v", eType, e) - } - case e := <-w.Errors(): - err = fmt.Errorf("got unexpected error waiting for events %v: %v", eType, e) - case <-time.After(watchWaitTime * 3): - err = fmt.Errorf("timeout waiting for event %v", eType) - } - return err -} diff --git a/gitutils/gitutils.go b/gitutils/gitutils.go deleted file mode 100644 index ded091f..0000000 --- a/gitutils/gitutils.go +++ /dev/null @@ -1,100 +0,0 @@ -package gitutils - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/docker/docker/pkg/symlink" - "github.com/docker/docker/pkg/urlutil" -) - -// Clone clones a repository into a newly created directory which -// will be under "docker-build-git" -func Clone(remoteURL string) (string, error) { - if !urlutil.IsGitTransport(remoteURL) { - remoteURL = "https://" + remoteURL - } - root, err := ioutil.TempDir("", "docker-build-git") - if err != nil { - return "", err - } - - u, err := url.Parse(remoteURL) - if err != nil { - return "", err - } - - fragment := u.Fragment - clone := cloneArgs(u, root) - - if output, err := git(clone...); err != nil { - return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output) - } - - return checkoutGit(fragment, root) -} - -func cloneArgs(remoteURL *url.URL, root string) []string { - args := []string{"clone", "--recursive"} - shallow := len(remoteURL.Fragment) == 0 - - if shallow && strings.HasPrefix(remoteURL.Scheme, "http") { - res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) - if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { - shallow = false - } - } - - if shallow { - args = append(args, "--depth", "1") - } - - if remoteURL.Fragment != "" { - remoteURL.Fragment = "" - } - - return append(args, remoteURL.String(), root) -} - -func checkoutGit(fragment, root string) (string, error) { - refAndDir := strings.SplitN(fragment, ":", 2) - - if len(refAndDir[0]) != 0 { - if output, err := gitWithinDir(root, "checkout", refAndDir[0]); err != nil { - return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output) - } - } - - if len(refAndDir) > 1 && len(refAndDir[1]) != 0 { - newCtx, err := symlink.FollowSymlinkInScope(filepath.Join(root, refAndDir[1]), root) - if err != nil { - return "", fmt.Errorf("Error setting git context, %q not within git root: %s", refAndDir[1], err) - } - - fi, err := os.Stat(newCtx) - if err != nil { - return "", err - } - if !fi.IsDir() { - return "", fmt.Errorf("Error setting git context, not a directory: %s", newCtx) - } - root = newCtx - } - - return root, nil -} - -func gitWithinDir(dir string, args ...string) ([]byte, error) { - a := []string{"--work-tree", dir, "--git-dir", filepath.Join(dir, ".git")} - return git(append(a, args...)...) -} - -func git(args ...string) ([]byte, error) { - return exec.Command("git", args...).CombinedOutput() -} diff --git a/gitutils/gitutils_test.go b/gitutils/gitutils_test.go deleted file mode 100644 index d197058..0000000 --- a/gitutils/gitutils_test.go +++ /dev/null @@ -1,220 +0,0 @@ -package gitutils - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "os" - "path/filepath" - "reflect" - "runtime" - "strings" - "testing" -) - -func TestCloneArgsSmartHttp(t *testing.T) { - mux := http.NewServeMux() - server := httptest.NewServer(mux) - serverURL, _ := url.Parse(server.URL) - - serverURL.Path = "/repo.git" - gitURL := serverURL.String() - - mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query().Get("service") - w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", q)) - }) - - args := cloneArgs(serverURL, "/tmp") - exp := []string{"clone", "--recursive", "--depth", "1", gitURL, "/tmp"} - if !reflect.DeepEqual(args, exp) { - t.Fatalf("Expected %v, got %v", exp, args) - } -} - -func TestCloneArgsDumbHttp(t *testing.T) { - mux := http.NewServeMux() - server := httptest.NewServer(mux) - serverURL, _ := url.Parse(server.URL) - - serverURL.Path = "/repo.git" - gitURL := serverURL.String() - - mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - }) - - args := cloneArgs(serverURL, "/tmp") - exp := []string{"clone", "--recursive", gitURL, "/tmp"} - if !reflect.DeepEqual(args, exp) { - t.Fatalf("Expected %v, got %v", exp, args) - } -} - -func TestCloneArgsGit(t *testing.T) { - u, _ := url.Parse("git://github.com/docker/docker") - args := cloneArgs(u, "/tmp") - exp := []string{"clone", "--recursive", "--depth", "1", "git://github.com/docker/docker", "/tmp"} - if !reflect.DeepEqual(args, exp) { - t.Fatalf("Expected %v, got %v", exp, args) - } -} - -func TestCloneArgsStripFragment(t *testing.T) { - u, _ := url.Parse("git://github.com/docker/docker#test") - args := cloneArgs(u, "/tmp") - exp := []string{"clone", "--recursive", "git://github.com/docker/docker", "/tmp"} - if !reflect.DeepEqual(args, exp) { - t.Fatalf("Expected %v, got %v", exp, args) - } -} - -func gitGetConfig(name string) string { - b, err := git([]string{"config", "--get", name}...) - if err != nil { - // since we are interested in empty or non empty string, - // we can safely ignore the err here. - return "" - } - return strings.TrimSpace(string(b)) -} - -func TestCheckoutGit(t *testing.T) { - root, err := ioutil.TempDir("", "docker-build-git-checkout") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(root) - - autocrlf := gitGetConfig("core.autocrlf") - if !(autocrlf == "true" || autocrlf == "false" || - autocrlf == "input" || autocrlf == "") { - t.Logf("unknown core.autocrlf value: \"%s\"", autocrlf) - } - eol := "\n" - if autocrlf == "true" { - eol = "\r\n" - } - - gitDir := filepath.Join(root, "repo") - _, err = git("init", gitDir) - if err != nil { - t.Fatal(err) - } - - if _, err = gitWithinDir(gitDir, "config", "user.email", "test@docker.com"); err != nil { - t.Fatal(err) - } - - if _, err = gitWithinDir(gitDir, "config", "user.name", "Docker test"); err != nil { - t.Fatal(err) - } - - if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch"), 0644); err != nil { - t.Fatal(err) - } - - subDir := filepath.Join(gitDir, "subdir") - if err = os.Mkdir(subDir, 0755); err != nil { - t.Fatal(err) - } - - if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 5000"), 0644); err != nil { - t.Fatal(err) - } - - if runtime.GOOS != "windows" { - if err = os.Symlink("../subdir", filepath.Join(gitDir, "parentlink")); err != nil { - t.Fatal(err) - } - - if err = os.Symlink("/subdir", filepath.Join(gitDir, "absolutelink")); err != nil { - t.Fatal(err) - } - } - - if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil { - t.Fatal(err) - } - - if _, err = gitWithinDir(gitDir, "commit", "-am", "First commit"); err != nil { - t.Fatal(err) - } - - if _, err = gitWithinDir(gitDir, "checkout", "-b", "test"); err != nil { - t.Fatal(err) - } - - if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 3000"), 0644); err != nil { - t.Fatal(err) - } - - if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM busybox\nEXPOSE 5000"), 0644); err != nil { - t.Fatal(err) - } - - if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil { - t.Fatal(err) - } - - if _, err = gitWithinDir(gitDir, "commit", "-am", "Branch commit"); err != nil { - t.Fatal(err) - } - - if _, err = gitWithinDir(gitDir, "checkout", "master"); err != nil { - t.Fatal(err) - } - - type singleCase struct { - frag string - exp string - fail bool - } - - cases := []singleCase{ - {"", "FROM scratch", false}, - {"master", "FROM scratch", false}, - {":subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, - {":nosubdir", "", true}, // missing directory error - {":Dockerfile", "", true}, // not a directory error - {"master:nosubdir", "", true}, - {"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, - {"master:../subdir", "", true}, - {"test", "FROM scratch" + eol + "EXPOSE 3000", false}, - {"test:", "FROM scratch" + eol + "EXPOSE 3000", false}, - {"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false}, - } - - if runtime.GOOS != "windows" { - // Windows GIT (2.7.1 x64) does not support parentlink/absolutelink. Sample output below - // git --work-tree .\repo --git-dir .\repo\.git add -A - // error: readlink("absolutelink"): Function not implemented - // error: unable to index file absolutelink - // fatal: adding files failed - cases = append(cases, singleCase{frag: "master:absolutelink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false}) - cases = append(cases, singleCase{frag: "master:parentlink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false}) - } - - for _, c := range cases { - r, err := checkoutGit(c.frag, gitDir) - - fail := err != nil - if fail != c.fail { - t.Fatalf("Expected %v failure, error was %v\n", c.fail, err) - } - if c.fail { - continue - } - - b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile")) - if err != nil { - t.Fatal(err) - } - - if string(b) != c.exp { - t.Fatalf("Expected %v, was %v\n", c.exp, string(b)) - } - } -} diff --git a/graphdb/conn_sqlite3.go b/graphdb/conn_sqlite3.go deleted file mode 100644 index dbcf44c..0000000 --- a/graphdb/conn_sqlite3.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build cgo - -package graphdb - -import "database/sql" - -// NewSqliteConn opens a connection to a sqlite -// database. -func NewSqliteConn(root string) (*Database, error) { - conn, err := sql.Open("sqlite3", root) - if err != nil { - return nil, err - } - return NewDatabase(conn) -} diff --git a/graphdb/conn_sqlite3_unix.go b/graphdb/conn_sqlite3_unix.go deleted file mode 100644 index f932fff..0000000 --- a/graphdb/conn_sqlite3_unix.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build cgo,!windows - -package graphdb - -import ( - _ "github.com/mattn/go-sqlite3" // registers sqlite -) diff --git a/graphdb/conn_sqlite3_windows.go b/graphdb/conn_sqlite3_windows.go deleted file mode 100644 index 5259030..0000000 --- a/graphdb/conn_sqlite3_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build cgo,windows - -package graphdb - -import ( - _ "github.com/mattn/go-sqlite3" // registers sqlite -) diff --git a/graphdb/conn_unsupported.go b/graphdb/conn_unsupported.go deleted file mode 100644 index cf97705..0000000 --- a/graphdb/conn_unsupported.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !cgo - -package graphdb - -// NewSqliteConn return a new sqlite connection. -func NewSqliteConn(root string) (*Database, error) { - panic("Not implemented") -} diff --git a/graphdb/graphdb.go b/graphdb/graphdb.go deleted file mode 100644 index eca433f..0000000 --- a/graphdb/graphdb.go +++ /dev/null @@ -1,551 +0,0 @@ -package graphdb - -import ( - "database/sql" - "fmt" - "path" - "strings" - "sync" -) - -const ( - createEntityTable = ` - CREATE TABLE IF NOT EXISTS entity ( - id text NOT NULL PRIMARY KEY - );` - - createEdgeTable = ` - CREATE TABLE IF NOT EXISTS edge ( - "entity_id" text NOT NULL, - "parent_id" text NULL, - "name" text NOT NULL, - CONSTRAINT "parent_fk" FOREIGN KEY ("parent_id") REFERENCES "entity" ("id"), - CONSTRAINT "entity_fk" FOREIGN KEY ("entity_id") REFERENCES "entity" ("id") - ); - ` - - createEdgeIndices = ` - CREATE UNIQUE INDEX IF NOT EXISTS "name_parent_ix" ON "edge" (parent_id, name); - ` -) - -// Entity with a unique id. -type Entity struct { - id string -} - -// An Edge connects two entities together. -type Edge struct { - EntityID string - Name string - ParentID string -} - -// Entities stores the list of entities. -type Entities map[string]*Entity - -// Edges stores the relationships between entities. -type Edges []*Edge - -// WalkFunc is a function invoked to process an individual entity. -type WalkFunc func(fullPath string, entity *Entity) error - -// Database is a graph database for storing entities and their relationships. -type Database struct { - conn *sql.DB - mux sync.RWMutex -} - -// IsNonUniqueNameError processes the error to check if it's caused by -// a constraint violation. -// This is necessary because the error isn't the same across various -// sqlite versions. -func IsNonUniqueNameError(err error) bool { - str := err.Error() - // sqlite 3.7.17-1ubuntu1 returns: - // Set failure: Abort due to constraint violation: columns parent_id, name are not unique - if strings.HasSuffix(str, "name are not unique") { - return true - } - // sqlite-3.8.3-1.fc20 returns: - // Set failure: Abort due to constraint violation: UNIQUE constraint failed: edge.parent_id, edge.name - if strings.Contains(str, "UNIQUE constraint failed") && strings.Contains(str, "edge.name") { - return true - } - // sqlite-3.6.20-1.el6 returns: - // Set failure: Abort due to constraint violation: constraint failed - if strings.HasSuffix(str, "constraint failed") { - return true - } - return false -} - -// NewDatabase creates a new graph database initialized with a root entity. -func NewDatabase(conn *sql.DB) (*Database, error) { - if conn == nil { - return nil, fmt.Errorf("Database connection cannot be nil") - } - db := &Database{conn: conn} - - // Create root entities - tx, err := conn.Begin() - if err != nil { - return nil, err - } - - if _, err := tx.Exec(createEntityTable); err != nil { - return nil, err - } - if _, err := tx.Exec(createEdgeTable); err != nil { - return nil, err - } - if _, err := tx.Exec(createEdgeIndices); err != nil { - return nil, err - } - - if _, err := tx.Exec("DELETE FROM entity where id = ?", "0"); err != nil { - tx.Rollback() - return nil, err - } - - if _, err := tx.Exec("INSERT INTO entity (id) VALUES (?);", "0"); err != nil { - tx.Rollback() - return nil, err - } - - if _, err := tx.Exec("DELETE FROM edge where entity_id=? and name=?", "0", "/"); err != nil { - tx.Rollback() - return nil, err - } - - if _, err := tx.Exec("INSERT INTO edge (entity_id, name) VALUES(?,?);", "0", "/"); err != nil { - tx.Rollback() - return nil, err - } - - if err := tx.Commit(); err != nil { - return nil, err - } - - return db, nil -} - -// Close the underlying connection to the database. -func (db *Database) Close() error { - return db.conn.Close() -} - -// Set the entity id for a given path. -func (db *Database) Set(fullPath, id string) (*Entity, error) { - db.mux.Lock() - defer db.mux.Unlock() - - tx, err := db.conn.Begin() - if err != nil { - return nil, err - } - - var entityID string - if err := tx.QueryRow("SELECT id FROM entity WHERE id = ?;", id).Scan(&entityID); err != nil { - if err == sql.ErrNoRows { - if _, err := tx.Exec("INSERT INTO entity (id) VALUES(?);", id); err != nil { - tx.Rollback() - return nil, err - } - } else { - tx.Rollback() - return nil, err - } - } - e := &Entity{id} - - parentPath, name := splitPath(fullPath) - if err := db.setEdge(parentPath, name, e, tx); err != nil { - tx.Rollback() - return nil, err - } - - if err := tx.Commit(); err != nil { - return nil, err - } - return e, nil -} - -// Exists returns true if a name already exists in the database. -func (db *Database) Exists(name string) bool { - db.mux.RLock() - defer db.mux.RUnlock() - - e, err := db.get(name) - if err != nil { - return false - } - return e != nil -} - -func (db *Database) setEdge(parentPath, name string, e *Entity, tx *sql.Tx) error { - parent, err := db.get(parentPath) - if err != nil { - return err - } - if parent.id == e.id { - return fmt.Errorf("Cannot set self as child") - } - - if _, err := tx.Exec("INSERT INTO edge (parent_id, name, entity_id) VALUES (?,?,?);", parent.id, name, e.id); err != nil { - return err - } - return nil -} - -// RootEntity returns the root "/" entity for the database. -func (db *Database) RootEntity() *Entity { - return &Entity{ - id: "0", - } -} - -// Get returns the entity for a given path. -func (db *Database) Get(name string) *Entity { - db.mux.RLock() - defer db.mux.RUnlock() - - e, err := db.get(name) - if err != nil { - return nil - } - return e -} - -func (db *Database) get(name string) (*Entity, error) { - e := db.RootEntity() - // We always know the root name so return it if - // it is requested - if name == "/" { - return e, nil - } - - parts := split(name) - for i := 1; i < len(parts); i++ { - p := parts[i] - if p == "" { - continue - } - - next := db.child(e, p) - if next == nil { - return nil, fmt.Errorf("Cannot find child for %s", name) - } - e = next - } - return e, nil - -} - -// List all entities by from the name. -// The key will be the full path of the entity. -func (db *Database) List(name string, depth int) Entities { - db.mux.RLock() - defer db.mux.RUnlock() - - out := Entities{} - e, err := db.get(name) - if err != nil { - return out - } - - children, err := db.children(e, name, depth, nil) - if err != nil { - return out - } - - for _, c := range children { - out[c.FullPath] = c.Entity - } - return out -} - -// Walk through the child graph of an entity, calling walkFunc for each child entity. -// It is safe for walkFunc to call graph functions. -func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error { - children, err := db.Children(name, depth) - if err != nil { - return err - } - - // Note: the database lock must not be held while calling walkFunc - for _, c := range children { - if err := walkFunc(c.FullPath, c.Entity); err != nil { - return err - } - } - return nil -} - -// Children returns the children of the specified entity. -func (db *Database) Children(name string, depth int) ([]WalkMeta, error) { - db.mux.RLock() - defer db.mux.RUnlock() - - e, err := db.get(name) - if err != nil { - return nil, err - } - - return db.children(e, name, depth, nil) -} - -// Parents returns the parents of a specified entity. -func (db *Database) Parents(name string) ([]string, error) { - db.mux.RLock() - defer db.mux.RUnlock() - - e, err := db.get(name) - if err != nil { - return nil, err - } - return db.parents(e) -} - -// Refs returns the reference count for a specified id. -func (db *Database) Refs(id string) int { - db.mux.RLock() - defer db.mux.RUnlock() - - var count int - if err := db.conn.QueryRow("SELECT COUNT(*) FROM edge WHERE entity_id = ?;", id).Scan(&count); err != nil { - return 0 - } - return count -} - -// RefPaths returns all the id's path references. -func (db *Database) RefPaths(id string) Edges { - db.mux.RLock() - defer db.mux.RUnlock() - - refs := Edges{} - - rows, err := db.conn.Query("SELECT name, parent_id FROM edge WHERE entity_id = ?;", id) - if err != nil { - return refs - } - defer rows.Close() - - for rows.Next() { - var name string - var parentID string - if err := rows.Scan(&name, &parentID); err != nil { - return refs - } - refs = append(refs, &Edge{ - EntityID: id, - Name: name, - ParentID: parentID, - }) - } - return refs -} - -// Delete the reference to an entity at a given path. -func (db *Database) Delete(name string) error { - db.mux.Lock() - defer db.mux.Unlock() - - if name == "/" { - return fmt.Errorf("Cannot delete root entity") - } - - parentPath, n := splitPath(name) - parent, err := db.get(parentPath) - if err != nil { - return err - } - - if _, err := db.conn.Exec("DELETE FROM edge WHERE parent_id = ? AND name = ?;", parent.id, n); err != nil { - return err - } - return nil -} - -// Purge removes the entity with the specified id -// Walk the graph to make sure all references to the entity -// are removed and return the number of references removed -func (db *Database) Purge(id string) (int, error) { - db.mux.Lock() - defer db.mux.Unlock() - - tx, err := db.conn.Begin() - if err != nil { - return -1, err - } - - // Delete all edges - rows, err := tx.Exec("DELETE FROM edge WHERE entity_id = ?;", id) - if err != nil { - tx.Rollback() - return -1, err - } - changes, err := rows.RowsAffected() - if err != nil { - return -1, err - } - - // Clear who's using this id as parent - refs, err := tx.Exec("DELETE FROM edge WHERE parent_id = ?;", id) - if err != nil { - tx.Rollback() - return -1, err - } - refsCount, err := refs.RowsAffected() - if err != nil { - return -1, err - } - - // Delete entity - if _, err := tx.Exec("DELETE FROM entity where id = ?;", id); err != nil { - tx.Rollback() - return -1, err - } - - if err := tx.Commit(); err != nil { - return -1, err - } - - return int(changes + refsCount), nil -} - -// Rename an edge for a given path -func (db *Database) Rename(currentName, newName string) error { - db.mux.Lock() - defer db.mux.Unlock() - - parentPath, name := splitPath(currentName) - newParentPath, newEdgeName := splitPath(newName) - - if parentPath != newParentPath { - return fmt.Errorf("Cannot rename when root paths do not match %s != %s", parentPath, newParentPath) - } - - parent, err := db.get(parentPath) - if err != nil { - return err - } - - rows, err := db.conn.Exec("UPDATE edge SET name = ? WHERE parent_id = ? AND name = ?;", newEdgeName, parent.id, name) - if err != nil { - return err - } - i, err := rows.RowsAffected() - if err != nil { - return err - } - if i == 0 { - return fmt.Errorf("Cannot locate edge for %s %s", parent.id, name) - } - return nil -} - -// WalkMeta stores the walk metadata. -type WalkMeta struct { - Parent *Entity - Entity *Entity - FullPath string - Edge *Edge -} - -func (db *Database) children(e *Entity, name string, depth int, entities []WalkMeta) ([]WalkMeta, error) { - if e == nil { - return entities, nil - } - - rows, err := db.conn.Query("SELECT entity_id, name FROM edge where parent_id = ?;", e.id) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var entityID, entityName string - if err := rows.Scan(&entityID, &entityName); err != nil { - return nil, err - } - child := &Entity{entityID} - edge := &Edge{ - ParentID: e.id, - Name: entityName, - EntityID: child.id, - } - - meta := WalkMeta{ - Parent: e, - Entity: child, - FullPath: path.Join(name, edge.Name), - Edge: edge, - } - - entities = append(entities, meta) - - if depth != 0 { - nDepth := depth - if depth != -1 { - nDepth-- - } - entities, err = db.children(child, meta.FullPath, nDepth, entities) - if err != nil { - return nil, err - } - } - } - - return entities, nil -} - -func (db *Database) parents(e *Entity) (parents []string, err error) { - if e == nil { - return parents, nil - } - - rows, err := db.conn.Query("SELECT parent_id FROM edge where entity_id = ?;", e.id) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var parentID string - if err := rows.Scan(&parentID); err != nil { - return nil, err - } - parents = append(parents, parentID) - } - - return parents, nil -} - -// Return the entity based on the parent path and name. -func (db *Database) child(parent *Entity, name string) *Entity { - var id string - if err := db.conn.QueryRow("SELECT entity_id FROM edge WHERE parent_id = ? AND name = ?;", parent.id, name).Scan(&id); err != nil { - return nil - } - return &Entity{id} -} - -// ID returns the id used to reference this entity. -func (e *Entity) ID() string { - return e.id -} - -// Paths returns the paths sorted by depth. -func (e Entities) Paths() []string { - out := make([]string, len(e)) - var i int - for k := range e { - out[i] = k - i++ - } - sortByDepth(out) - - return out -} diff --git a/graphdb/graphdb_test.go b/graphdb/graphdb_test.go deleted file mode 100644 index f0fb074..0000000 --- a/graphdb/graphdb_test.go +++ /dev/null @@ -1,721 +0,0 @@ -package graphdb - -import ( - "database/sql" - "fmt" - "os" - "path" - "runtime" - "strconv" - "testing" - - _ "github.com/mattn/go-sqlite3" -) - -func newTestDb(t *testing.T) (*Database, string) { - p := path.Join(os.TempDir(), "sqlite.db") - conn, err := sql.Open("sqlite3", p) - db, err := NewDatabase(conn) - if err != nil { - t.Fatal(err) - } - return db, p -} - -func destroyTestDb(dbPath string) { - os.Remove(dbPath) -} - -func TestNewDatabase(t *testing.T) { - db, dbpath := newTestDb(t) - if db == nil { - t.Fatal("Database should not be nil") - } - db.Close() - defer destroyTestDb(dbpath) -} - -func TestCreateRootEntity(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - root := db.RootEntity() - if root == nil { - t.Fatal("Root entity should not be nil") - } -} - -func TestGetRootEntity(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - e := db.Get("/") - if e == nil { - t.Fatal("Entity should not be nil") - } - if e.ID() != "0" { - t.Fatalf("Entity id should be 0, got %s", e.ID()) - } -} - -func TestSetEntityWithDifferentName(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/test", "1") - if _, err := db.Set("/other", "1"); err != nil { - t.Fatal(err) - } -} - -func TestSetDuplicateEntity(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - if _, err := db.Set("/foo", "42"); err != nil { - t.Fatal(err) - } - if _, err := db.Set("/foo", "43"); err == nil { - t.Fatalf("Creating an entry with a duplicate path did not cause an error") - } -} - -func TestCreateChild(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - child, err := db.Set("/db", "1") - if err != nil { - t.Fatal(err) - } - if child == nil { - t.Fatal("Child should not be nil") - } - if child.ID() != "1" { - t.Fail() - } -} - -func TestParents(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - for i := 1; i < 6; i++ { - a := strconv.Itoa(i) - if _, err := db.Set("/"+a, a); err != nil { - t.Fatal(err) - } - } - - for i := 6; i < 11; i++ { - a := strconv.Itoa(i) - p := strconv.Itoa(i - 5) - - key := fmt.Sprintf("/%s/%s", p, a) - - if _, err := db.Set(key, a); err != nil { - t.Fatal(err) - } - - parents, err := db.Parents(key) - if err != nil { - t.Fatal(err) - } - - if len(parents) != 1 { - t.Fatalf("Expected 1 entry for %s got %d", key, len(parents)) - } - - if parents[0] != p { - t.Fatalf("ID %s received, %s expected", parents[0], p) - } - } -} - -func TestChildren(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - str := "/" - for i := 1; i < 6; i++ { - a := strconv.Itoa(i) - if _, err := db.Set(str+a, a); err != nil { - t.Fatal(err) - } - - str = str + a + "/" - } - - str = "/" - for i := 10; i < 30; i++ { // 20 entities - a := strconv.Itoa(i) - if _, err := db.Set(str+a, a); err != nil { - t.Fatal(err) - } - - str = str + a + "/" - } - entries, err := db.Children("/", 5) - if err != nil { - t.Fatal(err) - } - - if len(entries) != 11 { - t.Fatalf("Expect 11 entries for / got %d", len(entries)) - } - - entries, err = db.Children("/", 20) - if err != nil { - t.Fatal(err) - } - - if len(entries) != 25 { - t.Fatalf("Expect 25 entries for / got %d", len(entries)) - } -} - -func TestListAllRootChildren(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - for i := 1; i < 6; i++ { - a := strconv.Itoa(i) - if _, err := db.Set("/"+a, a); err != nil { - t.Fatal(err) - } - } - entries := db.List("/", -1) - if len(entries) != 5 { - t.Fatalf("Expect 5 entries for / got %d", len(entries)) - } -} - -func TestListAllSubChildren(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - _, err := db.Set("/webapp", "1") - if err != nil { - t.Fatal(err) - } - child2, err := db.Set("/db", "2") - if err != nil { - t.Fatal(err) - } - child4, err := db.Set("/logs", "4") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/db/logs", child4.ID()); err != nil { - t.Fatal(err) - } - - child3, err := db.Set("/sentry", "3") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/db", child2.ID()); err != nil { - t.Fatal(err) - } - - entries := db.List("/webapp", 1) - if len(entries) != 3 { - t.Fatalf("Expect 3 entries for / got %d", len(entries)) - } - - entries = db.List("/webapp", 0) - if len(entries) != 2 { - t.Fatalf("Expect 2 entries for / got %d", len(entries)) - } -} - -func TestAddSelfAsChild(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - child, err := db.Set("/test", "1") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/test/other", child.ID()); err == nil { - t.Fatal("Error should not be nil") - } -} - -func TestAddChildToNonExistentRoot(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - if _, err := db.Set("/myapp", "1"); err != nil { - t.Fatal(err) - } - - if _, err := db.Set("/myapp/proxy/db", "2"); err == nil { - t.Fatal("Error should not be nil") - } -} - -func TestWalkAll(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - _, err := db.Set("/webapp", "1") - if err != nil { - t.Fatal(err) - } - child2, err := db.Set("/db", "2") - if err != nil { - t.Fatal(err) - } - child4, err := db.Set("/db/logs", "4") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/logs", child4.ID()); err != nil { - t.Fatal(err) - } - - child3, err := db.Set("/sentry", "3") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/db", child2.ID()); err != nil { - t.Fatal(err) - } - - child5, err := db.Set("/gograph", "5") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil { - t.Fatal(err) - } - - if err := db.Walk("/", func(p string, e *Entity) error { - t.Logf("Path: %s Entity: %s", p, e.ID()) - return nil - }, -1); err != nil { - t.Fatal(err) - } -} - -func TestGetEntityByPath(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - _, err := db.Set("/webapp", "1") - if err != nil { - t.Fatal(err) - } - child2, err := db.Set("/db", "2") - if err != nil { - t.Fatal(err) - } - child4, err := db.Set("/logs", "4") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/db/logs", child4.ID()); err != nil { - t.Fatal(err) - } - - child3, err := db.Set("/sentry", "3") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/db", child2.ID()); err != nil { - t.Fatal(err) - } - - child5, err := db.Set("/gograph", "5") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil { - t.Fatal(err) - } - - entity := db.Get("/webapp/db/logs") - if entity == nil { - t.Fatal("Entity should not be nil") - } - if entity.ID() != "4" { - t.Fatalf("Expected to get entity with id 4, got %s", entity.ID()) - } -} - -func TestEnitiesPaths(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - _, err := db.Set("/webapp", "1") - if err != nil { - t.Fatal(err) - } - child2, err := db.Set("/db", "2") - if err != nil { - t.Fatal(err) - } - child4, err := db.Set("/logs", "4") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/db/logs", child4.ID()); err != nil { - t.Fatal(err) - } - - child3, err := db.Set("/sentry", "3") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/db", child2.ID()); err != nil { - t.Fatal(err) - } - - child5, err := db.Set("/gograph", "5") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil { - t.Fatal(err) - } - - out := db.List("/", -1) - for _, p := range out.Paths() { - t.Log(p) - } -} - -func TestDeleteRootEntity(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - if err := db.Delete("/"); err == nil { - t.Fatal("Error should not be nil") - } -} - -func TestDeleteEntity(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - _, err := db.Set("/webapp", "1") - if err != nil { - t.Fatal(err) - } - child2, err := db.Set("/db", "2") - if err != nil { - t.Fatal(err) - } - child4, err := db.Set("/logs", "4") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/db/logs", child4.ID()); err != nil { - t.Fatal(err) - } - - child3, err := db.Set("/sentry", "3") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/db", child2.ID()); err != nil { - t.Fatal(err) - } - - child5, err := db.Set("/gograph", "5") - if err != nil { - t.Fatal(err) - } - if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil { - t.Fatal(err) - } - - if err := db.Delete("/webapp/sentry"); err != nil { - t.Fatal(err) - } - entity := db.Get("/webapp/sentry") - if entity != nil { - t.Fatal("Entity /webapp/sentry should be nil") - } -} - -func TestCountRefs(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/webapp", "1") - - if db.Refs("1") != 1 { - t.Fatal("Expect reference count to be 1") - } - - db.Set("/db", "2") - db.Set("/webapp/db", "2") - if db.Refs("2") != 2 { - t.Fatal("Expect reference count to be 2") - } -} - -func TestPurgeId(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/webapp", "1") - - if c := db.Refs("1"); c != 1 { - t.Fatalf("Expect reference count to be 1, got %d", c) - } - - db.Set("/db", "2") - db.Set("/webapp/db", "2") - - count, err := db.Purge("2") - if err != nil { - t.Fatal(err) - } - if count != 2 { - t.Fatalf("Expected 2 references to be removed, got %d", count) - } -} - -// Regression test https://github.com/docker/docker/issues/12334 -func TestPurgeIdRefPaths(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/webapp", "1") - db.Set("/db", "2") - - db.Set("/db/webapp", "1") - - if c := db.Refs("1"); c != 2 { - t.Fatalf("Expected 2 reference for webapp, got %d", c) - } - if c := db.Refs("2"); c != 1 { - t.Fatalf("Expected 1 reference for db, got %d", c) - } - - if rp := db.RefPaths("2"); len(rp) != 1 { - t.Fatalf("Expected 1 reference path for db, got %d", len(rp)) - } - - count, err := db.Purge("2") - if err != nil { - t.Fatal(err) - } - - if count != 2 { - t.Fatalf("Expected 2 rows to be removed, got %d", count) - } - - if c := db.Refs("2"); c != 0 { - t.Fatalf("Expected 0 reference for db, got %d", c) - } - if c := db.Refs("1"); c != 1 { - t.Fatalf("Expected 1 reference for webapp, got %d", c) - } -} - -func TestRename(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/webapp", "1") - - if db.Refs("1") != 1 { - t.Fatal("Expect reference count to be 1") - } - - db.Set("/db", "2") - db.Set("/webapp/db", "2") - - if db.Get("/webapp/db") == nil { - t.Fatal("Cannot find entity at path /webapp/db") - } - - if err := db.Rename("/webapp/db", "/webapp/newdb"); err != nil { - t.Fatal(err) - } - if db.Get("/webapp/db") != nil { - t.Fatal("Entity should not exist at /webapp/db") - } - if db.Get("/webapp/newdb") == nil { - t.Fatal("Cannot find entity at path /webapp/newdb") - } - -} - -func TestCreateMultipleNames(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/db", "1") - if _, err := db.Set("/myapp", "1"); err != nil { - t.Fatal(err) - } - - db.Walk("/", func(p string, e *Entity) error { - t.Logf("%s\n", p) - return nil - }, -1) -} - -func TestRefPaths(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/webapp", "1") - - db.Set("/db", "2") - db.Set("/webapp/db", "2") - - refs := db.RefPaths("2") - if len(refs) != 2 { - t.Fatalf("Expected reference count to be 2, got %d", len(refs)) - } -} - -func TestExistsTrue(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/testing", "1") - - if !db.Exists("/testing") { - t.Fatalf("/tesing should exist") - } -} - -func TestExistsFalse(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/toerhe", "1") - - if db.Exists("/testing") { - t.Fatalf("/tesing should not exist") - } - -} - -func TestGetNameWithTrailingSlash(t *testing.T) { - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - db.Set("/todo", "1") - - e := db.Get("/todo/") - if e == nil { - t.Fatalf("Entity should not be nil") - } -} - -func TestConcurrentWrites(t *testing.T) { - // TODO Windows: Port this test - if runtime.GOOS == "windows" { - t.Skip("Needs porting to Windows") - } - db, dbpath := newTestDb(t) - defer destroyTestDb(dbpath) - - errs := make(chan error, 2) - - save := func(name string, id string) { - if _, err := db.Set(fmt.Sprintf("/%s", name), id); err != nil { - errs <- err - } - errs <- nil - } - purge := func(id string) { - if _, err := db.Purge(id); err != nil { - errs <- err - } - errs <- nil - } - - save("/1", "1") - - go purge("1") - go save("/2", "2") - - any := false - for i := 0; i < 2; i++ { - if err := <-errs; err != nil { - any = true - t.Log(err) - } - } - if any { - t.Fail() - } -} diff --git a/graphdb/sort.go b/graphdb/sort.go deleted file mode 100644 index c07df07..0000000 --- a/graphdb/sort.go +++ /dev/null @@ -1,27 +0,0 @@ -package graphdb - -import "sort" - -type pathSorter struct { - paths []string - by func(i, j string) bool -} - -func sortByDepth(paths []string) { - s := &pathSorter{paths, func(i, j string) bool { - return PathDepth(i) > PathDepth(j) - }} - sort.Sort(s) -} - -func (s *pathSorter) Len() int { - return len(s.paths) -} - -func (s *pathSorter) Swap(i, j int) { - s.paths[i], s.paths[j] = s.paths[j], s.paths[i] -} - -func (s *pathSorter) Less(i, j int) bool { - return s.by(s.paths[i], s.paths[j]) -} diff --git a/graphdb/sort_test.go b/graphdb/sort_test.go deleted file mode 100644 index ddf2266..0000000 --- a/graphdb/sort_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package graphdb - -import ( - "testing" -) - -func TestSort(t *testing.T) { - paths := []string{ - "/", - "/myreallylongname", - "/app/db", - } - - sortByDepth(paths) - - if len(paths) != 3 { - t.Fatalf("Expected 3 parts got %d", len(paths)) - } - - if paths[0] != "/app/db" { - t.Fatalf("Expected /app/db got %s", paths[0]) - } - if paths[1] != "/myreallylongname" { - t.Fatalf("Expected /myreallylongname got %s", paths[1]) - } - if paths[2] != "/" { - t.Fatalf("Expected / got %s", paths[2]) - } -} diff --git a/graphdb/utils.go b/graphdb/utils.go deleted file mode 100644 index 9edd79c..0000000 --- a/graphdb/utils.go +++ /dev/null @@ -1,32 +0,0 @@ -package graphdb - -import ( - "path" - "strings" -) - -// Split p on / -func split(p string) []string { - return strings.Split(p, "/") -} - -// PathDepth returns the depth or number of / in a given path -func PathDepth(p string) int { - parts := split(p) - if len(parts) == 2 && parts[1] == "" { - return 1 - } - return len(parts) -} - -func splitPath(p string) (parent, name string) { - if p[0] != '/' { - p = "/" + p - } - parent, name = path.Split(p) - l := len(parent) - if parent[l-1] == '/' { - parent = parent[:l-1] - } - return -} diff --git a/httputils/httputils.go b/httputils/httputils.go deleted file mode 100644 index d7dc438..0000000 --- a/httputils/httputils.go +++ /dev/null @@ -1,56 +0,0 @@ -package httputils - -import ( - "errors" - "fmt" - "net/http" - "regexp" - "strings" - - "github.com/docker/docker/pkg/jsonmessage" -) - -var ( - headerRegexp = regexp.MustCompile(`^(?:(.+)/(.+?))\((.+)\).*$`) - errInvalidHeader = errors.New("Bad header, should be in format `docker/version (platform)`") -) - -// Download requests a given URL and returns an io.Reader. -func Download(url string) (resp *http.Response, err error) { - if resp, err = http.Get(url); err != nil { - return nil, err - } - if resp.StatusCode >= 400 { - return nil, fmt.Errorf("Got HTTP status code >= 400: %s", resp.Status) - } - return resp, nil -} - -// NewHTTPRequestError returns a JSON response error. -func NewHTTPRequestError(msg string, res *http.Response) error { - return &jsonmessage.JSONError{ - Message: msg, - Code: res.StatusCode, - } -} - -// ServerHeader contains the server information. -type ServerHeader struct { - App string // docker - Ver string // 1.8.0-dev - OS string // windows or linux -} - -// ParseServerHeader extracts pieces from an HTTP server header -// which is in the format "docker/version (os)" eg docker/1.8.0-dev (windows). -func ParseServerHeader(hdr string) (*ServerHeader, error) { - matches := headerRegexp.FindStringSubmatch(hdr) - if len(matches) != 4 { - return nil, errInvalidHeader - } - return &ServerHeader{ - App: strings.TrimSpace(matches[1]), - Ver: strings.TrimSpace(matches[2]), - OS: strings.TrimSpace(matches[3]), - }, nil -} diff --git a/httputils/httputils_test.go b/httputils/httputils_test.go deleted file mode 100644 index d35d082..0000000 --- a/httputils/httputils_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package httputils - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestDownload(t *testing.T) { - expected := "Hello, docker !" - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, expected) - })) - defer ts.Close() - response, err := Download(ts.URL) - if err != nil { - t.Fatal(err) - } - - actual, err := ioutil.ReadAll(response.Body) - response.Body.Close() - - if err != nil || string(actual) != expected { - t.Fatalf("Expected the response %q, got err:%v, response:%v, actual:%s", expected, err, response, string(actual)) - } -} - -func TestDownload400Errors(t *testing.T) { - expectedError := "Got HTTP status code >= 400: 403 Forbidden" - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // 403 - http.Error(w, "something failed (forbidden)", http.StatusForbidden) - })) - defer ts.Close() - // Expected status code = 403 - if _, err := Download(ts.URL); err == nil || err.Error() != expectedError { - t.Fatalf("Expected the the error %q, got %v", expectedError, err) - } -} - -func TestDownloadOtherErrors(t *testing.T) { - if _, err := Download("I'm not an url.."); err == nil || !strings.Contains(err.Error(), "unsupported protocol scheme") { - t.Fatalf("Expected an error with 'unsupported protocol scheme', got %v", err) - } -} - -func TestNewHTTPRequestError(t *testing.T) { - errorMessage := "Some error message" - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // 403 - http.Error(w, errorMessage, http.StatusForbidden) - })) - defer ts.Close() - httpResponse, err := http.Get(ts.URL) - if err != nil { - t.Fatal(err) - } - if err := NewHTTPRequestError(errorMessage, httpResponse); err.Error() != errorMessage { - t.Fatalf("Expected err to be %q, got %v", errorMessage, err) - } -} - -func TestParseServerHeader(t *testing.T) { - inputs := map[string][]string{ - "bad header": {"error"}, - "(bad header)": {"error"}, - "(without/spaces)": {"error"}, - "(header/with spaces)": {"error"}, - "foo/bar (baz)": {"foo", "bar", "baz"}, - "foo/bar": {"error"}, - "foo": {"error"}, - "foo/bar (baz space)": {"foo", "bar", "baz space"}, - " f f / b b ( b s ) ": {"f f", "b b", "b s"}, - "foo/bar (baz) ignore": {"foo", "bar", "baz"}, - "foo/bar ()": {"error"}, - "foo/bar()": {"error"}, - "foo/bar(baz)": {"foo", "bar", "baz"}, - "foo/bar/zzz(baz)": {"foo/bar", "zzz", "baz"}, - "foo/bar(baz/abc)": {"foo", "bar", "baz/abc"}, - "foo/bar(baz (abc))": {"foo", "bar", "baz (abc)"}, - } - - for header, values := range inputs { - serverHeader, err := ParseServerHeader(header) - if err != nil { - if err != errInvalidHeader { - t.Fatalf("Failed to parse %q, and got some unexpected error: %q", header, err) - } - if values[0] == "error" { - continue - } - t.Fatalf("Header %q failed to parse when it shouldn't have", header) - } - if values[0] == "error" { - t.Fatalf("Header %q parsed ok when it should have failed(%q).", header, serverHeader) - } - - if serverHeader.App != values[0] { - t.Fatalf("Expected serverHeader.App for %q to equal %q, got %q", header, values[0], serverHeader.App) - } - - if serverHeader.Ver != values[1] { - t.Fatalf("Expected serverHeader.Ver for %q to equal %q, got %q", header, values[1], serverHeader.Ver) - } - - if serverHeader.OS != values[2] { - t.Fatalf("Expected serverHeader.OS for %q to equal %q, got %q", header, values[2], serverHeader.OS) - } - - } - -} diff --git a/httputils/mimetype.go b/httputils/mimetype.go deleted file mode 100644 index d5cf34e..0000000 --- a/httputils/mimetype.go +++ /dev/null @@ -1,30 +0,0 @@ -package httputils - -import ( - "mime" - "net/http" -) - -// MimeTypes stores the MIME content type. -var MimeTypes = struct { - TextPlain string - Tar string - OctetStream string -}{"text/plain", "application/tar", "application/octet-stream"} - -// DetectContentType returns a best guess representation of the MIME -// content type for the bytes at c. The value detected by -// http.DetectContentType is guaranteed not be nil, defaulting to -// application/octet-stream when a better guess cannot be made. The -// result of this detection is then run through mime.ParseMediaType() -// which separates the actual MIME string from any parameters. -func DetectContentType(c []byte) (string, map[string]string, error) { - - ct := http.DetectContentType(c) - contentType, args, err := mime.ParseMediaType(ct) - if err != nil { - return "", nil, err - } - - return contentType, args, nil -} diff --git a/httputils/mimetype_test.go b/httputils/mimetype_test.go deleted file mode 100644 index 9de433e..0000000 --- a/httputils/mimetype_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package httputils - -import ( - "testing" -) - -func TestDetectContentType(t *testing.T) { - input := []byte("That is just a plain text") - - if contentType, _, err := DetectContentType(input); err != nil || contentType != "text/plain" { - t.Errorf("TestDetectContentType failed") - } -} diff --git a/httputils/resumablerequestreader.go b/httputils/resumablerequestreader.go deleted file mode 100644 index bebc860..0000000 --- a/httputils/resumablerequestreader.go +++ /dev/null @@ -1,95 +0,0 @@ -package httputils - -import ( - "fmt" - "io" - "net/http" - "time" - - "github.com/Sirupsen/logrus" -) - -type resumableRequestReader struct { - client *http.Client - request *http.Request - lastRange int64 - totalSize int64 - currentResponse *http.Response - failures uint32 - maxFailures uint32 -} - -// ResumableRequestReader makes it possible to resume reading a request's body transparently -// maxfail is the number of times we retry to make requests again (not resumes) -// totalsize is the total length of the body; auto detect if not provided -func ResumableRequestReader(c *http.Client, r *http.Request, maxfail uint32, totalsize int64) io.ReadCloser { - return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize} -} - -// ResumableRequestReaderWithInitialResponse makes it possible to resume -// reading the body of an already initiated request. -func ResumableRequestReaderWithInitialResponse(c *http.Client, r *http.Request, maxfail uint32, totalsize int64, initialResponse *http.Response) io.ReadCloser { - return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize, currentResponse: initialResponse} -} - -func (r *resumableRequestReader) Read(p []byte) (n int, err error) { - if r.client == nil || r.request == nil { - return 0, fmt.Errorf("client and request can't be nil\n") - } - isFreshRequest := false - if r.lastRange != 0 && r.currentResponse == nil { - readRange := fmt.Sprintf("bytes=%d-%d", r.lastRange, r.totalSize) - r.request.Header.Set("Range", readRange) - time.Sleep(5 * time.Second) - } - if r.currentResponse == nil { - r.currentResponse, err = r.client.Do(r.request) - isFreshRequest = true - } - if err != nil && r.failures+1 != r.maxFailures { - r.cleanUpResponse() - r.failures++ - time.Sleep(5 * time.Duration(r.failures) * time.Second) - return 0, nil - } else if err != nil { - r.cleanUpResponse() - return 0, err - } - if r.currentResponse.StatusCode == 416 && r.lastRange == r.totalSize && r.currentResponse.ContentLength == 0 { - r.cleanUpResponse() - return 0, io.EOF - } else if r.currentResponse.StatusCode != 206 && r.lastRange != 0 && isFreshRequest { - r.cleanUpResponse() - return 0, fmt.Errorf("the server doesn't support byte ranges") - } - if r.totalSize == 0 { - r.totalSize = r.currentResponse.ContentLength - } else if r.totalSize <= 0 { - r.cleanUpResponse() - return 0, fmt.Errorf("failed to auto detect content length") - } - n, err = r.currentResponse.Body.Read(p) - r.lastRange += int64(n) - if err != nil { - r.cleanUpResponse() - } - if err != nil && err != io.EOF { - logrus.Infof("encountered error during pull and clearing it before resume: %s", err) - err = nil - } - return n, err -} - -func (r *resumableRequestReader) Close() error { - r.cleanUpResponse() - r.client = nil - r.request = nil - return nil -} - -func (r *resumableRequestReader) cleanUpResponse() { - if r.currentResponse != nil { - r.currentResponse.Body.Close() - r.currentResponse = nil - } -} diff --git a/httputils/resumablerequestreader_test.go b/httputils/resumablerequestreader_test.go deleted file mode 100644 index 5a2906d..0000000 --- a/httputils/resumablerequestreader_test.go +++ /dev/null @@ -1,307 +0,0 @@ -package httputils - -import ( - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestResumableRequestHeaderSimpleErrors(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello, world !") - })) - defer ts.Close() - - client := &http.Client{} - - var req *http.Request - req, err := http.NewRequest("GET", ts.URL, nil) - if err != nil { - t.Fatal(err) - } - - expectedError := "client and request can't be nil\n" - resreq := &resumableRequestReader{} - _, err = resreq.Read([]byte{}) - if err == nil || err.Error() != expectedError { - t.Fatalf("Expected an error with '%s', got %v.", expectedError, err) - } - - resreq = &resumableRequestReader{ - client: client, - request: req, - totalSize: -1, - } - expectedError = "failed to auto detect content length" - _, err = resreq.Read([]byte{}) - if err == nil || err.Error() != expectedError { - t.Fatalf("Expected an error with '%s', got %v.", expectedError, err) - } - -} - -// Not too much failures, bails out after some wait -func TestResumableRequestHeaderNotTooMuchFailures(t *testing.T) { - client := &http.Client{} - - var badReq *http.Request - badReq, err := http.NewRequest("GET", "I'm not an url", nil) - if err != nil { - t.Fatal(err) - } - - resreq := &resumableRequestReader{ - client: client, - request: badReq, - failures: 0, - maxFailures: 2, - } - read, err := resreq.Read([]byte{}) - if err != nil || read != 0 { - t.Fatalf("Expected no error and no byte read, got err:%v, read:%v.", err, read) - } -} - -// Too much failures, returns the error -func TestResumableRequestHeaderTooMuchFailures(t *testing.T) { - client := &http.Client{} - - var badReq *http.Request - badReq, err := http.NewRequest("GET", "I'm not an url", nil) - if err != nil { - t.Fatal(err) - } - - resreq := &resumableRequestReader{ - client: client, - request: badReq, - failures: 0, - maxFailures: 1, - } - defer resreq.Close() - - expectedError := `Get I%27m%20not%20an%20url: unsupported protocol scheme ""` - read, err := resreq.Read([]byte{}) - if err == nil || err.Error() != expectedError || read != 0 { - t.Fatalf("Expected the error '%s', got err:%v, read:%v.", expectedError, err, read) - } -} - -type errorReaderCloser struct{} - -func (errorReaderCloser) Close() error { return nil } - -func (errorReaderCloser) Read(p []byte) (n int, err error) { - return 0, fmt.Errorf("An error occurred") -} - -// If an unknown error is encountered, return 0, nil and log it -func TestResumableRequestReaderWithReadError(t *testing.T) { - var req *http.Request - req, err := http.NewRequest("GET", "", nil) - if err != nil { - t.Fatal(err) - } - - client := &http.Client{} - - response := &http.Response{ - Status: "500 Internal Server", - StatusCode: 500, - ContentLength: 0, - Close: true, - Body: errorReaderCloser{}, - } - - resreq := &resumableRequestReader{ - client: client, - request: req, - currentResponse: response, - lastRange: 1, - totalSize: 1, - } - defer resreq.Close() - - buf := make([]byte, 1) - read, err := resreq.Read(buf) - if err != nil { - t.Fatal(err) - } - - if read != 0 { - t.Fatalf("Expected to have read nothing, but read %v", read) - } -} - -func TestResumableRequestReaderWithEOFWith416Response(t *testing.T) { - var req *http.Request - req, err := http.NewRequest("GET", "", nil) - if err != nil { - t.Fatal(err) - } - - client := &http.Client{} - - response := &http.Response{ - Status: "416 Requested Range Not Satisfiable", - StatusCode: 416, - ContentLength: 0, - Close: true, - Body: ioutil.NopCloser(strings.NewReader("")), - } - - resreq := &resumableRequestReader{ - client: client, - request: req, - currentResponse: response, - lastRange: 1, - totalSize: 1, - } - defer resreq.Close() - - buf := make([]byte, 1) - _, err = resreq.Read(buf) - if err == nil || err != io.EOF { - t.Fatalf("Expected an io.EOF error, got %v", err) - } -} - -func TestResumableRequestReaderWithServerDoesntSupportByteRanges(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Range") == "" { - t.Fatalf("Expected a Range HTTP header, got nothing") - } - })) - defer ts.Close() - - var req *http.Request - req, err := http.NewRequest("GET", ts.URL, nil) - if err != nil { - t.Fatal(err) - } - - client := &http.Client{} - - resreq := &resumableRequestReader{ - client: client, - request: req, - lastRange: 1, - } - defer resreq.Close() - - buf := make([]byte, 2) - _, err = resreq.Read(buf) - if err == nil || err.Error() != "the server doesn't support byte ranges" { - t.Fatalf("Expected an error 'the server doesn't support byte ranges', got %v", err) - } -} - -func TestResumableRequestReaderWithZeroTotalSize(t *testing.T) { - - srvtxt := "some response text data" - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, srvtxt) - })) - defer ts.Close() - - var req *http.Request - req, err := http.NewRequest("GET", ts.URL, nil) - if err != nil { - t.Fatal(err) - } - - client := &http.Client{} - retries := uint32(5) - - resreq := ResumableRequestReader(client, req, retries, 0) - defer resreq.Close() - - data, err := ioutil.ReadAll(resreq) - if err != nil { - t.Fatal(err) - } - - resstr := strings.TrimSuffix(string(data), "\n") - - if resstr != srvtxt { - t.Errorf("resstr != srvtxt") - } -} - -func TestResumableRequestReader(t *testing.T) { - - srvtxt := "some response text data" - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, srvtxt) - })) - defer ts.Close() - - var req *http.Request - req, err := http.NewRequest("GET", ts.URL, nil) - if err != nil { - t.Fatal(err) - } - - client := &http.Client{} - retries := uint32(5) - imgSize := int64(len(srvtxt)) - - resreq := ResumableRequestReader(client, req, retries, imgSize) - defer resreq.Close() - - data, err := ioutil.ReadAll(resreq) - if err != nil { - t.Fatal(err) - } - - resstr := strings.TrimSuffix(string(data), "\n") - - if resstr != srvtxt { - t.Errorf("resstr != srvtxt") - } -} - -func TestResumableRequestReaderWithInitialResponse(t *testing.T) { - - srvtxt := "some response text data" - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, srvtxt) - })) - defer ts.Close() - - var req *http.Request - req, err := http.NewRequest("GET", ts.URL, nil) - if err != nil { - t.Fatal(err) - } - - client := &http.Client{} - retries := uint32(5) - imgSize := int64(len(srvtxt)) - - res, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - - resreq := ResumableRequestReaderWithInitialResponse(client, req, retries, imgSize, res) - defer resreq.Close() - - data, err := ioutil.ReadAll(resreq) - if err != nil { - t.Fatal(err) - } - - resstr := strings.TrimSuffix(string(data), "\n") - - if resstr != srvtxt { - t.Errorf("resstr != srvtxt") - } -} diff --git a/jsonlog/jsonlog.go b/jsonlog/jsonlog.go deleted file mode 100644 index 4734c31..0000000 --- a/jsonlog/jsonlog.go +++ /dev/null @@ -1,42 +0,0 @@ -package jsonlog - -import ( - "encoding/json" - "fmt" - "time" -) - -// JSONLog represents a log message, typically a single entry from a given log stream. -// JSONLogs can be easily serialized to and from JSON and support custom formatting. -type JSONLog struct { - // Log is the log message - Log string `json:"log,omitempty"` - // Stream is the log source - Stream string `json:"stream,omitempty"` - // Created is the created timestamp of log - Created time.Time `json:"time"` - // Attrs is the list of extra attributes provided by the user - Attrs map[string]string `json:"attrs,omitempty"` -} - -// Format returns the log formatted according to format -// If format is nil, returns the log message -// If format is json, returns the log marshaled in json format -// By default, returns the log with the log time formatted according to format. -func (jl *JSONLog) Format(format string) (string, error) { - if format == "" { - return jl.Log, nil - } - if format == "json" { - m, err := json.Marshal(jl) - return string(m), err - } - return fmt.Sprintf("%s %s", jl.Created.Format(format), jl.Log), nil -} - -// Reset resets the log to nil. -func (jl *JSONLog) Reset() { - jl.Log = "" - jl.Stream = "" - jl.Created = time.Time{} -} diff --git a/jsonlog/jsonlog_marshalling.go b/jsonlog/jsonlog_marshalling.go deleted file mode 100644 index 83ce684..0000000 --- a/jsonlog/jsonlog_marshalling.go +++ /dev/null @@ -1,178 +0,0 @@ -// This code was initially generated by ffjson -// This code was generated via the following steps: -// $ go get -u github.com/pquerna/ffjson -// $ make BIND_DIR=. shell -// $ ffjson pkg/jsonlog/jsonlog.go -// $ mv pkg/jsonglog/jsonlog_ffjson.go pkg/jsonlog/jsonlog_marshalling.go -// -// It has been modified to improve the performance of time marshalling to JSON -// and to clean it up. -// Should this code need to be regenerated when the JSONLog struct is changed, -// the relevant changes which have been made are: -// import ( -// "bytes" -//- -// "unicode/utf8" -// ) -// -// func (mj *JSONLog) MarshalJSON() ([]byte, error) { -//@@ -20,13 +16,13 @@ func (mj *JSONLog) MarshalJSON() ([]byte, error) { -// } -// return buf.Bytes(), nil -// } -//+ -// func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { -//- var err error -//- var obj []byte -//- var first bool = true -//- _ = obj -//- _ = err -//- _ = first -//+ var ( -//+ err error -//+ timestamp string -//+ first bool = true -//+ ) -// buf.WriteString(`{`) -// if len(mj.Log) != 0 { -// if first == true { -//@@ -52,11 +48,11 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { -// buf.WriteString(`,`) -// } -// buf.WriteString(`"time":`) -//- obj, err = mj.Created.MarshalJSON() -//+ timestamp, err = FastTimeMarshalJSON(mj.Created) -// if err != nil { -// return err -// } -//- buf.Write(obj) -//+ buf.WriteString(timestamp) -// buf.WriteString(`}`) -// return nil -// } -// @@ -81,9 +81,10 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { -// if len(mj.Log) != 0 { -// - if first == true { -// - first = false -// - } else { -// - buf.WriteString(`,`) -// - } -// + first = false -// buf.WriteString(`"log":`) -// ffjsonWriteJSONString(buf, mj.Log) -// } - -package jsonlog - -import ( - "bytes" - "unicode/utf8" -) - -// MarshalJSON marshals the JSONLog. -func (mj *JSONLog) MarshalJSON() ([]byte, error) { - var buf bytes.Buffer - buf.Grow(1024) - if err := mj.MarshalJSONBuf(&buf); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// MarshalJSONBuf marshals the JSONLog and stores the result to a bytes.Buffer. -func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error { - var ( - err error - timestamp string - first = true - ) - buf.WriteString(`{`) - if len(mj.Log) != 0 { - first = false - buf.WriteString(`"log":`) - ffjsonWriteJSONString(buf, mj.Log) - } - if len(mj.Stream) != 0 { - if first { - first = false - } else { - buf.WriteString(`,`) - } - buf.WriteString(`"stream":`) - ffjsonWriteJSONString(buf, mj.Stream) - } - if !first { - buf.WriteString(`,`) - } - buf.WriteString(`"time":`) - timestamp, err = FastTimeMarshalJSON(mj.Created) - if err != nil { - return err - } - buf.WriteString(timestamp) - buf.WriteString(`}`) - return nil -} - -func ffjsonWriteJSONString(buf *bytes.Buffer, s string) { - const hex = "0123456789abcdef" - - buf.WriteByte('"') - start := 0 - for i := 0; i < len(s); { - if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { - i++ - continue - } - if start < i { - buf.WriteString(s[start:i]) - } - switch b { - case '\\', '"': - buf.WriteByte('\\') - buf.WriteByte(b) - case '\n': - buf.WriteByte('\\') - buf.WriteByte('n') - case '\r': - buf.WriteByte('\\') - buf.WriteByte('r') - default: - - buf.WriteString(`\u00`) - buf.WriteByte(hex[b>>4]) - buf.WriteByte(hex[b&0xF]) - } - i++ - start = i - continue - } - c, size := utf8.DecodeRuneInString(s[i:]) - if c == utf8.RuneError && size == 1 { - if start < i { - buf.WriteString(s[start:i]) - } - buf.WriteString(`\ufffd`) - i += size - start = i - continue - } - - if c == '\u2028' || c == '\u2029' { - if start < i { - buf.WriteString(s[start:i]) - } - buf.WriteString(`\u202`) - buf.WriteByte(hex[c&0xF]) - i += size - start = i - continue - } - i += size - } - if start < len(s) { - buf.WriteString(s[start:]) - } - buf.WriteByte('"') -} diff --git a/jsonlog/jsonlog_marshalling_test.go b/jsonlog/jsonlog_marshalling_test.go deleted file mode 100644 index 3edb271..0000000 --- a/jsonlog/jsonlog_marshalling_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package jsonlog - -import ( - "regexp" - "testing" -) - -func TestJSONLogMarshalJSON(t *testing.T) { - logs := map[*JSONLog]string{ - &JSONLog{Log: `"A log line with \\"`}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line"}: `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line with \r"}: `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line with & < >"}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`, - &JSONLog{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`, - &JSONLog{}: `^{\"time\":\".{20,}\"}$`, - // These ones are a little weird - &JSONLog{Log: "\u2028 \u2029"}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`, - } - for jsonLog, expression := range logs { - data, err := jsonLog.MarshalJSON() - if err != nil { - t.Fatal(err) - } - res := string(data) - t.Logf("Result of WriteLog: %q", res) - logRe := regexp.MustCompile(expression) - if !logRe.MatchString(res) { - t.Fatalf("Log line not in expected format [%v]: %q", expression, res) - } - } -} diff --git a/jsonlog/jsonlogbytes.go b/jsonlog/jsonlogbytes.go deleted file mode 100644 index df522c0..0000000 --- a/jsonlog/jsonlogbytes.go +++ /dev/null @@ -1,122 +0,0 @@ -package jsonlog - -import ( - "bytes" - "encoding/json" - "unicode/utf8" -) - -// JSONLogs is based on JSONLog. -// It allows marshalling JSONLog from Log as []byte -// and an already marshalled Created timestamp. -type JSONLogs struct { - Log []byte `json:"log,omitempty"` - Stream string `json:"stream,omitempty"` - Created string `json:"time"` - - // json-encoded bytes - RawAttrs json.RawMessage `json:"attrs,omitempty"` -} - -// MarshalJSONBuf is based on the same method from JSONLog -// It has been modified to take into account the necessary changes. -func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { - var first = true - - buf.WriteString(`{`) - if len(mj.Log) != 0 { - first = false - buf.WriteString(`"log":`) - ffjsonWriteJSONBytesAsString(buf, mj.Log) - } - if len(mj.Stream) != 0 { - if first == true { - first = false - } else { - buf.WriteString(`,`) - } - buf.WriteString(`"stream":`) - ffjsonWriteJSONString(buf, mj.Stream) - } - if len(mj.RawAttrs) > 0 { - if first { - first = false - } else { - buf.WriteString(`,`) - } - buf.WriteString(`"attrs":`) - buf.Write(mj.RawAttrs) - } - if !first { - buf.WriteString(`,`) - } - buf.WriteString(`"time":`) - buf.WriteString(mj.Created) - buf.WriteString(`}`) - return nil -} - -// This is based on ffjsonWriteJSONBytesAsString. It has been changed -// to accept a string passed as a slice of bytes. -func ffjsonWriteJSONBytesAsString(buf *bytes.Buffer, s []byte) { - const hex = "0123456789abcdef" - - buf.WriteByte('"') - start := 0 - for i := 0; i < len(s); { - if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { - i++ - continue - } - if start < i { - buf.Write(s[start:i]) - } - switch b { - case '\\', '"': - buf.WriteByte('\\') - buf.WriteByte(b) - case '\n': - buf.WriteByte('\\') - buf.WriteByte('n') - case '\r': - buf.WriteByte('\\') - buf.WriteByte('r') - default: - - buf.WriteString(`\u00`) - buf.WriteByte(hex[b>>4]) - buf.WriteByte(hex[b&0xF]) - } - i++ - start = i - continue - } - c, size := utf8.DecodeRune(s[i:]) - if c == utf8.RuneError && size == 1 { - if start < i { - buf.Write(s[start:i]) - } - buf.WriteString(`\ufffd`) - i += size - start = i - continue - } - - if c == '\u2028' || c == '\u2029' { - if start < i { - buf.Write(s[start:i]) - } - buf.WriteString(`\u202`) - buf.WriteByte(hex[c&0xF]) - i += size - start = i - continue - } - i += size - } - if start < len(s) { - buf.Write(s[start:]) - } - buf.WriteByte('"') -} diff --git a/jsonlog/jsonlogbytes_test.go b/jsonlog/jsonlogbytes_test.go deleted file mode 100644 index 6d6ad21..0000000 --- a/jsonlog/jsonlogbytes_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package jsonlog - -import ( - "bytes" - "regexp" - "testing" -) - -func TestJSONLogsMarshalJSONBuf(t *testing.T) { - logs := map[*JSONLogs]string{ - &JSONLogs{Log: []byte(`"A log line with \\"`)}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line with \r")}: `^{\"log\":\"A log line with \\r\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line with & < >")}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line with utf8 : 🚀 ψ ω β")}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":}$`, - &JSONLogs{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":}$`, - &JSONLogs{Stream: "stdout", Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":}$`, - &JSONLogs{Created: "time"}: `^{\"time\":time}$`, - &JSONLogs{}: `^{\"time\":}$`, - // These ones are a little weird - &JSONLogs{Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`, - &JSONLogs{Log: []byte{0xaF}}: `^{\"log\":\"\\ufffd\",\"time\":}$`, - &JSONLogs{Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":}$`, - // with raw attributes - &JSONLogs{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`, - } - for jsonLog, expression := range logs { - var buf bytes.Buffer - if err := jsonLog.MarshalJSONBuf(&buf); err != nil { - t.Fatal(err) - } - res := buf.String() - t.Logf("Result of WriteLog: %q", res) - logRe := regexp.MustCompile(expression) - if !logRe.MatchString(res) { - t.Fatalf("Log line not in expected format [%v]: %q", expression, res) - } - } -} diff --git a/jsonlog/time_marshalling.go b/jsonlog/time_marshalling.go deleted file mode 100644 index 2117338..0000000 --- a/jsonlog/time_marshalling.go +++ /dev/null @@ -1,27 +0,0 @@ -// Package jsonlog provides helper functions to parse and print time (time.Time) as JSON. -package jsonlog - -import ( - "errors" - "time" -) - -const ( - // RFC3339NanoFixed is our own version of RFC339Nano because we want one - // that pads the nano seconds part with zeros to ensure - // the timestamps are aligned in the logs. - RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" - // JSONFormat is the format used by FastMarshalJSON - JSONFormat = `"` + time.RFC3339Nano + `"` -) - -// FastTimeMarshalJSON avoids one of the extra allocations that -// time.MarshalJSON is making. -func FastTimeMarshalJSON(t time.Time) (string, error) { - if y := t.Year(); y < 0 || y >= 10000 { - // RFC 3339 is clear that years are 4 digits exactly. - // See golang.org/issue/4556#c15 for more discussion. - return "", errors.New("time.MarshalJSON: year outside of range [0,9999]") - } - return t.Format(JSONFormat), nil -} diff --git a/jsonlog/time_marshalling_test.go b/jsonlog/time_marshalling_test.go deleted file mode 100644 index 02d0302..0000000 --- a/jsonlog/time_marshalling_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package jsonlog - -import ( - "testing" - "time" -) - -// Testing to ensure 'year' fields is between 0 and 9999 -func TestFastTimeMarshalJSONWithInvalidDate(t *testing.T) { - aTime := time.Date(-1, 1, 1, 0, 0, 0, 0, time.Local) - json, err := FastTimeMarshalJSON(aTime) - if err == nil { - t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json) - } - anotherTime := time.Date(10000, 1, 1, 0, 0, 0, 0, time.Local) - json, err = FastTimeMarshalJSON(anotherTime) - if err == nil { - t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json) - } - -} - -func TestFastTimeMarshalJSON(t *testing.T) { - aTime := time.Date(2015, 5, 29, 11, 1, 2, 3, time.UTC) - json, err := FastTimeMarshalJSON(aTime) - if err != nil { - t.Fatal(err) - } - expected := "\"2015-05-29T11:01:02.000000003Z\"" - if json != expected { - t.Fatalf("Expected %v, got %v", expected, json) - } - - location, err := time.LoadLocation("Europe/Paris") - if err != nil { - t.Fatal(err) - } - aTime = time.Date(2015, 5, 29, 11, 1, 2, 3, location) - json, err = FastTimeMarshalJSON(aTime) - if err != nil { - t.Fatal(err) - } - expected = "\"2015-05-29T11:01:02.000000003+02:00\"" - if json != expected { - t.Fatalf("Expected %v, got %v", expected, json) - } -} diff --git a/listeners/listeners_solaris.go b/listeners/listeners_solaris.go deleted file mode 100644 index ff833e3..0000000 --- a/listeners/listeners_solaris.go +++ /dev/null @@ -1,31 +0,0 @@ -package listeners - -import ( - "crypto/tls" - "fmt" - "net" - - "github.com/docker/go-connections/sockets" -) - -// Init creates new listeners for the server. -func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) (ls []net.Listener, err error) { - switch proto { - case "tcp": - l, err := sockets.NewTCPSocket(addr, tlsConfig) - if err != nil { - return nil, err - } - ls = append(ls, l) - case "unix": - l, err := sockets.NewUnixSocket(addr, socketGroup) - if err != nil { - return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err) - } - ls = append(ls, l) - default: - return nil, fmt.Errorf("Invalid protocol format: %q", proto) - } - - return -} diff --git a/listeners/listeners_unix.go b/listeners/listeners_unix.go deleted file mode 100644 index 1bcae7a..0000000 --- a/listeners/listeners_unix.go +++ /dev/null @@ -1,94 +0,0 @@ -// +build !windows,!solaris - -package listeners - -import ( - "crypto/tls" - "fmt" - "net" - "strconv" - - "github.com/Sirupsen/logrus" - "github.com/coreos/go-systemd/activation" - "github.com/docker/go-connections/sockets" -) - -// Init creates new listeners for the server. -// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used. -func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) { - ls := []net.Listener{} - - switch proto { - case "fd": - fds, err := listenFD(addr, tlsConfig) - if err != nil { - return nil, err - } - ls = append(ls, fds...) - case "tcp": - l, err := sockets.NewTCPSocket(addr, tlsConfig) - if err != nil { - return nil, err - } - ls = append(ls, l) - case "unix": - l, err := sockets.NewUnixSocket(addr, socketGroup) - if err != nil { - return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err) - } - ls = append(ls, l) - default: - return nil, fmt.Errorf("invalid protocol format: %q", proto) - } - - return ls, nil -} - -// listenFD returns the specified socket activated files as a slice of -// net.Listeners or all of the activated files if "*" is given. -func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) { - var ( - err error - listeners []net.Listener - ) - // socket activation - if tlsConfig != nil { - listeners, err = activation.TLSListeners(false, tlsConfig) - } else { - listeners, err = activation.Listeners(false) - } - if err != nil { - return nil, err - } - - if len(listeners) == 0 { - return nil, fmt.Errorf("no sockets found via socket activation: make sure the service was started by systemd") - } - - // default to all fds just like unix:// and tcp:// - if addr == "" || addr == "*" { - return listeners, nil - } - - fdNum, err := strconv.Atoi(addr) - if err != nil { - return nil, fmt.Errorf("failed to parse systemd fd address: should be a number: %v", addr) - } - fdOffset := fdNum - 3 - if len(listeners) < int(fdOffset)+1 { - return nil, fmt.Errorf("too few socket activated files passed in by systemd") - } - if listeners[fdOffset] == nil { - return nil, fmt.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3) - } - for i, ls := range listeners { - if i == fdOffset || ls == nil { - continue - } - if err := ls.Close(); err != nil { - // TODO: We shouldn't log inside a library. Remove this or error out. - logrus.Errorf("failed to close systemd activated file: fd %d: %v", fdOffset+3, err) - } - } - return []net.Listener{listeners[fdOffset]}, nil -} diff --git a/listeners/listeners_windows.go b/listeners/listeners_windows.go deleted file mode 100644 index 5b5a470..0000000 --- a/listeners/listeners_windows.go +++ /dev/null @@ -1,54 +0,0 @@ -package listeners - -import ( - "crypto/tls" - "fmt" - "net" - "strings" - - "github.com/Microsoft/go-winio" - "github.com/docker/go-connections/sockets" -) - -// Init creates new listeners for the server. -func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) { - ls := []net.Listener{} - - switch proto { - case "tcp": - l, err := sockets.NewTCPSocket(addr, tlsConfig) - if err != nil { - return nil, err - } - ls = append(ls, l) - - case "npipe": - // allow Administrators and SYSTEM, plus whatever additional users or groups were specified - sddl := "D:P(A;;GA;;;BA)(A;;GA;;;SY)" - if socketGroup != "" { - for _, g := range strings.Split(socketGroup, ",") { - sid, err := winio.LookupSidByName(g) - if err != nil { - return nil, err - } - sddl += fmt.Sprintf("(A;;GRGW;;;%s)", sid) - } - } - c := winio.PipeConfig{ - SecurityDescriptor: sddl, - MessageMode: true, // Use message mode so that CloseWrite() is supported - InputBufferSize: 65536, // Use 64KB buffers to improve performance - OutputBufferSize: 65536, - } - l, err := winio.ListenPipe(addr, &c) - if err != nil { - return nil, err - } - ls = append(ls, l) - - default: - return nil, fmt.Errorf("invalid protocol format: windows only supports tcp and npipe") - } - - return ls, nil -} diff --git a/locker/README.md b/locker/README.md deleted file mode 100644 index e84a815..0000000 --- a/locker/README.md +++ /dev/null @@ -1,65 +0,0 @@ -Locker -===== - -locker provides a mechanism for creating finer-grained locking to help -free up more global locks to handle other tasks. - -The implementation looks close to a sync.Mutex, however the user must provide a -reference to use to refer to the underlying lock when locking and unlocking, -and unlock may generate an error. - -If a lock with a given name does not exist when `Lock` is called, one is -created. -Lock references are automatically cleaned up on `Unlock` if nothing else is -waiting for the lock. - - -## Usage - -```go -package important - -import ( - "sync" - "time" - - "github.com/docker/docker/pkg/locker" -) - -type important struct { - locks *locker.Locker - data map[string]interface{} - mu sync.Mutex -} - -func (i *important) Get(name string) interface{} { - i.locks.Lock(name) - defer i.locks.Unlock(name) - return data[name] -} - -func (i *important) Create(name string, data interface{}) { - i.locks.Lock(name) - defer i.locks.Unlock(name) - - i.createImportant(data) - - s.mu.Lock() - i.data[name] = data - s.mu.Unlock() -} - -func (i *important) createImportant(data interface{}) { - time.Sleep(10 * time.Second) -} -``` - -For functions dealing with a given name, always lock at the beginning of the -function (or before doing anything with the underlying state), this ensures any -other function that is dealing with the same name will block. - -When needing to modify the underlying data, use the global lock to ensure nothing -else is modfying it at the same time. -Since name lock is already in place, no reads will occur while the modification -is being performed. - diff --git a/locker/locker.go b/locker/locker.go deleted file mode 100644 index 0b22ddf..0000000 --- a/locker/locker.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Package locker provides a mechanism for creating finer-grained locking to help -free up more global locks to handle other tasks. - -The implementation looks close to a sync.Mutex, however the user must provide a -reference to use to refer to the underlying lock when locking and unlocking, -and unlock may generate an error. - -If a lock with a given name does not exist when `Lock` is called, one is -created. -Lock references are automatically cleaned up on `Unlock` if nothing else is -waiting for the lock. -*/ -package locker - -import ( - "errors" - "sync" - "sync/atomic" -) - -// ErrNoSuchLock is returned when the requested lock does not exist -var ErrNoSuchLock = errors.New("no such lock") - -// Locker provides a locking mechanism based on the passed in reference name -type Locker struct { - mu sync.Mutex - locks map[string]*lockCtr -} - -// lockCtr is used by Locker to represent a lock with a given name. -type lockCtr struct { - mu sync.Mutex - // waiters is the number of waiters waiting to acquire the lock - // this is int32 instead of uint32 so we can add `-1` in `dec()` - waiters int32 -} - -// inc increments the number of waiters waiting for the lock -func (l *lockCtr) inc() { - atomic.AddInt32(&l.waiters, 1) -} - -// dec decrements the number of waiters waiting on the lock -func (l *lockCtr) dec() { - atomic.AddInt32(&l.waiters, -1) -} - -// count gets the current number of waiters -func (l *lockCtr) count() int32 { - return atomic.LoadInt32(&l.waiters) -} - -// Lock locks the mutex -func (l *lockCtr) Lock() { - l.mu.Lock() -} - -// Unlock unlocks the mutex -func (l *lockCtr) Unlock() { - l.mu.Unlock() -} - -// New creates a new Locker -func New() *Locker { - return &Locker{ - locks: make(map[string]*lockCtr), - } -} - -// Lock locks a mutex with the given name. If it doesn't exist, one is created -func (l *Locker) Lock(name string) { - l.mu.Lock() - if l.locks == nil { - l.locks = make(map[string]*lockCtr) - } - - nameLock, exists := l.locks[name] - if !exists { - nameLock = &lockCtr{} - l.locks[name] = nameLock - } - - // increment the nameLock waiters while inside the main mutex - // this makes sure that the lock isn't deleted if `Lock` and `Unlock` are called concurrently - nameLock.inc() - l.mu.Unlock() - - // Lock the nameLock outside the main mutex so we don't block other operations - // once locked then we can decrement the number of waiters for this lock - nameLock.Lock() - nameLock.dec() -} - -// Unlock unlocks the mutex with the given name -// If the given lock is not being waited on by any other callers, it is deleted -func (l *Locker) Unlock(name string) error { - l.mu.Lock() - nameLock, exists := l.locks[name] - if !exists { - l.mu.Unlock() - return ErrNoSuchLock - } - - if nameLock.count() == 0 { - delete(l.locks, name) - } - nameLock.Unlock() - - l.mu.Unlock() - return nil -} diff --git a/locker/locker_test.go b/locker/locker_test.go deleted file mode 100644 index 5a297dd..0000000 --- a/locker/locker_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package locker - -import ( - "sync" - "testing" - "time" -) - -func TestLockCounter(t *testing.T) { - l := &lockCtr{} - l.inc() - - if l.waiters != 1 { - t.Fatal("counter inc failed") - } - - l.dec() - if l.waiters != 0 { - t.Fatal("counter dec failed") - } -} - -func TestLockerLock(t *testing.T) { - l := New() - l.Lock("test") - ctr := l.locks["test"] - - if ctr.count() != 0 { - t.Fatalf("expected waiters to be 0, got :%d", ctr.waiters) - } - - chDone := make(chan struct{}) - go func() { - l.Lock("test") - close(chDone) - }() - - chWaiting := make(chan struct{}) - go func() { - for range time.Tick(1 * time.Millisecond) { - if ctr.count() == 1 { - close(chWaiting) - break - } - } - }() - - select { - case <-chWaiting: - case <-time.After(3 * time.Second): - t.Fatal("timed out waiting for lock waiters to be incremented") - } - - select { - case <-chDone: - t.Fatal("lock should not have returned while it was still held") - default: - } - - if err := l.Unlock("test"); err != nil { - t.Fatal(err) - } - - select { - case <-chDone: - case <-time.After(3 * time.Second): - t.Fatalf("lock should have completed") - } - - if ctr.count() != 0 { - t.Fatalf("expected waiters to be 0, got: %d", ctr.count()) - } -} - -func TestLockerUnlock(t *testing.T) { - l := New() - - l.Lock("test") - l.Unlock("test") - - chDone := make(chan struct{}) - go func() { - l.Lock("test") - close(chDone) - }() - - select { - case <-chDone: - case <-time.After(3 * time.Second): - t.Fatalf("lock should not be blocked") - } -} - -func TestLockerConcurrency(t *testing.T) { - l := New() - - var wg sync.WaitGroup - for i := 0; i <= 10000; i++ { - wg.Add(1) - go func() { - l.Lock("test") - // if there is a concurrency issue, will very likely panic here - l.Unlock("test") - wg.Done() - }() - } - - chDone := make(chan struct{}) - go func() { - wg.Wait() - close(chDone) - }() - - select { - case <-chDone: - case <-time.After(10 * time.Second): - t.Fatal("timeout waiting for locks to complete") - } - - // Since everything has unlocked this should not exist anymore - if ctr, exists := l.locks["test"]; exists { - t.Fatalf("lock should not exist: %v", ctr) - } -} diff --git a/namesgenerator/cmd/names-generator/main.go b/namesgenerator/cmd/names-generator/main.go deleted file mode 100644 index 18a939b..0000000 --- a/namesgenerator/cmd/names-generator/main.go +++ /dev/null @@ -1,11 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/docker/docker/pkg/namesgenerator" -) - -func main() { - fmt.Println(namesgenerator.GetRandomName(0)) -} diff --git a/namesgenerator/names-generator.go b/namesgenerator/names-generator.go deleted file mode 100644 index 6109f1b..0000000 --- a/namesgenerator/names-generator.go +++ /dev/null @@ -1,552 +0,0 @@ -package namesgenerator - -import ( - "fmt" - - "github.com/docker/docker/pkg/random" -) - -var ( - left = [...]string{ - "admiring", - "adoring", - "agitated", - "amazing", - "angry", - "awesome", - "backstabbing", - "berserk", - "big", - "boring", - "clever", - "cocky", - "compassionate", - "condescending", - "cranky", - "desperate", - "determined", - "distracted", - "dreamy", - "drunk", - "ecstatic", - "elated", - "elegant", - "evil", - "fervent", - "focused", - "furious", - "gigantic", - "gloomy", - "goofy", - "grave", - "happy", - "high", - "hopeful", - "hungry", - "infallible", - "jolly", - "jovial", - "kickass", - "lonely", - "loving", - "mad", - "modest", - "naughty", - "nauseous", - "nostalgic", - "peaceful", - "pedantic", - "pensive", - "prickly", - "reverent", - "romantic", - "sad", - "serene", - "sharp", - "sick", - "silly", - "sleepy", - "small", - "stoic", - "stupefied", - "suspicious", - "tender", - "thirsty", - "tiny", - "trusting", - "zen", - } - - // Docker, starting from 0.7.x, generates names from notable scientists and hackers. - // Please, for any amazing man that you add to the list, consider adding an equally amazing woman to it, and vice versa. - right = [...]string{ - // Muhammad ibn Jābir al-Ḥarrānī al-Battānī was a founding father of astronomy. https://en.wikipedia.org/wiki/Mu%E1%B8%A5ammad_ibn_J%C4%81bir_al-%E1%B8%A4arr%C4%81n%C4%AB_al-Batt%C4%81n%C4%AB - "albattani", - - // Frances E. Allen, became the first female IBM Fellow in 1989. In 2006, she became the first female recipient of the ACM's Turing Award. https://en.wikipedia.org/wiki/Frances_E._Allen - "allen", - - // June Almeida - Scottish virologist who took the first pictures of the rubella virus - https://en.wikipedia.org/wiki/June_Almeida - "almeida", - - // Maria Gaetana Agnesi - Italian mathematician, philosopher, theologian and humanitarian. She was the first woman to write a mathematics handbook and the first woman appointed as a Mathematics Professor at a University. https://en.wikipedia.org/wiki/Maria_Gaetana_Agnesi - "agnesi", - - // Archimedes was a physicist, engineer and mathematician who invented too many things to list them here. https://en.wikipedia.org/wiki/Archimedes - "archimedes", - - // Maria Ardinghelli - Italian translator, mathematician and physicist - https://en.wikipedia.org/wiki/Maria_Ardinghelli - "ardinghelli", - - // Aryabhata - Ancient Indian mathematician-astronomer during 476-550 CE https://en.wikipedia.org/wiki/Aryabhata - "aryabhata", - - // Wanda Austin - Wanda Austin is the President and CEO of The Aerospace Corporation, a leading architect for the US security space programs. https://en.wikipedia.org/wiki/Wanda_Austin - "austin", - - // Charles Babbage invented the concept of a programmable computer. https://en.wikipedia.org/wiki/Charles_Babbage. - "babbage", - - // Stefan Banach - Polish mathematician, was one of the founders of modern functional analysis. https://en.wikipedia.org/wiki/Stefan_Banach - "banach", - - // John Bardeen co-invented the transistor - https://en.wikipedia.org/wiki/John_Bardeen - "bardeen", - - // Jean Bartik, born Betty Jean Jennings, was one of the original programmers for the ENIAC computer. https://en.wikipedia.org/wiki/Jean_Bartik - "bartik", - - // Laura Bassi, the world's first female professor https://en.wikipedia.org/wiki/Laura_Bassi - "bassi", - - // Alexander Graham Bell - an eminent Scottish-born scientist, inventor, engineer and innovator who is credited with inventing the first practical telephone - https://en.wikipedia.org/wiki/Alexander_Graham_Bell - "bell", - - // Homi J Bhabha - was an Indian nuclear physicist, founding director, and professor of physics at the Tata Institute of Fundamental Research. Colloquially known as "father of Indian nuclear programme"- https://en.wikipedia.org/wiki/Homi_J._Bhabha - "bhabha", - - // Bhaskara II - Ancient Indian mathematician-astronomer whose work on calculus predates Newton and Leibniz by over half a millennium - https://en.wikipedia.org/wiki/Bh%C4%81skara_II#Calculus - "bhaskara", - - // Elizabeth Blackwell - American doctor and first American woman to receive a medical degree - https://en.wikipedia.org/wiki/Elizabeth_Blackwell - "blackwell", - - // Niels Bohr is the father of quantum theory. https://en.wikipedia.org/wiki/Niels_Bohr. - "bohr", - - // Kathleen Booth, she's credited with writing the first assembly language. https://en.wikipedia.org/wiki/Kathleen_Booth - "booth", - - // Anita Borg - Anita Borg was the founding director of the Institute for Women and Technology (IWT). https://en.wikipedia.org/wiki/Anita_Borg - "borg", - - // Satyendra Nath Bose - He provided the foundation for Bose–Einstein statistics and the theory of the Bose–Einstein condensate. - https://en.wikipedia.org/wiki/Satyendra_Nath_Bose - "bose", - - // Evelyn Boyd Granville - She was one of the first African-American woman to receive a Ph.D. in mathematics; she earned it in 1949 from Yale University. https://en.wikipedia.org/wiki/Evelyn_Boyd_Granville - "boyd", - - // Brahmagupta - Ancient Indian mathematician during 598-670 CE who gave rules to compute with zero - https://en.wikipedia.org/wiki/Brahmagupta#Zero - "brahmagupta", - - // Walter Houser Brattain co-invented the transistor - https://en.wikipedia.org/wiki/Walter_Houser_Brattain - "brattain", - - // Emmett Brown invented time travel. https://en.wikipedia.org/wiki/Emmett_Brown (thanks Brian Goff) - "brown", - - // Rachel Carson - American marine biologist and conservationist, her book Silent Spring and other writings are credited with advancing the global environmental movement. https://en.wikipedia.org/wiki/Rachel_Carson - "carson", - - // Subrahmanyan Chandrasekhar - Astrophysicist known for his mathematical theory on different stages and evolution in structures of the stars. He has won nobel prize for physics - https://en.wikipedia.org/wiki/Subrahmanyan_Chandrasekhar - "chandrasekhar", - - //Claude Shannon - The father of information theory and founder of digital circuit design theory. (https://en.wikipedia.org/wiki/Claude_Shannon) - "shannon", - - // Jane Colden - American botanist widely considered the first female American botanist - https://en.wikipedia.org/wiki/Jane_Colden - "colden", - - // Gerty Theresa Cori - American biochemist who became the third woman—and first American woman—to win a Nobel Prize in science, and the first woman to be awarded the Nobel Prize in Physiology or Medicine. Cori was born in Prague. https://en.wikipedia.org/wiki/Gerty_Cori - "cori", - - // Seymour Roger Cray was an American electrical engineer and supercomputer architect who designed a series of computers that were the fastest in the world for decades. https://en.wikipedia.org/wiki/Seymour_Cray - "cray", - - // This entry reflects a husband and wife team who worked together: - // Joan Curran was a Welsh scientist who developed radar and invented chaff, a radar countermeasure. https://en.wikipedia.org/wiki/Joan_Curran - // Samuel Curran was an Irish physicist who worked alongside his wife during WWII and invented the proximity fuse. https://en.wikipedia.org/wiki/Samuel_Curran - "curran", - - // Marie Curie discovered radioactivity. https://en.wikipedia.org/wiki/Marie_Curie. - "curie", - - // Charles Darwin established the principles of natural evolution. https://en.wikipedia.org/wiki/Charles_Darwin. - "darwin", - - // Leonardo Da Vinci invented too many things to list here. https://en.wikipedia.org/wiki/Leonardo_da_Vinci. - "davinci", - - // Edsger Wybe Dijkstra was a Dutch computer scientist and mathematical scientist. https://en.wikipedia.org/wiki/Edsger_W._Dijkstra. - "dijkstra", - - // Donna Dubinsky - played an integral role in the development of personal digital assistants (PDAs) serving as CEO of Palm, Inc. and co-founding Handspring. https://en.wikipedia.org/wiki/Donna_Dubinsky - "dubinsky", - - // Annie Easley - She was a leading member of the team which developed software for the Centaur rocket stage and one of the first African-Americans in her field. https://en.wikipedia.org/wiki/Annie_Easley - "easley", - - // Thomas Alva Edison, prolific inventor https://en.wikipedia.org/wiki/Thomas_Edison - "edison", - - // Albert Einstein invented the general theory of relativity. https://en.wikipedia.org/wiki/Albert_Einstein - "einstein", - - // Gertrude Elion - American biochemist, pharmacologist and the 1988 recipient of the Nobel Prize in Medicine - https://en.wikipedia.org/wiki/Gertrude_Elion - "elion", - - // Douglas Engelbart gave the mother of all demos: https://en.wikipedia.org/wiki/Douglas_Engelbart - "engelbart", - - // Euclid invented geometry. https://en.wikipedia.org/wiki/Euclid - "euclid", - - // Leonhard Euler invented large parts of modern mathematics. https://de.wikipedia.org/wiki/Leonhard_Euler - "euler", - - // Pierre de Fermat pioneered several aspects of modern mathematics. https://en.wikipedia.org/wiki/Pierre_de_Fermat - "fermat", - - // Enrico Fermi invented the first nuclear reactor. https://en.wikipedia.org/wiki/Enrico_Fermi. - "fermi", - - // Richard Feynman was a key contributor to quantum mechanics and particle physics. https://en.wikipedia.org/wiki/Richard_Feynman - "feynman", - - // Benjamin Franklin is famous for his experiments in electricity and the invention of the lightning rod. - "franklin", - - // Galileo was a founding father of modern astronomy, and faced politics and obscurantism to establish scientific truth. https://en.wikipedia.org/wiki/Galileo_Galilei - "galileo", - - // William Henry "Bill" Gates III is an American business magnate, philanthropist, investor, computer programmer, and inventor. https://en.wikipedia.org/wiki/Bill_Gates - "gates", - - // Adele Goldberg, was one of the designers and developers of the Smalltalk language. https://en.wikipedia.org/wiki/Adele_Goldberg_(computer_scientist) - "goldberg", - - // Adele Goldstine, born Adele Katz, wrote the complete technical description for the first electronic digital computer, ENIAC. https://en.wikipedia.org/wiki/Adele_Goldstine - "goldstine", - - // Shafi Goldwasser is a computer scientist known for creating theoretical foundations of modern cryptography. Winner of 2012 ACM Turing Award. https://en.wikipedia.org/wiki/Shafi_Goldwasser - "goldwasser", - - // James Golick, all around gangster. - "golick", - - // Jane Goodall - British primatologist, ethologist, and anthropologist who is considered to be the world's foremost expert on chimpanzees - https://en.wikipedia.org/wiki/Jane_Goodall - "goodall", - - // Margaret Hamilton - Director of the Software Engineering Division of the MIT Instrumentation Laboratory, which developed on-board flight software for the Apollo space program. https://en.wikipedia.org/wiki/Margaret_Hamilton_(scientist) - "hamilton", - - // Stephen Hawking pioneered the field of cosmology by combining general relativity and quantum mechanics. https://en.wikipedia.org/wiki/Stephen_Hawking - "hawking", - - // Werner Heisenberg was a founding father of quantum mechanics. https://en.wikipedia.org/wiki/Werner_Heisenberg - "heisenberg", - - // Jaroslav Heyrovský was the inventor of the polarographic method, father of the electroanalytical method, and recipient of the Nobel Prize in 1959. His main field of work was polarography. https://en.wikipedia.org/wiki/Jaroslav_Heyrovsk%C3%BD - "heyrovsky", - - // Dorothy Hodgkin was a British biochemist, credited with the development of protein crystallography. She was awarded the Nobel Prize in Chemistry in 1964. https://en.wikipedia.org/wiki/Dorothy_Hodgkin - "hodgkin", - - // Erna Schneider Hoover revolutionized modern communication by inventing a computerized telephone switching method. https://en.wikipedia.org/wiki/Erna_Schneider_Hoover - "hoover", - - // Grace Hopper developed the first compiler for a computer programming language and is credited with popularizing the term "debugging" for fixing computer glitches. https://en.wikipedia.org/wiki/Grace_Hopper - "hopper", - - // Frances Hugle, she was an American scientist, engineer, and inventor who contributed to the understanding of semiconductors, integrated circuitry, and the unique electrical principles of microscopic materials. https://en.wikipedia.org/wiki/Frances_Hugle - "hugle", - - // Hypatia - Greek Alexandrine Neoplatonist philosopher in Egypt who was one of the earliest mothers of mathematics - https://en.wikipedia.org/wiki/Hypatia - "hypatia", - - // Yeong-Sil Jang was a Korean scientist and astronomer during the Joseon Dynasty; he invented the first metal printing press and water gauge. https://en.wikipedia.org/wiki/Jang_Yeong-sil - "jang", - - // Betty Jennings - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Jean_Bartik - "jennings", - - // Mary Lou Jepsen, was the founder and chief technology officer of One Laptop Per Child (OLPC), and the founder of Pixel Qi. https://en.wikipedia.org/wiki/Mary_Lou_Jepsen - "jepsen", - - // Irène Joliot-Curie - French scientist who was awarded the Nobel Prize for Chemistry in 1935. Daughter of Marie and Pierre Curie. https://en.wikipedia.org/wiki/Ir%C3%A8ne_Joliot-Curie - "joliot", - - // Karen Spärck Jones came up with the concept of inverse document frequency, which is used in most search engines today. https://en.wikipedia.org/wiki/Karen_Sp%C3%A4rck_Jones - "jones", - - // A. P. J. Abdul Kalam - is an Indian scientist aka Missile Man of India for his work on the development of ballistic missile and launch vehicle technology - https://en.wikipedia.org/wiki/A._P._J._Abdul_Kalam - "kalam", - - // Susan Kare, created the icons and many of the interface elements for the original Apple Macintosh in the 1980s, and was an original employee of NeXT, working as the Creative Director. https://en.wikipedia.org/wiki/Susan_Kare - "kare", - - // Mary Kenneth Keller, Sister Mary Kenneth Keller became the first American woman to earn a PhD in Computer Science in 1965. https://en.wikipedia.org/wiki/Mary_Kenneth_Keller - "keller", - - // Har Gobind Khorana - Indian-American biochemist who shared the 1968 Nobel Prize for Physiology - https://en.wikipedia.org/wiki/Har_Gobind_Khorana - "khorana", - - // Jack Kilby invented silicone integrated circuits and gave Silicon Valley its name. - https://en.wikipedia.org/wiki/Jack_Kilby - "kilby", - - // Maria Kirch - German astronomer and first woman to discover a comet - https://en.wikipedia.org/wiki/Maria_Margarethe_Kirch - "kirch", - - // Donald Knuth - American computer scientist, author of "The Art of Computer Programming" and creator of the TeX typesetting system. https://en.wikipedia.org/wiki/Donald_Knuth - "knuth", - - // Sophie Kowalevski - Russian mathematician responsible for important original contributions to analysis, differential equations and mechanics - https://en.wikipedia.org/wiki/Sofia_Kovalevskaya - "kowalevski", - - // Marie-Jeanne de Lalande - French astronomer, mathematician and cataloguer of stars - https://en.wikipedia.org/wiki/Marie-Jeanne_de_Lalande - "lalande", - - // Hedy Lamarr - Actress and inventor. The principles of her work are now incorporated into modern Wi-Fi, CDMA and Bluetooth technology. https://en.wikipedia.org/wiki/Hedy_Lamarr - "lamarr", - - // Leslie B. Lamport - American computer scientist. Lamport is best known for his seminal work in distributed systems and was the winner of the 2013 Turing Award. https://en.wikipedia.org/wiki/Leslie_Lamport - "lamport", - - // Mary Leakey - British paleoanthropologist who discovered the first fossilized Proconsul skull - https://en.wikipedia.org/wiki/Mary_Leakey - "leakey", - - // Henrietta Swan Leavitt - she was an American astronomer who discovered the relation between the luminosity and the period of Cepheid variable stars. https://en.wikipedia.org/wiki/Henrietta_Swan_Leavitt - "leavitt", - - // Ruth Lichterman - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Ruth_Teitelbaum - "lichterman", - - // Barbara Liskov - co-developed the Liskov substitution principle. Liskov was also the winner of the Turing Prize in 2008. - https://en.wikipedia.org/wiki/Barbara_Liskov - "liskov", - - // Ada Lovelace invented the first algorithm. https://en.wikipedia.org/wiki/Ada_Lovelace (thanks James Turnbull) - "lovelace", - - // Auguste and Louis Lumière - the first filmmakers in history - https://en.wikipedia.org/wiki/Auguste_and_Louis_Lumi%C3%A8re - "lumiere", - - // Mahavira - Ancient Indian mathematician during 9th century AD who discovered basic algebraic identities - https://en.wikipedia.org/wiki/Mah%C4%81v%C4%ABra_(mathematician) - "mahavira", - - // Maria Mayer - American theoretical physicist and Nobel laureate in Physics for proposing the nuclear shell model of the atomic nucleus - https://en.wikipedia.org/wiki/Maria_Mayer - "mayer", - - // John McCarthy invented LISP: https://en.wikipedia.org/wiki/John_McCarthy_(computer_scientist) - "mccarthy", - - // Barbara McClintock - a distinguished American cytogeneticist, 1983 Nobel Laureate in Physiology or Medicine for discovering transposons. https://en.wikipedia.org/wiki/Barbara_McClintock - "mcclintock", - - // Malcolm McLean invented the modern shipping container: https://en.wikipedia.org/wiki/Malcom_McLean - "mclean", - - // Kay McNulty - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Kathleen_Antonelli - "mcnulty", - - // Lise Meitner - Austrian/Swedish physicist who was involved in the discovery of nuclear fission. The element meitnerium is named after her - https://en.wikipedia.org/wiki/Lise_Meitner - "meitner", - - // Carla Meninsky, was the game designer and programmer for Atari 2600 games Dodge 'Em and Warlords. https://en.wikipedia.org/wiki/Carla_Meninsky - "meninsky", - - // Johanna Mestorf - German prehistoric archaeologist and first female museum director in Germany - https://en.wikipedia.org/wiki/Johanna_Mestorf - "mestorf", - - // Marvin Minsky - Pioneer in Artificial Intelligence, co-founder of the MIT's AI Lab, won the Turing Award in 1969. https://en.wikipedia.org/wiki/Marvin_Minsky - "minsky", - - // Maryam Mirzakhani - an Iranian mathematician and the first woman to win the Fields Medal. https://en.wikipedia.org/wiki/Maryam_Mirzakhani - "mirzakhani", - - // Samuel Morse - contributed to the invention of a single-wire telegraph system based on European telegraphs and was a co-developer of the Morse code - https://en.wikipedia.org/wiki/Samuel_Morse - "morse", - - // Ian Murdock - founder of the Debian project - https://en.wikipedia.org/wiki/Ian_Murdock - "murdock", - - // Isaac Newton invented classic mechanics and modern optics. https://en.wikipedia.org/wiki/Isaac_Newton - "newton", - - // Florence Nightingale, more prominently known as a nurse, was also the first female member of the Royal Statistical Society and a pioneer in statistical graphics https://en.wikipedia.org/wiki/Florence_Nightingale#Statistics_and_sanitary_reform - "nightingale", - - // Alfred Nobel - a Swedish chemist, engineer, innovator, and armaments manufacturer (inventor of dynamite) - https://en.wikipedia.org/wiki/Alfred_Nobel - "nobel", - - // Emmy Noether, German mathematician. Noether's Theorem is named after her. https://en.wikipedia.org/wiki/Emmy_Noether - "noether", - - // Poppy Northcutt. Poppy Northcutt was the first woman to work as part of NASA’s Mission Control. http://www.businessinsider.com/poppy-northcutt-helped-apollo-astronauts-2014-12?op=1 - "northcutt", - - // Robert Noyce invented silicone integrated circuits and gave Silicon Valley its name. - https://en.wikipedia.org/wiki/Robert_Noyce - "noyce", - - // Panini - Ancient Indian linguist and grammarian from 4th century CE who worked on the world's first formal system - https://en.wikipedia.org/wiki/P%C4%81%E1%B9%87ini#Comparison_with_modern_formal_systems - "panini", - - // Ambroise Pare invented modern surgery. https://en.wikipedia.org/wiki/Ambroise_Par%C3%A9 - "pare", - - // Louis Pasteur discovered vaccination, fermentation and pasteurization. https://en.wikipedia.org/wiki/Louis_Pasteur. - "pasteur", - - // Cecilia Payne-Gaposchkin was an astronomer and astrophysicist who, in 1925, proposed in her Ph.D. thesis an explanation for the composition of stars in terms of the relative abundances of hydrogen and helium. https://en.wikipedia.org/wiki/Cecilia_Payne-Gaposchkin - "payne", - - // Radia Perlman is a software designer and network engineer and most famous for her invention of the spanning-tree protocol (STP). https://en.wikipedia.org/wiki/Radia_Perlman - "perlman", - - // Rob Pike was a key contributor to Unix, Plan 9, the X graphic system, utf-8, and the Go programming language. https://en.wikipedia.org/wiki/Rob_Pike - "pike", - - // Henri Poincaré made fundamental contributions in several fields of mathematics. https://en.wikipedia.org/wiki/Henri_Poincar%C3%A9 - "poincare", - - // Laura Poitras is a director and producer whose work, made possible by open source crypto tools, advances the causes of truth and freedom of information by reporting disclosures by whistleblowers such as Edward Snowden. https://en.wikipedia.org/wiki/Laura_Poitras - "poitras", - - // Claudius Ptolemy - a Greco-Egyptian writer of Alexandria, known as a mathematician, astronomer, geographer, astrologer, and poet of a single epigram in the Greek Anthology - https://en.wikipedia.org/wiki/Ptolemy - "ptolemy", - - // C. V. Raman - Indian physicist who won the Nobel Prize in 1930 for proposing the Raman effect. - https://en.wikipedia.org/wiki/C._V._Raman - "raman", - - // Srinivasa Ramanujan - Indian mathematician and autodidact who made extraordinary contributions to mathematical analysis, number theory, infinite series, and continued fractions. - https://en.wikipedia.org/wiki/Srinivasa_Ramanujan - "ramanujan", - - // Sally Kristen Ride was an American physicist and astronaut. She was the first American woman in space, and the youngest American astronaut. https://en.wikipedia.org/wiki/Sally_Ride - "ride", - - // Rita Levi-Montalcini - Won Nobel Prize in Physiology or Medicine jointly with colleague Stanley Cohen for the discovery of nerve growth factor (https://en.wikipedia.org/wiki/Rita_Levi-Montalcini) - "montalcini", - - // Dennis Ritchie - co-creator of UNIX and the C programming language. - https://en.wikipedia.org/wiki/Dennis_Ritchie - "ritchie", - - // Wilhelm Conrad Röntgen - German physicist who was awarded the first Nobel Prize in Physics in 1901 for the discovery of X-rays (Röntgen rays). https://en.wikipedia.org/wiki/Wilhelm_R%C3%B6ntgen - "roentgen", - - // Rosalind Franklin - British biophysicist and X-ray crystallographer whose research was critical to the understanding of DNA - https://en.wikipedia.org/wiki/Rosalind_Franklin - "rosalind", - - // Meghnad Saha - Indian astrophysicist best known for his development of the Saha equation, used to describe chemical and physical conditions in stars - https://en.wikipedia.org/wiki/Meghnad_Saha - "saha", - - // Jean E. Sammet developed FORMAC, the first widely used computer language for symbolic manipulation of mathematical formulas. https://en.wikipedia.org/wiki/Jean_E._Sammet - "sammet", - - // Carol Shaw - Originally an Atari employee, Carol Shaw is said to be the first female video game designer. https://en.wikipedia.org/wiki/Carol_Shaw_(video_game_designer) - "shaw", - - // Dame Stephanie "Steve" Shirley - Founded a software company in 1962 employing women working from home. https://en.wikipedia.org/wiki/Steve_Shirley - "shirley", - - // William Shockley co-invented the transistor - https://en.wikipedia.org/wiki/William_Shockley - "shockley", - - // Françoise Barré-Sinoussi - French virologist and Nobel Prize Laureate in Physiology or Medicine; her work was fundamental in identifying HIV as the cause of AIDS. https://en.wikipedia.org/wiki/Fran%C3%A7oise_Barr%C3%A9-Sinoussi - "sinoussi", - - // Betty Snyder - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Betty_Holberton - "snyder", - - // Frances Spence - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Frances_Spence - "spence", - - // Richard Matthew Stallman - the founder of the Free Software movement, the GNU project, the Free Software Foundation, and the League for Programming Freedom. He also invented the concept of copyleft to protect the ideals of this movement, and enshrined this concept in the widely-used GPL (General Public License) for software. https://en.wikiquote.org/wiki/Richard_Stallman - "stallman", - - // Michael Stonebraker is a database research pioneer and architect of Ingres, Postgres, VoltDB and SciDB. Winner of 2014 ACM Turing Award. https://en.wikipedia.org/wiki/Michael_Stonebraker - "stonebraker", - - // Janese Swanson (with others) developed the first of the Carmen Sandiego games. She went on to found Girl Tech. https://en.wikipedia.org/wiki/Janese_Swanson - "swanson", - - // Aaron Swartz was influential in creating RSS, Markdown, Creative Commons, Reddit, and much of the internet as we know it today. He was devoted to freedom of information on the web. https://en.wikiquote.org/wiki/Aaron_Swartz - "swartz", - - // Bertha Swirles was a theoretical physicist who made a number of contributions to early quantum theory. https://en.wikipedia.org/wiki/Bertha_Swirles - "swirles", - - // Nikola Tesla invented the AC electric system and every gadget ever used by a James Bond villain. https://en.wikipedia.org/wiki/Nikola_Tesla - "tesla", - - // Ken Thompson - co-creator of UNIX and the C programming language - https://en.wikipedia.org/wiki/Ken_Thompson - "thompson", - - // Linus Torvalds invented Linux and Git. https://en.wikipedia.org/wiki/Linus_Torvalds - "torvalds", - - // Alan Turing was a founding father of computer science. https://en.wikipedia.org/wiki/Alan_Turing. - "turing", - - // Varahamihira - Ancient Indian mathematician who discovered trigonometric formulae during 505-587 CE - https://en.wikipedia.org/wiki/Var%C4%81hamihira#Contributions - "varahamihira", - - // Sir Mokshagundam Visvesvaraya - is a notable Indian engineer. He is a recipient of the Indian Republic's highest honour, the Bharat Ratna, in 1955. On his birthday, 15 September is celebrated as Engineer's Day in India in his memory - https://en.wikipedia.org/wiki/Visvesvaraya - "visvesvaraya", - - // Christiane Nüsslein-Volhard - German biologist, won Nobel Prize in Physiology or Medicine in 1995 for research on the genetic control of embryonic development. https://en.wikipedia.org/wiki/Christiane_N%C3%BCsslein-Volhard - "volhard", - - // Marlyn Wescoff - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Marlyn_Meltzer - "wescoff", - - // Andrew Wiles - Notable British mathematician who proved the enigmatic Fermat's Last Theorem - https://en.wikipedia.org/wiki/Andrew_Wiles - "wiles", - - // Roberta Williams, did pioneering work in graphical adventure games for personal computers, particularly the King's Quest series. https://en.wikipedia.org/wiki/Roberta_Williams - "williams", - - // Sophie Wilson designed the first Acorn Micro-Computer and the instruction set for ARM processors. https://en.wikipedia.org/wiki/Sophie_Wilson - "wilson", - - // Jeannette Wing - co-developed the Liskov substitution principle. - https://en.wikipedia.org/wiki/Jeannette_Wing - "wing", - - // Steve Wozniak invented the Apple I and Apple II. https://en.wikipedia.org/wiki/Steve_Wozniak - "wozniak", - - // The Wright brothers, Orville and Wilbur - credited with inventing and building the world's first successful airplane and making the first controlled, powered and sustained heavier-than-air human flight - https://en.wikipedia.org/wiki/Wright_brothers - "wright", - - // Rosalyn Sussman Yalow - Rosalyn Sussman Yalow was an American medical physicist, and a co-winner of the 1977 Nobel Prize in Physiology or Medicine for development of the radioimmunoassay technique. https://en.wikipedia.org/wiki/Rosalyn_Sussman_Yalow - "yalow", - - // Ada Yonath - an Israeli crystallographer, the first woman from the Middle East to win a Nobel prize in the sciences. https://en.wikipedia.org/wiki/Ada_Yonath - "yonath", - } -) - -// GetRandomName generates a random name from the list of adjectives and surnames in this package -// formatted as "adjective_surname". For example 'focused_turing'. If retry is non-zero, a random -// integer between 0 and 10 will be added to the end of the name, e.g `focused_turing3` -func GetRandomName(retry int) string { - rnd := random.Rand -begin: - name := fmt.Sprintf("%s_%s", left[rnd.Intn(len(left))], right[rnd.Intn(len(right))]) - if name == "boring_wozniak" /* Steve Wozniak is not boring */ { - goto begin - } - - if retry > 0 { - name = fmt.Sprintf("%s%d", name, rnd.Intn(10)) - } - return name -} diff --git a/namesgenerator/names-generator_test.go b/namesgenerator/names-generator_test.go deleted file mode 100644 index d1a9497..0000000 --- a/namesgenerator/names-generator_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package namesgenerator - -import ( - "strings" - "testing" -) - -func TestNameFormat(t *testing.T) { - name := GetRandomName(0) - if !strings.Contains(name, "_") { - t.Fatalf("Generated name does not contain an underscore") - } - if strings.ContainsAny(name, "0123456789") { - t.Fatalf("Generated name contains numbers!") - } -} - -func TestNameRetries(t *testing.T) { - name := GetRandomName(1) - if !strings.Contains(name, "_") { - t.Fatalf("Generated name does not contain an underscore") - } - if !strings.ContainsAny(name, "0123456789") { - t.Fatalf("Generated name doesn't contain a number") - } - -} diff --git a/platform/architecture_linux.go b/platform/architecture_linux.go deleted file mode 100644 index 2cdc2c5..0000000 --- a/platform/architecture_linux.go +++ /dev/null @@ -1,16 +0,0 @@ -// Package platform provides helper function to get the runtime architecture -// for different platforms. -package platform - -import ( - "syscall" -) - -// runtimeArchitecture gets the name of the current architecture (x86, x86_64, …) -func runtimeArchitecture() (string, error) { - utsname := &syscall.Utsname{} - if err := syscall.Uname(utsname); err != nil { - return "", err - } - return charsToString(utsname.Machine), nil -} diff --git a/platform/architecture_unix.go b/platform/architecture_unix.go deleted file mode 100644 index 45bbcf1..0000000 --- a/platform/architecture_unix.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build freebsd solaris darwin - -// Package platform provides helper function to get the runtime architecture -// for different platforms. -package platform - -import ( - "os/exec" - "strings" -) - -// runtimeArchitecture gets the name of the current architecture (x86, x86_64, i86pc, sun4v, ...) -func runtimeArchitecture() (string, error) { - cmd := exec.Command("/usr/bin/uname", "-m") - machine, err := cmd.Output() - if err != nil { - return "", err - } - return strings.TrimSpace(string(machine)), nil -} diff --git a/platform/architecture_windows.go b/platform/architecture_windows.go deleted file mode 100644 index 0dd8a2e..0000000 --- a/platform/architecture_windows.go +++ /dev/null @@ -1,52 +0,0 @@ -package platform - -import ( - "fmt" - "syscall" - "unsafe" -) - -var ( - modkernel32 = syscall.NewLazyDLL("kernel32.dll") - procGetSystemInfo = modkernel32.NewProc("GetSystemInfo") -) - -// see http://msdn.microsoft.com/en-us/library/windows/desktop/ms724958(v=vs.85).aspx -type systeminfo struct { - wProcessorArchitecture uint16 - wReserved uint16 - dwPageSize uint32 - lpMinimumApplicationAddress uintptr - lpMaximumApplicationAddress uintptr - dwActiveProcessorMask uintptr - dwNumberOfProcessors uint32 - dwProcessorType uint32 - dwAllocationGranularity uint32 - wProcessorLevel uint16 - wProcessorRevision uint16 -} - -// Constants -const ( - ProcessorArchitecture64 = 9 // PROCESSOR_ARCHITECTURE_AMD64 - ProcessorArchitectureIA64 = 6 // PROCESSOR_ARCHITECTURE_IA64 - ProcessorArchitecture32 = 0 // PROCESSOR_ARCHITECTURE_INTEL - ProcessorArchitectureArm = 5 // PROCESSOR_ARCHITECTURE_ARM -) - -var sysinfo systeminfo - -// runtimeArchitecture gets the name of the current architecture (x86, x86_64, …) -func runtimeArchitecture() (string, error) { - syscall.Syscall(procGetSystemInfo.Addr(), 1, uintptr(unsafe.Pointer(&sysinfo)), 0, 0) - switch sysinfo.wProcessorArchitecture { - case ProcessorArchitecture64, ProcessorArchitectureIA64: - return "x86_64", nil - case ProcessorArchitecture32: - return "i686", nil - case ProcessorArchitectureArm: - return "arm", nil - default: - return "", fmt.Errorf("Unknown processor architecture") - } -} diff --git a/platform/platform.go b/platform/platform.go deleted file mode 100644 index e4b0312..0000000 --- a/platform/platform.go +++ /dev/null @@ -1,23 +0,0 @@ -package platform - -import ( - "runtime" - - "github.com/Sirupsen/logrus" -) - -var ( - // Architecture holds the runtime architecture of the process. - Architecture string - // OSType holds the runtime operating system type (Linux, …) of the process. - OSType string -) - -func init() { - var err error - Architecture, err = runtimeArchitecture() - if err != nil { - logrus.Errorf("Could not read system architecture info: %v", err) - } - OSType = runtime.GOOS -} diff --git a/platform/utsname_int8.go b/platform/utsname_int8.go deleted file mode 100644 index 5dcbadf..0000000 --- a/platform/utsname_int8.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build linux,386 linux,amd64 linux,arm64 -// see golang's sources src/syscall/ztypes_linux_*.go that use int8 - -package platform - -// Convert the OS/ARCH-specific utsname.Machine to string -// given as an array of signed int8 -func charsToString(ca [65]int8) string { - s := make([]byte, len(ca)) - var lens int - for ; lens < len(ca); lens++ { - if ca[lens] == 0 { - break - } - s[lens] = uint8(ca[lens]) - } - return string(s[0:lens]) -} diff --git a/platform/utsname_uint8.go b/platform/utsname_uint8.go deleted file mode 100644 index c9875cf..0000000 --- a/platform/utsname_uint8.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build linux,arm linux,ppc64 linux,ppc64le s390x -// see golang's sources src/syscall/ztypes_linux_*.go that use uint8 - -package platform - -// Convert the OS/ARCH-specific utsname.Machine to string -// given as an array of unsigned uint8 -func charsToString(ca [65]uint8) string { - s := make([]byte, len(ca)) - var lens int - for ; lens < len(ca); lens++ { - if ca[lens] == 0 { - break - } - s[lens] = ca[lens] - } - return string(s[0:lens]) -} diff --git a/progress/progress.go b/progress/progress.go deleted file mode 100644 index df3c2ba..0000000 --- a/progress/progress.go +++ /dev/null @@ -1,73 +0,0 @@ -package progress - -import ( - "fmt" -) - -// Progress represents the progress of a transfer. -type Progress struct { - ID string - - // Progress contains a Message or... - Message string - - // ...progress of an action - Action string - Current int64 - Total int64 - - // Aux contains extra information not presented to the user, such as - // digests for push signing. - Aux interface{} - - LastUpdate bool -} - -// Output is an interface for writing progress information. It's -// like a writer for progress, but we don't call it Writer because -// that would be confusing next to ProgressReader (also, because it -// doesn't implement the io.Writer interface). -type Output interface { - WriteProgress(Progress) error -} - -type chanOutput chan<- Progress - -func (out chanOutput) WriteProgress(p Progress) error { - out <- p - return nil -} - -// ChanOutput returns an Output that writes progress updates to the -// supplied channel. -func ChanOutput(progressChan chan<- Progress) Output { - return chanOutput(progressChan) -} - -// Update is a convenience function to write a progress update to the channel. -func Update(out Output, id, action string) { - out.WriteProgress(Progress{ID: id, Action: action}) -} - -// Updatef is a convenience function to write a printf-formatted progress update -// to the channel. -func Updatef(out Output, id, format string, a ...interface{}) { - Update(out, id, fmt.Sprintf(format, a...)) -} - -// Message is a convenience function to write a progress message to the channel. -func Message(out Output, id, message string) { - out.WriteProgress(Progress{ID: id, Message: message}) -} - -// Messagef is a convenience function to write a printf-formatted progress -// message to the channel. -func Messagef(out Output, id, format string, a ...interface{}) { - Message(out, id, fmt.Sprintf(format, a...)) -} - -// Aux sends auxiliary information over a progress interface, which will not be -// formatted for the UI. This is used for things such as push signing. -func Aux(out Output, a interface{}) { - out.WriteProgress(Progress{Aux: a}) -} diff --git a/progress/progressreader.go b/progress/progressreader.go deleted file mode 100644 index c39e2b6..0000000 --- a/progress/progressreader.go +++ /dev/null @@ -1,59 +0,0 @@ -package progress - -import ( - "io" -) - -// Reader is a Reader with progress bar. -type Reader struct { - in io.ReadCloser // Stream to read from - out Output // Where to send progress bar to - size int64 - current int64 - lastUpdate int64 - id string - action string -} - -// NewProgressReader creates a new ProgressReader. -func NewProgressReader(in io.ReadCloser, out Output, size int64, id, action string) *Reader { - return &Reader{ - in: in, - out: out, - size: size, - id: id, - action: action, - } -} - -func (p *Reader) Read(buf []byte) (n int, err error) { - read, err := p.in.Read(buf) - p.current += int64(read) - updateEvery := int64(1024 * 512) //512kB - if p.size > 0 { - // Update progress for every 1% read if 1% < 512kB - if increment := int64(0.01 * float64(p.size)); increment < updateEvery { - updateEvery = increment - } - } - if p.current-p.lastUpdate > updateEvery || err != nil { - p.updateProgress(err != nil && read == 0) - p.lastUpdate = p.current - } - - return read, err -} - -// Close closes the progress reader and its underlying reader. -func (p *Reader) Close() error { - if p.current < p.size { - // print a full progress bar when closing prematurely - p.current = p.size - p.updateProgress(false) - } - return p.in.Close() -} - -func (p *Reader) updateProgress(last bool) { - p.out.WriteProgress(Progress{ID: p.id, Action: p.action, Current: p.current, Total: p.size, LastUpdate: last}) -} diff --git a/progress/progressreader_test.go b/progress/progressreader_test.go deleted file mode 100644 index b14d401..0000000 --- a/progress/progressreader_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package progress - -import ( - "bytes" - "io" - "io/ioutil" - "testing" -) - -func TestOutputOnPrematureClose(t *testing.T) { - content := []byte("TESTING") - reader := ioutil.NopCloser(bytes.NewReader(content)) - progressChan := make(chan Progress, 10) - - pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read") - - part := make([]byte, 4, 4) - _, err := io.ReadFull(pr, part) - if err != nil { - pr.Close() - t.Fatal(err) - } - -drainLoop: - for { - select { - case <-progressChan: - default: - break drainLoop - } - } - - pr.Close() - - select { - case <-progressChan: - default: - t.Fatalf("Expected some output when closing prematurely") - } -} - -func TestCompleteSilently(t *testing.T) { - content := []byte("TESTING") - reader := ioutil.NopCloser(bytes.NewReader(content)) - progressChan := make(chan Progress, 10) - - pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read") - - out, err := ioutil.ReadAll(pr) - if err != nil { - pr.Close() - t.Fatal(err) - } - if string(out) != "TESTING" { - pr.Close() - t.Fatalf("Unexpected output %q from reader", string(out)) - } - -drainLoop: - for { - select { - case <-progressChan: - default: - break drainLoop - } - } - - pr.Close() - - select { - case <-progressChan: - t.Fatalf("Should have closed silently when read is complete") - default: - } -} diff --git a/pubsub/publisher.go b/pubsub/publisher.go deleted file mode 100644 index 0936461..0000000 --- a/pubsub/publisher.go +++ /dev/null @@ -1,111 +0,0 @@ -package pubsub - -import ( - "sync" - "time" -) - -var wgPool = sync.Pool{New: func() interface{} { return new(sync.WaitGroup) }} - -// NewPublisher creates a new pub/sub publisher to broadcast messages. -// The duration is used as the send timeout as to not block the publisher publishing -// messages to other clients if one client is slow or unresponsive. -// The buffer is used when creating new channels for subscribers. -func NewPublisher(publishTimeout time.Duration, buffer int) *Publisher { - return &Publisher{ - buffer: buffer, - timeout: publishTimeout, - subscribers: make(map[subscriber]topicFunc), - } -} - -type subscriber chan interface{} -type topicFunc func(v interface{}) bool - -// Publisher is basic pub/sub structure. Allows to send events and subscribe -// to them. Can be safely used from multiple goroutines. -type Publisher struct { - m sync.RWMutex - buffer int - timeout time.Duration - subscribers map[subscriber]topicFunc -} - -// Len returns the number of subscribers for the publisher -func (p *Publisher) Len() int { - p.m.RLock() - i := len(p.subscribers) - p.m.RUnlock() - return i -} - -// Subscribe adds a new subscriber to the publisher returning the channel. -func (p *Publisher) Subscribe() chan interface{} { - return p.SubscribeTopic(nil) -} - -// SubscribeTopic adds a new subscriber that filters messages sent by a topic. -func (p *Publisher) SubscribeTopic(topic topicFunc) chan interface{} { - ch := make(chan interface{}, p.buffer) - p.m.Lock() - p.subscribers[ch] = topic - p.m.Unlock() - return ch -} - -// Evict removes the specified subscriber from receiving any more messages. -func (p *Publisher) Evict(sub chan interface{}) { - p.m.Lock() - delete(p.subscribers, sub) - close(sub) - p.m.Unlock() -} - -// Publish sends the data in v to all subscribers currently registered with the publisher. -func (p *Publisher) Publish(v interface{}) { - p.m.RLock() - if len(p.subscribers) == 0 { - p.m.RUnlock() - return - } - - wg := wgPool.Get().(*sync.WaitGroup) - for sub, topic := range p.subscribers { - wg.Add(1) - go p.sendTopic(sub, topic, v, wg) - } - wg.Wait() - wgPool.Put(wg) - p.m.RUnlock() -} - -// Close closes the channels to all subscribers registered with the publisher. -func (p *Publisher) Close() { - p.m.Lock() - for sub := range p.subscribers { - delete(p.subscribers, sub) - close(sub) - } - p.m.Unlock() -} - -func (p *Publisher) sendTopic(sub subscriber, topic topicFunc, v interface{}, wg *sync.WaitGroup) { - defer wg.Done() - if topic != nil && !topic(v) { - return - } - - // send under a select as to not block if the receiver is unavailable - if p.timeout > 0 { - select { - case sub <- v: - case <-time.After(p.timeout): - } - return - } - - select { - case sub <- v: - default: - } -} diff --git a/pubsub/publisher_test.go b/pubsub/publisher_test.go deleted file mode 100644 index d6b0a1d..0000000 --- a/pubsub/publisher_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package pubsub - -import ( - "fmt" - "testing" - "time" -) - -func TestSendToOneSub(t *testing.T) { - p := NewPublisher(100*time.Millisecond, 10) - c := p.Subscribe() - - p.Publish("hi") - - msg := <-c - if msg.(string) != "hi" { - t.Fatalf("expected message hi but received %v", msg) - } -} - -func TestSendToMultipleSubs(t *testing.T) { - p := NewPublisher(100*time.Millisecond, 10) - subs := []chan interface{}{} - subs = append(subs, p.Subscribe(), p.Subscribe(), p.Subscribe()) - - p.Publish("hi") - - for _, c := range subs { - msg := <-c - if msg.(string) != "hi" { - t.Fatalf("expected message hi but received %v", msg) - } - } -} - -func TestEvictOneSub(t *testing.T) { - p := NewPublisher(100*time.Millisecond, 10) - s1 := p.Subscribe() - s2 := p.Subscribe() - - p.Evict(s1) - p.Publish("hi") - if _, ok := <-s1; ok { - t.Fatal("expected s1 to not receive the published message") - } - - msg := <-s2 - if msg.(string) != "hi" { - t.Fatalf("expected message hi but received %v", msg) - } -} - -func TestClosePublisher(t *testing.T) { - p := NewPublisher(100*time.Millisecond, 10) - subs := []chan interface{}{} - subs = append(subs, p.Subscribe(), p.Subscribe(), p.Subscribe()) - p.Close() - - for _, c := range subs { - if _, ok := <-c; ok { - t.Fatal("expected all subscriber channels to be closed") - } - } -} - -const sampleText = "test" - -type testSubscriber struct { - dataCh chan interface{} - ch chan error -} - -func (s *testSubscriber) Wait() error { - return <-s.ch -} - -func newTestSubscriber(p *Publisher) *testSubscriber { - ts := &testSubscriber{ - dataCh: p.Subscribe(), - ch: make(chan error), - } - go func() { - for data := range ts.dataCh { - s, ok := data.(string) - if !ok { - ts.ch <- fmt.Errorf("Unexpected type %T", data) - break - } - if s != sampleText { - ts.ch <- fmt.Errorf("Unexpected text %s", s) - break - } - } - close(ts.ch) - }() - return ts -} - -// for testing with -race -func TestPubSubRace(t *testing.T) { - p := NewPublisher(0, 1024) - var subs [](*testSubscriber) - for j := 0; j < 50; j++ { - subs = append(subs, newTestSubscriber(p)) - } - for j := 0; j < 1000; j++ { - p.Publish(sampleText) - } - time.AfterFunc(1*time.Second, func() { - for _, s := range subs { - p.Evict(s.dataCh) - } - }) - for _, s := range subs { - s.Wait() - } -} - -func BenchmarkPubSub(b *testing.B) { - for i := 0; i < b.N; i++ { - b.StopTimer() - p := NewPublisher(0, 1024) - var subs [](*testSubscriber) - for j := 0; j < 50; j++ { - subs = append(subs, newTestSubscriber(p)) - } - b.StartTimer() - for j := 0; j < 1000; j++ { - p.Publish(sampleText) - } - time.AfterFunc(1*time.Second, func() { - for _, s := range subs { - p.Evict(s.dataCh) - } - }) - for _, s := range subs { - if err := s.Wait(); err != nil { - b.Fatal(err) - } - } - } -} diff --git a/registrar/registrar.go b/registrar/registrar.go deleted file mode 100644 index 1e75ee9..0000000 --- a/registrar/registrar.go +++ /dev/null @@ -1,127 +0,0 @@ -// Package registrar provides name registration. It reserves a name to a given key. -package registrar - -import ( - "errors" - "sync" -) - -var ( - // ErrNameReserved is an error which is returned when a name is requested to be reserved that already is reserved - ErrNameReserved = errors.New("name is reserved") - // ErrNameNotReserved is an error which is returned when trying to find a name that is not reserved - ErrNameNotReserved = errors.New("name is not reserved") - // ErrNoSuchKey is returned when trying to find the names for a key which is not known - ErrNoSuchKey = errors.New("provided key does not exist") -) - -// Registrar stores indexes a list of keys and their registered names as well as indexes names and the key that they are registered to -// Names must be unique. -// Registrar is safe for concurrent access. -type Registrar struct { - idx map[string][]string - names map[string]string - mu sync.Mutex -} - -// NewRegistrar creates a new Registrar with the an empty index -func NewRegistrar() *Registrar { - return &Registrar{ - idx: make(map[string][]string), - names: make(map[string]string), - } -} - -// Reserve registers a key to a name -// Reserve is idempotent -// Attempting to reserve a key to a name that already exists results in an `ErrNameReserved` -// A name reservation is globally unique -func (r *Registrar) Reserve(name, key string) error { - r.mu.Lock() - defer r.mu.Unlock() - - if k, exists := r.names[name]; exists { - if k != key { - return ErrNameReserved - } - return nil - } - - r.idx[key] = append(r.idx[key], name) - r.names[name] = key - return nil -} - -// Release releases the reserved name -// Once released, a name can be reserved again -func (r *Registrar) Release(name string) { - r.mu.Lock() - defer r.mu.Unlock() - - key, exists := r.names[name] - if !exists { - return - } - - for i, n := range r.idx[key] { - if n != name { - continue - } - r.idx[key] = append(r.idx[key][:i], r.idx[key][i+1:]...) - break - } - - delete(r.names, name) - - if len(r.idx[key]) == 0 { - delete(r.idx, key) - } -} - -// Delete removes all reservations for the passed in key. -// All names reserved to this key are released. -func (r *Registrar) Delete(key string) { - r.mu.Lock() - for _, name := range r.idx[key] { - delete(r.names, name) - } - delete(r.idx, key) - r.mu.Unlock() -} - -// GetNames lists all the reserved names for the given key -func (r *Registrar) GetNames(key string) ([]string, error) { - r.mu.Lock() - defer r.mu.Unlock() - - names, exists := r.idx[key] - if !exists { - return nil, ErrNoSuchKey - } - return names, nil -} - -// Get returns the key that the passed in name is reserved to -func (r *Registrar) Get(name string) (string, error) { - r.mu.Lock() - key, exists := r.names[name] - r.mu.Unlock() - - if !exists { - return "", ErrNameNotReserved - } - return key, nil -} - -// GetAll returns all registered names -func (r *Registrar) GetAll() map[string][]string { - out := make(map[string][]string) - - r.mu.Lock() - // copy index into out - for id, names := range r.idx { - out[id] = names - } - r.mu.Unlock() - return out -} diff --git a/registrar/registrar_test.go b/registrar/registrar_test.go deleted file mode 100644 index 0c1ef31..0000000 --- a/registrar/registrar_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package registrar - -import ( - "reflect" - "testing" -) - -func TestReserve(t *testing.T) { - r := NewRegistrar() - - obj := "test1" - if err := r.Reserve("test", obj); err != nil { - t.Fatal(err) - } - - if err := r.Reserve("test", obj); err != nil { - t.Fatal(err) - } - - obj2 := "test2" - err := r.Reserve("test", obj2) - if err == nil { - t.Fatalf("expected error when reserving an already reserved name to another object") - } - if err != ErrNameReserved { - t.Fatal("expected `ErrNameReserved` error when attempting to reserve an already reserved name") - } -} - -func TestRelease(t *testing.T) { - r := NewRegistrar() - obj := "testing" - - if err := r.Reserve("test", obj); err != nil { - t.Fatal(err) - } - r.Release("test") - r.Release("test") // Ensure there is no panic here - - if err := r.Reserve("test", obj); err != nil { - t.Fatal(err) - } -} - -func TestGetNames(t *testing.T) { - r := NewRegistrar() - obj := "testing" - names := []string{"test1", "test2"} - - for _, name := range names { - if err := r.Reserve(name, obj); err != nil { - t.Fatal(err) - } - } - r.Reserve("test3", "other") - - names2, err := r.GetNames(obj) - if err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(names, names2) { - t.Fatalf("Exepected: %v, Got: %v", names, names2) - } -} - -func TestDelete(t *testing.T) { - r := NewRegistrar() - obj := "testing" - names := []string{"test1", "test2"} - for _, name := range names { - if err := r.Reserve(name, obj); err != nil { - t.Fatal(err) - } - } - - r.Reserve("test3", "other") - r.Delete(obj) - - _, err := r.GetNames(obj) - if err == nil { - t.Fatal("expected error getting names for deleted key") - } - - if err != ErrNoSuchKey { - t.Fatal("expected `ErrNoSuchKey`") - } -} - -func TestGet(t *testing.T) { - r := NewRegistrar() - obj := "testing" - name := "test" - - _, err := r.Get(name) - if err == nil { - t.Fatal("expected error when key does not exist") - } - if err != ErrNameNotReserved { - t.Fatal(err) - } - - if err := r.Reserve(name, obj); err != nil { - t.Fatal(err) - } - - if _, err = r.Get(name); err != nil { - t.Fatal(err) - } - - r.Delete(obj) - _, err = r.Get(name) - if err == nil { - t.Fatal("expected error when key does not exist") - } - if err != ErrNameNotReserved { - t.Fatal(err) - } -} diff --git a/signal/README.md b/signal/README.md deleted file mode 100644 index 2b237a5..0000000 --- a/signal/README.md +++ /dev/null @@ -1 +0,0 @@ -This package provides helper functions for dealing with signals across various operating systems \ No newline at end of file diff --git a/signal/signal.go b/signal/signal.go deleted file mode 100644 index 68bb77c..0000000 --- a/signal/signal.go +++ /dev/null @@ -1,54 +0,0 @@ -// Package signal provides helper functions for dealing with signals across -// various operating systems. -package signal - -import ( - "fmt" - "os" - "os/signal" - "strconv" - "strings" - "syscall" -) - -// CatchAll catches all signals and relays them to the specified channel. -func CatchAll(sigc chan os.Signal) { - handledSigs := []os.Signal{} - for _, s := range SignalMap { - handledSigs = append(handledSigs, s) - } - signal.Notify(sigc, handledSigs...) -} - -// StopCatch stops catching the signals and closes the specified channel. -func StopCatch(sigc chan os.Signal) { - signal.Stop(sigc) - close(sigc) -} - -// ParseSignal translates a string to a valid syscall signal. -// It returns an error if the signal map doesn't include the given signal. -func ParseSignal(rawSignal string) (syscall.Signal, error) { - s, err := strconv.Atoi(rawSignal) - if err == nil { - if s == 0 { - return -1, fmt.Errorf("Invalid signal: %s", rawSignal) - } - return syscall.Signal(s), nil - } - signal, ok := SignalMap[strings.TrimPrefix(strings.ToUpper(rawSignal), "SIG")] - if !ok { - return -1, fmt.Errorf("Invalid signal: %s", rawSignal) - } - return signal, nil -} - -// ValidSignalForPlatform returns true if a signal is valid on the platform -func ValidSignalForPlatform(sig syscall.Signal) bool { - for _, v := range SignalMap { - if v == sig { - return true - } - } - return false -} diff --git a/signal/signal_darwin.go b/signal/signal_darwin.go deleted file mode 100644 index 946de87..0000000 --- a/signal/signal_darwin.go +++ /dev/null @@ -1,41 +0,0 @@ -package signal - -import ( - "syscall" -) - -// SignalMap is a map of Darwin signals. -var SignalMap = map[string]syscall.Signal{ - "ABRT": syscall.SIGABRT, - "ALRM": syscall.SIGALRM, - "BUG": syscall.SIGBUS, - "CHLD": syscall.SIGCHLD, - "CONT": syscall.SIGCONT, - "EMT": syscall.SIGEMT, - "FPE": syscall.SIGFPE, - "HUP": syscall.SIGHUP, - "ILL": syscall.SIGILL, - "INFO": syscall.SIGINFO, - "INT": syscall.SIGINT, - "IO": syscall.SIGIO, - "IOT": syscall.SIGIOT, - "KILL": syscall.SIGKILL, - "PIPE": syscall.SIGPIPE, - "PROF": syscall.SIGPROF, - "QUIT": syscall.SIGQUIT, - "SEGV": syscall.SIGSEGV, - "STOP": syscall.SIGSTOP, - "SYS": syscall.SIGSYS, - "TERM": syscall.SIGTERM, - "TRAP": syscall.SIGTRAP, - "TSTP": syscall.SIGTSTP, - "TTIN": syscall.SIGTTIN, - "TTOU": syscall.SIGTTOU, - "URG": syscall.SIGURG, - "USR1": syscall.SIGUSR1, - "USR2": syscall.SIGUSR2, - "VTALRM": syscall.SIGVTALRM, - "WINCH": syscall.SIGWINCH, - "XCPU": syscall.SIGXCPU, - "XFSZ": syscall.SIGXFSZ, -} diff --git a/signal/signal_freebsd.go b/signal/signal_freebsd.go deleted file mode 100644 index 6b9569b..0000000 --- a/signal/signal_freebsd.go +++ /dev/null @@ -1,43 +0,0 @@ -package signal - -import ( - "syscall" -) - -// SignalMap is a map of FreeBSD signals. -var SignalMap = map[string]syscall.Signal{ - "ABRT": syscall.SIGABRT, - "ALRM": syscall.SIGALRM, - "BUF": syscall.SIGBUS, - "CHLD": syscall.SIGCHLD, - "CONT": syscall.SIGCONT, - "EMT": syscall.SIGEMT, - "FPE": syscall.SIGFPE, - "HUP": syscall.SIGHUP, - "ILL": syscall.SIGILL, - "INFO": syscall.SIGINFO, - "INT": syscall.SIGINT, - "IO": syscall.SIGIO, - "IOT": syscall.SIGIOT, - "KILL": syscall.SIGKILL, - "LWP": syscall.SIGLWP, - "PIPE": syscall.SIGPIPE, - "PROF": syscall.SIGPROF, - "QUIT": syscall.SIGQUIT, - "SEGV": syscall.SIGSEGV, - "STOP": syscall.SIGSTOP, - "SYS": syscall.SIGSYS, - "TERM": syscall.SIGTERM, - "THR": syscall.SIGTHR, - "TRAP": syscall.SIGTRAP, - "TSTP": syscall.SIGTSTP, - "TTIN": syscall.SIGTTIN, - "TTOU": syscall.SIGTTOU, - "URG": syscall.SIGURG, - "USR1": syscall.SIGUSR1, - "USR2": syscall.SIGUSR2, - "VTALRM": syscall.SIGVTALRM, - "WINCH": syscall.SIGWINCH, - "XCPU": syscall.SIGXCPU, - "XFSZ": syscall.SIGXFSZ, -} diff --git a/signal/signal_linux.go b/signal/signal_linux.go deleted file mode 100644 index d418cbe..0000000 --- a/signal/signal_linux.go +++ /dev/null @@ -1,80 +0,0 @@ -package signal - -import ( - "syscall" -) - -const ( - sigrtmin = 34 - sigrtmax = 64 -) - -// SignalMap is a map of Linux signals. -var SignalMap = map[string]syscall.Signal{ - "ABRT": syscall.SIGABRT, - "ALRM": syscall.SIGALRM, - "BUS": syscall.SIGBUS, - "CHLD": syscall.SIGCHLD, - "CLD": syscall.SIGCLD, - "CONT": syscall.SIGCONT, - "FPE": syscall.SIGFPE, - "HUP": syscall.SIGHUP, - "ILL": syscall.SIGILL, - "INT": syscall.SIGINT, - "IO": syscall.SIGIO, - "IOT": syscall.SIGIOT, - "KILL": syscall.SIGKILL, - "PIPE": syscall.SIGPIPE, - "POLL": syscall.SIGPOLL, - "PROF": syscall.SIGPROF, - "PWR": syscall.SIGPWR, - "QUIT": syscall.SIGQUIT, - "SEGV": syscall.SIGSEGV, - "STKFLT": syscall.SIGSTKFLT, - "STOP": syscall.SIGSTOP, - "SYS": syscall.SIGSYS, - "TERM": syscall.SIGTERM, - "TRAP": syscall.SIGTRAP, - "TSTP": syscall.SIGTSTP, - "TTIN": syscall.SIGTTIN, - "TTOU": syscall.SIGTTOU, - "UNUSED": syscall.SIGUNUSED, - "URG": syscall.SIGURG, - "USR1": syscall.SIGUSR1, - "USR2": syscall.SIGUSR2, - "VTALRM": syscall.SIGVTALRM, - "WINCH": syscall.SIGWINCH, - "XCPU": syscall.SIGXCPU, - "XFSZ": syscall.SIGXFSZ, - "RTMIN": sigrtmin, - "RTMIN+1": sigrtmin + 1, - "RTMIN+2": sigrtmin + 2, - "RTMIN+3": sigrtmin + 3, - "RTMIN+4": sigrtmin + 4, - "RTMIN+5": sigrtmin + 5, - "RTMIN+6": sigrtmin + 6, - "RTMIN+7": sigrtmin + 7, - "RTMIN+8": sigrtmin + 8, - "RTMIN+9": sigrtmin + 9, - "RTMIN+10": sigrtmin + 10, - "RTMIN+11": sigrtmin + 11, - "RTMIN+12": sigrtmin + 12, - "RTMIN+13": sigrtmin + 13, - "RTMIN+14": sigrtmin + 14, - "RTMIN+15": sigrtmin + 15, - "RTMAX-14": sigrtmax - 14, - "RTMAX-13": sigrtmax - 13, - "RTMAX-12": sigrtmax - 12, - "RTMAX-11": sigrtmax - 11, - "RTMAX-10": sigrtmax - 10, - "RTMAX-9": sigrtmax - 9, - "RTMAX-8": sigrtmax - 8, - "RTMAX-7": sigrtmax - 7, - "RTMAX-6": sigrtmax - 6, - "RTMAX-5": sigrtmax - 5, - "RTMAX-4": sigrtmax - 4, - "RTMAX-3": sigrtmax - 3, - "RTMAX-2": sigrtmax - 2, - "RTMAX-1": sigrtmax - 1, - "RTMAX": sigrtmax, -} diff --git a/signal/signal_solaris.go b/signal/signal_solaris.go deleted file mode 100644 index 89576b9..0000000 --- a/signal/signal_solaris.go +++ /dev/null @@ -1,42 +0,0 @@ -package signal - -import ( - "syscall" -) - -// SignalMap is a map of Solaris signals. -// SIGINFO and SIGTHR not defined for Solaris -var SignalMap = map[string]syscall.Signal{ - "ABRT": syscall.SIGABRT, - "ALRM": syscall.SIGALRM, - "BUF": syscall.SIGBUS, - "CHLD": syscall.SIGCHLD, - "CONT": syscall.SIGCONT, - "EMT": syscall.SIGEMT, - "FPE": syscall.SIGFPE, - "HUP": syscall.SIGHUP, - "ILL": syscall.SIGILL, - "INT": syscall.SIGINT, - "IO": syscall.SIGIO, - "IOT": syscall.SIGIOT, - "KILL": syscall.SIGKILL, - "LWP": syscall.SIGLWP, - "PIPE": syscall.SIGPIPE, - "PROF": syscall.SIGPROF, - "QUIT": syscall.SIGQUIT, - "SEGV": syscall.SIGSEGV, - "STOP": syscall.SIGSTOP, - "SYS": syscall.SIGSYS, - "TERM": syscall.SIGTERM, - "TRAP": syscall.SIGTRAP, - "TSTP": syscall.SIGTSTP, - "TTIN": syscall.SIGTTIN, - "TTOU": syscall.SIGTTOU, - "URG": syscall.SIGURG, - "USR1": syscall.SIGUSR1, - "USR2": syscall.SIGUSR2, - "VTALRM": syscall.SIGVTALRM, - "WINCH": syscall.SIGWINCH, - "XCPU": syscall.SIGXCPU, - "XFSZ": syscall.SIGXFSZ, -} diff --git a/signal/signal_unix.go b/signal/signal_unix.go deleted file mode 100644 index 6621d37..0000000 --- a/signal/signal_unix.go +++ /dev/null @@ -1,21 +0,0 @@ -// +build !windows - -package signal - -import ( - "syscall" -) - -// Signals used in api/client (no windows equivalent, use -// invalid signals so they don't get handled) - -const ( - // SIGCHLD is a signal sent to a process when a child process terminates, is interrupted, or resumes after being interrupted. - SIGCHLD = syscall.SIGCHLD - // SIGWINCH is a signal sent to a process when its controlling terminal changes its size - SIGWINCH = syscall.SIGWINCH - // SIGPIPE is a signal sent to a process when a pipe is written to before the other end is open for reading - SIGPIPE = syscall.SIGPIPE - // DefaultStopSignal is the syscall signal used to stop a container in unix systems. - DefaultStopSignal = "SIGTERM" -) diff --git a/signal/signal_unsupported.go b/signal/signal_unsupported.go deleted file mode 100644 index c592d37..0000000 --- a/signal/signal_unsupported.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build !linux,!darwin,!freebsd,!windows,!solaris - -package signal - -import ( - "syscall" -) - -// SignalMap is an empty map of signals for unsupported platform. -var SignalMap = map[string]syscall.Signal{} diff --git a/signal/signal_windows.go b/signal/signal_windows.go deleted file mode 100644 index 698cbf2..0000000 --- a/signal/signal_windows.go +++ /dev/null @@ -1,28 +0,0 @@ -// +build windows - -package signal - -import ( - "syscall" -) - -// Signals used in api/client (no windows equivalent, use -// invalid signals so they don't get handled) -const ( - SIGCHLD = syscall.Signal(0xff) - SIGWINCH = syscall.Signal(0xff) - SIGPIPE = syscall.Signal(0xff) - // DefaultStopSignal is the syscall signal used to stop a container in windows systems. - DefaultStopSignal = "15" -) - -// SignalMap is a map of "supported" signals. As per the comment in GOLang's -// ztypes_windows.go: "More invented values for signals". Windows doesn't -// really support signals in any way, shape or form that Unix does. -// -// We have these so that docker kill can be used to gracefully (TERM) and -// forcibly (KILL) terminate a container on Windows. -var SignalMap = map[string]syscall.Signal{ - "KILL": syscall.SIGKILL, - "TERM": syscall.SIGTERM, -} diff --git a/signal/trap.go b/signal/trap.go deleted file mode 100644 index d35ef0e..0000000 --- a/signal/trap.go +++ /dev/null @@ -1,81 +0,0 @@ -package signal - -import ( - "os" - gosignal "os/signal" - "runtime" - "sync/atomic" - "syscall" - - "github.com/Sirupsen/logrus" -) - -// Trap sets up a simplified signal "trap", appropriate for common -// behavior expected from a vanilla unix command-line tool in general -// (and the Docker engine in particular). -// -// * If SIGINT or SIGTERM are received, `cleanup` is called, then the process is terminated. -// * If SIGINT or SIGTERM are received 3 times before cleanup is complete, then cleanup is -// skipped and the process is terminated immediately (allows force quit of stuck daemon) -// * A SIGQUIT always causes an exit without cleanup, with a goroutine dump preceding exit. -// * Ignore SIGPIPE events. These are generated by systemd when journald is restarted while -// the docker daemon is not restarted and also running under systemd. -// Fixes https://github.com/docker/docker/issues/19728 -// -func Trap(cleanup func()) { - c := make(chan os.Signal, 1) - // we will handle INT, TERM, QUIT, SIGPIPE here - signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGPIPE} - gosignal.Notify(c, signals...) - go func() { - interruptCount := uint32(0) - for sig := range c { - if sig == syscall.SIGPIPE { - continue - } - - go func(sig os.Signal) { - logrus.Infof("Processing signal '%v'", sig) - switch sig { - case os.Interrupt, syscall.SIGTERM: - if atomic.LoadUint32(&interruptCount) < 3 { - // Initiate the cleanup only once - if atomic.AddUint32(&interruptCount, 1) == 1 { - // Call the provided cleanup handler - cleanup() - os.Exit(0) - } else { - return - } - } else { - // 3 SIGTERM/INT signals received; force exit without cleanup - logrus.Info("Forcing docker daemon shutdown without cleanup; 3 interrupts received") - } - case syscall.SIGQUIT: - DumpStacks() - logrus.Info("Forcing docker daemon shutdown without cleanup on SIGQUIT") - } - //for the SIGINT/TERM, and SIGQUIT non-clean shutdown case, exit with 128 + signal # - os.Exit(128 + int(sig.(syscall.Signal))) - }(sig) - } - }() -} - -// DumpStacks dumps the runtime stack. -func DumpStacks() { - var ( - buf []byte - stackSize int - ) - bufferLen := 16384 - for stackSize == len(buf) { - buf = make([]byte, bufferLen) - stackSize = runtime.Stack(buf, true) - bufferLen *= 2 - } - buf = buf[:stackSize] - // Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine - // traces won't show up in the log. - logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) -} diff --git a/stdcopy/stdcopy.go b/stdcopy/stdcopy.go deleted file mode 100644 index 8f67ece..0000000 --- a/stdcopy/stdcopy.go +++ /dev/null @@ -1,185 +0,0 @@ -package stdcopy - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" - "sync" - - "github.com/Sirupsen/logrus" -) - -// StdType is the type of standard stream -// a writer can multiplex to. -type StdType byte - -const ( - // Stdin represents standard input stream type. - Stdin StdType = iota - // Stdout represents standard output stream type. - Stdout - // Stderr represents standard error steam type. - Stderr - - stdWriterPrefixLen = 8 - stdWriterFdIndex = 0 - stdWriterSizeIndex = 4 - - startingBufLen = 32*1024 + stdWriterPrefixLen + 1 -) - -var bufPool = &sync.Pool{New: func() interface{} { return bytes.NewBuffer(nil) }} - -// stdWriter is wrapper of io.Writer with extra customized info. -type stdWriter struct { - io.Writer - prefix byte -} - -// Write sends the buffer to the underneath writer. -// It inserts the prefix header before the buffer, -// so stdcopy.StdCopy knows where to multiplex the output. -// It makes stdWriter to implement io.Writer. -func (w *stdWriter) Write(p []byte) (n int, err error) { - if w == nil || w.Writer == nil { - return 0, errors.New("Writer not instantiated") - } - if p == nil { - return 0, nil - } - - header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix} - binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(p))) - buf := bufPool.Get().(*bytes.Buffer) - buf.Write(header[:]) - buf.Write(p) - - n, err = w.Writer.Write(buf.Bytes()) - n -= stdWriterPrefixLen - if n < 0 { - n = 0 - } - - buf.Reset() - bufPool.Put(buf) - return -} - -// NewStdWriter instantiates a new Writer. -// Everything written to it will be encapsulated using a custom format, -// and written to the underlying `w` stream. -// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection. -// `t` indicates the id of the stream to encapsulate. -// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr. -func NewStdWriter(w io.Writer, t StdType) io.Writer { - return &stdWriter{ - Writer: w, - prefix: byte(t), - } -} - -// StdCopy is a modified version of io.Copy. -// -// StdCopy will demultiplex `src`, assuming that it contains two streams, -// previously multiplexed together using a StdWriter instance. -// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`. -// -// StdCopy will read until it hits EOF on `src`. It will then return a nil error. -// In other words: if `err` is non nil, it indicates a real underlying error. -// -// `written` will hold the total number of bytes written to `dstout` and `dsterr`. -func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { - var ( - buf = make([]byte, startingBufLen) - bufLen = len(buf) - nr, nw int - er, ew error - out io.Writer - frameSize int - ) - - for { - // Make sure we have at least a full header - for nr < stdWriterPrefixLen { - var nr2 int - nr2, er = src.Read(buf[nr:]) - nr += nr2 - if er == io.EOF { - if nr < stdWriterPrefixLen { - logrus.Debugf("Corrupted prefix: %v", buf[:nr]) - return written, nil - } - break - } - if er != nil { - logrus.Debugf("Error reading header: %s", er) - return 0, er - } - } - - // Check the first byte to know where to write - switch StdType(buf[stdWriterFdIndex]) { - case Stdin: - fallthrough - case Stdout: - // Write on stdout - out = dstout - case Stderr: - // Write on stderr - out = dsterr - default: - logrus.Debugf("Error selecting output fd: (%d)", buf[stdWriterFdIndex]) - return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex]) - } - - // Retrieve the size of the frame - frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4])) - logrus.Debugf("framesize: %d", frameSize) - - // Check if the buffer is big enough to read the frame. - // Extend it if necessary. - if frameSize+stdWriterPrefixLen > bufLen { - logrus.Debugf("Extending buffer cap by %d (was %d)", frameSize+stdWriterPrefixLen-bufLen+1, len(buf)) - buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...) - bufLen = len(buf) - } - - // While the amount of bytes read is less than the size of the frame + header, we keep reading - for nr < frameSize+stdWriterPrefixLen { - var nr2 int - nr2, er = src.Read(buf[nr:]) - nr += nr2 - if er == io.EOF { - if nr < frameSize+stdWriterPrefixLen { - logrus.Debugf("Corrupted frame: %v", buf[stdWriterPrefixLen:nr]) - return written, nil - } - break - } - if er != nil { - logrus.Debugf("Error reading frame: %s", er) - return 0, er - } - } - - // Write the retrieved frame (without header) - nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen]) - if ew != nil { - logrus.Debugf("Error writing frame: %s", ew) - return 0, ew - } - // If the frame has not been fully written: error - if nw != frameSize { - logrus.Debugf("Error Short Write: (%d on %d)", nw, frameSize) - return 0, io.ErrShortWrite - } - written += int64(nw) - - // Move the rest of the buffer to the beginning - copy(buf, buf[frameSize+stdWriterPrefixLen:]) - // Move the index - nr -= frameSize + stdWriterPrefixLen - } -} diff --git a/stdcopy/stdcopy_test.go b/stdcopy/stdcopy_test.go deleted file mode 100644 index 3137a75..0000000 --- a/stdcopy/stdcopy_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package stdcopy - -import ( - "bytes" - "errors" - "io" - "io/ioutil" - "strings" - "testing" -) - -func TestNewStdWriter(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - if writer == nil { - t.Fatalf("NewStdWriter with an invalid StdType should not return nil.") - } -} - -func TestWriteWithUnitializedStdWriter(t *testing.T) { - writer := stdWriter{ - Writer: nil, - prefix: byte(Stdout), - } - n, err := writer.Write([]byte("Something here")) - if n != 0 || err == nil { - t.Fatalf("Should fail when given an uncomplete or uninitialized StdWriter") - } -} - -func TestWriteWithNilBytes(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - n, err := writer.Write(nil) - if err != nil { - t.Fatalf("Shouldn't have fail when given no data") - } - if n > 0 { - t.Fatalf("Write should have written 0 byte, but has written %d", n) - } -} - -func TestWrite(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - data := []byte("Test StdWrite.Write") - n, err := writer.Write(data) - if err != nil { - t.Fatalf("Error while writing with StdWrite") - } - if n != len(data) { - t.Fatalf("Write should have written %d byte but wrote %d.", len(data), n) - } -} - -type errWriter struct { - n int - err error -} - -func (f *errWriter) Write(buf []byte) (int, error) { - return f.n, f.err -} - -func TestWriteWithWriterError(t *testing.T) { - expectedError := errors.New("expected") - expectedReturnedBytes := 10 - writer := NewStdWriter(&errWriter{ - n: stdWriterPrefixLen + expectedReturnedBytes, - err: expectedError}, Stdout) - data := []byte("This won't get written, sigh") - n, err := writer.Write(data) - if err != expectedError { - t.Fatalf("Didn't get expected error.") - } - if n != expectedReturnedBytes { - t.Fatalf("Didn't get expected written bytes %d, got %d.", - expectedReturnedBytes, n) - } -} - -func TestWriteDoesNotReturnNegativeWrittenBytes(t *testing.T) { - writer := NewStdWriter(&errWriter{n: -1}, Stdout) - data := []byte("This won't get written, sigh") - actual, _ := writer.Write(data) - if actual != 0 { - t.Fatalf("Expected returned written bytes equal to 0, got %d", actual) - } -} - -func getSrcBuffer(stdOutBytes, stdErrBytes []byte) (buffer *bytes.Buffer, err error) { - buffer = new(bytes.Buffer) - dstOut := NewStdWriter(buffer, Stdout) - _, err = dstOut.Write(stdOutBytes) - if err != nil { - return - } - dstErr := NewStdWriter(buffer, Stderr) - _, err = dstErr.Write(stdErrBytes) - return -} - -func TestStdCopyWriteAndRead(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - written, err := StdCopy(ioutil.Discard, ioutil.Discard, buffer) - if err != nil { - t.Fatal(err) - } - expectedTotalWritten := len(stdOutBytes) + len(stdErrBytes) - if written != int64(expectedTotalWritten) { - t.Fatalf("Expected to have total of %d bytes written, got %d", expectedTotalWritten, written) - } -} - -type customReader struct { - n int - err error - totalCalls int - correctCalls int - src *bytes.Buffer -} - -func (f *customReader) Read(buf []byte) (int, error) { - f.totalCalls++ - if f.totalCalls <= f.correctCalls { - return f.src.Read(buf) - } - return f.n, f.err -} - -func TestStdCopyReturnsErrorReadingHeader(t *testing.T) { - expectedError := errors.New("error") - reader := &customReader{ - err: expectedError} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != 0 { - t.Fatalf("Expected 0 bytes read, got %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error") - } -} - -func TestStdCopyReturnsErrorReadingFrame(t *testing.T) { - expectedError := errors.New("error") - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - reader := &customReader{ - correctCalls: 1, - n: stdWriterPrefixLen + 1, - err: expectedError, - src: buffer} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != 0 { - t.Fatalf("Expected 0 bytes read, got %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error") - } -} - -func TestStdCopyDetectsCorruptedFrame(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - reader := &customReader{ - correctCalls: 1, - n: stdWriterPrefixLen + 1, - err: io.EOF, - src: buffer} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != startingBufLen { - t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written) - } - if err != nil { - t.Fatal("Didn't get nil error") - } -} - -func TestStdCopyWithInvalidInputHeader(t *testing.T) { - dstOut := NewStdWriter(ioutil.Discard, Stdout) - dstErr := NewStdWriter(ioutil.Discard, Stderr) - src := strings.NewReader("Invalid input") - _, err := StdCopy(dstOut, dstErr, src) - if err == nil { - t.Fatal("StdCopy with invalid input header should fail.") - } -} - -func TestStdCopyWithCorruptedPrefix(t *testing.T) { - data := []byte{0x01, 0x02, 0x03} - src := bytes.NewReader(data) - written, err := StdCopy(nil, nil, src) - if err != nil { - t.Fatalf("StdCopy should not return an error with corrupted prefix.") - } - if written != 0 { - t.Fatalf("StdCopy should have written 0, but has written %d", written) - } -} - -func TestStdCopyReturnsWriteErrors(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - expectedError := errors.New("expected") - - dstOut := &errWriter{err: expectedError} - - written, err := StdCopy(dstOut, ioutil.Discard, buffer) - if written != 0 { - t.Fatalf("StdCopy should have written 0, but has written %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error, got %v", err) - } -} - -func TestStdCopyDetectsNotFullyWrittenFrames(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - dstOut := &errWriter{n: startingBufLen - 10} - - written, err := StdCopy(dstOut, ioutil.Discard, buffer) - if written != 0 { - t.Fatalf("StdCopy should have return 0 written bytes, but returned %d", written) - } - if err != io.ErrShortWrite { - t.Fatalf("Didn't get expected io.ErrShortWrite error") - } -} - -func BenchmarkWrite(b *testing.B) { - w := NewStdWriter(ioutil.Discard, Stdout) - data := []byte("Test line for testing stdwriter performance\n") - data = bytes.Repeat(data, 100) - b.SetBytes(int64(len(data))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := w.Write(data); err != nil { - b.Fatal(err) - } - } -} diff --git a/streamformatter/streamformatter.go b/streamformatter/streamformatter.go deleted file mode 100644 index ce6ea79..0000000 --- a/streamformatter/streamformatter.go +++ /dev/null @@ -1,172 +0,0 @@ -// Package streamformatter provides helper functions to format a stream. -package streamformatter - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/docker/docker/pkg/jsonmessage" - "github.com/docker/docker/pkg/progress" -) - -// StreamFormatter formats a stream, optionally using JSON. -type StreamFormatter struct { - json bool -} - -// NewStreamFormatter returns a simple StreamFormatter -func NewStreamFormatter() *StreamFormatter { - return &StreamFormatter{} -} - -// NewJSONStreamFormatter returns a StreamFormatter configured to stream json -func NewJSONStreamFormatter() *StreamFormatter { - return &StreamFormatter{true} -} - -const streamNewline = "\r\n" - -var streamNewlineBytes = []byte(streamNewline) - -// FormatStream formats the specified stream. -func (sf *StreamFormatter) FormatStream(str string) []byte { - if sf.json { - b, err := json.Marshal(&jsonmessage.JSONMessage{Stream: str}) - if err != nil { - return sf.FormatError(err) - } - return append(b, streamNewlineBytes...) - } - return []byte(str + "\r") -} - -// FormatStatus formats the specified objects according to the specified format (and id). -func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte { - str := fmt.Sprintf(format, a...) - if sf.json { - b, err := json.Marshal(&jsonmessage.JSONMessage{ID: id, Status: str}) - if err != nil { - return sf.FormatError(err) - } - return append(b, streamNewlineBytes...) - } - return []byte(str + streamNewline) -} - -// FormatError formats the specified error. -func (sf *StreamFormatter) FormatError(err error) []byte { - if sf.json { - jsonError, ok := err.(*jsonmessage.JSONError) - if !ok { - jsonError = &jsonmessage.JSONError{Message: err.Error()} - } - if b, err := json.Marshal(&jsonmessage.JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { - return append(b, streamNewlineBytes...) - } - return []byte("{\"error\":\"format error\"}" + streamNewline) - } - return []byte("Error: " + err.Error() + streamNewline) -} - -// FormatProgress formats the progress information for a specified action. -func (sf *StreamFormatter) FormatProgress(id, action string, progress *jsonmessage.JSONProgress, aux interface{}) []byte { - if progress == nil { - progress = &jsonmessage.JSONProgress{} - } - if sf.json { - var auxJSON *json.RawMessage - if aux != nil { - auxJSONBytes, err := json.Marshal(aux) - if err != nil { - return nil - } - auxJSON = new(json.RawMessage) - *auxJSON = auxJSONBytes - } - b, err := json.Marshal(&jsonmessage.JSONMessage{ - Status: action, - ProgressMessage: progress.String(), - Progress: progress, - ID: id, - Aux: auxJSON, - }) - if err != nil { - return nil - } - return append(b, streamNewlineBytes...) - } - endl := "\r" - if progress.String() == "" { - endl += "\n" - } - return []byte(action + " " + progress.String() + endl) -} - -// NewProgressOutput returns a progress.Output object that can be passed to -// progress.NewProgressReader. -func (sf *StreamFormatter) NewProgressOutput(out io.Writer, newLines bool) progress.Output { - return &progressOutput{ - sf: sf, - out: out, - newLines: newLines, - } -} - -type progressOutput struct { - sf *StreamFormatter - out io.Writer - newLines bool -} - -// WriteProgress formats progress information from a ProgressReader. -func (out *progressOutput) WriteProgress(prog progress.Progress) error { - var formatted []byte - if prog.Message != "" { - formatted = out.sf.FormatStatus(prog.ID, prog.Message) - } else { - jsonProgress := jsonmessage.JSONProgress{Current: prog.Current, Total: prog.Total} - formatted = out.sf.FormatProgress(prog.ID, prog.Action, &jsonProgress, prog.Aux) - } - _, err := out.out.Write(formatted) - if err != nil { - return err - } - - if out.newLines && prog.LastUpdate { - _, err = out.out.Write(out.sf.FormatStatus("", "")) - return err - } - - return nil -} - -// StdoutFormatter is a streamFormatter that writes to the standard output. -type StdoutFormatter struct { - io.Writer - *StreamFormatter -} - -func (sf *StdoutFormatter) Write(buf []byte) (int, error) { - formattedBuf := sf.StreamFormatter.FormatStream(string(buf)) - n, err := sf.Writer.Write(formattedBuf) - if n != len(formattedBuf) { - return n, io.ErrShortWrite - } - return len(buf), err -} - -// StderrFormatter is a streamFormatter that writes to the standard error. -type StderrFormatter struct { - io.Writer - *StreamFormatter -} - -func (sf *StderrFormatter) Write(buf []byte) (int, error) { - formattedBuf := sf.StreamFormatter.FormatStream("\033[91m" + string(buf) + "\033[0m") - n, err := sf.Writer.Write(formattedBuf) - if n != len(formattedBuf) { - return n, io.ErrShortWrite - } - return len(buf), err -} diff --git a/streamformatter/streamformatter_test.go b/streamformatter/streamformatter_test.go deleted file mode 100644 index 93ec90f..0000000 --- a/streamformatter/streamformatter_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package streamformatter - -import ( - "encoding/json" - "errors" - "reflect" - "strings" - "testing" - - "github.com/docker/docker/pkg/jsonmessage" -) - -func TestFormatStream(t *testing.T) { - sf := NewStreamFormatter() - res := sf.FormatStream("stream") - if string(res) != "stream"+"\r" { - t.Fatalf("%q", res) - } -} - -func TestFormatJSONStatus(t *testing.T) { - sf := NewStreamFormatter() - res := sf.FormatStatus("ID", "%s%d", "a", 1) - if string(res) != "a1\r\n" { - t.Fatalf("%q", res) - } -} - -func TestFormatSimpleError(t *testing.T) { - sf := NewStreamFormatter() - res := sf.FormatError(errors.New("Error for formatter")) - if string(res) != "Error: Error for formatter\r\n" { - t.Fatalf("%q", res) - } -} - -func TestJSONFormatStream(t *testing.T) { - sf := NewJSONStreamFormatter() - res := sf.FormatStream("stream") - if string(res) != `{"stream":"stream"}`+"\r\n" { - t.Fatalf("%q", res) - } -} - -func TestJSONFormatStatus(t *testing.T) { - sf := NewJSONStreamFormatter() - res := sf.FormatStatus("ID", "%s%d", "a", 1) - if string(res) != `{"status":"a1","id":"ID"}`+"\r\n" { - t.Fatalf("%q", res) - } -} - -func TestJSONFormatSimpleError(t *testing.T) { - sf := NewJSONStreamFormatter() - res := sf.FormatError(errors.New("Error for formatter")) - if string(res) != `{"errorDetail":{"message":"Error for formatter"},"error":"Error for formatter"}`+"\r\n" { - t.Fatalf("%q", res) - } -} - -func TestJSONFormatJSONError(t *testing.T) { - sf := NewJSONStreamFormatter() - err := &jsonmessage.JSONError{Code: 50, Message: "Json error"} - res := sf.FormatError(err) - if string(res) != `{"errorDetail":{"code":50,"message":"Json error"},"error":"Json error"}`+"\r\n" { - t.Fatalf("%q", res) - } -} - -func TestJSONFormatProgress(t *testing.T) { - sf := NewJSONStreamFormatter() - progress := &jsonmessage.JSONProgress{ - Current: 15, - Total: 30, - Start: 1, - } - res := sf.FormatProgress("id", "action", progress, nil) - msg := &jsonmessage.JSONMessage{} - if err := json.Unmarshal(res, msg); err != nil { - t.Fatal(err) - } - if msg.ID != "id" { - t.Fatalf("ID must be 'id', got: %s", msg.ID) - } - if msg.Status != "action" { - t.Fatalf("Status must be 'action', got: %s", msg.Status) - } - - // The progress will always be in the format of: - // [=========================> ] 15 B/30 B 404933h7m11s - // The last entry '404933h7m11s' is the timeLeftBox. - // However, the timeLeftBox field may change as progress.String() depends on time.Now(). - // Therefore, we have to strip the timeLeftBox from the strings to do the comparison. - - // Compare the progress strings before the timeLeftBox - expectedProgress := "[=========================> ] 15 B/30 B" - // if terminal column is <= 110, expectedProgressShort is expected. - expectedProgressShort := " 15 B/30 B" - if !(strings.HasPrefix(msg.ProgressMessage, expectedProgress) || - strings.HasPrefix(msg.ProgressMessage, expectedProgressShort)) { - t.Fatalf("ProgressMessage without the timeLeftBox must be %s or %s, got: %s", - expectedProgress, expectedProgressShort, msg.ProgressMessage) - } - - if !reflect.DeepEqual(msg.Progress, progress) { - t.Fatal("Original progress not equals progress from FormatProgress") - } -} diff --git a/symlink/LICENSE.APACHE b/symlink/LICENSE.APACHE deleted file mode 100644 index 34c4ea7..0000000 --- a/symlink/LICENSE.APACHE +++ /dev/null @@ -1,191 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2014-2016 Docker, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/symlink/LICENSE.BSD b/symlink/LICENSE.BSD deleted file mode 100644 index 9b4f4a2..0000000 --- a/symlink/LICENSE.BSD +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2014-2016 The Docker & Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/symlink/README.md b/symlink/README.md deleted file mode 100644 index 8dba54f..0000000 --- a/symlink/README.md +++ /dev/null @@ -1,6 +0,0 @@ -Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks, -as well as a Windows long-path aware version of filepath.EvalSymlinks -from the [Go standard library](https://golang.org/pkg/path/filepath). - -The code from filepath.EvalSymlinks has been adapted in fs.go. -Please read the LICENSE.BSD file that governs fs.go and LICENSE.APACHE for fs_test.go. diff --git a/symlink/fs.go b/symlink/fs.go deleted file mode 100644 index dcf707f..0000000 --- a/symlink/fs.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.BSD file. - -// This code is a modified version of path/filepath/symlink.go from the Go standard library. - -package symlink - -import ( - "bytes" - "errors" - "os" - "path/filepath" - "strings" - - "github.com/docker/docker/pkg/system" -) - -// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an -// absolute path. This function handles paths in a platform-agnostic manner. -func FollowSymlinkInScope(path, root string) (string, error) { - path, err := filepath.Abs(filepath.FromSlash(path)) - if err != nil { - return "", err - } - root, err = filepath.Abs(filepath.FromSlash(root)) - if err != nil { - return "", err - } - return evalSymlinksInScope(path, root) -} - -// evalSymlinksInScope will evaluate symlinks in `path` within a scope `root` and return -// a result guaranteed to be contained within the scope `root`, at the time of the call. -// Symlinks in `root` are not evaluated and left as-is. -// Errors encountered while attempting to evaluate symlinks in path will be returned. -// Non-existing paths are valid and do not constitute an error. -// `path` has to contain `root` as a prefix, or else an error will be returned. -// Trying to break out from `root` does not constitute an error. -// -// Example: -// If /foo/bar -> /outside, -// FollowSymlinkInScope("/foo/bar", "/foo") == "/foo/outside" instead of "/oustide" -// -// IMPORTANT: it is the caller's responsibility to call evalSymlinksInScope *after* relevant symlinks -// are created and not to create subsequently, additional symlinks that could potentially make a -// previously-safe path, unsafe. Example: if /foo/bar does not exist, evalSymlinksInScope("/foo/bar", "/foo") -// would return "/foo/bar". If one makes /foo/bar a symlink to /baz subsequently, then "/foo/bar" should -// no longer be considered safely contained in "/foo". -func evalSymlinksInScope(path, root string) (string, error) { - root = filepath.Clean(root) - if path == root { - return path, nil - } - if !strings.HasPrefix(path, root) { - return "", errors.New("evalSymlinksInScope: " + path + " is not in " + root) - } - const maxIter = 255 - originalPath := path - // given root of "/a" and path of "/a/b/../../c" we want path to be "/b/../../c" - path = path[len(root):] - if root == string(filepath.Separator) { - path = string(filepath.Separator) + path - } - if !strings.HasPrefix(path, string(filepath.Separator)) { - return "", errors.New("evalSymlinksInScope: " + path + " is not in " + root) - } - path = filepath.Clean(path) - // consume path by taking each frontmost path element, - // expanding it if it's a symlink, and appending it to b - var b bytes.Buffer - // b here will always be considered to be the "current absolute path inside - // root" when we append paths to it, we also append a slash and use - // filepath.Clean after the loop to trim the trailing slash - for n := 0; path != ""; n++ { - if n > maxIter { - return "", errors.New("evalSymlinksInScope: too many links in " + originalPath) - } - - // find next path component, p - i := strings.IndexRune(path, filepath.Separator) - var p string - if i == -1 { - p, path = path, "" - } else { - p, path = path[:i], path[i+1:] - } - - if p == "" { - continue - } - - // this takes a b.String() like "b/../" and a p like "c" and turns it - // into "/b/../c" which then gets filepath.Cleaned into "/c" and then - // root gets prepended and we Clean again (to remove any trailing slash - // if the first Clean gave us just "/") - cleanP := filepath.Clean(string(filepath.Separator) + b.String() + p) - if cleanP == string(filepath.Separator) { - // never Lstat "/" itself - b.Reset() - continue - } - fullP := filepath.Clean(root + cleanP) - - fi, err := os.Lstat(fullP) - if os.IsNotExist(err) { - // if p does not exist, accept it - b.WriteString(p) - b.WriteRune(filepath.Separator) - continue - } - if err != nil { - return "", err - } - if fi.Mode()&os.ModeSymlink == 0 { - b.WriteString(p + string(filepath.Separator)) - continue - } - - // it's a symlink, put it at the front of path - dest, err := os.Readlink(fullP) - if err != nil { - return "", err - } - if system.IsAbs(dest) { - b.Reset() - } - path = dest + string(filepath.Separator) + path - } - - // see note above on "fullP := ..." for why this is double-cleaned and - // what's happening here - return filepath.Clean(root + filepath.Clean(string(filepath.Separator)+b.String())), nil -} - -// EvalSymlinks returns the path name after the evaluation of any symbolic -// links. -// If path is relative the result will be relative to the current directory, -// unless one of the components is an absolute symbolic link. -// This version has been updated to support long paths prepended with `\\?\`. -func EvalSymlinks(path string) (string, error) { - return evalSymlinks(path) -} diff --git a/symlink/fs_unix.go b/symlink/fs_unix.go deleted file mode 100644 index 818004f..0000000 --- a/symlink/fs_unix.go +++ /dev/null @@ -1,11 +0,0 @@ -// +build !windows - -package symlink - -import ( - "path/filepath" -) - -func evalSymlinks(path string) (string, error) { - return filepath.EvalSymlinks(path) -} diff --git a/symlink/fs_unix_test.go b/symlink/fs_unix_test.go deleted file mode 100644 index 7085c0b..0000000 --- a/symlink/fs_unix_test.go +++ /dev/null @@ -1,407 +0,0 @@ -// +build !windows - -// Licensed under the Apache License, Version 2.0; See LICENSE.APACHE - -package symlink - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" -) - -// TODO Windows: This needs some serious work to port to Windows. For now, -// turning off testing in this package. - -type dirOrLink struct { - path string - target string -} - -func makeFs(tmpdir string, fs []dirOrLink) error { - for _, s := range fs { - s.path = filepath.Join(tmpdir, s.path) - if s.target == "" { - os.MkdirAll(s.path, 0755) - continue - } - if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { - return err - } - if err := os.Symlink(s.target, s.path); err != nil && !os.IsExist(err) { - return err - } - } - return nil -} - -func testSymlink(tmpdir, path, expected, scope string) error { - rewrite, err := FollowSymlinkInScope(filepath.Join(tmpdir, path), filepath.Join(tmpdir, scope)) - if err != nil { - return err - } - expected, err = filepath.Abs(filepath.Join(tmpdir, expected)) - if err != nil { - return err - } - if expected != rewrite { - return fmt.Errorf("Expected %q got %q", expected, rewrite) - } - return nil -} - -func TestFollowSymlinkAbsolute(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkAbsolute") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/d", target: "/b"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "testdata/fs/a/d/c/data", "testdata/b/c/data", "testdata"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkRelativePath(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativePath") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/i", target: "a"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "testdata/fs/i", "testdata/fs/a", "testdata"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkSkipSymlinksOutsideScope(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkSkipSymlinksOutsideScope") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - if err := makeFs(tmpdir, []dirOrLink{ - {path: "linkdir", target: "realdir"}, - {path: "linkdir/foo/bar"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "linkdir/foo/bar", "linkdir/foo/bar", "linkdir/foo"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkInvalidScopePathPair(t *testing.T) { - if _, err := FollowSymlinkInScope("toto", "testdata"); err == nil { - t.Fatal("expected an error") - } -} - -func TestFollowSymlinkLastLink(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkLastLink") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/d", target: "/b"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "testdata/fs/a/d", "testdata/b", "testdata"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkRelativeLinkChangeScope(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkChangeScope") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/e", target: "../b"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "testdata/fs/a/e/c/data", "testdata/fs/b/c/data", "testdata"); err != nil { - t.Fatal(err) - } - // avoid letting allowing symlink e lead us to ../b - // normalize to the "testdata/fs/a" - if err := testSymlink(tmpdir, "testdata/fs/a/e", "testdata/fs/a/b", "testdata/fs/a"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkDeepRelativeLinkChangeScope(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkDeepRelativeLinkChangeScope") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/f", target: "../../../../test"}}); err != nil { - t.Fatal(err) - } - // avoid letting symlink f lead us out of the "testdata" scope - // we don't normalize because symlink f is in scope and there is no - // information leak - if err := testSymlink(tmpdir, "testdata/fs/a/f", "testdata/test", "testdata"); err != nil { - t.Fatal(err) - } - // avoid letting symlink f lead us out of the "testdata/fs" scope - // we don't normalize because symlink f is in scope and there is no - // information leak - if err := testSymlink(tmpdir, "testdata/fs/a/f", "testdata/fs/test", "testdata/fs"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkRelativeLinkChain(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkChain") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - // avoid letting symlink g (pointed at by symlink h) take out of scope - // TODO: we should probably normalize to scope here because ../[....]/root - // is out of scope and we leak information - if err := makeFs(tmpdir, []dirOrLink{ - {path: "testdata/fs/b/h", target: "../g"}, - {path: "testdata/fs/g", target: "../../../../../../../../../../../../root"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "testdata/fs/b/h", "testdata/root", "testdata"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkBreakoutPath(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkBreakoutPath") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - // avoid letting symlink -> ../directory/file escape from scope - // normalize to "testdata/fs/j" - if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/j/k", target: "../i/a"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "testdata/fs/j/k", "testdata/fs/j/i/a", "testdata/fs/j"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkToRoot(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkToRoot") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - // make sure we don't allow escaping to / - // normalize to dir - if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "/"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "foo", "", ""); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkSlashDotdot(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkSlashDotdot") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - tmpdir = filepath.Join(tmpdir, "dir", "subdir") - - // make sure we don't allow escaping to / - // normalize to dir - if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "/../../"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "foo", "", ""); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkDotdot(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkDotdot") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - tmpdir = filepath.Join(tmpdir, "dir", "subdir") - - // make sure we stay in scope without leaking information - // this also checks for escaping to / - // normalize to dir - if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "../../"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "foo", "", ""); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkRelativePath2(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativePath2") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - if err := makeFs(tmpdir, []dirOrLink{{path: "bar/foo", target: "baz/target"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "bar/foo", "bar/baz/target", ""); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkScopeLink(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkScopeLink") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - if err := makeFs(tmpdir, []dirOrLink{ - {path: "root2"}, - {path: "root", target: "root2"}, - {path: "root2/foo", target: "../bar"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "root/foo", "root/bar", "root"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkRootScope(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRootScope") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - expected, err := filepath.EvalSymlinks(tmpdir) - if err != nil { - t.Fatal(err) - } - rewrite, err := FollowSymlinkInScope(tmpdir, "/") - if err != nil { - t.Fatal(err) - } - if rewrite != expected { - t.Fatalf("expected %q got %q", expected, rewrite) - } -} - -func TestFollowSymlinkEmpty(t *testing.T) { - res, err := FollowSymlinkInScope("", "") - if err != nil { - t.Fatal(err) - } - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - if res != wd { - t.Fatalf("expected %q got %q", wd, res) - } -} - -func TestFollowSymlinkCircular(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkCircular") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - if err := makeFs(tmpdir, []dirOrLink{{path: "root/foo", target: "foo"}}); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "root/foo", "", "root"); err == nil { - t.Fatal("expected an error for foo -> foo") - } - - if err := makeFs(tmpdir, []dirOrLink{ - {path: "root/bar", target: "baz"}, - {path: "root/baz", target: "../bak"}, - {path: "root/bak", target: "/bar"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "root/foo", "", "root"); err == nil { - t.Fatal("expected an error for bar -> baz -> bak -> bar") - } -} - -func TestFollowSymlinkComplexChainWithTargetPathsContainingLinks(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkComplexChainWithTargetPathsContainingLinks") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - if err := makeFs(tmpdir, []dirOrLink{ - {path: "root2"}, - {path: "root", target: "root2"}, - {path: "root/a", target: "r/s"}, - {path: "root/r", target: "../root/t"}, - {path: "root/root/t/s/b", target: "/../u"}, - {path: "root/u/c", target: "."}, - {path: "root/u/x/y", target: "../v"}, - {path: "root/u/v", target: "/../w"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "root/a/b/c/x/y/z", "root/w/z", "root"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkBreakoutNonExistent(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkBreakoutNonExistent") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - if err := makeFs(tmpdir, []dirOrLink{ - {path: "root/slash", target: "/"}, - {path: "root/sym", target: "/idontexist/../slash"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "root/sym/file", "root/file", "root"); err != nil { - t.Fatal(err) - } -} - -func TestFollowSymlinkNoLexicalCleaning(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkNoLexicalCleaning") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - if err := makeFs(tmpdir, []dirOrLink{ - {path: "root/sym", target: "/foo/bar"}, - {path: "root/hello", target: "/sym/../baz"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "root/hello", "root/foo/baz", "root"); err != nil { - t.Fatal(err) - } -} diff --git a/symlink/fs_windows.go b/symlink/fs_windows.go deleted file mode 100644 index 449fe56..0000000 --- a/symlink/fs_windows.go +++ /dev/null @@ -1,155 +0,0 @@ -package symlink - -import ( - "bytes" - "errors" - "os" - "path/filepath" - "strings" - "syscall" - - "github.com/docker/docker/pkg/longpath" -) - -func toShort(path string) (string, error) { - p, err := syscall.UTF16FromString(path) - if err != nil { - return "", err - } - b := p // GetShortPathName says we can reuse buffer - n, err := syscall.GetShortPathName(&p[0], &b[0], uint32(len(b))) - if err != nil { - return "", err - } - if n > uint32(len(b)) { - b = make([]uint16, n) - if _, err = syscall.GetShortPathName(&p[0], &b[0], uint32(len(b))); err != nil { - return "", err - } - } - return syscall.UTF16ToString(b), nil -} - -func toLong(path string) (string, error) { - p, err := syscall.UTF16FromString(path) - if err != nil { - return "", err - } - b := p // GetLongPathName says we can reuse buffer - n, err := syscall.GetLongPathName(&p[0], &b[0], uint32(len(b))) - if err != nil { - return "", err - } - if n > uint32(len(b)) { - b = make([]uint16, n) - n, err = syscall.GetLongPathName(&p[0], &b[0], uint32(len(b))) - if err != nil { - return "", err - } - } - b = b[:n] - return syscall.UTF16ToString(b), nil -} - -func evalSymlinks(path string) (string, error) { - path, err := walkSymlinks(path) - if err != nil { - return "", err - } - - p, err := toShort(path) - if err != nil { - return "", err - } - p, err = toLong(p) - if err != nil { - return "", err - } - // syscall.GetLongPathName does not change the case of the drive letter, - // but the result of EvalSymlinks must be unique, so we have - // EvalSymlinks(`c:\a`) == EvalSymlinks(`C:\a`). - // Make drive letter upper case. - if len(p) >= 2 && p[1] == ':' && 'a' <= p[0] && p[0] <= 'z' { - p = string(p[0]+'A'-'a') + p[1:] - } else if len(p) >= 6 && p[5] == ':' && 'a' <= p[4] && p[4] <= 'z' { - p = p[:3] + string(p[4]+'A'-'a') + p[5:] - } - return filepath.Clean(p), nil -} - -const utf8RuneSelf = 0x80 - -func walkSymlinks(path string) (string, error) { - const maxIter = 255 - originalPath := path - // consume path by taking each frontmost path element, - // expanding it if it's a symlink, and appending it to b - var b bytes.Buffer - for n := 0; path != ""; n++ { - if n > maxIter { - return "", errors.New("EvalSymlinks: too many links in " + originalPath) - } - - // A path beginning with `\\?\` represents the root, so automatically - // skip that part and begin processing the next segment. - if strings.HasPrefix(path, longpath.Prefix) { - b.WriteString(longpath.Prefix) - path = path[4:] - continue - } - - // find next path component, p - var i = -1 - for j, c := range path { - if c < utf8RuneSelf && os.IsPathSeparator(uint8(c)) { - i = j - break - } - } - var p string - if i == -1 { - p, path = path, "" - } else { - p, path = path[:i], path[i+1:] - } - - if p == "" { - if b.Len() == 0 { - // must be absolute path - b.WriteRune(filepath.Separator) - } - continue - } - - // If this is the first segment after the long path prefix, accept the - // current segment as a volume root or UNC share and move on to the next. - if b.String() == longpath.Prefix { - b.WriteString(p) - b.WriteRune(filepath.Separator) - continue - } - - fi, err := os.Lstat(b.String() + p) - if err != nil { - return "", err - } - if fi.Mode()&os.ModeSymlink == 0 { - b.WriteString(p) - if path != "" || (b.Len() == 2 && len(p) == 2 && p[1] == ':') { - b.WriteRune(filepath.Separator) - } - continue - } - - // it's a symlink, put it at the front of path - dest, err := os.Readlink(b.String() + p) - if err != nil { - return "", err - } - if filepath.IsAbs(dest) || os.IsPathSeparator(dest[0]) { - b.Reset() - } - path = dest + string(filepath.Separator) + path - } - return filepath.Clean(b.String()), nil -} diff --git a/tailfile/tailfile.go b/tailfile/tailfile.go deleted file mode 100644 index d580584..0000000 --- a/tailfile/tailfile.go +++ /dev/null @@ -1,66 +0,0 @@ -// Package tailfile provides helper functions to read the nth lines of any -// ReadSeeker. -package tailfile - -import ( - "bytes" - "errors" - "io" - "os" -) - -const blockSize = 1024 - -var eol = []byte("\n") - -// ErrNonPositiveLinesNumber is an error returned if the lines number was negative. -var ErrNonPositiveLinesNumber = errors.New("The number of lines to extract from the file must be positive") - -//TailFile returns last n lines of reader f (could be a fil). -func TailFile(f io.ReadSeeker, n int) ([][]byte, error) { - if n <= 0 { - return nil, ErrNonPositiveLinesNumber - } - size, err := f.Seek(0, os.SEEK_END) - if err != nil { - return nil, err - } - block := -1 - var data []byte - var cnt int - for { - var b []byte - step := int64(block * blockSize) - left := size + step // how many bytes to beginning - if left < 0 { - if _, err := f.Seek(0, os.SEEK_SET); err != nil { - return nil, err - } - b = make([]byte, blockSize+left) - if _, err := f.Read(b); err != nil { - return nil, err - } - data = append(b, data...) - break - } else { - b = make([]byte, blockSize) - if _, err := f.Seek(step, os.SEEK_END); err != nil { - return nil, err - } - if _, err := f.Read(b); err != nil { - return nil, err - } - data = append(b, data...) - } - cnt += bytes.Count(b, eol) - if cnt > n { - break - } - block-- - } - lines := bytes.Split(data, eol) - if n < len(lines) { - return lines[len(lines)-n-1 : len(lines)-1], nil - } - return lines[:len(lines)-1], nil -} diff --git a/tailfile/tailfile_test.go b/tailfile/tailfile_test.go deleted file mode 100644 index 31217c0..0000000 --- a/tailfile/tailfile_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package tailfile - -import ( - "io/ioutil" - "os" - "testing" -) - -func TestTailFile(t *testing.T) { - f, err := ioutil.TempFile("", "tail-test") - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.RemoveAll(f.Name()) - testFile := []byte(`first line -second line -third line -fourth line -fifth line -next first line -next second line -next third line -next fourth line -next fifth line -last first line -next first line -next second line -next third line -next fourth line -next fifth line -next first line -next second line -next third line -next fourth line -next fifth line -last second line -last third line -last fourth line -last fifth line -truncated line`) - if _, err := f.Write(testFile); err != nil { - t.Fatal(err) - } - if _, err := f.Seek(0, os.SEEK_SET); err != nil { - t.Fatal(err) - } - expected := []string{"last fourth line", "last fifth line"} - res, err := TailFile(f, 2) - if err != nil { - t.Fatal(err) - } - for i, l := range res { - t.Logf("%s", l) - if expected[i] != string(l) { - t.Fatalf("Expected line %s, got %s", expected[i], l) - } - } -} - -func TestTailFileManyLines(t *testing.T) { - f, err := ioutil.TempFile("", "tail-test") - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.RemoveAll(f.Name()) - testFile := []byte(`first line -second line -truncated line`) - if _, err := f.Write(testFile); err != nil { - t.Fatal(err) - } - if _, err := f.Seek(0, os.SEEK_SET); err != nil { - t.Fatal(err) - } - expected := []string{"first line", "second line"} - res, err := TailFile(f, 10000) - if err != nil { - t.Fatal(err) - } - for i, l := range res { - t.Logf("%s", l) - if expected[i] != string(l) { - t.Fatalf("Expected line %s, got %s", expected[i], l) - } - } -} - -func TestTailEmptyFile(t *testing.T) { - f, err := ioutil.TempFile("", "tail-test") - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.RemoveAll(f.Name()) - res, err := TailFile(f, 10000) - if err != nil { - t.Fatal(err) - } - if len(res) != 0 { - t.Fatal("Must be empty slice from empty file") - } -} - -func TestTailNegativeN(t *testing.T) { - f, err := ioutil.TempFile("", "tail-test") - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.RemoveAll(f.Name()) - testFile := []byte(`first line -second line -truncated line`) - if _, err := f.Write(testFile); err != nil { - t.Fatal(err) - } - if _, err := f.Seek(0, os.SEEK_SET); err != nil { - t.Fatal(err) - } - if _, err := TailFile(f, -1); err != ErrNonPositiveLinesNumber { - t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err) - } - if _, err := TailFile(f, 0); err != ErrNonPositiveLinesNumber { - t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err) - } -} - -func BenchmarkTail(b *testing.B) { - f, err := ioutil.TempFile("", "tail-test") - if err != nil { - b.Fatal(err) - } - defer f.Close() - defer os.RemoveAll(f.Name()) - for i := 0; i < 10000; i++ { - if _, err := f.Write([]byte("tailfile pretty interesting line\n")); err != nil { - b.Fatal(err) - } - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := TailFile(f, 1000); err != nil { - b.Fatal(err) - } - } -} diff --git a/tarsum/builder_context.go b/tarsum/builder_context.go deleted file mode 100644 index b42983e..0000000 --- a/tarsum/builder_context.go +++ /dev/null @@ -1,21 +0,0 @@ -package tarsum - -// BuilderContext is an interface extending TarSum by adding the Remove method. -// In general there was concern about adding this method to TarSum itself -// so instead it is being added just to "BuilderContext" which will then -// only be used during the .dockerignore file processing -// - see builder/evaluator.go -type BuilderContext interface { - TarSum - Remove(string) -} - -func (bc *tarSum) Remove(filename string) { - for i, fis := range bc.sums { - if fis.Name() == filename { - bc.sums = append(bc.sums[:i], bc.sums[i+1:]...) - // Note, we don't just return because there could be - // more than one with this name - } - } -} diff --git a/tarsum/builder_context_test.go b/tarsum/builder_context_test.go deleted file mode 100644 index 719f728..0000000 --- a/tarsum/builder_context_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package tarsum - -import ( - "io" - "io/ioutil" - "os" - "testing" -) - -// Try to remove tarsum (in the BuilderContext) that do not exists, won't change a thing -func TestTarSumRemoveNonExistent(t *testing.T) { - filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar" - reader, err := os.Open(filename) - if err != nil { - t.Fatal(err) - } - ts, err := NewTarSum(reader, false, Version0) - if err != nil { - t.Fatal(err) - } - - // Read and discard bytes so that it populates sums - _, err = io.Copy(ioutil.Discard, ts) - if err != nil { - t.Errorf("failed to read from %s: %s", filename, err) - } - - expected := len(ts.GetSums()) - - ts.(BuilderContext).Remove("") - ts.(BuilderContext).Remove("Anything") - - if len(ts.GetSums()) != expected { - t.Fatalf("Expected %v sums, go %v.", expected, ts.GetSums()) - } -} - -// Remove a tarsum (in the BuilderContext) -func TestTarSumRemove(t *testing.T) { - filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar" - reader, err := os.Open(filename) - if err != nil { - t.Fatal(err) - } - ts, err := NewTarSum(reader, false, Version0) - if err != nil { - t.Fatal(err) - } - - // Read and discard bytes so that it populates sums - _, err = io.Copy(ioutil.Discard, ts) - if err != nil { - t.Errorf("failed to read from %s: %s", filename, err) - } - - expected := len(ts.GetSums()) - 1 - - ts.(BuilderContext).Remove("etc/sudoers") - - if len(ts.GetSums()) != expected { - t.Fatalf("Expected %v sums, go %v.", expected, len(ts.GetSums())) - } -} diff --git a/tarsum/fileinfosums.go b/tarsum/fileinfosums.go deleted file mode 100644 index 5abf5e7..0000000 --- a/tarsum/fileinfosums.go +++ /dev/null @@ -1,126 +0,0 @@ -package tarsum - -import "sort" - -// FileInfoSumInterface provides an interface for accessing file checksum -// information within a tar file. This info is accessed through interface -// so the actual name and sum cannot be melded with. -type FileInfoSumInterface interface { - // File name - Name() string - // Checksum of this particular file and its headers - Sum() string - // Position of file in the tar - Pos() int64 -} - -type fileInfoSum struct { - name string - sum string - pos int64 -} - -func (fis fileInfoSum) Name() string { - return fis.name -} -func (fis fileInfoSum) Sum() string { - return fis.sum -} -func (fis fileInfoSum) Pos() int64 { - return fis.pos -} - -// FileInfoSums provides a list of FileInfoSumInterfaces. -type FileInfoSums []FileInfoSumInterface - -// GetFile returns the first FileInfoSumInterface with a matching name. -func (fis FileInfoSums) GetFile(name string) FileInfoSumInterface { - for i := range fis { - if fis[i].Name() == name { - return fis[i] - } - } - return nil -} - -// GetAllFile returns a FileInfoSums with all matching names. -func (fis FileInfoSums) GetAllFile(name string) FileInfoSums { - f := FileInfoSums{} - for i := range fis { - if fis[i].Name() == name { - f = append(f, fis[i]) - } - } - return f -} - -// GetDuplicatePaths returns a FileInfoSums with all duplicated paths. -func (fis FileInfoSums) GetDuplicatePaths() (dups FileInfoSums) { - seen := make(map[string]int, len(fis)) // allocate earl. no need to grow this map. - for i := range fis { - f := fis[i] - if _, ok := seen[f.Name()]; ok { - dups = append(dups, f) - } else { - seen[f.Name()] = 0 - } - } - return dups -} - -// Len returns the size of the FileInfoSums. -func (fis FileInfoSums) Len() int { return len(fis) } - -// Swap swaps two FileInfoSum values if a FileInfoSums list. -func (fis FileInfoSums) Swap(i, j int) { fis[i], fis[j] = fis[j], fis[i] } - -// SortByPos sorts FileInfoSums content by position. -func (fis FileInfoSums) SortByPos() { - sort.Sort(byPos{fis}) -} - -// SortByNames sorts FileInfoSums content by name. -func (fis FileInfoSums) SortByNames() { - sort.Sort(byName{fis}) -} - -// SortBySums sorts FileInfoSums content by sums. -func (fis FileInfoSums) SortBySums() { - dups := fis.GetDuplicatePaths() - if len(dups) > 0 { - sort.Sort(bySum{fis, dups}) - } else { - sort.Sort(bySum{fis, nil}) - } -} - -// byName is a sort.Sort helper for sorting by file names. -// If names are the same, order them by their appearance in the tar archive -type byName struct{ FileInfoSums } - -func (bn byName) Less(i, j int) bool { - if bn.FileInfoSums[i].Name() == bn.FileInfoSums[j].Name() { - return bn.FileInfoSums[i].Pos() < bn.FileInfoSums[j].Pos() - } - return bn.FileInfoSums[i].Name() < bn.FileInfoSums[j].Name() -} - -// bySum is a sort.Sort helper for sorting by the sums of all the fileinfos in the tar archive -type bySum struct { - FileInfoSums - dups FileInfoSums -} - -func (bs bySum) Less(i, j int) bool { - if bs.dups != nil && bs.FileInfoSums[i].Name() == bs.FileInfoSums[j].Name() { - return bs.FileInfoSums[i].Pos() < bs.FileInfoSums[j].Pos() - } - return bs.FileInfoSums[i].Sum() < bs.FileInfoSums[j].Sum() -} - -// byPos is a sort.Sort helper for sorting by the sums of all the fileinfos by their original order -type byPos struct{ FileInfoSums } - -func (bp byPos) Less(i, j int) bool { - return bp.FileInfoSums[i].Pos() < bp.FileInfoSums[j].Pos() -} diff --git a/tarsum/fileinfosums_test.go b/tarsum/fileinfosums_test.go deleted file mode 100644 index bb700d8..0000000 --- a/tarsum/fileinfosums_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package tarsum - -import "testing" - -func newFileInfoSums() FileInfoSums { - return FileInfoSums{ - fileInfoSum{name: "file3", sum: "2abcdef1234567890", pos: 2}, - fileInfoSum{name: "dup1", sum: "deadbeef1", pos: 5}, - fileInfoSum{name: "file1", sum: "0abcdef1234567890", pos: 0}, - fileInfoSum{name: "file4", sum: "3abcdef1234567890", pos: 3}, - fileInfoSum{name: "dup1", sum: "deadbeef0", pos: 4}, - fileInfoSum{name: "file2", sum: "1abcdef1234567890", pos: 1}, - } -} - -func TestSortFileInfoSums(t *testing.T) { - dups := newFileInfoSums().GetAllFile("dup1") - if len(dups) != 2 { - t.Errorf("expected length 2, got %d", len(dups)) - } - dups.SortByNames() - if dups[0].Pos() != 4 { - t.Errorf("sorted dups should be ordered by position. Expected 4, got %d", dups[0].Pos()) - } - - fis := newFileInfoSums() - expected := "0abcdef1234567890" - fis.SortBySums() - got := fis[0].Sum() - if got != expected { - t.Errorf("Expected %q, got %q", expected, got) - } - - fis = newFileInfoSums() - expected = "dup1" - fis.SortByNames() - gotFis := fis[0] - if gotFis.Name() != expected { - t.Errorf("Expected %q, got %q", expected, gotFis.Name()) - } - // since a duplicate is first, ensure it is ordered first by position too - if gotFis.Pos() != 4 { - t.Errorf("Expected %d, got %d", 4, gotFis.Pos()) - } - - fis = newFileInfoSums() - fis.SortByPos() - if fis[0].Pos() != 0 { - t.Errorf("sorted fileInfoSums by Pos should order them by position.") - } - - fis = newFileInfoSums() - expected = "deadbeef1" - gotFileInfoSum := fis.GetFile("dup1") - if gotFileInfoSum.Sum() != expected { - t.Errorf("Expected %q, got %q", expected, gotFileInfoSum) - } - if fis.GetFile("noPresent") != nil { - t.Errorf("Should have return nil if name not found.") - } - -} diff --git a/tarsum/tarsum.go b/tarsum/tarsum.go deleted file mode 100644 index 154788d..0000000 --- a/tarsum/tarsum.go +++ /dev/null @@ -1,295 +0,0 @@ -// Package tarsum provides algorithms to perform checksum calculation on -// filesystem layers. -// -// The transportation of filesystems, regarding Docker, is done with tar(1) -// archives. There are a variety of tar serialization formats [2], and a key -// concern here is ensuring a repeatable checksum given a set of inputs from a -// generic tar archive. Types of transportation include distribution to and from a -// registry endpoint, saving and loading through commands or Docker daemon APIs, -// transferring the build context from client to Docker daemon, and committing the -// filesystem of a container to become an image. -// -// As tar archives are used for transit, but not preserved in many situations, the -// focus of the algorithm is to ensure the integrity of the preserved filesystem, -// while maintaining a deterministic accountability. This includes neither -// constraining the ordering or manipulation of the files during the creation or -// unpacking of the archive, nor include additional metadata state about the file -// system attributes. -package tarsum - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "crypto" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "hash" - "io" - "path" - "strings" -) - -const ( - buf8K = 8 * 1024 - buf16K = 16 * 1024 - buf32K = 32 * 1024 -) - -// NewTarSum creates a new interface for calculating a fixed time checksum of a -// tar archive. -// -// This is used for calculating checksums of layers of an image, in some cases -// including the byte payload of the image's json metadata as well, and for -// calculating the checksums for buildcache. -func NewTarSum(r io.Reader, dc bool, v Version) (TarSum, error) { - return NewTarSumHash(r, dc, v, DefaultTHash) -} - -// NewTarSumHash creates a new TarSum, providing a THash to use rather than -// the DefaultTHash. -func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) { - headerSelector, err := getTarHeaderSelector(v) - if err != nil { - return nil, err - } - ts := &tarSum{Reader: r, DisableCompression: dc, tarSumVersion: v, headerSelector: headerSelector, tHash: tHash} - err = ts.initTarSum() - return ts, err -} - -// NewTarSumForLabel creates a new TarSum using the provided TarSum version+hash label. -func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) { - parts := strings.SplitN(label, "+", 2) - if len(parts) != 2 { - return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}") - } - - versionName, hashName := parts[0], parts[1] - - version, ok := tarSumVersionsByName[versionName] - if !ok { - return nil, fmt.Errorf("unknown TarSum version name: %q", versionName) - } - - hashConfig, ok := standardHashConfigs[hashName] - if !ok { - return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName) - } - - tHash := NewTHash(hashConfig.name, hashConfig.hash.New) - - return NewTarSumHash(r, disableCompression, version, tHash) -} - -// TarSum is the generic interface for calculating fixed time -// checksums of a tar archive. -type TarSum interface { - io.Reader - GetSums() FileInfoSums - Sum([]byte) string - Version() Version - Hash() THash -} - -// tarSum struct is the structure for a Version0 checksum calculation. -type tarSum struct { - io.Reader - tarR *tar.Reader - tarW *tar.Writer - writer writeCloseFlusher - bufTar *bytes.Buffer - bufWriter *bytes.Buffer - bufData []byte - h hash.Hash - tHash THash - sums FileInfoSums - fileCounter int64 - currentFile string - finished bool - first bool - DisableCompression bool // false by default. When false, the output gzip compressed. - tarSumVersion Version // this field is not exported so it can not be mutated during use - headerSelector tarHeaderSelector // handles selecting and ordering headers for files in the archive -} - -func (ts tarSum) Hash() THash { - return ts.tHash -} - -func (ts tarSum) Version() Version { - return ts.tarSumVersion -} - -// THash provides a hash.Hash type generator and its name. -type THash interface { - Hash() hash.Hash - Name() string -} - -// NewTHash is a convenience method for creating a THash. -func NewTHash(name string, h func() hash.Hash) THash { - return simpleTHash{n: name, h: h} -} - -type tHashConfig struct { - name string - hash crypto.Hash -} - -var ( - // NOTE: DO NOT include MD5 or SHA1, which are considered insecure. - standardHashConfigs = map[string]tHashConfig{ - "sha256": {name: "sha256", hash: crypto.SHA256}, - "sha512": {name: "sha512", hash: crypto.SHA512}, - } -) - -// DefaultTHash is default TarSum hashing algorithm - "sha256". -var DefaultTHash = NewTHash("sha256", sha256.New) - -type simpleTHash struct { - n string - h func() hash.Hash -} - -func (sth simpleTHash) Name() string { return sth.n } -func (sth simpleTHash) Hash() hash.Hash { return sth.h() } - -func (ts *tarSum) encodeHeader(h *tar.Header) error { - for _, elem := range ts.headerSelector.selectHeaders(h) { - if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil { - return err - } - } - return nil -} - -func (ts *tarSum) initTarSum() error { - ts.bufTar = bytes.NewBuffer([]byte{}) - ts.bufWriter = bytes.NewBuffer([]byte{}) - ts.tarR = tar.NewReader(ts.Reader) - ts.tarW = tar.NewWriter(ts.bufTar) - if !ts.DisableCompression { - ts.writer = gzip.NewWriter(ts.bufWriter) - } else { - ts.writer = &nopCloseFlusher{Writer: ts.bufWriter} - } - if ts.tHash == nil { - ts.tHash = DefaultTHash - } - ts.h = ts.tHash.Hash() - ts.h.Reset() - ts.first = true - ts.sums = FileInfoSums{} - return nil -} - -func (ts *tarSum) Read(buf []byte) (int, error) { - if ts.finished { - return ts.bufWriter.Read(buf) - } - if len(ts.bufData) < len(buf) { - switch { - case len(buf) <= buf8K: - ts.bufData = make([]byte, buf8K) - case len(buf) <= buf16K: - ts.bufData = make([]byte, buf16K) - case len(buf) <= buf32K: - ts.bufData = make([]byte, buf32K) - default: - ts.bufData = make([]byte, len(buf)) - } - } - buf2 := ts.bufData[:len(buf)] - - n, err := ts.tarR.Read(buf2) - if err != nil { - if err == io.EOF { - if _, err := ts.h.Write(buf2[:n]); err != nil { - return 0, err - } - if !ts.first { - ts.sums = append(ts.sums, fileInfoSum{name: ts.currentFile, sum: hex.EncodeToString(ts.h.Sum(nil)), pos: ts.fileCounter}) - ts.fileCounter++ - ts.h.Reset() - } else { - ts.first = false - } - - currentHeader, err := ts.tarR.Next() - if err != nil { - if err == io.EOF { - if err := ts.tarW.Close(); err != nil { - return 0, err - } - if _, err := io.Copy(ts.writer, ts.bufTar); err != nil { - return 0, err - } - if err := ts.writer.Close(); err != nil { - return 0, err - } - ts.finished = true - return n, nil - } - return n, err - } - ts.currentFile = path.Clean(currentHeader.Name) - if err := ts.encodeHeader(currentHeader); err != nil { - return 0, err - } - if err := ts.tarW.WriteHeader(currentHeader); err != nil { - return 0, err - } - if _, err := ts.tarW.Write(buf2[:n]); err != nil { - return 0, err - } - ts.tarW.Flush() - if _, err := io.Copy(ts.writer, ts.bufTar); err != nil { - return 0, err - } - ts.writer.Flush() - - return ts.bufWriter.Read(buf) - } - return n, err - } - - // Filling the hash buffer - if _, err = ts.h.Write(buf2[:n]); err != nil { - return 0, err - } - - // Filling the tar writer - if _, err = ts.tarW.Write(buf2[:n]); err != nil { - return 0, err - } - ts.tarW.Flush() - - // Filling the output writer - if _, err = io.Copy(ts.writer, ts.bufTar); err != nil { - return 0, err - } - ts.writer.Flush() - - return ts.bufWriter.Read(buf) -} - -func (ts *tarSum) Sum(extra []byte) string { - ts.sums.SortBySums() - h := ts.tHash.Hash() - if extra != nil { - h.Write(extra) - } - for _, fis := range ts.sums { - h.Write([]byte(fis.Sum())) - } - checksum := ts.Version().String() + "+" + ts.tHash.Name() + ":" + hex.EncodeToString(h.Sum(nil)) - return checksum -} - -func (ts *tarSum) GetSums() FileInfoSums { - return ts.sums -} diff --git a/tarsum/tarsum_spec.md b/tarsum/tarsum_spec.md deleted file mode 100644 index 89b2e49..0000000 --- a/tarsum/tarsum_spec.md +++ /dev/null @@ -1,230 +0,0 @@ -page_title: TarSum checksum specification -page_description: Documentation for algorithms used in the TarSum checksum calculation -page_keywords: docker, checksum, validation, tarsum - -# TarSum Checksum Specification - -## Abstract - -This document describes the algorithms used in performing the TarSum checksum -calculation on filesystem layers, the need for this method over existing -methods, and the versioning of this calculation. - -## Warning - -This checksum algorithm is for best-effort comparison of file trees with fuzzy logic. - -This is _not_ a cryptographic attestation, and should not be considered secure. - -## Introduction - -The transportation of filesystems, regarding Docker, is done with tar(1) -archives. There are a variety of tar serialization formats [2], and a key -concern here is ensuring a repeatable checksum given a set of inputs from a -generic tar archive. Types of transportation include distribution to and from a -registry endpoint, saving and loading through commands or Docker daemon APIs, -transferring the build context from client to Docker daemon, and committing the -filesystem of a container to become an image. - -As tar archives are used for transit, but not preserved in many situations, the -focus of the algorithm is to ensure the integrity of the preserved filesystem, -while maintaining a deterministic accountability. This includes neither -constraining the ordering or manipulation of the files during the creation or -unpacking of the archive, nor include additional metadata state about the file -system attributes. - -## Intended Audience - -This document is outlining the methods used for consistent checksum calculation -for filesystems transported via tar archives. - -Auditing these methodologies is an open and iterative process. This document -should accommodate the review of source code. Ultimately, this document should -be the starting point of further refinements to the algorithm and its future -versions. - -## Concept - -The checksum mechanism must ensure the integrity and assurance of the -filesystem payload. - -## Checksum Algorithm Profile - -A checksum mechanism must define the following operations and attributes: - -* Associated hashing cipher - used to checksum each file payload and attribute - information. -* Checksum list - each file of the filesystem archive has its checksum - calculated from the payload and attributes of the file. The final checksum is - calculated from this list, with specific ordering. -* Version - as the algorithm adapts to requirements, there are behaviors of the - algorithm to manage by versioning. -* Archive being calculated - the tar archive having its checksum calculated - -## Elements of TarSum checksum - -The calculated sum output is a text string. The elements included in the output -of the calculated sum comprise the information needed for validation of the sum -(TarSum version and hashing cipher used) and the expected checksum in hexadecimal -form. - -There are two delimiters used: -* '+' separates TarSum version from hashing cipher -* ':' separates calculation mechanics from expected hash - -Example: - -``` - "tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e" - | | \ | - | | \ | - |_version_|_cipher__|__ | - | \ | - |_calculation_mechanics_|______________________expected_sum_______________________| -``` - -## Versioning - -Versioning was introduced [0] to accommodate differences in calculation needed, -and ability to maintain reverse compatibility. - -The general algorithm will be describe further in the 'Calculation'. - -### Version0 - -This is the initial version of TarSum. - -Its element in the TarSum checksum string is `tarsum`. - -### Version1 - -Its element in the TarSum checksum is `tarsum.v1`. - -The notable changes in this version: -* Exclusion of file `mtime` from the file information headers, in each file - checksum calculation -* Inclusion of extended attributes (`xattrs`. Also seen as `SCHILY.xattr.` prefixed Pax - tar file info headers) keys and values in each file checksum calculation - -### VersionDev - -*Do not use unless validating refinements to the checksum algorithm* - -Its element in the TarSum checksum is `tarsum.dev`. - -This is a floating place holder for a next version and grounds for testing -changes. The methods used for calculation are subject to change without notice, -and this version is for testing and not for production use. - -## Ciphers - -The official default and standard hashing cipher used in the calculation mechanic -is `sha256`. This refers to SHA256 hash algorithm as defined in FIPS 180-4. - -Though the TarSum algorithm itself is not exclusively bound to the single -hashing cipher `sha256`, support for alternate hashing ciphers was later added -[1]. Use cases for alternate cipher could include future-proofing TarSum -checksum format and using faster cipher hashes for tar filesystem checksums. - -## Calculation - -### Requirement - -As mentioned earlier, the calculation is such that it takes into consideration -the lifecycle of the tar archive. In that the tar archive is not an immutable, -permanent artifact. Otherwise options like relying on a known hashing cipher -checksum of the archive itself would be reliable enough. The tar archive of the -filesystem is used as a transportation medium for Docker images, and the -archive is discarded once its contents are extracted. Therefore, for consistent -validation items such as order of files in the tar archive and time stamps are -subject to change once an image is received. - -### Process - -The method is typically iterative due to reading tar info headers from the -archive stream, though this is not a strict requirement. - -#### Files - -Each file in the tar archive have their contents (headers and body) checksummed -individually using the designated associated hashing cipher. The ordered -headers of the file are written to the checksum calculation first, and then the -payload of the file body. - -The resulting checksum of the file is appended to the list of file sums. The -sum is encoded as a string of the hexadecimal digest. Additionally, the file -name and position in the archive is kept as reference for special ordering. - -#### Headers - -The following headers are read, in this -order ( and the corresponding representation of its value): -* 'name' - string -* 'mode' - string of the base10 integer -* 'uid' - string of the integer -* 'gid' - string of the integer -* 'size' - string of the integer -* 'mtime' (_Version0 only_) - string of integer of the seconds since 1970-01-01 00:00:00 UTC -* 'typeflag' - string of the char -* 'linkname' - string -* 'uname' - string -* 'gname' - string -* 'devmajor' - string of the integer -* 'devminor' - string of the integer - -For >= Version1, the extended attribute headers ("SCHILY.xattr." prefixed pax -headers) included after the above list. These xattrs key/values are first -sorted by the keys. - -#### Header Format - -The ordered headers are written to the hash in the format of - - "{.key}{.value}" - -with no newline. - -#### Body - -After the order headers of the file have been added to the checksum for the -file, the body of the file is written to the hash. - -#### List of file sums - -The list of file sums is sorted by the string of the hexadecimal digest. - -If there are two files in the tar with matching paths, the order of occurrence -for that path is reflected for the sums of the corresponding file header and -body. - -#### Final Checksum - -Begin with a fresh or initial state of the associated hash cipher. If there is -additional payload to include in the TarSum calculation for the archive, it is -written first. Then each checksum from the ordered list of file sums is written -to the hash. - -The resulting digest is formatted per the Elements of TarSum checksum, -including the TarSum version, the associated hash cipher and the hexadecimal -encoded checksum digest. - -## Security Considerations - -The initial version of TarSum has undergone one update that could invalidate -handcrafted tar archives. The tar archive format supports appending of files -with same names as prior files in the archive. The latter file will clobber the -prior file of the same path. Due to this the algorithm now accounts for files -with matching paths, and orders the list of file sums accordingly [3]. - -## Footnotes - -* [0] Versioning https://github.com/docker/docker/commit/747f89cd327db9d50251b17797c4d825162226d0 -* [1] Alternate ciphers https://github.com/docker/docker/commit/4e9925d780665149b8bc940d5ba242ada1973c4e -* [2] Tar http://en.wikipedia.org/wiki/Tar_%28computing%29 -* [3] Name collision https://github.com/docker/docker/commit/c5e6362c53cbbc09ddbabd5a7323e04438b57d31 - -## Acknowledgments - -Joffrey F (shin-) and Guillaume J. Charmes (creack) on the initial work of the -TarSum calculation. - diff --git a/tarsum/tarsum_test.go b/tarsum/tarsum_test.go deleted file mode 100644 index 54bec53..0000000 --- a/tarsum/tarsum_test.go +++ /dev/null @@ -1,656 +0,0 @@ -package tarsum - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "crypto/md5" - "crypto/rand" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" - "encoding/hex" - "fmt" - "io" - "io/ioutil" - "os" - "strings" - "testing" -) - -type testLayer struct { - filename string - options *sizedOptions - jsonfile string - gzip bool - tarsum string - version Version - hash THash -} - -var testLayers = []testLayer{ - { - filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar", - jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json", - version: Version0, - tarsum: "tarsum+sha256:4095cc12fa5fdb1ab2760377e1cd0c4ecdd3e61b4f9b82319d96fcea6c9a41c6"}, - { - filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar", - jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json", - version: VersionDev, - tarsum: "tarsum.dev+sha256:db56e35eec6ce65ba1588c20ba6b1ea23743b59e81fb6b7f358ccbde5580345c"}, - { - filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar", - jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json", - gzip: true, - tarsum: "tarsum+sha256:4095cc12fa5fdb1ab2760377e1cd0c4ecdd3e61b4f9b82319d96fcea6c9a41c6"}, - { - // Tests existing version of TarSum when xattrs are present - filename: "testdata/xattr/layer.tar", - jsonfile: "testdata/xattr/json", - version: Version0, - tarsum: "tarsum+sha256:07e304a8dbcb215b37649fde1a699f8aeea47e60815707f1cdf4d55d25ff6ab4"}, - { - // Tests next version of TarSum when xattrs are present - filename: "testdata/xattr/layer.tar", - jsonfile: "testdata/xattr/json", - version: VersionDev, - tarsum: "tarsum.dev+sha256:6c58917892d77b3b357b0f9ad1e28e1f4ae4de3a8006bd3beb8beda214d8fd16"}, - { - filename: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar", - jsonfile: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json", - tarsum: "tarsum+sha256:c66bd5ec9f87b8f4c6135ca37684618f486a3dd1d113b138d0a177bfa39c2571"}, - { - options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory) - tarsum: "tarsum+sha256:8bf12d7e67c51ee2e8306cba569398b1b9f419969521a12ffb9d8875e8836738"}, - { - // this tar has two files with the same path - filename: "testdata/collision/collision-0.tar", - tarsum: "tarsum+sha256:08653904a68d3ab5c59e65ef58c49c1581caa3c34744f8d354b3f575ea04424a"}, - { - // this tar has the same two files (with the same path), but reversed order. ensuring is has different hash than above - filename: "testdata/collision/collision-1.tar", - tarsum: "tarsum+sha256:b51c13fbefe158b5ce420d2b930eef54c5cd55c50a2ee4abdddea8fa9f081e0d"}, - { - // this tar has newer of collider-0.tar, ensuring is has different hash - filename: "testdata/collision/collision-2.tar", - tarsum: "tarsum+sha256:381547080919bb82691e995508ae20ed33ce0f6948d41cafbeb70ce20c73ee8e"}, - { - // this tar has newer of collider-1.tar, ensuring is has different hash - filename: "testdata/collision/collision-3.tar", - tarsum: "tarsum+sha256:f886e431c08143164a676805205979cd8fa535dfcef714db5515650eea5a7c0f"}, - { - options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory) - tarsum: "tarsum+md5:0d7529ec7a8360155b48134b8e599f53", - hash: md5THash, - }, - { - options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory) - tarsum: "tarsum+sha1:f1fee39c5925807ff75ef1925e7a23be444ba4df", - hash: sha1Hash, - }, - { - options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory) - tarsum: "tarsum+sha224:6319390c0b061d639085d8748b14cd55f697cf9313805218b21cf61c", - hash: sha224Hash, - }, - { - options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory) - tarsum: "tarsum+sha384:a578ce3ce29a2ae03b8ed7c26f47d0f75b4fc849557c62454be4b5ffd66ba021e713b48ce71e947b43aab57afd5a7636", - hash: sha384Hash, - }, - { - options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory) - tarsum: "tarsum+sha512:e9bfb90ca5a4dfc93c46ee061a5cf9837de6d2fdf82544d6460d3147290aecfabf7b5e415b9b6e72db9b8941f149d5d69fb17a394cbfaf2eac523bd9eae21855", - hash: sha512Hash, - }, -} - -type sizedOptions struct { - num int64 - size int64 - isRand bool - realFile bool -} - -// make a tar: -// * num is the number of files the tar should have -// * size is the bytes per file -// * isRand is whether the contents of the files should be a random chunk (otherwise it's all zeros) -// * realFile will write to a TempFile, instead of an in memory buffer -func sizedTar(opts sizedOptions) io.Reader { - var ( - fh io.ReadWriter - err error - ) - if opts.realFile { - fh, err = ioutil.TempFile("", "tarsum") - if err != nil { - return nil - } - } else { - fh = bytes.NewBuffer([]byte{}) - } - tarW := tar.NewWriter(fh) - defer tarW.Close() - for i := int64(0); i < opts.num; i++ { - err := tarW.WriteHeader(&tar.Header{ - Name: fmt.Sprintf("/testdata%d", i), - Mode: 0755, - Uid: 0, - Gid: 0, - Size: opts.size, - }) - if err != nil { - return nil - } - var rBuf []byte - if opts.isRand { - rBuf = make([]byte, 8) - _, err = rand.Read(rBuf) - if err != nil { - return nil - } - } else { - rBuf = []byte{0, 0, 0, 0, 0, 0, 0, 0} - } - - for i := int64(0); i < opts.size/int64(8); i++ { - tarW.Write(rBuf) - } - } - return fh -} - -func emptyTarSum(gzip bool) (TarSum, error) { - reader, writer := io.Pipe() - tarWriter := tar.NewWriter(writer) - - // Immediately close tarWriter and write-end of the - // Pipe in a separate goroutine so we don't block. - go func() { - tarWriter.Close() - writer.Close() - }() - - return NewTarSum(reader, !gzip, Version0) -} - -// Test errors on NewTarsumForLabel -func TestNewTarSumForLabelInvalid(t *testing.T) { - reader := strings.NewReader("") - - if _, err := NewTarSumForLabel(reader, true, "invalidlabel"); err == nil { - t.Fatalf("Expected an error, got nothing.") - } - - if _, err := NewTarSumForLabel(reader, true, "invalid+sha256"); err == nil { - t.Fatalf("Expected an error, got nothing.") - } - if _, err := NewTarSumForLabel(reader, true, "tarsum.v1+invalid"); err == nil { - t.Fatalf("Expected an error, got nothing.") - } -} - -func TestNewTarSumForLabel(t *testing.T) { - - layer := testLayers[0] - - reader, err := os.Open(layer.filename) - if err != nil { - t.Fatal(err) - } - label := strings.Split(layer.tarsum, ":")[0] - ts, err := NewTarSumForLabel(reader, false, label) - if err != nil { - t.Fatal(err) - } - - // Make sure it actually worked by reading a little bit of it - nbByteToRead := 8 * 1024 - dBuf := make([]byte, nbByteToRead) - _, err = ts.Read(dBuf) - if err != nil { - t.Errorf("failed to read %vKB from %s: %s", nbByteToRead, layer.filename, err) - } -} - -// TestEmptyTar tests that tarsum does not fail to read an empty tar -// and correctly returns the hex digest of an empty hash. -func TestEmptyTar(t *testing.T) { - // Test without gzip. - ts, err := emptyTarSum(false) - if err != nil { - t.Fatal(err) - } - - zeroBlock := make([]byte, 1024) - buf := new(bytes.Buffer) - - n, err := io.Copy(buf, ts) - if err != nil { - t.Fatal(err) - } - - if n != int64(len(zeroBlock)) || !bytes.Equal(buf.Bytes(), zeroBlock) { - t.Fatalf("tarSum did not write the correct number of zeroed bytes: %d", n) - } - - expectedSum := ts.Version().String() + "+sha256:" + hex.EncodeToString(sha256.New().Sum(nil)) - resultSum := ts.Sum(nil) - - if resultSum != expectedSum { - t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum) - } - - // Test with gzip. - ts, err = emptyTarSum(true) - if err != nil { - t.Fatal(err) - } - buf.Reset() - - n, err = io.Copy(buf, ts) - if err != nil { - t.Fatal(err) - } - - bufgz := new(bytes.Buffer) - gz := gzip.NewWriter(bufgz) - n, err = io.Copy(gz, bytes.NewBuffer(zeroBlock)) - gz.Close() - gzBytes := bufgz.Bytes() - - if n != int64(len(zeroBlock)) || !bytes.Equal(buf.Bytes(), gzBytes) { - t.Fatalf("tarSum did not write the correct number of gzipped-zeroed bytes: %d", n) - } - - resultSum = ts.Sum(nil) - - if resultSum != expectedSum { - t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum) - } - - // Test without ever actually writing anything. - if ts, err = NewTarSum(bytes.NewReader([]byte{}), true, Version0); err != nil { - t.Fatal(err) - } - - resultSum = ts.Sum(nil) - - if resultSum != expectedSum { - t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum) - } -} - -var ( - md5THash = NewTHash("md5", md5.New) - sha1Hash = NewTHash("sha1", sha1.New) - sha224Hash = NewTHash("sha224", sha256.New224) - sha384Hash = NewTHash("sha384", sha512.New384) - sha512Hash = NewTHash("sha512", sha512.New) -) - -// Test all the build-in read size : buf8K, buf16K, buf32K and more -func TestTarSumsReadSize(t *testing.T) { - // Test always on the same layer (that is big enough) - layer := testLayers[0] - - for i := 0; i < 5; i++ { - - reader, err := os.Open(layer.filename) - if err != nil { - t.Fatal(err) - } - ts, err := NewTarSum(reader, false, layer.version) - if err != nil { - t.Fatal(err) - } - - // Read and discard bytes so that it populates sums - nbByteToRead := (i + 1) * 8 * 1024 - dBuf := make([]byte, nbByteToRead) - _, err = ts.Read(dBuf) - if err != nil { - t.Errorf("failed to read %vKB from %s: %s", nbByteToRead, layer.filename, err) - continue - } - } -} - -func TestTarSums(t *testing.T) { - for _, layer := range testLayers { - var ( - fh io.Reader - err error - ) - if len(layer.filename) > 0 { - fh, err = os.Open(layer.filename) - if err != nil { - t.Errorf("failed to open %s: %s", layer.filename, err) - continue - } - } else if layer.options != nil { - fh = sizedTar(*layer.options) - } else { - // What else is there to test? - t.Errorf("what to do with %#v", layer) - continue - } - if file, ok := fh.(*os.File); ok { - defer file.Close() - } - - var ts TarSum - if layer.hash == nil { - // double negatives! - ts, err = NewTarSum(fh, !layer.gzip, layer.version) - } else { - ts, err = NewTarSumHash(fh, !layer.gzip, layer.version, layer.hash) - } - if err != nil { - t.Errorf("%q :: %q", err, layer.filename) - continue - } - - // Read variable number of bytes to test dynamic buffer - dBuf := make([]byte, 1) - _, err = ts.Read(dBuf) - if err != nil { - t.Errorf("failed to read 1B from %s: %s", layer.filename, err) - continue - } - dBuf = make([]byte, 16*1024) - _, err = ts.Read(dBuf) - if err != nil { - t.Errorf("failed to read 16KB from %s: %s", layer.filename, err) - continue - } - - // Read and discard remaining bytes - _, err = io.Copy(ioutil.Discard, ts) - if err != nil { - t.Errorf("failed to copy from %s: %s", layer.filename, err) - continue - } - var gotSum string - if len(layer.jsonfile) > 0 { - jfh, err := os.Open(layer.jsonfile) - if err != nil { - t.Errorf("failed to open %s: %s", layer.jsonfile, err) - continue - } - buf, err := ioutil.ReadAll(jfh) - if err != nil { - t.Errorf("failed to readAll %s: %s", layer.jsonfile, err) - continue - } - gotSum = ts.Sum(buf) - } else { - gotSum = ts.Sum(nil) - } - - if layer.tarsum != gotSum { - t.Errorf("expecting [%s], but got [%s]", layer.tarsum, gotSum) - } - var expectedHashName string - if layer.hash != nil { - expectedHashName = layer.hash.Name() - } else { - expectedHashName = DefaultTHash.Name() - } - if expectedHashName != ts.Hash().Name() { - t.Errorf("expecting hash [%v], but got [%s]", expectedHashName, ts.Hash().Name()) - } - } -} - -func TestIteration(t *testing.T) { - headerTests := []struct { - expectedSum string // TODO(vbatts) it would be nice to get individual sums of each - version Version - hdr *tar.Header - data []byte - }{ - { - "tarsum+sha256:626c4a2e9a467d65c33ae81f7f3dedd4de8ccaee72af73223c4bc4718cbc7bbd", - Version0, - &tar.Header{ - Name: "file.txt", - Size: 0, - Typeflag: tar.TypeReg, - Devminor: 0, - Devmajor: 0, - }, - []byte(""), - }, - { - "tarsum.dev+sha256:6ffd43a1573a9913325b4918e124ee982a99c0f3cba90fc032a65f5e20bdd465", - VersionDev, - &tar.Header{ - Name: "file.txt", - Size: 0, - Typeflag: tar.TypeReg, - Devminor: 0, - Devmajor: 0, - }, - []byte(""), - }, - { - "tarsum.dev+sha256:b38166c059e11fb77bef30bf16fba7584446e80fcc156ff46d47e36c5305d8ef", - VersionDev, - &tar.Header{ - Name: "another.txt", - Uid: 1000, - Gid: 1000, - Uname: "slartibartfast", - Gname: "users", - Size: 4, - Typeflag: tar.TypeReg, - Devminor: 0, - Devmajor: 0, - }, - []byte("test"), - }, - { - "tarsum.dev+sha256:4cc2e71ac5d31833ab2be9b4f7842a14ce595ec96a37af4ed08f87bc374228cd", - VersionDev, - &tar.Header{ - Name: "xattrs.txt", - Uid: 1000, - Gid: 1000, - Uname: "slartibartfast", - Gname: "users", - Size: 4, - Typeflag: tar.TypeReg, - Xattrs: map[string]string{ - "user.key1": "value1", - "user.key2": "value2", - }, - }, - []byte("test"), - }, - { - "tarsum.dev+sha256:65f4284fa32c0d4112dd93c3637697805866415b570587e4fd266af241503760", - VersionDev, - &tar.Header{ - Name: "xattrs.txt", - Uid: 1000, - Gid: 1000, - Uname: "slartibartfast", - Gname: "users", - Size: 4, - Typeflag: tar.TypeReg, - Xattrs: map[string]string{ - "user.KEY1": "value1", // adding different case to ensure different sum - "user.key2": "value2", - }, - }, - []byte("test"), - }, - { - "tarsum+sha256:c12bb6f1303a9ddbf4576c52da74973c00d14c109bcfa76b708d5da1154a07fa", - Version0, - &tar.Header{ - Name: "xattrs.txt", - Uid: 1000, - Gid: 1000, - Uname: "slartibartfast", - Gname: "users", - Size: 4, - Typeflag: tar.TypeReg, - Xattrs: map[string]string{ - "user.NOT": "CALCULATED", - }, - }, - []byte("test"), - }, - } - for _, htest := range headerTests { - s, err := renderSumForHeader(htest.version, htest.hdr, htest.data) - if err != nil { - t.Fatal(err) - } - - if s != htest.expectedSum { - t.Errorf("expected sum: %q, got: %q", htest.expectedSum, s) - } - } - -} - -func renderSumForHeader(v Version, h *tar.Header, data []byte) (string, error) { - buf := bytes.NewBuffer(nil) - // first build our test tar - tw := tar.NewWriter(buf) - if err := tw.WriteHeader(h); err != nil { - return "", err - } - if _, err := tw.Write(data); err != nil { - return "", err - } - tw.Close() - - ts, err := NewTarSum(buf, true, v) - if err != nil { - return "", err - } - tr := tar.NewReader(ts) - for { - hdr, err := tr.Next() - if hdr == nil || err == io.EOF { - // Signals the end of the archive. - break - } - if err != nil { - return "", err - } - if _, err = io.Copy(ioutil.Discard, tr); err != nil { - return "", err - } - } - return ts.Sum(nil), nil -} - -func Benchmark9kTar(b *testing.B) { - buf := bytes.NewBuffer([]byte{}) - fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar") - if err != nil { - b.Error(err) - return - } - n, err := io.Copy(buf, fh) - if err != nil { - b.Error(err) - return - } - fh.Close() - - reader := bytes.NewReader(buf.Bytes()) - - b.SetBytes(n) - b.ResetTimer() - for i := 0; i < b.N; i++ { - reader.Seek(0, 0) - ts, err := NewTarSum(reader, true, Version0) - if err != nil { - b.Error(err) - return - } - io.Copy(ioutil.Discard, ts) - ts.Sum(nil) - } -} - -func Benchmark9kTarGzip(b *testing.B) { - buf := bytes.NewBuffer([]byte{}) - fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar") - if err != nil { - b.Error(err) - return - } - n, err := io.Copy(buf, fh) - if err != nil { - b.Error(err) - return - } - fh.Close() - - reader := bytes.NewReader(buf.Bytes()) - - b.SetBytes(n) - b.ResetTimer() - for i := 0; i < b.N; i++ { - reader.Seek(0, 0) - ts, err := NewTarSum(reader, false, Version0) - if err != nil { - b.Error(err) - return - } - io.Copy(ioutil.Discard, ts) - ts.Sum(nil) - } -} - -// this is a single big file in the tar archive -func Benchmark1mbSingleFileTar(b *testing.B) { - benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, false) -} - -// this is a single big file in the tar archive -func Benchmark1mbSingleFileTarGzip(b *testing.B) { - benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, true) -} - -// this is 1024 1k files in the tar archive -func Benchmark1kFilesTar(b *testing.B) { - benchmarkTar(b, sizedOptions{1024, 1024, true, true}, false) -} - -// this is 1024 1k files in the tar archive -func Benchmark1kFilesTarGzip(b *testing.B) { - benchmarkTar(b, sizedOptions{1024, 1024, true, true}, true) -} - -func benchmarkTar(b *testing.B, opts sizedOptions, isGzip bool) { - var fh *os.File - tarReader := sizedTar(opts) - if br, ok := tarReader.(*os.File); ok { - fh = br - } - defer os.Remove(fh.Name()) - defer fh.Close() - - b.SetBytes(opts.size * opts.num) - b.ResetTimer() - for i := 0; i < b.N; i++ { - ts, err := NewTarSum(fh, !isGzip, Version0) - if err != nil { - b.Error(err) - return - } - io.Copy(ioutil.Discard, ts) - ts.Sum(nil) - fh.Seek(0, 0) - } -} diff --git a/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json b/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json deleted file mode 100644 index 48e2af3..0000000 --- a/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json +++ /dev/null @@ -1 +0,0 @@ -{"id":"46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457","parent":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","created":"2014-04-07T02:45:52.610504484Z","container":"e0f07f8d72cae171a3dcc35859960e7e956e0628bce6fedc4122bf55b2c287c7","container_config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","sed -ri 's/^(%wheel.*)(ALL)$/\\1NOPASSWD: \\2/' /etc/sudoers"],"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.9.1-dev","config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":3425} \ No newline at end of file diff --git a/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar b/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar deleted file mode 100644 index dfd5c20..0000000 Binary files a/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar and /dev/null differ diff --git a/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json b/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json deleted file mode 100644 index af57be0..0000000 --- a/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json +++ /dev/null @@ -1 +0,0 @@ -{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0} \ No newline at end of file diff --git a/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar b/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar deleted file mode 100644 index 880b3f2..0000000 Binary files a/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar and /dev/null differ diff --git a/tarsum/testdata/collision/collision-0.tar b/tarsum/testdata/collision/collision-0.tar deleted file mode 100644 index 1c636b3..0000000 Binary files a/tarsum/testdata/collision/collision-0.tar and /dev/null differ diff --git a/tarsum/testdata/collision/collision-1.tar b/tarsum/testdata/collision/collision-1.tar deleted file mode 100644 index b411be9..0000000 Binary files a/tarsum/testdata/collision/collision-1.tar and /dev/null differ diff --git a/tarsum/testdata/collision/collision-2.tar b/tarsum/testdata/collision/collision-2.tar deleted file mode 100644 index 7b5c04a..0000000 Binary files a/tarsum/testdata/collision/collision-2.tar and /dev/null differ diff --git a/tarsum/testdata/collision/collision-3.tar b/tarsum/testdata/collision/collision-3.tar deleted file mode 100644 index f8c6458..0000000 Binary files a/tarsum/testdata/collision/collision-3.tar and /dev/null differ diff --git a/tarsum/testdata/xattr/json b/tarsum/testdata/xattr/json deleted file mode 100644 index 288441a..0000000 --- a/tarsum/testdata/xattr/json +++ /dev/null @@ -1 +0,0 @@ -{"id":"4439c3c7f847954100b42b267e7e5529cac1d6934db082f65795c5ca2e594d93","parent":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","created":"2014-05-16T17:19:44.091534414Z","container":"5f92fb06cc58f357f0cde41394e2bbbb664e663974b2ac1693ab07b7a306749b","container_config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","setcap 'cap_setgid,cap_setuid+ep' ./file \u0026\u0026 getcap ./file"],"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.11.1-dev","config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":0} \ No newline at end of file diff --git a/tarsum/testdata/xattr/layer.tar b/tarsum/testdata/xattr/layer.tar deleted file mode 100644 index 819351d..0000000 Binary files a/tarsum/testdata/xattr/layer.tar and /dev/null differ diff --git a/tarsum/versioning.go b/tarsum/versioning.go deleted file mode 100644 index 2882286..0000000 --- a/tarsum/versioning.go +++ /dev/null @@ -1,150 +0,0 @@ -package tarsum - -import ( - "archive/tar" - "errors" - "sort" - "strconv" - "strings" -) - -// Version is used for versioning of the TarSum algorithm -// based on the prefix of the hash used -// i.e. "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b" -type Version int - -// Prefix of "tarsum" -const ( - Version0 Version = iota - Version1 - // VersionDev this constant will be either the latest or an unsettled next-version of the TarSum calculation - VersionDev -) - -// VersionLabelForChecksum returns the label for the given tarsum -// checksum, i.e., everything before the first `+` character in -// the string or an empty string if no label separator is found. -func VersionLabelForChecksum(checksum string) string { - // Checksums are in the form: {versionLabel}+{hashID}:{hex} - sepIndex := strings.Index(checksum, "+") - if sepIndex < 0 { - return "" - } - return checksum[:sepIndex] -} - -// GetVersions gets a list of all known tarsum versions. -func GetVersions() []Version { - v := []Version{} - for k := range tarSumVersions { - v = append(v, k) - } - return v -} - -var ( - tarSumVersions = map[Version]string{ - Version0: "tarsum", - Version1: "tarsum.v1", - VersionDev: "tarsum.dev", - } - tarSumVersionsByName = map[string]Version{ - "tarsum": Version0, - "tarsum.v1": Version1, - "tarsum.dev": VersionDev, - } -) - -func (tsv Version) String() string { - return tarSumVersions[tsv] -} - -// GetVersionFromTarsum returns the Version from the provided string. -func GetVersionFromTarsum(tarsum string) (Version, error) { - tsv := tarsum - if strings.Contains(tarsum, "+") { - tsv = strings.SplitN(tarsum, "+", 2)[0] - } - for v, s := range tarSumVersions { - if s == tsv { - return v, nil - } - } - return -1, ErrNotVersion -} - -// Errors that may be returned by functions in this package -var ( - ErrNotVersion = errors.New("string does not include a TarSum Version") - ErrVersionNotImplemented = errors.New("TarSum Version is not yet implemented") -) - -// tarHeaderSelector is the interface which different versions -// of tarsum should use for selecting and ordering tar headers -// for each item in the archive. -type tarHeaderSelector interface { - selectHeaders(h *tar.Header) (orderedHeaders [][2]string) -} - -type tarHeaderSelectFunc func(h *tar.Header) (orderedHeaders [][2]string) - -func (f tarHeaderSelectFunc) selectHeaders(h *tar.Header) (orderedHeaders [][2]string) { - return f(h) -} - -func v0TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) { - return [][2]string{ - {"name", h.Name}, - {"mode", strconv.FormatInt(h.Mode, 10)}, - {"uid", strconv.Itoa(h.Uid)}, - {"gid", strconv.Itoa(h.Gid)}, - {"size", strconv.FormatInt(h.Size, 10)}, - {"mtime", strconv.FormatInt(h.ModTime.UTC().Unix(), 10)}, - {"typeflag", string([]byte{h.Typeflag})}, - {"linkname", h.Linkname}, - {"uname", h.Uname}, - {"gname", h.Gname}, - {"devmajor", strconv.FormatInt(h.Devmajor, 10)}, - {"devminor", strconv.FormatInt(h.Devminor, 10)}, - } -} - -func v1TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) { - // Get extended attributes. - xAttrKeys := make([]string, len(h.Xattrs)) - for k := range h.Xattrs { - xAttrKeys = append(xAttrKeys, k) - } - sort.Strings(xAttrKeys) - - // Make the slice with enough capacity to hold the 11 basic headers - // we want from the v0 selector plus however many xattrs we have. - orderedHeaders = make([][2]string, 0, 11+len(xAttrKeys)) - - // Copy all headers from v0 excluding the 'mtime' header (the 5th element). - v0headers := v0TarHeaderSelect(h) - orderedHeaders = append(orderedHeaders, v0headers[0:5]...) - orderedHeaders = append(orderedHeaders, v0headers[6:]...) - - // Finally, append the sorted xattrs. - for _, k := range xAttrKeys { - orderedHeaders = append(orderedHeaders, [2]string{k, h.Xattrs[k]}) - } - - return -} - -var registeredHeaderSelectors = map[Version]tarHeaderSelectFunc{ - Version0: v0TarHeaderSelect, - Version1: v1TarHeaderSelect, - VersionDev: v1TarHeaderSelect, -} - -func getTarHeaderSelector(v Version) (tarHeaderSelector, error) { - headerSelector, ok := registeredHeaderSelectors[v] - if !ok { - return nil, ErrVersionNotImplemented - } - - return headerSelector, nil -} diff --git a/tarsum/versioning_test.go b/tarsum/versioning_test.go deleted file mode 100644 index 88e0a57..0000000 --- a/tarsum/versioning_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package tarsum - -import ( - "testing" -) - -func TestVersionLabelForChecksum(t *testing.T) { - version := VersionLabelForChecksum("tarsum+sha256:deadbeef") - if version != "tarsum" { - t.Fatalf("Version should have been 'tarsum', was %v", version) - } - version = VersionLabelForChecksum("tarsum.v1+sha256:deadbeef") - if version != "tarsum.v1" { - t.Fatalf("Version should have been 'tarsum.v1', was %v", version) - } - version = VersionLabelForChecksum("something+somethingelse") - if version != "something" { - t.Fatalf("Version should have been 'something', was %v", version) - } - version = VersionLabelForChecksum("invalidChecksum") - if version != "" { - t.Fatalf("Version should have been empty, was %v", version) - } -} - -func TestVersion(t *testing.T) { - expected := "tarsum" - var v Version - if v.String() != expected { - t.Errorf("expected %q, got %q", expected, v.String()) - } - - expected = "tarsum.v1" - v = 1 - if v.String() != expected { - t.Errorf("expected %q, got %q", expected, v.String()) - } - - expected = "tarsum.dev" - v = 2 - if v.String() != expected { - t.Errorf("expected %q, got %q", expected, v.String()) - } -} - -func TestGetVersion(t *testing.T) { - testSet := []struct { - Str string - Expected Version - }{ - {"tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", Version0}, - {"tarsum+sha256", Version0}, - {"tarsum", Version0}, - {"tarsum.dev", VersionDev}, - {"tarsum.dev+sha256:deadbeef", VersionDev}, - } - - for _, ts := range testSet { - v, err := GetVersionFromTarsum(ts.Str) - if err != nil { - t.Fatalf("%q : %s", err, ts.Str) - } - if v != ts.Expected { - t.Errorf("expected %d (%q), got %d (%q)", ts.Expected, ts.Expected, v, v) - } - } - - // test one that does not exist, to ensure it errors - str := "weak+md5:abcdeabcde" - _, err := GetVersionFromTarsum(str) - if err != ErrNotVersion { - t.Fatalf("%q : %s", err, str) - } -} - -func TestGetVersions(t *testing.T) { - expected := []Version{ - Version0, - Version1, - VersionDev, - } - versions := GetVersions() - if len(versions) != len(expected) { - t.Fatalf("Expected %v versions, got %v", len(expected), len(versions)) - } - if !containsVersion(versions, expected[0]) || !containsVersion(versions, expected[1]) || !containsVersion(versions, expected[2]) { - t.Fatalf("Expected [%v], got [%v]", expected, versions) - } -} - -func containsVersion(versions []Version, version Version) bool { - for _, v := range versions { - if v == version { - return true - } - } - return false -} diff --git a/tarsum/writercloser.go b/tarsum/writercloser.go deleted file mode 100644 index 9727ecd..0000000 --- a/tarsum/writercloser.go +++ /dev/null @@ -1,22 +0,0 @@ -package tarsum - -import ( - "io" -) - -type writeCloseFlusher interface { - io.WriteCloser - Flush() error -} - -type nopCloseFlusher struct { - io.Writer -} - -func (n *nopCloseFlusher) Close() error { - return nil -} - -func (n *nopCloseFlusher) Flush() error { - return nil -} diff --git a/term/ascii.go b/term/ascii.go deleted file mode 100644 index f5262bc..0000000 --- a/term/ascii.go +++ /dev/null @@ -1,66 +0,0 @@ -package term - -import ( - "fmt" - "strings" -) - -// ASCII list the possible supported ASCII key sequence -var ASCII = []string{ - "ctrl-@", - "ctrl-a", - "ctrl-b", - "ctrl-c", - "ctrl-d", - "ctrl-e", - "ctrl-f", - "ctrl-g", - "ctrl-h", - "ctrl-i", - "ctrl-j", - "ctrl-k", - "ctrl-l", - "ctrl-m", - "ctrl-n", - "ctrl-o", - "ctrl-p", - "ctrl-q", - "ctrl-r", - "ctrl-s", - "ctrl-t", - "ctrl-u", - "ctrl-v", - "ctrl-w", - "ctrl-x", - "ctrl-y", - "ctrl-z", - "ctrl-[", - "ctrl-\\", - "ctrl-]", - "ctrl-^", - "ctrl-_", -} - -// ToBytes converts a string representing a suite of key-sequence to the corresponding ASCII code. -func ToBytes(keys string) ([]byte, error) { - codes := []byte{} -next: - for _, key := range strings.Split(keys, ",") { - if len(key) != 1 { - for code, ctrl := range ASCII { - if ctrl == key { - codes = append(codes, byte(code)) - continue next - } - } - if key == "DEL" { - codes = append(codes, 127) - } else { - return nil, fmt.Errorf("Unknown character: '%s'", key) - } - } else { - codes = append(codes, byte(key[0])) - } - } - return codes, nil -} diff --git a/term/ascii_test.go b/term/ascii_test.go deleted file mode 100644 index 4a1e7f3..0000000 --- a/term/ascii_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package term - -import "testing" - -func TestToBytes(t *testing.T) { - codes, err := ToBytes("ctrl-a,a") - if err != nil { - t.Fatal(err) - } - if len(codes) != 2 { - t.Fatalf("Expected 2 codes, got %d", len(codes)) - } - if codes[0] != 1 || codes[1] != 97 { - t.Fatalf("Expected '1' '97', got '%d' '%d'", codes[0], codes[1]) - } - - codes, err = ToBytes("shift-z") - if err == nil { - t.Fatalf("Expected error, got none") - } - - codes, err = ToBytes("ctrl-@,ctrl-[,~,ctrl-o") - if err != nil { - t.Fatal(err) - } - if len(codes) != 4 { - t.Fatalf("Expected 4 codes, got %d", len(codes)) - } - if codes[0] != 0 || codes[1] != 27 || codes[2] != 126 || codes[3] != 15 { - t.Fatalf("Expected '0' '27' '126', '15', got '%d' '%d' '%d' '%d'", codes[0], codes[1], codes[2], codes[3]) - } - - codes, err = ToBytes("DEL,+") - if err != nil { - t.Fatal(err) - } - if len(codes) != 2 { - t.Fatalf("Expected 2 codes, got %d", len(codes)) - } - if codes[0] != 127 || codes[1] != 43 { - t.Fatalf("Expected '127 '43'', got '%d' '%d'", codes[0], codes[1]) - } -} diff --git a/term/tc_linux_cgo.go b/term/tc_linux_cgo.go deleted file mode 100644 index 59dac5b..0000000 --- a/term/tc_linux_cgo.go +++ /dev/null @@ -1,50 +0,0 @@ -// +build linux,cgo - -package term - -import ( - "syscall" - "unsafe" -) - -// #include -import "C" - -// Termios is the Unix API for terminal I/O. -// It is passthrough for syscall.Termios in order to make it portable with -// other platforms where it is not available or handled differently. -type Termios syscall.Termios - -// MakeRaw put the terminal connected to the given file descriptor into raw -// mode and returns the previous state of the terminal so that it can be -// restored. -func MakeRaw(fd uintptr) (*State, error) { - var oldState State - if err := tcget(fd, &oldState.termios); err != 0 { - return nil, err - } - - newState := oldState.termios - - C.cfmakeraw((*C.struct_termios)(unsafe.Pointer(&newState))) - if err := tcset(fd, &newState); err != 0 { - return nil, err - } - return &oldState, nil -} - -func tcget(fd uintptr, p *Termios) syscall.Errno { - ret, err := C.tcgetattr(C.int(fd), (*C.struct_termios)(unsafe.Pointer(p))) - if ret != 0 { - return err.(syscall.Errno) - } - return 0 -} - -func tcset(fd uintptr, p *Termios) syscall.Errno { - ret, err := C.tcsetattr(C.int(fd), C.TCSANOW, (*C.struct_termios)(unsafe.Pointer(p))) - if ret != 0 { - return err.(syscall.Errno) - } - return 0 -} diff --git a/term/tc_other.go b/term/tc_other.go deleted file mode 100644 index 750d7c3..0000000 --- a/term/tc_other.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build !windows -// +build !linux !cgo -// +build !solaris !cgo - -package term - -import ( - "syscall" - "unsafe" -) - -func tcget(fd uintptr, p *Termios) syscall.Errno { - _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(getTermios), uintptr(unsafe.Pointer(p))) - return err -} - -func tcset(fd uintptr, p *Termios) syscall.Errno { - _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, setTermios, uintptr(unsafe.Pointer(p))) - return err -} diff --git a/term/tc_solaris_cgo.go b/term/tc_solaris_cgo.go deleted file mode 100644 index c9139d0..0000000 --- a/term/tc_solaris_cgo.go +++ /dev/null @@ -1,63 +0,0 @@ -// +build solaris,cgo - -package term - -import ( - "syscall" - "unsafe" -) - -// #include -import "C" - -// Termios is the Unix API for terminal I/O. -// It is passthrough for syscall.Termios in order to make it portable with -// other platforms where it is not available or handled differently. -type Termios syscall.Termios - -// MakeRaw put the terminal connected to the given file descriptor into raw -// mode and returns the previous state of the terminal so that it can be -// restored. -func MakeRaw(fd uintptr) (*State, error) { - var oldState State - if err := tcget(fd, &oldState.termios); err != 0 { - return nil, err - } - - newState := oldState.termios - - newState.Iflag &^= (syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON | syscall.IXANY) - newState.Oflag &^= syscall.OPOST - newState.Lflag &^= (syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN) - newState.Cflag &^= (syscall.CSIZE | syscall.PARENB) - newState.Cflag |= syscall.CS8 - - /* - VMIN is the minimum number of characters that needs to be read in non-canonical mode for it to be returned - Since VMIN is overloaded with another element in canonical mode when we switch modes it defaults to 4. It - needs to be explicitly set to 1. - */ - newState.Cc[C.VMIN] = 1 - newState.Cc[C.VTIME] = 0 - - if err := tcset(fd, &newState); err != 0 { - return nil, err - } - return &oldState, nil -} - -func tcget(fd uintptr, p *Termios) syscall.Errno { - ret, err := C.tcgetattr(C.int(fd), (*C.struct_termios)(unsafe.Pointer(p))) - if ret != 0 { - return err.(syscall.Errno) - } - return 0 -} - -func tcset(fd uintptr, p *Termios) syscall.Errno { - ret, err := C.tcsetattr(C.int(fd), C.TCSANOW, (*C.struct_termios)(unsafe.Pointer(p))) - if ret != 0 { - return err.(syscall.Errno) - } - return 0 -} diff --git a/term/term.go b/term/term.go deleted file mode 100644 index 8f55484..0000000 --- a/term/term.go +++ /dev/null @@ -1,109 +0,0 @@ -// +build !windows - -// Package term provides provides structures and helper functions to work with -// terminal (state, sizes). -package term - -import ( - "errors" - "io" - "os" - "os/signal" - "syscall" -) - -var ( - // ErrInvalidState is returned if the state of the terminal is invalid. - ErrInvalidState = errors.New("Invalid terminal state") -) - -// State represents the state of the terminal. -type State struct { - termios Termios -} - -// Winsize represents the size of the terminal window. -type Winsize struct { - Height uint16 - Width uint16 - x uint16 - y uint16 -} - -// StdStreams returns the standard streams (stdin, stdout, stedrr). -func StdStreams() (stdIn io.ReadCloser, stdOut, stdErr io.Writer) { - return os.Stdin, os.Stdout, os.Stderr -} - -// GetFdInfo returns the file descriptor for an os.File and indicates whether the file represents a terminal. -func GetFdInfo(in interface{}) (uintptr, bool) { - var inFd uintptr - var isTerminalIn bool - if file, ok := in.(*os.File); ok { - inFd = file.Fd() - isTerminalIn = IsTerminal(inFd) - } - return inFd, isTerminalIn -} - -// IsTerminal returns true if the given file descriptor is a terminal. -func IsTerminal(fd uintptr) bool { - var termios Termios - return tcget(fd, &termios) == 0 -} - -// RestoreTerminal restores the terminal connected to the given file descriptor -// to a previous state. -func RestoreTerminal(fd uintptr, state *State) error { - if state == nil { - return ErrInvalidState - } - if err := tcset(fd, &state.termios); err != 0 { - return err - } - return nil -} - -// SaveState saves the state of the terminal connected to the given file descriptor. -func SaveState(fd uintptr) (*State, error) { - var oldState State - if err := tcget(fd, &oldState.termios); err != 0 { - return nil, err - } - - return &oldState, nil -} - -// DisableEcho applies the specified state to the terminal connected to the file -// descriptor, with echo disabled. -func DisableEcho(fd uintptr, state *State) error { - newState := state.termios - newState.Lflag &^= syscall.ECHO - - if err := tcset(fd, &newState); err != 0 { - return err - } - handleInterrupt(fd, state) - return nil -} - -// SetRawTerminal puts the terminal connected to the given file descriptor into -// raw mode and returns the previous state. -func SetRawTerminal(fd uintptr) (*State, error) { - oldState, err := MakeRaw(fd) - if err != nil { - return nil, err - } - handleInterrupt(fd, oldState) - return oldState, err -} - -func handleInterrupt(fd uintptr, state *State) { - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, os.Interrupt) - - go func() { - _ = <-sigchan - RestoreTerminal(fd, state) - }() -} diff --git a/term/term_solaris.go b/term/term_solaris.go deleted file mode 100644 index 112debb..0000000 --- a/term/term_solaris.go +++ /dev/null @@ -1,41 +0,0 @@ -// +build solaris - -package term - -import ( - "syscall" - "unsafe" -) - -/* -#include -#include -#include - -// Small wrapper to get rid of variadic args of ioctl() -int my_ioctl(int fd, int cmd, struct winsize *ws) { - return ioctl(fd, cmd, ws); -} -*/ -import "C" - -// GetWinsize returns the window size based on the specified file descriptor. -func GetWinsize(fd uintptr) (*Winsize, error) { - ws := &Winsize{} - ret, err := C.my_ioctl(C.int(fd), C.int(syscall.TIOCGWINSZ), (*C.struct_winsize)(unsafe.Pointer(ws))) - // Skip retval = 0 - if ret == 0 { - return ws, nil - } - return ws, err -} - -// SetWinsize tries to set the specified window size for the specified file descriptor. -func SetWinsize(fd uintptr, ws *Winsize) error { - ret, err := C.my_ioctl(C.int(fd), C.int(syscall.TIOCSWINSZ), (*C.struct_winsize)(unsafe.Pointer(ws))) - // Skip retval = 0 - if ret == 0 { - return nil - } - return err -} diff --git a/term/term_unix.go b/term/term_unix.go deleted file mode 100644 index ddf87a0..0000000 --- a/term/term_unix.go +++ /dev/null @@ -1,29 +0,0 @@ -// +build !solaris,!windows - -package term - -import ( - "syscall" - "unsafe" -) - -// GetWinsize returns the window size based on the specified file descriptor. -func GetWinsize(fd uintptr) (*Winsize, error) { - ws := &Winsize{} - _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(ws))) - // Skipp errno = 0 - if err == 0 { - return ws, nil - } - return ws, err -} - -// SetWinsize tries to set the specified window size for the specified file descriptor. -func SetWinsize(fd uintptr, ws *Winsize) error { - _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TIOCSWINSZ), uintptr(unsafe.Pointer(ws))) - // Skipp errno = 0 - if err == 0 { - return nil - } - return err -} diff --git a/term/term_windows.go b/term/term_windows.go deleted file mode 100644 index 9bc52a8..0000000 --- a/term/term_windows.go +++ /dev/null @@ -1,303 +0,0 @@ -// +build windows - -package term - -import ( - "io" - "os" - "os/signal" - "syscall" - - "github.com/Azure/go-ansiterm/winterm" - "github.com/docker/docker/pkg/system" - "github.com/docker/docker/pkg/term/windows" -) - -// State holds the console mode for the terminal. -type State struct { - inMode, outMode uint32 - inHandle, outHandle syscall.Handle -} - -// Winsize is used for window size. -type Winsize struct { - Height uint16 - Width uint16 -} - -const ( - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms683167(v=vs.85).aspx - enableVirtualTerminalInput = 0x0200 - enableVirtualTerminalProcessing = 0x0004 - disableNewlineAutoReturn = 0x0008 -) - -// usingNativeConsole is true if we are using the Windows native console -var usingNativeConsole bool - -// StdStreams returns the standard streams (stdin, stdout, stedrr). -func StdStreams() (stdIn io.ReadCloser, stdOut, stdErr io.Writer) { - switch { - case os.Getenv("ConEmuANSI") == "ON": - // The ConEmu terminal emulates ANSI on output streams well. - return windows.ConEmuStreams() - case os.Getenv("MSYSTEM") != "": - // MSYS (mingw) does not emulate ANSI well. - return windows.ConsoleStreams() - default: - if useNativeConsole() { - usingNativeConsole = true - return os.Stdin, os.Stdout, os.Stderr - } - return windows.ConsoleStreams() - } -} - -// useNativeConsole determines if the docker client should use the built-in -// console which supports ANSI emulation, or fall-back to the golang emulator -// (github.com/azure/go-ansiterm). -func useNativeConsole() bool { - osv := system.GetOSVersion() - - // Native console is not available before major version 10 - if osv.MajorVersion < 10 { - return false - } - - // Get the console modes. If this fails, we can't use the native console - state, err := getNativeConsole() - if err != nil { - return false - } - - // Probe the console to see if it can be enabled. - if nil != probeNativeConsole(state) { - return false - } - - // Environment variable override - if e := os.Getenv("USE_NATIVE_CONSOLE"); e != "" { - if e == "1" { - return true - } - return false - } - - // Must have a post-TP5 RS1 build of Windows Server 2016/Windows 10 for - // the native console to be usable. - if osv.Build < 14350 { - return false - } - - return true -} - -// getNativeConsole returns the console modes ('state') for the native Windows console -func getNativeConsole() (State, error) { - var ( - err error - state State - ) - - // Get the handle to stdout - if state.outHandle, err = syscall.GetStdHandle(syscall.STD_OUTPUT_HANDLE); err != nil { - return state, err - } - - // Get the console mode from the consoles stdout handle - if err = syscall.GetConsoleMode(state.outHandle, &state.outMode); err != nil { - return state, err - } - - // Get the handle to stdin - if state.inHandle, err = syscall.GetStdHandle(syscall.STD_INPUT_HANDLE); err != nil { - return state, err - } - - // Get the console mode from the consoles stdin handle - if err = syscall.GetConsoleMode(state.inHandle, &state.inMode); err != nil { - return state, err - } - - return state, nil -} - -// probeNativeConsole probes the console to determine if native can be supported, -func probeNativeConsole(state State) error { - if err := winterm.SetConsoleMode(uintptr(state.outHandle), state.outMode|enableVirtualTerminalProcessing); err != nil { - return err - } - defer winterm.SetConsoleMode(uintptr(state.outHandle), state.outMode) - - if err := winterm.SetConsoleMode(uintptr(state.inHandle), state.inMode|enableVirtualTerminalInput); err != nil { - return err - } - defer winterm.SetConsoleMode(uintptr(state.inHandle), state.inMode) - - return nil -} - -// enableNativeConsole turns on native console mode -func enableNativeConsole(state State) error { - // First attempt both enableVirtualTerminalProcessing and disableNewlineAutoReturn - if err := winterm.SetConsoleMode(uintptr(state.outHandle), - state.outMode|(enableVirtualTerminalProcessing|disableNewlineAutoReturn)); err != nil { - - // That may fail, so fallback to trying just enableVirtualTerminalProcessing - if err := winterm.SetConsoleMode(uintptr(state.outHandle), state.outMode|enableVirtualTerminalProcessing); err != nil { - return err - } - } - - if err := winterm.SetConsoleMode(uintptr(state.inHandle), state.inMode|enableVirtualTerminalInput); err != nil { - winterm.SetConsoleMode(uintptr(state.outHandle), state.outMode) // restore out if we can - return err - } - - return nil -} - -// disableNativeConsole turns off native console mode -func disableNativeConsole(state *State) error { - // Try and restore both in an out before error checking. - errout := winterm.SetConsoleMode(uintptr(state.outHandle), state.outMode) - errin := winterm.SetConsoleMode(uintptr(state.inHandle), state.inMode) - if errout != nil { - return errout - } - if errin != nil { - return errin - } - return nil -} - -// GetFdInfo returns the file descriptor for an os.File and indicates whether the file represents a terminal. -func GetFdInfo(in interface{}) (uintptr, bool) { - return windows.GetHandleInfo(in) -} - -// GetWinsize returns the window size based on the specified file descriptor. -func GetWinsize(fd uintptr) (*Winsize, error) { - info, err := winterm.GetConsoleScreenBufferInfo(fd) - if err != nil { - return nil, err - } - - winsize := &Winsize{ - Width: uint16(info.Window.Right - info.Window.Left + 1), - Height: uint16(info.Window.Bottom - info.Window.Top + 1), - } - - return winsize, nil -} - -// IsTerminal returns true if the given file descriptor is a terminal. -func IsTerminal(fd uintptr) bool { - return windows.IsConsole(fd) -} - -// RestoreTerminal restores the terminal connected to the given file descriptor -// to a previous state. -func RestoreTerminal(fd uintptr, state *State) error { - if usingNativeConsole { - return disableNativeConsole(state) - } - return winterm.SetConsoleMode(fd, state.outMode) -} - -// SaveState saves the state of the terminal connected to the given file descriptor. -func SaveState(fd uintptr) (*State, error) { - if usingNativeConsole { - state, err := getNativeConsole() - if err != nil { - return nil, err - } - return &state, nil - } - - mode, e := winterm.GetConsoleMode(fd) - if e != nil { - return nil, e - } - - return &State{outMode: mode}, nil -} - -// DisableEcho disables echo for the terminal connected to the given file descriptor. -// -- See https://msdn.microsoft.com/en-us/library/windows/desktop/ms683462(v=vs.85).aspx -func DisableEcho(fd uintptr, state *State) error { - mode := state.inMode - mode &^= winterm.ENABLE_ECHO_INPUT - mode |= winterm.ENABLE_PROCESSED_INPUT | winterm.ENABLE_LINE_INPUT - err := winterm.SetConsoleMode(fd, mode) - if err != nil { - return err - } - - // Register an interrupt handler to catch and restore prior state - restoreAtInterrupt(fd, state) - return nil -} - -// SetRawTerminal puts the terminal connected to the given file descriptor into raw -// mode and returns the previous state. -func SetRawTerminal(fd uintptr) (*State, error) { - state, err := MakeRaw(fd) - if err != nil { - return nil, err - } - - // Register an interrupt handler to catch and restore prior state - restoreAtInterrupt(fd, state) - return state, err -} - -// MakeRaw puts the terminal (Windows Console) connected to the given file descriptor into raw -// mode and returns the previous state of the terminal so that it can be restored. -func MakeRaw(fd uintptr) (*State, error) { - state, err := SaveState(fd) - if err != nil { - return nil, err - } - - mode := state.inMode - if usingNativeConsole { - if err := enableNativeConsole(*state); err != nil { - return nil, err - } - mode |= enableVirtualTerminalInput - } - - // See - // -- https://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx - // -- https://msdn.microsoft.com/en-us/library/windows/desktop/ms683462(v=vs.85).aspx - - // Disable these modes - mode &^= winterm.ENABLE_ECHO_INPUT - mode &^= winterm.ENABLE_LINE_INPUT - mode &^= winterm.ENABLE_MOUSE_INPUT - mode &^= winterm.ENABLE_WINDOW_INPUT - mode &^= winterm.ENABLE_PROCESSED_INPUT - - // Enable these modes - mode |= winterm.ENABLE_EXTENDED_FLAGS - mode |= winterm.ENABLE_INSERT_MODE - mode |= winterm.ENABLE_QUICK_EDIT_MODE - - err = winterm.SetConsoleMode(fd, mode) - if err != nil { - return nil, err - } - return state, nil -} - -func restoreAtInterrupt(fd uintptr, state *State) { - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, os.Interrupt) - - go func() { - _ = <-sigchan - RestoreTerminal(fd, state) - os.Exit(0) - }() -} diff --git a/term/termios_darwin.go b/term/termios_darwin.go deleted file mode 100644 index 480db90..0000000 --- a/term/termios_darwin.go +++ /dev/null @@ -1,69 +0,0 @@ -package term - -import ( - "syscall" - "unsafe" -) - -const ( - getTermios = syscall.TIOCGETA - setTermios = syscall.TIOCSETA -) - -// Termios magic numbers, passthrough to the ones defined in syscall. -const ( - IGNBRK = syscall.IGNBRK - PARMRK = syscall.PARMRK - INLCR = syscall.INLCR - IGNCR = syscall.IGNCR - ECHONL = syscall.ECHONL - CSIZE = syscall.CSIZE - ICRNL = syscall.ICRNL - ISTRIP = syscall.ISTRIP - PARENB = syscall.PARENB - ECHO = syscall.ECHO - ICANON = syscall.ICANON - ISIG = syscall.ISIG - IXON = syscall.IXON - BRKINT = syscall.BRKINT - INPCK = syscall.INPCK - OPOST = syscall.OPOST - CS8 = syscall.CS8 - IEXTEN = syscall.IEXTEN -) - -// Termios is the Unix API for terminal I/O. -type Termios struct { - Iflag uint64 - Oflag uint64 - Cflag uint64 - Lflag uint64 - Cc [20]byte - Ispeed uint64 - Ospeed uint64 -} - -// MakeRaw put the terminal connected to the given file descriptor into raw -// mode and returns the previous state of the terminal so that it can be -// restored. -func MakeRaw(fd uintptr) (*State, error) { - var oldState State - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios))); err != 0 { - return nil, err - } - - newState := oldState.termios - newState.Iflag &^= (IGNBRK | BRKINT | PARMRK | ISTRIP | INLCR | IGNCR | ICRNL | IXON) - newState.Oflag &^= OPOST - newState.Lflag &^= (ECHO | ECHONL | ICANON | ISIG | IEXTEN) - newState.Cflag &^= (CSIZE | PARENB) - newState.Cflag |= CS8 - newState.Cc[syscall.VMIN] = 1 - newState.Cc[syscall.VTIME] = 0 - - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(setTermios), uintptr(unsafe.Pointer(&newState))); err != 0 { - return nil, err - } - - return &oldState, nil -} diff --git a/term/termios_freebsd.go b/term/termios_freebsd.go deleted file mode 100644 index ed843ad..0000000 --- a/term/termios_freebsd.go +++ /dev/null @@ -1,69 +0,0 @@ -package term - -import ( - "syscall" - "unsafe" -) - -const ( - getTermios = syscall.TIOCGETA - setTermios = syscall.TIOCSETA -) - -// Termios magic numbers, passthrough to the ones defined in syscall. -const ( - IGNBRK = syscall.IGNBRK - PARMRK = syscall.PARMRK - INLCR = syscall.INLCR - IGNCR = syscall.IGNCR - ECHONL = syscall.ECHONL - CSIZE = syscall.CSIZE - ICRNL = syscall.ICRNL - ISTRIP = syscall.ISTRIP - PARENB = syscall.PARENB - ECHO = syscall.ECHO - ICANON = syscall.ICANON - ISIG = syscall.ISIG - IXON = syscall.IXON - BRKINT = syscall.BRKINT - INPCK = syscall.INPCK - OPOST = syscall.OPOST - CS8 = syscall.CS8 - IEXTEN = syscall.IEXTEN -) - -// Termios is the Unix API for terminal I/O. -type Termios struct { - Iflag uint32 - Oflag uint32 - Cflag uint32 - Lflag uint32 - Cc [20]byte - Ispeed uint32 - Ospeed uint32 -} - -// MakeRaw put the terminal connected to the given file descriptor into raw -// mode and returns the previous state of the terminal so that it can be -// restored. -func MakeRaw(fd uintptr) (*State, error) { - var oldState State - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios))); err != 0 { - return nil, err - } - - newState := oldState.termios - newState.Iflag &^= (IGNBRK | BRKINT | PARMRK | ISTRIP | INLCR | IGNCR | ICRNL | IXON) - newState.Oflag &^= OPOST - newState.Lflag &^= (ECHO | ECHONL | ICANON | ISIG | IEXTEN) - newState.Cflag &^= (CSIZE | PARENB) - newState.Cflag |= CS8 - newState.Cc[syscall.VMIN] = 1 - newState.Cc[syscall.VTIME] = 0 - - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(setTermios), uintptr(unsafe.Pointer(&newState))); err != 0 { - return nil, err - } - - return &oldState, nil -} diff --git a/term/termios_linux.go b/term/termios_linux.go deleted file mode 100644 index 22921b6..0000000 --- a/term/termios_linux.go +++ /dev/null @@ -1,47 +0,0 @@ -// +build !cgo - -package term - -import ( - "syscall" - "unsafe" -) - -const ( - getTermios = syscall.TCGETS - setTermios = syscall.TCSETS -) - -// Termios is the Unix API for terminal I/O. -type Termios struct { - Iflag uint32 - Oflag uint32 - Cflag uint32 - Lflag uint32 - Cc [20]byte - Ispeed uint32 - Ospeed uint32 -} - -// MakeRaw put the terminal connected to the given file descriptor into raw -// mode and returns the previous state of the terminal so that it can be -// restored. -func MakeRaw(fd uintptr) (*State, error) { - var oldState State - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, getTermios, uintptr(unsafe.Pointer(&oldState.termios))); err != 0 { - return nil, err - } - - newState := oldState.termios - - newState.Iflag &^= (syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON) - newState.Oflag &^= syscall.OPOST - newState.Lflag &^= (syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN) - newState.Cflag &^= (syscall.CSIZE | syscall.PARENB) - newState.Cflag |= syscall.CS8 - - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, setTermios, uintptr(unsafe.Pointer(&newState))); err != 0 { - return nil, err - } - return &oldState, nil -} diff --git a/term/termios_openbsd.go b/term/termios_openbsd.go deleted file mode 100644 index ed843ad..0000000 --- a/term/termios_openbsd.go +++ /dev/null @@ -1,69 +0,0 @@ -package term - -import ( - "syscall" - "unsafe" -) - -const ( - getTermios = syscall.TIOCGETA - setTermios = syscall.TIOCSETA -) - -// Termios magic numbers, passthrough to the ones defined in syscall. -const ( - IGNBRK = syscall.IGNBRK - PARMRK = syscall.PARMRK - INLCR = syscall.INLCR - IGNCR = syscall.IGNCR - ECHONL = syscall.ECHONL - CSIZE = syscall.CSIZE - ICRNL = syscall.ICRNL - ISTRIP = syscall.ISTRIP - PARENB = syscall.PARENB - ECHO = syscall.ECHO - ICANON = syscall.ICANON - ISIG = syscall.ISIG - IXON = syscall.IXON - BRKINT = syscall.BRKINT - INPCK = syscall.INPCK - OPOST = syscall.OPOST - CS8 = syscall.CS8 - IEXTEN = syscall.IEXTEN -) - -// Termios is the Unix API for terminal I/O. -type Termios struct { - Iflag uint32 - Oflag uint32 - Cflag uint32 - Lflag uint32 - Cc [20]byte - Ispeed uint32 - Ospeed uint32 -} - -// MakeRaw put the terminal connected to the given file descriptor into raw -// mode and returns the previous state of the terminal so that it can be -// restored. -func MakeRaw(fd uintptr) (*State, error) { - var oldState State - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios))); err != 0 { - return nil, err - } - - newState := oldState.termios - newState.Iflag &^= (IGNBRK | BRKINT | PARMRK | ISTRIP | INLCR | IGNCR | ICRNL | IXON) - newState.Oflag &^= OPOST - newState.Lflag &^= (ECHO | ECHONL | ICANON | ISIG | IEXTEN) - newState.Cflag &^= (CSIZE | PARENB) - newState.Cflag |= CS8 - newState.Cc[syscall.VMIN] = 1 - newState.Cc[syscall.VTIME] = 0 - - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(setTermios), uintptr(unsafe.Pointer(&newState))); err != 0 { - return nil, err - } - - return &oldState, nil -} diff --git a/term/windows/ansi_reader.go b/term/windows/ansi_reader.go deleted file mode 100644 index 1db920d..0000000 --- a/term/windows/ansi_reader.go +++ /dev/null @@ -1,258 +0,0 @@ -// +build windows - -package windows - -import ( - "bytes" - "errors" - "fmt" - "os" - "strings" - "unsafe" - - ansiterm "github.com/Azure/go-ansiterm" - "github.com/Azure/go-ansiterm/winterm" -) - -const ( - escapeSequence = ansiterm.KEY_ESC_CSI -) - -// ansiReader wraps a standard input file (e.g., os.Stdin) providing ANSI sequence translation. -type ansiReader struct { - file *os.File - fd uintptr - buffer []byte - cbBuffer int - command []byte -} - -func newAnsiReader(nFile int) *ansiReader { - initLogger() - file, fd := winterm.GetStdFile(nFile) - return &ansiReader{ - file: file, - fd: fd, - command: make([]byte, 0, ansiterm.ANSI_MAX_CMD_LENGTH), - buffer: make([]byte, 0), - } -} - -// Close closes the wrapped file. -func (ar *ansiReader) Close() (err error) { - return ar.file.Close() -} - -// Fd returns the file descriptor of the wrapped file. -func (ar *ansiReader) Fd() uintptr { - return ar.fd -} - -// Read reads up to len(p) bytes of translated input events into p. -func (ar *ansiReader) Read(p []byte) (int, error) { - if len(p) == 0 { - return 0, nil - } - - // Previously read bytes exist, read as much as we can and return - if len(ar.buffer) > 0 { - logger.Debugf("Reading previously cached bytes") - - originalLength := len(ar.buffer) - copiedLength := copy(p, ar.buffer) - - if copiedLength == originalLength { - ar.buffer = make([]byte, 0, len(p)) - } else { - ar.buffer = ar.buffer[copiedLength:] - } - - logger.Debugf("Read from cache p[%d]: % x", copiedLength, p) - return copiedLength, nil - } - - // Read and translate key events - events, err := readInputEvents(ar.fd, len(p)) - if err != nil { - return 0, err - } else if len(events) == 0 { - logger.Debug("No input events detected") - return 0, nil - } - - keyBytes := translateKeyEvents(events, []byte(escapeSequence)) - - // Save excess bytes and right-size keyBytes - if len(keyBytes) > len(p) { - logger.Debugf("Received %d keyBytes, only room for %d bytes", len(keyBytes), len(p)) - ar.buffer = keyBytes[len(p):] - keyBytes = keyBytes[:len(p)] - } else if len(keyBytes) == 0 { - logger.Debug("No key bytes returned from the translator") - return 0, nil - } - - copiedLength := copy(p, keyBytes) - if copiedLength != len(keyBytes) { - return 0, errors.New("Unexpected copy length encountered.") - } - - logger.Debugf("Read p[%d]: % x", copiedLength, p) - logger.Debugf("Read keyBytes[%d]: % x", copiedLength, keyBytes) - return copiedLength, nil -} - -// readInputEvents polls until at least one event is available. -func readInputEvents(fd uintptr, maxBytes int) ([]winterm.INPUT_RECORD, error) { - // Determine the maximum number of records to retrieve - // -- Cast around the type system to obtain the size of a single INPUT_RECORD. - // unsafe.Sizeof requires an expression vs. a type-reference; the casting - // tricks the type system into believing it has such an expression. - recordSize := int(unsafe.Sizeof(*((*winterm.INPUT_RECORD)(unsafe.Pointer(&maxBytes))))) - countRecords := maxBytes / recordSize - if countRecords > ansiterm.MAX_INPUT_EVENTS { - countRecords = ansiterm.MAX_INPUT_EVENTS - } - logger.Debugf("[windows] readInputEvents: Reading %v records (buffer size %v, record size %v)", countRecords, maxBytes, recordSize) - - // Wait for and read input events - events := make([]winterm.INPUT_RECORD, countRecords) - nEvents := uint32(0) - eventsExist, err := winterm.WaitForSingleObject(fd, winterm.WAIT_INFINITE) - if err != nil { - return nil, err - } - - if eventsExist { - err = winterm.ReadConsoleInput(fd, events, &nEvents) - if err != nil { - return nil, err - } - } - - // Return a slice restricted to the number of returned records - logger.Debugf("[windows] readInputEvents: Read %v events", nEvents) - return events[:nEvents], nil -} - -// KeyEvent Translation Helpers - -var arrowKeyMapPrefix = map[uint16]string{ - winterm.VK_UP: "%s%sA", - winterm.VK_DOWN: "%s%sB", - winterm.VK_RIGHT: "%s%sC", - winterm.VK_LEFT: "%s%sD", -} - -var keyMapPrefix = map[uint16]string{ - winterm.VK_UP: "\x1B[%sA", - winterm.VK_DOWN: "\x1B[%sB", - winterm.VK_RIGHT: "\x1B[%sC", - winterm.VK_LEFT: "\x1B[%sD", - winterm.VK_HOME: "\x1B[1%s~", // showkey shows ^[[1 - winterm.VK_END: "\x1B[4%s~", // showkey shows ^[[4 - winterm.VK_INSERT: "\x1B[2%s~", - winterm.VK_DELETE: "\x1B[3%s~", - winterm.VK_PRIOR: "\x1B[5%s~", - winterm.VK_NEXT: "\x1B[6%s~", - winterm.VK_F1: "", - winterm.VK_F2: "", - winterm.VK_F3: "\x1B[13%s~", - winterm.VK_F4: "\x1B[14%s~", - winterm.VK_F5: "\x1B[15%s~", - winterm.VK_F6: "\x1B[17%s~", - winterm.VK_F7: "\x1B[18%s~", - winterm.VK_F8: "\x1B[19%s~", - winterm.VK_F9: "\x1B[20%s~", - winterm.VK_F10: "\x1B[21%s~", - winterm.VK_F11: "\x1B[23%s~", - winterm.VK_F12: "\x1B[24%s~", -} - -// translateKeyEvents converts the input events into the appropriate ANSI string. -func translateKeyEvents(events []winterm.INPUT_RECORD, escapeSequence []byte) []byte { - var buffer bytes.Buffer - for _, event := range events { - if event.EventType == winterm.KEY_EVENT && event.KeyEvent.KeyDown != 0 { - buffer.WriteString(keyToString(&event.KeyEvent, escapeSequence)) - } - } - - return buffer.Bytes() -} - -// keyToString maps the given input event record to the corresponding string. -func keyToString(keyEvent *winterm.KEY_EVENT_RECORD, escapeSequence []byte) string { - if keyEvent.UnicodeChar == 0 { - return formatVirtualKey(keyEvent.VirtualKeyCode, keyEvent.ControlKeyState, escapeSequence) - } - - _, alt, control := getControlKeys(keyEvent.ControlKeyState) - if control { - // TODO(azlinux): Implement following control sequences - // -D Signals the end of input from the keyboard; also exits current shell. - // -H Deletes the first character to the left of the cursor. Also called the ERASE key. - // -Q Restarts printing after it has been stopped with -s. - // -S Suspends printing on the screen (does not stop the program). - // -U Deletes all characters on the current line. Also called the KILL key. - // -E Quits current command and creates a core - - } - - // +Key generates ESC N Key - if !control && alt { - return ansiterm.KEY_ESC_N + strings.ToLower(string(keyEvent.UnicodeChar)) - } - - return string(keyEvent.UnicodeChar) -} - -// formatVirtualKey converts a virtual key (e.g., up arrow) into the appropriate ANSI string. -func formatVirtualKey(key uint16, controlState uint32, escapeSequence []byte) string { - shift, alt, control := getControlKeys(controlState) - modifier := getControlKeysModifier(shift, alt, control) - - if format, ok := arrowKeyMapPrefix[key]; ok { - return fmt.Sprintf(format, escapeSequence, modifier) - } - - if format, ok := keyMapPrefix[key]; ok { - return fmt.Sprintf(format, modifier) - } - - return "" -} - -// getControlKeys extracts the shift, alt, and ctrl key states. -func getControlKeys(controlState uint32) (shift, alt, control bool) { - shift = 0 != (controlState & winterm.SHIFT_PRESSED) - alt = 0 != (controlState & (winterm.LEFT_ALT_PRESSED | winterm.RIGHT_ALT_PRESSED)) - control = 0 != (controlState & (winterm.LEFT_CTRL_PRESSED | winterm.RIGHT_CTRL_PRESSED)) - return shift, alt, control -} - -// getControlKeysModifier returns the ANSI modifier for the given combination of control keys. -func getControlKeysModifier(shift, alt, control bool) string { - if shift && alt && control { - return ansiterm.KEY_CONTROL_PARAM_8 - } - if alt && control { - return ansiterm.KEY_CONTROL_PARAM_7 - } - if shift && control { - return ansiterm.KEY_CONTROL_PARAM_6 - } - if control { - return ansiterm.KEY_CONTROL_PARAM_5 - } - if shift && alt { - return ansiterm.KEY_CONTROL_PARAM_4 - } - if alt { - return ansiterm.KEY_CONTROL_PARAM_3 - } - if shift { - return ansiterm.KEY_CONTROL_PARAM_2 - } - return "" -} diff --git a/term/windows/ansi_writer.go b/term/windows/ansi_writer.go deleted file mode 100644 index 0299a88..0000000 --- a/term/windows/ansi_writer.go +++ /dev/null @@ -1,61 +0,0 @@ -// +build windows - -package windows - -import ( - "os" - - ansiterm "github.com/Azure/go-ansiterm" - "github.com/Azure/go-ansiterm/winterm" -) - -// ansiWriter wraps a standard output file (e.g., os.Stdout) providing ANSI sequence translation. -type ansiWriter struct { - file *os.File - fd uintptr - infoReset *winterm.CONSOLE_SCREEN_BUFFER_INFO - command []byte - escapeSequence []byte - inAnsiSequence bool - parser *ansiterm.AnsiParser -} - -func newAnsiWriter(nFile int) *ansiWriter { - initLogger() - file, fd := winterm.GetStdFile(nFile) - info, err := winterm.GetConsoleScreenBufferInfo(fd) - if err != nil { - return nil - } - - parser := ansiterm.CreateParser("Ground", winterm.CreateWinEventHandler(fd, file)) - logger.Infof("newAnsiWriter: parser %p", parser) - - aw := &ansiWriter{ - file: file, - fd: fd, - infoReset: info, - command: make([]byte, 0, ansiterm.ANSI_MAX_CMD_LENGTH), - escapeSequence: []byte(ansiterm.KEY_ESC_CSI), - parser: parser, - } - - logger.Infof("newAnsiWriter: aw.parser %p", aw.parser) - logger.Infof("newAnsiWriter: %v", aw) - return aw -} - -func (aw *ansiWriter) Fd() uintptr { - return aw.fd -} - -// Write writes len(p) bytes from p to the underlying data stream. -func (aw *ansiWriter) Write(p []byte) (total int, err error) { - if len(p) == 0 { - return 0, nil - } - - logger.Infof("Write: % x", p) - logger.Infof("Write: %s", string(p)) - return aw.parser.Parse(p) -} diff --git a/term/windows/console.go b/term/windows/console.go deleted file mode 100644 index 3036a04..0000000 --- a/term/windows/console.go +++ /dev/null @@ -1,97 +0,0 @@ -// +build windows - -package windows - -import ( - "io" - "os" - "syscall" - - "github.com/Azure/go-ansiterm/winterm" - - ansiterm "github.com/Azure/go-ansiterm" - "github.com/Sirupsen/logrus" - "io/ioutil" -) - -// ConEmuStreams returns prepared versions of console streams, -// for proper use in ConEmu terminal. -// The ConEmu terminal emulates ANSI on output streams well by default. -func ConEmuStreams() (stdIn io.ReadCloser, stdOut, stdErr io.Writer) { - if IsConsole(os.Stdin.Fd()) { - stdIn = newAnsiReader(syscall.STD_INPUT_HANDLE) - } else { - stdIn = os.Stdin - } - - stdOut = os.Stdout - stdErr = os.Stderr - - // WARNING (BEGIN): sourced from newAnsiWriter - - logFile := ioutil.Discard - - if isDebugEnv := os.Getenv(ansiterm.LogEnv); isDebugEnv == "1" { - logFile, _ = os.Create("ansiReaderWriter.log") - } - - logger = &logrus.Logger{ - Out: logFile, - Formatter: new(logrus.TextFormatter), - Level: logrus.DebugLevel, - } - - // WARNING (END): sourced from newAnsiWriter - - return stdIn, stdOut, stdErr -} - -// ConsoleStreams returns a wrapped version for each standard stream referencing a console, -// that handles ANSI character sequences. -func ConsoleStreams() (stdIn io.ReadCloser, stdOut, stdErr io.Writer) { - if IsConsole(os.Stdin.Fd()) { - stdIn = newAnsiReader(syscall.STD_INPUT_HANDLE) - } else { - stdIn = os.Stdin - } - - if IsConsole(os.Stdout.Fd()) { - stdOut = newAnsiWriter(syscall.STD_OUTPUT_HANDLE) - } else { - stdOut = os.Stdout - } - - if IsConsole(os.Stderr.Fd()) { - stdErr = newAnsiWriter(syscall.STD_ERROR_HANDLE) - } else { - stdErr = os.Stderr - } - - return stdIn, stdOut, stdErr -} - -// GetHandleInfo returns file descriptor and bool indicating whether the file is a console. -func GetHandleInfo(in interface{}) (uintptr, bool) { - switch t := in.(type) { - case *ansiReader: - return t.Fd(), true - case *ansiWriter: - return t.Fd(), true - } - - var inFd uintptr - var isTerminal bool - - if file, ok := in.(*os.File); ok { - inFd = file.Fd() - isTerminal = IsConsole(inFd) - } - return inFd, isTerminal -} - -// IsConsole returns true if the given file descriptor is a Windows Console. -// The code assumes that GetConsoleMode will return an error for file descriptors that are not a console. -func IsConsole(fd uintptr) bool { - _, e := winterm.GetConsoleMode(fd) - return e == nil -} diff --git a/term/windows/windows.go b/term/windows/windows.go deleted file mode 100644 index ce4cb59..0000000 --- a/term/windows/windows.go +++ /dev/null @@ -1,33 +0,0 @@ -// These files implement ANSI-aware input and output streams for use by the Docker Windows client. -// When asked for the set of standard streams (e.g., stdin, stdout, stderr), the code will create -// and return pseudo-streams that convert ANSI sequences to / from Windows Console API calls. - -package windows - -import ( - "io/ioutil" - "os" - "sync" - - ansiterm "github.com/Azure/go-ansiterm" - "github.com/Sirupsen/logrus" -) - -var logger *logrus.Logger -var initOnce sync.Once - -func initLogger() { - initOnce.Do(func() { - logFile := ioutil.Discard - - if isDebugEnv := os.Getenv(ansiterm.LogEnv); isDebugEnv == "1" { - logFile, _ = os.Create("ansiReaderWriter.log") - } - - logger = &logrus.Logger{ - Out: logFile, - Formatter: new(logrus.TextFormatter), - Level: logrus.DebugLevel, - } - }) -} diff --git a/term/windows/windows_test.go b/term/windows/windows_test.go deleted file mode 100644 index 52aeab5..0000000 --- a/term/windows/windows_test.go +++ /dev/null @@ -1,3 +0,0 @@ -// This file is necessary to pass the Docker tests. - -package windows diff --git a/tlsconfig/config.go b/tlsconfig/config.go deleted file mode 100644 index e3dfad1..0000000 --- a/tlsconfig/config.go +++ /dev/null @@ -1,133 +0,0 @@ -// Package tlsconfig provides primitives to retrieve secure-enough TLS configurations for both clients and servers. -// -// As a reminder from https://golang.org/pkg/crypto/tls/#Config: -// A Config structure is used to configure a TLS client or server. After one has been passed to a TLS function it must not be modified. -// A Config may be reused; the tls package will also not modify it. -package tlsconfig - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" - "os" - - "github.com/Sirupsen/logrus" -) - -// Options represents the information needed to create client and server TLS configurations. -type Options struct { - CAFile string - - // If either CertFile or KeyFile is empty, Client() will not load them - // preventing the client from authenticating to the server. - // However, Server() requires them and will error out if they are empty. - CertFile string - KeyFile string - - // client-only option - InsecureSkipVerify bool - // server-only option - ClientAuth tls.ClientAuthType -} - -// Extra (server-side) accepted CBC cipher suites - will phase out in the future -var acceptedCBCCiphers = []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, -} - -// Client TLS cipher suites (dropping CBC ciphers for client preferred suite set) -var clientCipherSuites = []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -} - -// DefaultServerAcceptedCiphers should be uses by code which already has a crypto/tls -// options struct but wants to use a commonly accepted set of TLS cipher suites, with -// known weak algorithms removed. -var DefaultServerAcceptedCiphers = append(clientCipherSuites, acceptedCBCCiphers...) - -// ServerDefault is a secure-enough TLS configuration for the server TLS configuration. -var ServerDefault = tls.Config{ - // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - PreferServerCipherSuites: true, - CipherSuites: DefaultServerAcceptedCiphers, -} - -// ClientDefault is a secure-enough TLS configuration for the client TLS configuration. -var ClientDefault = tls.Config{ - // Prefer TLS1.2 as the client minimum - MinVersion: tls.VersionTLS12, - CipherSuites: clientCipherSuites, -} - -// certPool returns an X.509 certificate pool from `caFile`, the certificate file. -func certPool(caFile string) (*x509.CertPool, error) { - // If we should verify the server, we need to load a trusted ca - certPool := x509.NewCertPool() - pem, err := ioutil.ReadFile(caFile) - if err != nil { - return nil, fmt.Errorf("Could not read CA certificate %q: %v", caFile, err) - } - if !certPool.AppendCertsFromPEM(pem) { - return nil, fmt.Errorf("failed to append certificates from PEM file: %q", caFile) - } - s := certPool.Subjects() - subjects := make([]string, len(s)) - for i, subject := range s { - subjects[i] = string(subject) - } - logrus.Debugf("Trusting certs with subjects: %v", subjects) - return certPool, nil -} - -// Client returns a TLS configuration meant to be used by a client. -func Client(options Options) (*tls.Config, error) { - tlsConfig := ClientDefault - tlsConfig.InsecureSkipVerify = options.InsecureSkipVerify - if !options.InsecureSkipVerify { - CAs, err := certPool(options.CAFile) - if err != nil { - return nil, err - } - tlsConfig.RootCAs = CAs - } - - if options.CertFile != "" && options.KeyFile != "" { - tlsCert, err := tls.LoadX509KeyPair(options.CertFile, options.KeyFile) - if err != nil { - return nil, fmt.Errorf("Could not load X509 key pair: %v. Make sure the key is not encrypted", err) - } - tlsConfig.Certificates = []tls.Certificate{tlsCert} - } - - return &tlsConfig, nil -} - -// Server returns a TLS configuration meant to be used by a server. -func Server(options Options) (*tls.Config, error) { - tlsConfig := ServerDefault - tlsConfig.ClientAuth = options.ClientAuth - tlsCert, err := tls.LoadX509KeyPair(options.CertFile, options.KeyFile) - if err != nil { - if os.IsNotExist(err) { - return nil, fmt.Errorf("Could not load X509 key pair (cert: %q, key: %q): %v", options.CertFile, options.KeyFile, err) - } - return nil, fmt.Errorf("Error reading X509 key pair (cert: %q, key: %q): %v. Make sure the key is not encrypted.", options.CertFile, options.KeyFile, err) - } - tlsConfig.Certificates = []tls.Certificate{tlsCert} - if options.ClientAuth >= tls.VerifyClientCertIfGiven { - CAs, err := certPool(options.CAFile) - if err != nil { - return nil, err - } - tlsConfig.ClientCAs = CAs - } - return &tlsConfig, nil -} diff --git a/truncindex/truncindex.go b/truncindex/truncindex.go deleted file mode 100644 index 02610b8..0000000 --- a/truncindex/truncindex.go +++ /dev/null @@ -1,137 +0,0 @@ -// Package truncindex provides a general 'index tree', used by Docker -// in order to be able to reference containers by only a few unambiguous -// characters of their id. -package truncindex - -import ( - "errors" - "fmt" - "strings" - "sync" - - "github.com/tchap/go-patricia/patricia" -) - -var ( - // ErrEmptyPrefix is an error returned if the prefix was empty. - ErrEmptyPrefix = errors.New("Prefix can't be empty") - - // ErrIllegalChar is returned when a space is in the ID - ErrIllegalChar = errors.New("illegal character: ' '") - - // ErrNotExist is returned when ID or its prefix not found in index. - ErrNotExist = errors.New("ID does not exist") -) - -// ErrAmbiguousPrefix is returned if the prefix was ambiguous -// (multiple ids for the prefix). -type ErrAmbiguousPrefix struct { - prefix string -} - -func (e ErrAmbiguousPrefix) Error() string { - return fmt.Sprintf("Multiple IDs found with provided prefix: %s", e.prefix) -} - -// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes. -// This is used to retrieve image and container IDs by more convenient shorthand prefixes. -type TruncIndex struct { - sync.RWMutex - trie *patricia.Trie - ids map[string]struct{} -} - -// NewTruncIndex creates a new TruncIndex and initializes with a list of IDs. -func NewTruncIndex(ids []string) (idx *TruncIndex) { - idx = &TruncIndex{ - ids: make(map[string]struct{}), - - // Change patricia max prefix per node length, - // because our len(ID) always 64 - trie: patricia.NewTrie(patricia.MaxPrefixPerNode(64)), - } - for _, id := range ids { - idx.addID(id) - } - return -} - -func (idx *TruncIndex) addID(id string) error { - if strings.Contains(id, " ") { - return ErrIllegalChar - } - if id == "" { - return ErrEmptyPrefix - } - if _, exists := idx.ids[id]; exists { - return fmt.Errorf("id already exists: '%s'", id) - } - idx.ids[id] = struct{}{} - if inserted := idx.trie.Insert(patricia.Prefix(id), struct{}{}); !inserted { - return fmt.Errorf("failed to insert id: %s", id) - } - return nil -} - -// Add adds a new ID to the TruncIndex. -func (idx *TruncIndex) Add(id string) error { - idx.Lock() - defer idx.Unlock() - if err := idx.addID(id); err != nil { - return err - } - return nil -} - -// Delete removes an ID from the TruncIndex. If there are multiple IDs -// with the given prefix, an error is thrown. -func (idx *TruncIndex) Delete(id string) error { - idx.Lock() - defer idx.Unlock() - if _, exists := idx.ids[id]; !exists || id == "" { - return fmt.Errorf("no such id: '%s'", id) - } - delete(idx.ids, id) - if deleted := idx.trie.Delete(patricia.Prefix(id)); !deleted { - return fmt.Errorf("no such id: '%s'", id) - } - return nil -} - -// Get retrieves an ID from the TruncIndex. If there are multiple IDs -// with the given prefix, an error is thrown. -func (idx *TruncIndex) Get(s string) (string, error) { - if s == "" { - return "", ErrEmptyPrefix - } - var ( - id string - ) - subTreeVisitFunc := func(prefix patricia.Prefix, item patricia.Item) error { - if id != "" { - // we haven't found the ID if there are two or more IDs - id = "" - return ErrAmbiguousPrefix{prefix: string(prefix)} - } - id = string(prefix) - return nil - } - - idx.RLock() - defer idx.RUnlock() - if err := idx.trie.VisitSubtree(patricia.Prefix(s), subTreeVisitFunc); err != nil { - return "", err - } - if id != "" { - return id, nil - } - return "", ErrNotExist -} - -// Iterate iterates over all stored IDs, and passes each of them to the given handler. -func (idx *TruncIndex) Iterate(handler func(id string)) { - idx.trie.Visit(func(prefix patricia.Prefix, item patricia.Item) error { - handler(string(prefix)) - return nil - }) -} diff --git a/truncindex/truncindex_test.go b/truncindex/truncindex_test.go deleted file mode 100644 index 8197baf..0000000 --- a/truncindex/truncindex_test.go +++ /dev/null @@ -1,429 +0,0 @@ -package truncindex - -import ( - "math/rand" - "testing" - - "github.com/docker/docker/pkg/stringid" -) - -// Test the behavior of TruncIndex, an index for querying IDs from a non-conflicting prefix. -func TestTruncIndex(t *testing.T) { - ids := []string{} - index := NewTruncIndex(ids) - // Get on an empty index - if _, err := index.Get("foobar"); err == nil { - t.Fatal("Get on an empty index should return an error") - } - - // Spaces should be illegal in an id - if err := index.Add("I have a space"); err == nil { - t.Fatalf("Adding an id with ' ' should return an error") - } - - id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96" - // Add an id - if err := index.Add(id); err != nil { - t.Fatal(err) - } - - // Add an empty id (should fail) - if err := index.Add(""); err == nil { - t.Fatalf("Adding an empty id should return an error") - } - - // Get a non-existing id - assertIndexGet(t, index, "abracadabra", "", true) - // Get an empty id - assertIndexGet(t, index, "", "", true) - // Get the exact id - assertIndexGet(t, index, id, id, false) - // The first letter should match - assertIndexGet(t, index, id[:1], id, false) - // The first half should match - assertIndexGet(t, index, id[:len(id)/2], id, false) - // The second half should NOT match - assertIndexGet(t, index, id[len(id)/2:], "", true) - - id2 := id[:6] + "blabla" - // Add an id - if err := index.Add(id2); err != nil { - t.Fatal(err) - } - // Both exact IDs should work - assertIndexGet(t, index, id, id, false) - assertIndexGet(t, index, id2, id2, false) - - // 6 characters or less should conflict - assertIndexGet(t, index, id[:6], "", true) - assertIndexGet(t, index, id[:4], "", true) - assertIndexGet(t, index, id[:1], "", true) - - // An ambiguous id prefix should return an error - if _, err := index.Get(id[:4]); err == nil { - t.Fatal("An ambiguous id prefix should return an error") - } - - // 7 characters should NOT conflict - assertIndexGet(t, index, id[:7], id, false) - assertIndexGet(t, index, id2[:7], id2, false) - - // Deleting a non-existing id should return an error - if err := index.Delete("non-existing"); err == nil { - t.Fatalf("Deleting a non-existing id should return an error") - } - - // Deleting an empty id should return an error - if err := index.Delete(""); err == nil { - t.Fatal("Deleting an empty id should return an error") - } - - // Deleting id2 should remove conflicts - if err := index.Delete(id2); err != nil { - t.Fatal(err) - } - // id2 should no longer work - assertIndexGet(t, index, id2, "", true) - assertIndexGet(t, index, id2[:7], "", true) - assertIndexGet(t, index, id2[:11], "", true) - - // conflicts between id and id2 should be gone - assertIndexGet(t, index, id[:6], id, false) - assertIndexGet(t, index, id[:4], id, false) - assertIndexGet(t, index, id[:1], id, false) - - // non-conflicting substrings should still not conflict - assertIndexGet(t, index, id[:7], id, false) - assertIndexGet(t, index, id[:15], id, false) - assertIndexGet(t, index, id, id, false) - - assertIndexIterate(t) -} - -func assertIndexIterate(t *testing.T) { - ids := []string{ - "19b36c2c326ccc11e726eee6ee78a0baf166ef96", - "28b36c2c326ccc11e726eee6ee78a0baf166ef96", - "37b36c2c326ccc11e726eee6ee78a0baf166ef96", - "46b36c2c326ccc11e726eee6ee78a0baf166ef96", - } - - index := NewTruncIndex(ids) - - index.Iterate(func(targetId string) { - for _, id := range ids { - if targetId == id { - return - } - } - - t.Fatalf("An unknown ID '%s'", targetId) - }) -} - -func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult string, expectError bool) { - if result, err := index.Get(input); err != nil && !expectError { - t.Fatalf("Unexpected error getting '%s': %s", input, err) - } else if err == nil && expectError { - t.Fatalf("Getting '%s' should return an error, not '%s'", input, result) - } else if result != expectedResult { - t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult) - } -} - -func BenchmarkTruncIndexAdd100(b *testing.B) { - var testSet []string - for i := 0; i < 100; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - } -} - -func BenchmarkTruncIndexAdd250(b *testing.B) { - var testSet []string - for i := 0; i < 250; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - } -} - -func BenchmarkTruncIndexAdd500(b *testing.B) { - var testSet []string - for i := 0; i < 500; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - } -} - -func BenchmarkTruncIndexGet100(b *testing.B) { - var testSet []string - var testKeys []string - for i := 0; i < 100; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - l := rand.Intn(12) + 12 - testKeys = append(testKeys, id[:l]) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, id := range testKeys { - if res, err := index.Get(id); err != nil { - b.Fatal(res, err) - } - } - } -} - -func BenchmarkTruncIndexGet250(b *testing.B) { - var testSet []string - var testKeys []string - for i := 0; i < 250; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - l := rand.Intn(12) + 12 - testKeys = append(testKeys, id[:l]) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, id := range testKeys { - if res, err := index.Get(id); err != nil { - b.Fatal(res, err) - } - } - } -} - -func BenchmarkTruncIndexGet500(b *testing.B) { - var testSet []string - var testKeys []string - for i := 0; i < 500; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - l := rand.Intn(12) + 12 - testKeys = append(testKeys, id[:l]) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, id := range testKeys { - if res, err := index.Get(id); err != nil { - b.Fatal(res, err) - } - } - } -} - -func BenchmarkTruncIndexDelete100(b *testing.B) { - var testSet []string - for i := 0; i < 100; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - b.StartTimer() - for _, id := range testSet { - if err := index.Delete(id); err != nil { - b.Fatal(err) - } - } - } -} - -func BenchmarkTruncIndexDelete250(b *testing.B) { - var testSet []string - for i := 0; i < 250; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - b.StartTimer() - for _, id := range testSet { - if err := index.Delete(id); err != nil { - b.Fatal(err) - } - } - } -} - -func BenchmarkTruncIndexDelete500(b *testing.B) { - var testSet []string - for i := 0; i < 500; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - b.StartTimer() - for _, id := range testSet { - if err := index.Delete(id); err != nil { - b.Fatal(err) - } - } - } -} - -func BenchmarkTruncIndexNew100(b *testing.B) { - var testSet []string - for i := 0; i < 100; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - NewTruncIndex(testSet) - } -} - -func BenchmarkTruncIndexNew250(b *testing.B) { - var testSet []string - for i := 0; i < 250; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - NewTruncIndex(testSet) - } -} - -func BenchmarkTruncIndexNew500(b *testing.B) { - var testSet []string - for i := 0; i < 500; i++ { - testSet = append(testSet, stringid.GenerateNonCryptoID()) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - NewTruncIndex(testSet) - } -} - -func BenchmarkTruncIndexAddGet100(b *testing.B) { - var testSet []string - var testKeys []string - for i := 0; i < 500; i++ { - id := stringid.GenerateNonCryptoID() - testSet = append(testSet, id) - l := rand.Intn(12) + 12 - testKeys = append(testKeys, id[:l]) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - for _, id := range testKeys { - if res, err := index.Get(id); err != nil { - b.Fatal(res, err) - } - } - } -} - -func BenchmarkTruncIndexAddGet250(b *testing.B) { - var testSet []string - var testKeys []string - for i := 0; i < 500; i++ { - id := stringid.GenerateNonCryptoID() - testSet = append(testSet, id) - l := rand.Intn(12) + 12 - testKeys = append(testKeys, id[:l]) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - for _, id := range testKeys { - if res, err := index.Get(id); err != nil { - b.Fatal(res, err) - } - } - } -} - -func BenchmarkTruncIndexAddGet500(b *testing.B) { - var testSet []string - var testKeys []string - for i := 0; i < 500; i++ { - id := stringid.GenerateNonCryptoID() - testSet = append(testSet, id) - l := rand.Intn(12) + 12 - testKeys = append(testKeys, id[:l]) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - index := NewTruncIndex([]string{}) - for _, id := range testSet { - if err := index.Add(id); err != nil { - b.Fatal(err) - } - } - for _, id := range testKeys { - if res, err := index.Get(id); err != nil { - b.Fatal(res, err) - } - } - } -} diff --git a/urlutil/urlutil.go b/urlutil/urlutil.go deleted file mode 100644 index 4415287..0000000 --- a/urlutil/urlutil.go +++ /dev/null @@ -1,50 +0,0 @@ -// Package urlutil provides helper function to check urls kind. -// It supports http urls, git urls and transport url (tcp://, …) -package urlutil - -import ( - "regexp" - "strings" -) - -var ( - validPrefixes = map[string][]string{ - "url": {"http://", "https://"}, - "git": {"git://", "github.com/", "git@"}, - "transport": {"tcp://", "tcp+tls://", "udp://", "unix://", "unixgram://"}, - } - urlPathWithFragmentSuffix = regexp.MustCompile(".git(?:#.+)?$") -) - -// IsURL returns true if the provided str is an HTTP(S) URL. -func IsURL(str string) bool { - return checkURL(str, "url") -} - -// IsGitURL returns true if the provided str is a git repository URL. -func IsGitURL(str string) bool { - if IsURL(str) && urlPathWithFragmentSuffix.MatchString(str) { - return true - } - return checkURL(str, "git") -} - -// IsGitTransport returns true if the provided str is a git transport by inspecting -// the prefix of the string for known protocols used in git. -func IsGitTransport(str string) bool { - return IsURL(str) || strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "git@") -} - -// IsTransportURL returns true if the provided str is a transport (tcp, tcp+tls, udp, unix) URL. -func IsTransportURL(str string) bool { - return checkURL(str, "transport") -} - -func checkURL(str, kind string) bool { - for _, prefix := range validPrefixes[kind] { - if strings.HasPrefix(str, prefix) { - return true - } - } - return false -} diff --git a/urlutil/urlutil_test.go b/urlutil/urlutil_test.go deleted file mode 100644 index 75eb464..0000000 --- a/urlutil/urlutil_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package urlutil - -import "testing" - -var ( - gitUrls = []string{ - "git://github.com/docker/docker", - "git@github.com:docker/docker.git", - "git@bitbucket.org:atlassianlabs/atlassian-docker.git", - "https://github.com/docker/docker.git", - "http://github.com/docker/docker.git", - "http://github.com/docker/docker.git#branch", - "http://github.com/docker/docker.git#:dir", - } - incompleteGitUrls = []string{ - "github.com/docker/docker", - } - invalidGitUrls = []string{ - "http://github.com/docker/docker.git:#branch", - } - transportUrls = []string{ - "tcp://example.com", - "tcp+tls://example.com", - "udp://example.com", - "unix:///example", - "unixgram:///example", - } -) - -func TestValidGitTransport(t *testing.T) { - for _, url := range gitUrls { - if IsGitTransport(url) == false { - t.Fatalf("%q should be detected as valid Git prefix", url) - } - } - - for _, url := range incompleteGitUrls { - if IsGitTransport(url) == true { - t.Fatalf("%q should not be detected as valid Git prefix", url) - } - } -} - -func TestIsGIT(t *testing.T) { - for _, url := range gitUrls { - if IsGitURL(url) == false { - t.Fatalf("%q should be detected as valid Git url", url) - } - } - - for _, url := range incompleteGitUrls { - if IsGitURL(url) == false { - t.Fatalf("%q should be detected as valid Git url", url) - } - } - - for _, url := range invalidGitUrls { - if IsGitURL(url) == true { - t.Fatalf("%q should not be detected as valid Git prefix", url) - } - } -} - -func TestIsTransport(t *testing.T) { - for _, url := range transportUrls { - if IsTransportURL(url) == false { - t.Fatalf("%q should be detected as valid Transport url", url) - } - } -} diff --git a/useragent/README.md b/useragent/README.md deleted file mode 100644 index d9cb367..0000000 --- a/useragent/README.md +++ /dev/null @@ -1 +0,0 @@ -This package provides helper functions to pack version information into a single User-Agent header. diff --git a/useragent/useragent.go b/useragent/useragent.go deleted file mode 100644 index 1137db5..0000000 --- a/useragent/useragent.go +++ /dev/null @@ -1,55 +0,0 @@ -// Package useragent provides helper functions to pack -// version information into a single User-Agent header. -package useragent - -import ( - "strings" -) - -// VersionInfo is used to model UserAgent versions. -type VersionInfo struct { - Name string - Version string -} - -func (vi *VersionInfo) isValid() bool { - const stopChars = " \t\r\n/" - name := vi.Name - vers := vi.Version - if len(name) == 0 || strings.ContainsAny(name, stopChars) { - return false - } - if len(vers) == 0 || strings.ContainsAny(vers, stopChars) { - return false - } - return true -} - -// AppendVersions converts versions to a string and appends the string to the string base. -// -// Each VersionInfo will be converted to a string in the format of -// "product/version", where the "product" is get from the name field, while -// version is get from the version field. Several pieces of version information -// will be concatenated and separated by space. -// -// Example: -// AppendVersions("base", VersionInfo{"foo", "1.0"}, VersionInfo{"bar", "2.0"}) -// results in "base foo/1.0 bar/2.0". -func AppendVersions(base string, versions ...VersionInfo) string { - if len(versions) == 0 { - return base - } - - verstrs := make([]string, 0, 1+len(versions)) - if len(base) > 0 { - verstrs = append(verstrs, base) - } - - for _, v := range versions { - if !v.isValid() { - continue - } - verstrs = append(verstrs, v.Name+"/"+v.Version) - } - return strings.Join(verstrs, " ") -} diff --git a/useragent/useragent_test.go b/useragent/useragent_test.go deleted file mode 100644 index 0ad7243..0000000 --- a/useragent/useragent_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package useragent - -import "testing" - -func TestVersionInfo(t *testing.T) { - vi := VersionInfo{"foo", "bar"} - if !vi.isValid() { - t.Fatalf("VersionInfo should be valid") - } - vi = VersionInfo{"", "bar"} - if vi.isValid() { - t.Fatalf("Expected VersionInfo to be invalid") - } - vi = VersionInfo{"foo", ""} - if vi.isValid() { - t.Fatalf("Expected VersionInfo to be invalid") - } -} - -func TestAppendVersions(t *testing.T) { - vis := []VersionInfo{ - {"foo", "1.0"}, - {"bar", "0.1"}, - {"pi", "3.1.4"}, - } - v := AppendVersions("base", vis...) - expect := "base foo/1.0 bar/0.1 pi/3.1.4" - if v != expect { - t.Fatalf("expected %q, got %q", expect, v) - } -}