From 3aa2a282f76b7d1f71d28d602cfb4f81b1388b94 Mon Sep 17 00:00:00 2001 From: Shawnpku Date: Mon, 11 Feb 2019 15:07:36 +0800 Subject: [PATCH 1/7] support alicdn middleware Signed-off-by: Shawnpku --- docs/configuration.md | 11 ++ .../driver/middleware/alicdn/middleware.go | 116 ++++++++++++++++++ .../aliyungo/cdn/auth/random_uuid.go | 97 +++++++++++++++ .../aliyungo/cdn/auth/random_uuid_test.go | 21 ++++ .../denverdino/aliyungo/cdn/auth/sign_url.go | 80 ++++++++++++ .../aliyungo/cdn/auth/sign_url_test.go | 53 ++++++++ 6 files changed, 378 insertions(+) create mode 100755 registry/storage/driver/middleware/alicdn/middleware.go create mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go create mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go create mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go create mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go diff --git a/docs/configuration.md b/docs/configuration.md index c26fa16d..10245bf1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -715,6 +715,17 @@ Then value of ipfilteredby: `aws`: IP from AWS goes to S3 directly `awsregion`: IP from certain AWS regions goes to S3 directly, use together with `awsregion` +### `alicdn` + +`alicdn` storage middleware allows the registry to serve layers via a content delivery network provided by Alibaba Cloud. Alicdn requires the OSS storage driver. + +| Parameter | Required | Description | +|-----------|----------|-------------------------------------------------------| +| `baseurl` | yes | The `SCHEME://HOST` at which Alicdn is served. | +| `authtype` | yes | The URL authentication type for Alicdn, which should be `a`, `b` or `c`. | +| `privatekey` | yes | The URL authentication key for Alicdn. | +| `duration` | no | An integer and unit for the duration of the Alicdn session. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`.| + ### `redirect` You can use the `redirect` storage middleware to specify a custom URL to a diff --git a/registry/storage/driver/middleware/alicdn/middleware.go b/registry/storage/driver/middleware/alicdn/middleware.go new file mode 100755 index 00000000..828d88c2 --- /dev/null +++ b/registry/storage/driver/middleware/alicdn/middleware.go @@ -0,0 +1,116 @@ +package middleware + +import ( + "fmt" + "net/url" + "strings" + "time" + + "github.com/docker/distribution/context" + storagedriver "github.com/docker/distribution/registry/storage/driver" + storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" + + "github.com/denverdino/aliyungo/cdn/auth" +) + +// aliCdnStorageMiddleware provides a simple implementation of layerHandler that +// constructs temporary signed AliCDN URLs from the storagedriver layer URL, +// then issues HTTP Temporary Redirects to this AliCDN content URL. +type aliCdnStorageMiddleware struct { + storagedriver.StorageDriver + baseURL string + urlSigner *auth.URLSigner + duration time.Duration +} + +var _ storagedriver.StorageDriver = &aliCdnStorageMiddleware{} + +// newAliCdnLayerHandler constructs and returns a new AliCDN +// LayerHandler implementation. +// Required options: baseurl, authtype, privatekey +// Optional options: duration +func newAliCdnStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) { + // parse baseurl + base, ok := options["baseurl"] + if !ok { + return nil, fmt.Errorf("no baseurl provided") + } + baseURL, ok := base.(string) + if !ok { + return nil, fmt.Errorf("baseurl must be a string") + } + if !strings.Contains(baseURL, "://") { + baseURL = "https://" + baseURL + } + if _, err := url.Parse(baseURL); err != nil { + return nil, fmt.Errorf("invalid baseurl: %v", err) + } + + // parse authtype + at, ok := options["authtype"] + if !ok { + return nil, fmt.Errorf("no authtype provided") + } + authType, ok := at.(string) + if !ok { + return nil, fmt.Errorf("authtype must be a string") + } + if authType != "a" && authType != "b" && authType != "c" { + return nil, fmt.Errorf("invalid authentication type") + } + + // parse privatekey + pk, ok := options["privatekey"] + if !ok { + return nil, fmt.Errorf("no privatekey provided") + } + privateKey, ok := pk.(string) + if !ok { + return nil, fmt.Errorf("privatekey must be a string") + } + + urlSigner := auth.NewURLSigner(authType, privateKey) + + // parse duration + duration := 60 * time.Minute + d, ok := options["duration"] + if ok { + switch d := d.(type) { + case time.Duration: + duration = d + case string: + dur, err := time.ParseDuration(d) + if err != nil { + return nil, fmt.Errorf("invalid duration: %s", err) + } + duration = dur + } + } + + return &aliCdnStorageMiddleware{ + StorageDriver: storageDriver, + baseURL: baseURL, + urlSigner: urlSigner, + duration: duration, + }, nil +} + +// URLFor attempts to find a url which may be used to retrieve the file at the given path. +// Returns an error if the file cannot be found. +func (ac *aliCdnStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { + + if ac.StorageDriver.Name() != "oss" { + context.GetLogger(ctx).Warn("the AliCdn middleware does not support this backend storage driver") + return ac.StorageDriver.URLFor(ctx, path, options) + } + acURL, err := ac.urlSigner.Sign(ac.baseURL+path, time.Now().Add(ac.duration)) + if err != nil { + return "", err + } + return acURL, nil +} + +// init registers the alicdn layerHandler backend. +func init() { + storagemiddleware.Register("alicdn", storagemiddleware.InitFunc(newAliCdnStorageMiddleware)) +} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go new file mode 100644 index 00000000..169a2b6c --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go @@ -0,0 +1,97 @@ +package auth + +import ( + "crypto/rand" + "fmt" + "io" + "os" + "syscall" + "time" +) + +const ( + // Bits is the number of bits in a UUID + Bits = 128 + + // Size is the number of bytes in a UUID + Size = Bits / 8 + + format = "%08x%04x%04x%04x%012x" +) + +var ( + // Loggerf can be used to override the default logging destination. Such + // log messages in this library should be logged at warning or higher. + Loggerf = func(format string, args ...interface{}) {} +) + +// UUID represents a UUID value. UUIDs can be compared and set to other values +// and accessed by byte. +type UUID [Size]byte + +// GenerateUUID creates a new, version 4 uuid. +func GenerateUUID() (u UUID) { + const ( + // ensures we backoff for less than 450ms total. Use the following to + // select new value, in units of 10ms: + // n*(n+1)/2 = d -> n^2 + n - 2d -> n = (sqrt(8d + 1) - 1)/2 + maxretries = 9 + backoff = time.Millisecond * 10 + ) + + var ( + totalBackoff time.Duration + count int + retries int + ) + + for { + // This should never block but the read may fail. Because of this, + // we just try to read the random number generator until we get + // something. This is a very rare condition but may happen. + b := time.Duration(retries) * backoff + time.Sleep(b) + totalBackoff += b + + n, err := io.ReadFull(rand.Reader, u[count:]) + if err != nil { + if retryOnError(err) && retries < maxretries { + count += n + retries++ + Loggerf("error generating version 4 uuid, retrying: %v", err) + continue + } + + // Any other errors represent a system problem. What did someone + // do to /dev/urandom? + panic(fmt.Errorf("error reading random number generator, retried for %v: %v", totalBackoff.String(), err)) + } + + break + } + + u[6] = (u[6] & 0x0f) | 0x40 // set version byte + u[8] = (u[8] & 0x3f) | 0x80 // set high order byte 0b10{8,9,a,b} + + return u +} + +func (u UUID) String() string { + return fmt.Sprintf(format, u[:4], u[4:6], u[6:8], u[8:10], u[10:]) +} + +// retryOnError tries to detect whether or not retrying would be fruitful. +func retryOnError(err error) bool { + switch err := err.(type) { + case *os.PathError: + return retryOnError(err.Err) // unpack the target error + case syscall.Errno: + if err == syscall.EPERM { + // EPERM represents an entropy pool exhaustion, a condition under + // which we backoff and retry. + return true + } + } + + return false +} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go new file mode 100644 index 00000000..d2cb1a4e --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go @@ -0,0 +1,21 @@ +package auth + +import ( + "testing" +) + +const iterations = 1000 + +func TestUUID4Generation(t *testing.T) { + for i := 0; i < iterations; i++ { + u := GenerateUUID() + + if u[6]&0xf0 != 0x40 { + t.Fatalf("version byte not correctly set: %v, %08b %08b", u, u[6], u[6]&0xf0) + } + + if u[8]&0xc0 != 0x80 { + t.Fatalf("top order 8th byte not correctly set: %v, %b", u, u[8]) + } + } +} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go new file mode 100644 index 00000000..211bd04f --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go @@ -0,0 +1,80 @@ +package auth + +import ( + "crypto/md5" + "fmt" + "net/url" + "time" +) + +// An URLSigner provides URL signing utilities to sign URLs for Aliyun CDN +// resources. +// authentication document: https://help.aliyun.com/document_detail/85117.html +type URLSigner struct { + authType string + privKey string +} + +// NewURLSigner returns a new signer object. +func NewURLSigner(authType string, privKey string) *URLSigner { + return &URLSigner{ + authType: authType, + privKey: privKey, + } +} + +// Sign returns a signed aliyuncdn url based on authentication type +func (s URLSigner) Sign(uri string, expires time.Time) (string, error) { + r, err := url.Parse(uri) + if err != nil { + return "", fmt.Errorf("unable to parse url: %s", uri) + } + + switch s.authType { + case "a": + return aTypeSign(r, s.privKey, expires), nil + case "b": + return bTypeSign(r, s.privKey, expires), nil + case "c": + return cTypeSign(r, s.privKey, expires), nil + default: + return "", fmt.Errorf("invalid authentication type") + } +} + +// sign by A type authentication method. +// authentication document: https://help.aliyun.com/document_detail/85113.html +func aTypeSign(r *url.URL, privateKey string, expires time.Time) string { + //rand is a random uuid without "-" + rand := GenerateUUID().String() + // not use, "0" by default + uid := "0" + secret := fmt.Sprintf("%s-%d-%s-%s-%s", r.Path, expires.Unix(), rand, uid, privateKey) + hashValue := md5.Sum([]byte(secret)) + authKey := fmt.Sprintf("%d-%s-%s-%x", expires.Unix(), rand, uid, hashValue) + if r.RawQuery == "" { + return fmt.Sprintf("%s?auth_key=%s", r.String(), authKey) + } + return fmt.Sprintf("%s&auth_key=%s", r.String(), authKey) + +} + +// sign by B type authentication method. +// authentication document: https://help.aliyun.com/document_detail/85114.html +func bTypeSign(r *url.URL, privateKey string, expires time.Time) string { + formatExp := expires.Format("200601021504") + secret := privateKey + formatExp + r.Path + hashValue := md5.Sum([]byte(secret)) + signURL := fmt.Sprintf("%s://%s/%s/%x%s?%s", r.Scheme, r.Host, formatExp, hashValue, r.Path, r.RawQuery) + return signURL +} + +// sign by C type authentication method. +// authentication document: https://help.aliyun.com/document_detail/85115.html +func cTypeSign(r *url.URL, privateKey string, expires time.Time) string { + hexExp := fmt.Sprintf("%x", expires.Unix()) + secret := privateKey + r.Path + hexExp + hashValue := md5.Sum([]byte(secret)) + signURL := fmt.Sprintf("%s://%s/%x/%s%s?%s", r.Scheme, r.Host, hashValue, hexExp, r.Path, r.RawQuery) + return signURL +} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go new file mode 100644 index 00000000..46291d48 --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go @@ -0,0 +1,53 @@ +package auth + +import ( + "crypto/md5" + "fmt" + "net/url" + "reflect" + "testing" + "time" +) + +var ( + testSignTime = time.Unix(1541064730, 0) + testPrivKey = "12345678" +) + +func assertEqual(t *testing.T, name string, x, y interface{}) { + if !reflect.DeepEqual(x, y) { + t.Errorf("%s: Not equal! Expected='%v', Actual='%v'\n", name, x, y) + t.FailNow() + } +} + +func TestAtypeAuth(t *testing.T) { + r, _ := url.Parse("https://example.com/a?foo=bar") + url := aTypeTest(r, testPrivKey, testSignTime) + assertEqual(t, "testTypeA", "https://example.com/a?foo=bar&auth_key=1541064730-0-0-f9dd5ed1e274ab4b1d5f5745344bf28b", url) +} + +func TestBtypeAuth(t *testing.T) { + signer := NewURLSigner("b", testPrivKey) + url, _ := signer.Sign("https://example.com/a?foo=bar", testSignTime) + assertEqual(t, "testTypeB", "https://example.com/201811011732/3a19d83a89ccb00a73212420791b0123/a?foo=bar", url) +} + +func TestCtypeAuth(t *testing.T) { + signer := NewURLSigner("c", testPrivKey) + url, _ := signer.Sign("https://example.com/a?foo=bar", testSignTime) + assertEqual(t, "testTypeC", "https://example.com/7d6b308ce87beb16d9dba32d741220f6/5bdac81a/a?foo=bar", url) +} + +func aTypeTest(r *url.URL, privateKey string, expires time.Time) string { + //rand equals "0" in test case + rand := "0" + uid := "0" + secret := fmt.Sprintf("%s-%d-%s-%s-%s", r.Path, expires.Unix(), rand, uid, privateKey) + hashValue := md5.Sum([]byte(secret)) + authKey := fmt.Sprintf("%d-%s-%s-%x", expires.Unix(), rand, uid, hashValue) + if r.RawQuery == "" { + return fmt.Sprintf("%s?auth_key=%s", r.String(), authKey) + } + return fmt.Sprintf("%s&auth_key=%s", r.String(), authKey) +} From bbc9885aa216988e7a3332f9da197933de2849e3 Mon Sep 17 00:00:00 2001 From: Shawnpku Date: Mon, 11 Feb 2019 19:20:47 +0800 Subject: [PATCH 2/7] fix func name Signed-off-by: Shawnpku --- .../driver/middleware/alicdn/middleware.go | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/registry/storage/driver/middleware/alicdn/middleware.go b/registry/storage/driver/middleware/alicdn/middleware.go index 828d88c2..1609a9f6 100755 --- a/registry/storage/driver/middleware/alicdn/middleware.go +++ b/registry/storage/driver/middleware/alicdn/middleware.go @@ -13,23 +13,23 @@ import ( "github.com/denverdino/aliyungo/cdn/auth" ) -// aliCdnStorageMiddleware provides a simple implementation of layerHandler that +// aliCDNStorageMiddleware provides a simple implementation of layerHandler that // constructs temporary signed AliCDN URLs from the storagedriver layer URL, // then issues HTTP Temporary Redirects to this AliCDN content URL. -type aliCdnStorageMiddleware struct { +type aliCDNStorageMiddleware struct { storagedriver.StorageDriver baseURL string urlSigner *auth.URLSigner duration time.Duration } -var _ storagedriver.StorageDriver = &aliCdnStorageMiddleware{} +var _ storagedriver.StorageDriver = &aliCDNStorageMiddleware{} -// newAliCdnLayerHandler constructs and returns a new AliCDN -// LayerHandler implementation. +// newAliCDNStorageMiddleware constructs and returns a new AliCDN +// layerHandler implementation. // Required options: baseurl, authtype, privatekey // Optional options: duration -func newAliCdnStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) { +func newAliCDNStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) { // parse baseurl base, ok := options["baseurl"] if !ok { @@ -87,7 +87,7 @@ func newAliCdnStorageMiddleware(storageDriver storagedriver.StorageDriver, optio } } - return &aliCdnStorageMiddleware{ + return &aliCDNStorageMiddleware{ StorageDriver: storageDriver, baseURL: baseURL, urlSigner: urlSigner, @@ -96,11 +96,10 @@ func newAliCdnStorageMiddleware(storageDriver storagedriver.StorageDriver, optio } // URLFor attempts to find a url which may be used to retrieve the file at the given path. -// Returns an error if the file cannot be found. -func (ac *aliCdnStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { +func (ac *aliCDNStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { if ac.StorageDriver.Name() != "oss" { - context.GetLogger(ctx).Warn("the AliCdn middleware does not support this backend storage driver") + context.GetLogger(ctx).Warn("the AliCDN middleware does not support this backend storage driver") return ac.StorageDriver.URLFor(ctx, path, options) } acURL, err := ac.urlSigner.Sign(ac.baseURL+path, time.Now().Add(ac.duration)) @@ -112,5 +111,5 @@ func (ac *aliCdnStorageMiddleware) URLFor(ctx context.Context, path string, opti // init registers the alicdn layerHandler backend. func init() { - storagemiddleware.Register("alicdn", storagemiddleware.InitFunc(newAliCdnStorageMiddleware)) + storagemiddleware.Register("alicdn", storagemiddleware.InitFunc(newAliCDNStorageMiddleware)) } From 6e10631d9c8b28038468c9759df2cde15fc4b674 Mon Sep 17 00:00:00 2001 From: Shawnpku Date: Mon, 4 Mar 2019 14:53:48 +0800 Subject: [PATCH 3/7] fix default cdn auth duration Signed-off-by: Shawnpku --- .../driver/middleware/alicdn/middleware.go | 2 +- .../aliyungo/cdn/auth/random_uuid.go | 97 ------------------- .../aliyungo/cdn/auth/random_uuid_test.go | 21 ---- .../denverdino/aliyungo/cdn/auth/sign_url.go | 80 --------------- .../aliyungo/cdn/auth/sign_url_test.go | 53 ---------- 5 files changed, 1 insertion(+), 252 deletions(-) delete mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go delete mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go delete mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go delete mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go diff --git a/registry/storage/driver/middleware/alicdn/middleware.go b/registry/storage/driver/middleware/alicdn/middleware.go index 1609a9f6..7c5fc732 100755 --- a/registry/storage/driver/middleware/alicdn/middleware.go +++ b/registry/storage/driver/middleware/alicdn/middleware.go @@ -72,7 +72,7 @@ func newAliCDNStorageMiddleware(storageDriver storagedriver.StorageDriver, optio urlSigner := auth.NewURLSigner(authType, privateKey) // parse duration - duration := 60 * time.Minute + duration := 20 * time.Minute d, ok := options["duration"] if ok { switch d := d.(type) { diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go deleted file mode 100644 index 169a2b6c..00000000 --- a/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go +++ /dev/null @@ -1,97 +0,0 @@ -package auth - -import ( - "crypto/rand" - "fmt" - "io" - "os" - "syscall" - "time" -) - -const ( - // Bits is the number of bits in a UUID - Bits = 128 - - // Size is the number of bytes in a UUID - Size = Bits / 8 - - format = "%08x%04x%04x%04x%012x" -) - -var ( - // Loggerf can be used to override the default logging destination. Such - // log messages in this library should be logged at warning or higher. - Loggerf = func(format string, args ...interface{}) {} -) - -// UUID represents a UUID value. UUIDs can be compared and set to other values -// and accessed by byte. -type UUID [Size]byte - -// GenerateUUID creates a new, version 4 uuid. -func GenerateUUID() (u UUID) { - const ( - // ensures we backoff for less than 450ms total. Use the following to - // select new value, in units of 10ms: - // n*(n+1)/2 = d -> n^2 + n - 2d -> n = (sqrt(8d + 1) - 1)/2 - maxretries = 9 - backoff = time.Millisecond * 10 - ) - - var ( - totalBackoff time.Duration - count int - retries int - ) - - for { - // This should never block but the read may fail. Because of this, - // we just try to read the random number generator until we get - // something. This is a very rare condition but may happen. - b := time.Duration(retries) * backoff - time.Sleep(b) - totalBackoff += b - - n, err := io.ReadFull(rand.Reader, u[count:]) - if err != nil { - if retryOnError(err) && retries < maxretries { - count += n - retries++ - Loggerf("error generating version 4 uuid, retrying: %v", err) - continue - } - - // Any other errors represent a system problem. What did someone - // do to /dev/urandom? - panic(fmt.Errorf("error reading random number generator, retried for %v: %v", totalBackoff.String(), err)) - } - - break - } - - u[6] = (u[6] & 0x0f) | 0x40 // set version byte - u[8] = (u[8] & 0x3f) | 0x80 // set high order byte 0b10{8,9,a,b} - - return u -} - -func (u UUID) String() string { - return fmt.Sprintf(format, u[:4], u[4:6], u[6:8], u[8:10], u[10:]) -} - -// retryOnError tries to detect whether or not retrying would be fruitful. -func retryOnError(err error) bool { - switch err := err.(type) { - case *os.PathError: - return retryOnError(err.Err) // unpack the target error - case syscall.Errno: - if err == syscall.EPERM { - // EPERM represents an entropy pool exhaustion, a condition under - // which we backoff and retry. - return true - } - } - - return false -} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go deleted file mode 100644 index d2cb1a4e..00000000 --- a/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package auth - -import ( - "testing" -) - -const iterations = 1000 - -func TestUUID4Generation(t *testing.T) { - for i := 0; i < iterations; i++ { - u := GenerateUUID() - - if u[6]&0xf0 != 0x40 { - t.Fatalf("version byte not correctly set: %v, %08b %08b", u, u[6], u[6]&0xf0) - } - - if u[8]&0xc0 != 0x80 { - t.Fatalf("top order 8th byte not correctly set: %v, %b", u, u[8]) - } - } -} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go deleted file mode 100644 index 211bd04f..00000000 --- a/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go +++ /dev/null @@ -1,80 +0,0 @@ -package auth - -import ( - "crypto/md5" - "fmt" - "net/url" - "time" -) - -// An URLSigner provides URL signing utilities to sign URLs for Aliyun CDN -// resources. -// authentication document: https://help.aliyun.com/document_detail/85117.html -type URLSigner struct { - authType string - privKey string -} - -// NewURLSigner returns a new signer object. -func NewURLSigner(authType string, privKey string) *URLSigner { - return &URLSigner{ - authType: authType, - privKey: privKey, - } -} - -// Sign returns a signed aliyuncdn url based on authentication type -func (s URLSigner) Sign(uri string, expires time.Time) (string, error) { - r, err := url.Parse(uri) - if err != nil { - return "", fmt.Errorf("unable to parse url: %s", uri) - } - - switch s.authType { - case "a": - return aTypeSign(r, s.privKey, expires), nil - case "b": - return bTypeSign(r, s.privKey, expires), nil - case "c": - return cTypeSign(r, s.privKey, expires), nil - default: - return "", fmt.Errorf("invalid authentication type") - } -} - -// sign by A type authentication method. -// authentication document: https://help.aliyun.com/document_detail/85113.html -func aTypeSign(r *url.URL, privateKey string, expires time.Time) string { - //rand is a random uuid without "-" - rand := GenerateUUID().String() - // not use, "0" by default - uid := "0" - secret := fmt.Sprintf("%s-%d-%s-%s-%s", r.Path, expires.Unix(), rand, uid, privateKey) - hashValue := md5.Sum([]byte(secret)) - authKey := fmt.Sprintf("%d-%s-%s-%x", expires.Unix(), rand, uid, hashValue) - if r.RawQuery == "" { - return fmt.Sprintf("%s?auth_key=%s", r.String(), authKey) - } - return fmt.Sprintf("%s&auth_key=%s", r.String(), authKey) - -} - -// sign by B type authentication method. -// authentication document: https://help.aliyun.com/document_detail/85114.html -func bTypeSign(r *url.URL, privateKey string, expires time.Time) string { - formatExp := expires.Format("200601021504") - secret := privateKey + formatExp + r.Path - hashValue := md5.Sum([]byte(secret)) - signURL := fmt.Sprintf("%s://%s/%s/%x%s?%s", r.Scheme, r.Host, formatExp, hashValue, r.Path, r.RawQuery) - return signURL -} - -// sign by C type authentication method. -// authentication document: https://help.aliyun.com/document_detail/85115.html -func cTypeSign(r *url.URL, privateKey string, expires time.Time) string { - hexExp := fmt.Sprintf("%x", expires.Unix()) - secret := privateKey + r.Path + hexExp - hashValue := md5.Sum([]byte(secret)) - signURL := fmt.Sprintf("%s://%s/%x/%s%s?%s", r.Scheme, r.Host, hashValue, hexExp, r.Path, r.RawQuery) - return signURL -} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go deleted file mode 100644 index 46291d48..00000000 --- a/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package auth - -import ( - "crypto/md5" - "fmt" - "net/url" - "reflect" - "testing" - "time" -) - -var ( - testSignTime = time.Unix(1541064730, 0) - testPrivKey = "12345678" -) - -func assertEqual(t *testing.T, name string, x, y interface{}) { - if !reflect.DeepEqual(x, y) { - t.Errorf("%s: Not equal! Expected='%v', Actual='%v'\n", name, x, y) - t.FailNow() - } -} - -func TestAtypeAuth(t *testing.T) { - r, _ := url.Parse("https://example.com/a?foo=bar") - url := aTypeTest(r, testPrivKey, testSignTime) - assertEqual(t, "testTypeA", "https://example.com/a?foo=bar&auth_key=1541064730-0-0-f9dd5ed1e274ab4b1d5f5745344bf28b", url) -} - -func TestBtypeAuth(t *testing.T) { - signer := NewURLSigner("b", testPrivKey) - url, _ := signer.Sign("https://example.com/a?foo=bar", testSignTime) - assertEqual(t, "testTypeB", "https://example.com/201811011732/3a19d83a89ccb00a73212420791b0123/a?foo=bar", url) -} - -func TestCtypeAuth(t *testing.T) { - signer := NewURLSigner("c", testPrivKey) - url, _ := signer.Sign("https://example.com/a?foo=bar", testSignTime) - assertEqual(t, "testTypeC", "https://example.com/7d6b308ce87beb16d9dba32d741220f6/5bdac81a/a?foo=bar", url) -} - -func aTypeTest(r *url.URL, privateKey string, expires time.Time) string { - //rand equals "0" in test case - rand := "0" - uid := "0" - secret := fmt.Sprintf("%s-%d-%s-%s-%s", r.Path, expires.Unix(), rand, uid, privateKey) - hashValue := md5.Sum([]byte(secret)) - authKey := fmt.Sprintf("%d-%s-%s-%x", expires.Unix(), rand, uid, hashValue) - if r.RawQuery == "" { - return fmt.Sprintf("%s?auth_key=%s", r.String(), authKey) - } - return fmt.Sprintf("%s&auth_key=%s", r.String(), authKey) -} From ae91d1f429a2006ce21b4419a3f15b52592ee4da Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Mon, 4 Mar 2019 17:17:57 +0800 Subject: [PATCH 4/7] fix ci issue Signed-off-by: Shawn Chen --- vendor.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor.conf b/vendor.conf index 12f71672..235f97fb 100644 --- a/vendor.conf +++ b/vendor.conf @@ -7,7 +7,7 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274 github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702 github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782 -github.com/denverdino/aliyungo 6df11717a253d9c7d4141f9af4deaa7c580cd531 +github.com/denverdino/aliyungo a747050bb1baf06cdd65de7cddc281a2b1c2fde5 github.com/dgrijalva/jwt-go a601269ab70c205d26370c16f7c81e9017c14e04 github.com/docker/go-metrics 399ea8c73916000c64c2c76e8da00ca82f8387ab github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21 From 3390f32aecd35b9c17eb110c9327f175048a5580 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Mon, 4 Mar 2019 17:48:32 +0800 Subject: [PATCH 5/7] fix Context issue Signed-off-by: Shawn Chen --- .../driver/middleware/alicdn/middleware.go | 5 +- .../aliyungo/cdn/auth/random_uuid.go | 97 +++++++++++++++++++ .../denverdino/aliyungo/cdn/auth/sign_url.go | 80 +++++++++++++++ .../denverdino/aliyungo/oss/client.go | 14 ++- .../denverdino/aliyungo/oss/signature.go | 48 ++++++--- .../denverdino/aliyungo/util/util.go | 39 ++++++-- 6 files changed, 259 insertions(+), 24 deletions(-) create mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go create mode 100644 vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go diff --git a/registry/storage/driver/middleware/alicdn/middleware.go b/registry/storage/driver/middleware/alicdn/middleware.go index 7c5fc732..2e281712 100755 --- a/registry/storage/driver/middleware/alicdn/middleware.go +++ b/registry/storage/driver/middleware/alicdn/middleware.go @@ -1,12 +1,13 @@ package middleware import ( + "context" "fmt" "net/url" "strings" "time" - "github.com/docker/distribution/context" + dcontext "github.com/docker/distribution/context" storagedriver "github.com/docker/distribution/registry/storage/driver" storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" @@ -99,7 +100,7 @@ func newAliCDNStorageMiddleware(storageDriver storagedriver.StorageDriver, optio func (ac *aliCDNStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { if ac.StorageDriver.Name() != "oss" { - context.GetLogger(ctx).Warn("the AliCDN middleware does not support this backend storage driver") + dcontext.GetLogger(ctx).Warn("the AliCDN middleware does not support this backend storage driver") return ac.StorageDriver.URLFor(ctx, path, options) } acURL, err := ac.urlSigner.Sign(ac.baseURL+path, time.Now().Add(ac.duration)) diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go new file mode 100644 index 00000000..169a2b6c --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/cdn/auth/random_uuid.go @@ -0,0 +1,97 @@ +package auth + +import ( + "crypto/rand" + "fmt" + "io" + "os" + "syscall" + "time" +) + +const ( + // Bits is the number of bits in a UUID + Bits = 128 + + // Size is the number of bytes in a UUID + Size = Bits / 8 + + format = "%08x%04x%04x%04x%012x" +) + +var ( + // Loggerf can be used to override the default logging destination. Such + // log messages in this library should be logged at warning or higher. + Loggerf = func(format string, args ...interface{}) {} +) + +// UUID represents a UUID value. UUIDs can be compared and set to other values +// and accessed by byte. +type UUID [Size]byte + +// GenerateUUID creates a new, version 4 uuid. +func GenerateUUID() (u UUID) { + const ( + // ensures we backoff for less than 450ms total. Use the following to + // select new value, in units of 10ms: + // n*(n+1)/2 = d -> n^2 + n - 2d -> n = (sqrt(8d + 1) - 1)/2 + maxretries = 9 + backoff = time.Millisecond * 10 + ) + + var ( + totalBackoff time.Duration + count int + retries int + ) + + for { + // This should never block but the read may fail. Because of this, + // we just try to read the random number generator until we get + // something. This is a very rare condition but may happen. + b := time.Duration(retries) * backoff + time.Sleep(b) + totalBackoff += b + + n, err := io.ReadFull(rand.Reader, u[count:]) + if err != nil { + if retryOnError(err) && retries < maxretries { + count += n + retries++ + Loggerf("error generating version 4 uuid, retrying: %v", err) + continue + } + + // Any other errors represent a system problem. What did someone + // do to /dev/urandom? + panic(fmt.Errorf("error reading random number generator, retried for %v: %v", totalBackoff.String(), err)) + } + + break + } + + u[6] = (u[6] & 0x0f) | 0x40 // set version byte + u[8] = (u[8] & 0x3f) | 0x80 // set high order byte 0b10{8,9,a,b} + + return u +} + +func (u UUID) String() string { + return fmt.Sprintf(format, u[:4], u[4:6], u[6:8], u[8:10], u[10:]) +} + +// retryOnError tries to detect whether or not retrying would be fruitful. +func retryOnError(err error) bool { + switch err := err.(type) { + case *os.PathError: + return retryOnError(err.Err) // unpack the target error + case syscall.Errno: + if err == syscall.EPERM { + // EPERM represents an entropy pool exhaustion, a condition under + // which we backoff and retry. + return true + } + } + + return false +} diff --git a/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go new file mode 100644 index 00000000..211bd04f --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/cdn/auth/sign_url.go @@ -0,0 +1,80 @@ +package auth + +import ( + "crypto/md5" + "fmt" + "net/url" + "time" +) + +// An URLSigner provides URL signing utilities to sign URLs for Aliyun CDN +// resources. +// authentication document: https://help.aliyun.com/document_detail/85117.html +type URLSigner struct { + authType string + privKey string +} + +// NewURLSigner returns a new signer object. +func NewURLSigner(authType string, privKey string) *URLSigner { + return &URLSigner{ + authType: authType, + privKey: privKey, + } +} + +// Sign returns a signed aliyuncdn url based on authentication type +func (s URLSigner) Sign(uri string, expires time.Time) (string, error) { + r, err := url.Parse(uri) + if err != nil { + return "", fmt.Errorf("unable to parse url: %s", uri) + } + + switch s.authType { + case "a": + return aTypeSign(r, s.privKey, expires), nil + case "b": + return bTypeSign(r, s.privKey, expires), nil + case "c": + return cTypeSign(r, s.privKey, expires), nil + default: + return "", fmt.Errorf("invalid authentication type") + } +} + +// sign by A type authentication method. +// authentication document: https://help.aliyun.com/document_detail/85113.html +func aTypeSign(r *url.URL, privateKey string, expires time.Time) string { + //rand is a random uuid without "-" + rand := GenerateUUID().String() + // not use, "0" by default + uid := "0" + secret := fmt.Sprintf("%s-%d-%s-%s-%s", r.Path, expires.Unix(), rand, uid, privateKey) + hashValue := md5.Sum([]byte(secret)) + authKey := fmt.Sprintf("%d-%s-%s-%x", expires.Unix(), rand, uid, hashValue) + if r.RawQuery == "" { + return fmt.Sprintf("%s?auth_key=%s", r.String(), authKey) + } + return fmt.Sprintf("%s&auth_key=%s", r.String(), authKey) + +} + +// sign by B type authentication method. +// authentication document: https://help.aliyun.com/document_detail/85114.html +func bTypeSign(r *url.URL, privateKey string, expires time.Time) string { + formatExp := expires.Format("200601021504") + secret := privateKey + formatExp + r.Path + hashValue := md5.Sum([]byte(secret)) + signURL := fmt.Sprintf("%s://%s/%s/%x%s?%s", r.Scheme, r.Host, formatExp, hashValue, r.Path, r.RawQuery) + return signURL +} + +// sign by C type authentication method. +// authentication document: https://help.aliyun.com/document_detail/85115.html +func cTypeSign(r *url.URL, privateKey string, expires time.Time) string { + hexExp := fmt.Sprintf("%x", expires.Unix()) + secret := privateKey + r.Path + hexExp + hashValue := md5.Sum([]byte(secret)) + signURL := fmt.Sprintf("%s://%s/%x/%s%s?%s", r.Scheme, r.Host, hashValue, hexExp, r.Path, r.RawQuery) + return signURL +} diff --git a/vendor/github.com/denverdino/aliyungo/oss/client.go b/vendor/github.com/denverdino/aliyungo/oss/client.go index 7b5a55b8..c0919194 100644 --- a/vendor/github.com/denverdino/aliyungo/oss/client.go +++ b/vendor/github.com/denverdino/aliyungo/oss/client.go @@ -851,6 +851,17 @@ func (b *Bucket) SignedURLWithArgs(path string, expires time.Time, params url.Va return b.SignedURLWithMethod("GET", path, expires, params, headers) } +func (b *Bucket) SignedURLWithMethodForAssumeRole(method, path string, expires time.Time, params url.Values, headers http.Header) string { + var uv = url.Values{} + if params != nil { + uv = params + } + if len(b.Client.SecurityToken) != 0 { + uv.Set("security-token", b.Client.SecurityToken) + } + return b.SignedURLWithMethod(method, path, expires, params, headers) +} + // SignedURLWithMethod returns a signed URL that allows anyone holding the URL // to either retrieve the object at path or make a HEAD request against it. The signature is valid until expires. func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, params url.Values, headers http.Header) string { @@ -1039,7 +1050,8 @@ func partiallyEscapedPath(path string) string { func (client *Client) prepare(req *request) error { // Copy so they can be mutated without affecting on retries. headers := copyHeader(req.headers) - if len(client.SecurityToken) != 0 { + // security-token should be in either Params or Header, cannot be in both + if len(req.params.Get("security-token")) == 0 && len(client.SecurityToken) != 0 { headers.Set("x-oss-security-token", client.SecurityToken) } diff --git a/vendor/github.com/denverdino/aliyungo/oss/signature.go b/vendor/github.com/denverdino/aliyungo/oss/signature.go index 12677175..4a0f4d97 100644 --- a/vendor/github.com/denverdino/aliyungo/oss/signature.go +++ b/vendor/github.com/denverdino/aliyungo/oss/signature.go @@ -13,26 +13,44 @@ const HeaderOSSPrefix = "x-oss-" var ossParamsToSign = map[string]bool{ "acl": true, + "append": true, + "bucketInfo": true, + "cname": true, + "comp": true, + "cors": true, "delete": true, + "endTime": true, + "img": true, + "lifecycle": true, + "live": true, "location": true, "logging": true, - "notification": true, + "objectMeta": true, "partNumber": true, - "policy": true, - "requestPayment": true, - "torrent": true, - "uploadId": true, - "uploads": true, - "versionId": true, - "versioning": true, - "versions": true, - "response-content-type": true, - "response-content-language": true, - "response-expires": true, + "position": true, + "qos": true, + "referer": true, + "replication": true, + "replicationLocation": true, + "replicationProgress": true, "response-cache-control": true, "response-content-disposition": true, "response-content-encoding": true, - "bucketInfo": true, + "response-content-language": true, + "response-content-type": true, + "response-expires": true, + "security-token": true, + "startTime": true, + "status": true, + "style": true, + "styleName": true, + "symlink": true, + "tagging": true, + "uploadId": true, + "uploads": true, + "vod": true, + "website": true, + "x-oss-process": true, } func (client *Client) signRequest(request *request) { @@ -62,7 +80,7 @@ func (client *Client) signRequest(request *request) { } if len(params) > 0 { - resource = resource + "?" + util.Encode(params) + resource = resource + "?" + util.EncodeWithoutEscape(params) } canonicalizedResource := resource @@ -74,7 +92,7 @@ func (client *Client) signRequest(request *request) { //log.Println("stringToSign: ", stringToSign) signature := util.CreateSignature(stringToSign, client.AccessKeySecret) - if query.Get("OSSAccessKeyId") != "" { + if urlSignature { query.Set("Signature", signature) } else { headers.Set("Authorization", "OSS "+client.AccessKeyId+":"+signature) diff --git a/vendor/github.com/denverdino/aliyungo/util/util.go b/vendor/github.com/denverdino/aliyungo/util/util.go index ff49f2d8..15a990da 100644 --- a/vendor/github.com/denverdino/aliyungo/util/util.go +++ b/vendor/github.com/denverdino/aliyungo/util/util.go @@ -4,13 +4,13 @@ import ( "bytes" srand "crypto/rand" "encoding/binary" + "encoding/json" + "fmt" "math/rand" "net/http" "net/url" "sort" "time" - "fmt" - "encoding/json" ) const dictionary = "_0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" @@ -66,6 +66,34 @@ func Encode(v url.Values) string { return buf.String() } +// Like Encode, but key and value are not escaped +func EncodeWithoutEscape(v url.Values) string { + if v == nil { + return "" + } + var buf bytes.Buffer + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + vs := v[k] + prefix := k + for _, v := range vs { + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(prefix) + if v != "" { + buf.WriteString("=") + buf.WriteString(v) + } + } + } + return buf.String() +} + func GetGMTime() string { return time.Now().UTC().Format(http.TimeFormat) } @@ -148,11 +176,10 @@ func GenerateRandomECSPassword() string { } - func PrettyJson(object interface{}) string { - b,err := json.MarshalIndent(object,"", " ") + b, err := json.MarshalIndent(object, "", " ") if err != nil { - fmt.Printf("ERROR: PrettyJson, %v\n %s\n",err,b) + fmt.Printf("ERROR: PrettyJson, %v\n %s\n", err, b) } return string(b) -} \ No newline at end of file +} From fd77cf43a6920459c59bb7d5df4e2bdd0e2159f9 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Mon, 18 Mar 2019 11:25:35 +0800 Subject: [PATCH 6/7] change package name & format document Signed-off-by: Shawn Chen --- docs/configuration.md | 12 ++++++------ .../storage/driver/middleware/alicdn/middleware.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 10245bf1..e1a708d0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -719,12 +719,12 @@ Then value of ipfilteredby: `alicdn` storage middleware allows the registry to serve layers via a content delivery network provided by Alibaba Cloud. Alicdn requires the OSS storage driver. -| Parameter | Required | Description | -|-----------|----------|-------------------------------------------------------| -| `baseurl` | yes | The `SCHEME://HOST` at which Alicdn is served. | -| `authtype` | yes | The URL authentication type for Alicdn, which should be `a`, `b` or `c`. | -| `privatekey` | yes | The URL authentication key for Alicdn. | -| `duration` | no | An integer and unit for the duration of the Alicdn session. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`.| +| Parameter | Required | Description | +|--------------|----------|-------------------------------------------------------------------------| +| `baseurl` | yes | The `SCHEME://HOST` at which Alicdn is served. | +| `authtype` | yes | The URL authentication type for Alicdn, which should be `a`, `b` or `c`. See the [Authentication configuration](https://www.alibabacloud.com/help/doc-detail/85117.htm).| +| `privatekey` | yes | The URL authentication key for Alicdn. | +| `duration` | no | An integer and unit for the duration of the Alicdn session. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`.| ### `redirect` diff --git a/registry/storage/driver/middleware/alicdn/middleware.go b/registry/storage/driver/middleware/alicdn/middleware.go index 2e281712..51993161 100755 --- a/registry/storage/driver/middleware/alicdn/middleware.go +++ b/registry/storage/driver/middleware/alicdn/middleware.go @@ -1,4 +1,4 @@ -package middleware +package alicdn import ( "context" @@ -27,7 +27,7 @@ type aliCDNStorageMiddleware struct { var _ storagedriver.StorageDriver = &aliCDNStorageMiddleware{} // newAliCDNStorageMiddleware constructs and returns a new AliCDN -// layerHandler implementation. +// StorageDriver implementation. // Required options: baseurl, authtype, privatekey // Optional options: duration func newAliCDNStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) { From 51bb5cee5b6ad10b0c7856164e7fce0823eb8374 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Tue, 19 Mar 2019 15:06:02 +0800 Subject: [PATCH 7/7] import alicdn package Signed-off-by: Shawn Chen --- cmd/registry/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/registry/main.go b/cmd/registry/main.go index 5beaa563..06a3cf19 100644 --- a/cmd/registry/main.go +++ b/cmd/registry/main.go @@ -12,6 +12,7 @@ import ( _ "github.com/docker/distribution/registry/storage/driver/filesystem" _ "github.com/docker/distribution/registry/storage/driver/gcs" _ "github.com/docker/distribution/registry/storage/driver/inmemory" + _ "github.com/docker/distribution/registry/storage/driver/middleware/alicdn" _ "github.com/docker/distribution/registry/storage/driver/middleware/cloudfront" _ "github.com/docker/distribution/registry/storage/driver/middleware/redirect" _ "github.com/docker/distribution/registry/storage/driver/oss"