From 4cb31d96154678f83401457eeae0ddec97e6b64e Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Wed, 15 Feb 2017 13:13:12 -0800 Subject: [PATCH 1/4] vendor: update gogo/protobuf dependency Updates to the gogo/protobuf dependency are required to correctly generate time types. We also remove an unused windows dependency. Signed-off-by: Stephen J Day --- api/types/container/container.pb.go | 36 +-- vendor.conf | 4 +- .../gogo/protobuf/gogoproto/gogo.pb.go | 187 ++++++++------ .../gogo/protobuf/gogoproto/gogo.proto | 5 + .../gogo/protobuf/gogoproto/helper.go | 8 + .../gogo/protobuf/plugin/gostring/gostring.go | 57 +++-- .../protobuf/plugin/marshalto/marshalto.go | 2 +- .../gogo/protobuf/plugin/populate/populate.go | 35 ++- .../gogo/protobuf/plugin/size/size.go | 2 +- .../gogo/protobuf/plugin/stringer/stringer.go | 2 +- .../protobuf/plugin/unmarshal/unmarshal.go | 26 +- .../protoc-gen-gogo/descriptor/descriptor.go | 26 ++ .../protoc-gen-gogo/descriptor/helper.go | 33 +++ .../protoc-gen-gogo/generator/generator.go | 236 ++++++++++-------- .../github.com/gogo/protobuf/types/any.pb.go | 21 -- .../gogo/protobuf/types/duration.pb.go | 21 -- .../gogo/protobuf/types/empty.pb.go | 21 -- .../gogo/protobuf/types/field_mask.pb.go | 21 -- .../gogo/protobuf/types/struct.pb.go | 20 -- .../gogo/protobuf/types/timestamp.pb.go | 21 -- .../gogo/protobuf/types/wrappers.pb.go | 21 -- 21 files changed, 402 insertions(+), 403 deletions(-) diff --git a/api/types/container/container.pb.go b/api/types/container/container.pb.go index 652ad2e..de8c232 100644 --- a/api/types/container/container.pb.go +++ b/api/types/container/container.pb.go @@ -984,7 +984,24 @@ func (m *User) Unmarshal(dAtA []byte) error { } } case 3: - if wireType == 2 { + if wireType == 0 { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContainer + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.AdditionalGids = append(m.AdditionalGids, v) + } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { @@ -1025,23 +1042,6 @@ func (m *User) Unmarshal(dAtA []byte) error { } m.AdditionalGids = append(m.AdditionalGids, v) } - } else if wireType == 0 { - var v uint32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowContainer - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (uint32(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.AdditionalGids = append(m.AdditionalGids, v) } else { return fmt.Errorf("proto: wrong wireType = %d for field AdditionalGids", wireType) } diff --git a/vendor.conf b/vendor.conf index 26f0e7e..e077760 100644 --- a/vendor.conf +++ b/vendor.conf @@ -18,8 +18,8 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 github.com/matttproud/golang_protobuf_extensions v1.0.0 # go-units from Docker; latest release as of 12/16/2016 github.com/docker/go-units v0.3.1 -# gogo/protobuf - master as of 12/16/2016 (latest tagged release doesn't have needed change) -github.com/gogo/protobuf 06ec6c31ff1bac6ed4e205a547a3d72934813ef3 +# gogo/protobuf - master as of 2/15/2016 (latest tagged release doesn't have needed change) +github.com/gogo/protobuf d2e1ade2d719b78fe5b061b4c18a9f7111b5bdc8 # golang support for protobufs - master as of 12/16/2016 github.com/golang/protobuf 8ee79997227bf9b34611aee7946ae64735e6fd93 # runc, latest release as of 12/16/2016 diff --git a/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go b/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go index d88ba80..f6a2483 100644 --- a/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go +++ b/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go @@ -64,6 +64,15 @@ var E_EnumCustomname = &proto.ExtensionDesc{ Filename: "gogo.proto", } +var E_Enumdecl = &proto.ExtensionDesc{ + ExtendedType: (*google_protobuf.EnumOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 62024, + Name: "gogoproto.enumdecl", + Tag: "varint,62024,opt,name=enumdecl", + Filename: "gogo.proto", +} + var E_EnumvalueCustomname = &proto.ExtensionDesc{ ExtendedType: (*google_protobuf.EnumValueOptions)(nil), ExtensionType: (*string)(nil), @@ -307,6 +316,24 @@ var E_CompareAll = &proto.ExtensionDesc{ Filename: "gogo.proto", } +var E_TypedeclAll = &proto.ExtensionDesc{ + ExtendedType: (*google_protobuf.FileOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 63030, + Name: "gogoproto.typedecl_all", + Tag: "varint,63030,opt,name=typedecl_all,json=typedeclAll", + Filename: "gogo.proto", +} + +var E_EnumdeclAll = &proto.ExtensionDesc{ + ExtendedType: (*google_protobuf.FileOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 63031, + Name: "gogoproto.enumdecl_all", + Tag: "varint,63031,opt,name=enumdecl_all,json=enumdeclAll", + Filename: "gogo.proto", +} + var E_GoprotoGetters = &proto.ExtensionDesc{ ExtendedType: (*google_protobuf.MessageOptions)(nil), ExtensionType: (*bool)(nil), @@ -505,6 +532,15 @@ var E_Compare = &proto.ExtensionDesc{ Filename: "gogo.proto", } +var E_Typedecl = &proto.ExtensionDesc{ + ExtendedType: (*google_protobuf.MessageOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 64030, + Name: "gogoproto.typedecl", + Tag: "varint,64030,opt,name=typedecl", + Filename: "gogo.proto", +} + var E_Nullable = &proto.ExtensionDesc{ ExtendedType: (*google_protobuf.FieldOptions)(nil), ExtensionType: (*bool)(nil), @@ -609,6 +645,7 @@ func init() { proto.RegisterExtension(E_GoprotoEnumStringer) proto.RegisterExtension(E_EnumStringer) proto.RegisterExtension(E_EnumCustomname) + proto.RegisterExtension(E_Enumdecl) proto.RegisterExtension(E_EnumvalueCustomname) proto.RegisterExtension(E_GoprotoGettersAll) proto.RegisterExtension(E_GoprotoEnumPrefixAll) @@ -636,6 +673,8 @@ func init() { proto.RegisterExtension(E_GogoprotoImport) proto.RegisterExtension(E_ProtosizerAll) proto.RegisterExtension(E_CompareAll) + proto.RegisterExtension(E_TypedeclAll) + proto.RegisterExtension(E_EnumdeclAll) proto.RegisterExtension(E_GoprotoGetters) proto.RegisterExtension(E_GoprotoStringer) proto.RegisterExtension(E_VerboseEqual) @@ -658,6 +697,7 @@ func init() { proto.RegisterExtension(E_GoprotoUnrecognized) proto.RegisterExtension(E_Protosizer) proto.RegisterExtension(E_Compare) + proto.RegisterExtension(E_Typedecl) proto.RegisterExtension(E_Nullable) proto.RegisterExtension(E_Embed) proto.RegisterExtension(E_Customtype) @@ -674,76 +714,79 @@ func init() { func init() { proto.RegisterFile("gogo.proto", fileDescriptorGogo) } var fileDescriptorGogo = []byte{ - // 1129 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x97, 0xc9, 0x6f, 0x1c, 0x45, - 0x14, 0x87, 0x85, 0x70, 0x64, 0xcf, 0xf3, 0x86, 0xc7, 0xc6, 0x84, 0x08, 0x44, 0x72, 0xe3, 0xe4, - 0x9c, 0x22, 0x94, 0xb2, 0x22, 0xcb, 0xb1, 0x9c, 0x51, 0x10, 0x86, 0x91, 0x89, 0x03, 0x88, 0xc3, - 0xa8, 0x67, 0xa6, 0xdc, 0x69, 0xe8, 0xee, 0x6a, 0xba, 0xaa, 0xa3, 0x38, 0x37, 0x14, 0x16, 0x21, - 0xc4, 0x8e, 0x04, 0x09, 0x09, 0xcb, 0x81, 0x7d, 0x0d, 0xcb, 0x9d, 0x0b, 0x70, 0xe6, 0x7f, 0xe0, - 0x02, 0x84, 0xdd, 0x37, 0x5f, 0x50, 0x75, 0xbf, 0xd7, 0x53, 0xdd, 0x1e, 0xa9, 0x6a, 0x6e, 0xe3, - 0x71, 0x7d, 0xdf, 0x54, 0xbf, 0x37, 0xf5, 0x7e, 0x53, 0x00, 0xbe, 0xf0, 0xc5, 0x52, 0x92, 0x0a, - 0x25, 0x9a, 0x0d, 0xfd, 0x3a, 0x7f, 0x79, 0xe8, 0xb0, 0x2f, 0x84, 0x1f, 0xf2, 0xa3, 0xf9, 0x5f, - 0xdd, 0x6c, 0xfb, 0x68, 0x9f, 0xcb, 0x5e, 0x1a, 0x24, 0x4a, 0xa4, 0xc5, 0x62, 0x76, 0x3f, 0xcc, - 0xe3, 0xe2, 0x0e, 0x8f, 0xb3, 0xa8, 0x93, 0xa4, 0x7c, 0x3b, 0xb8, 0xd0, 0xbc, 0x63, 0xa9, 0x20, - 0x97, 0x88, 0x5c, 0x5a, 0x8f, 0xb3, 0xe8, 0x81, 0x44, 0x05, 0x22, 0x96, 0x07, 0xaf, 0xff, 0x72, - 0xf3, 0xe1, 0x9b, 0xee, 0x9e, 0xd8, 0x9c, 0x43, 0x54, 0xff, 0xaf, 0x9d, 0x83, 0x6c, 0x13, 0x6e, - 0xad, 0xf8, 0xa4, 0x4a, 0x83, 0xd8, 0xe7, 0xa9, 0xc5, 0xf8, 0x03, 0x1a, 0xe7, 0x0d, 0xe3, 0x83, - 0x88, 0xb2, 0x35, 0x98, 0x1e, 0xc5, 0xf5, 0x23, 0xba, 0xa6, 0xb8, 0x29, 0x69, 0xc1, 0x6c, 0x2e, - 0xe9, 0x65, 0x52, 0x89, 0x28, 0xf6, 0x22, 0x6e, 0xd1, 0xfc, 0x94, 0x6b, 0x1a, 0x9b, 0x33, 0x1a, - 0x5b, 0x2b, 0x29, 0x76, 0x16, 0x16, 0xf4, 0x3b, 0xe7, 0xbd, 0x30, 0xe3, 0xa6, 0xed, 0xc8, 0x50, - 0xdb, 0x59, 0xbd, 0x8c, 0x94, 0x3f, 0x5f, 0x1a, 0xcb, 0x95, 0xf3, 0xa5, 0xc0, 0xf0, 0x1a, 0x9d, - 0xf0, 0xb9, 0x52, 0x3c, 0x95, 0x1d, 0x2f, 0x0c, 0x87, 0x6c, 0xf2, 0x54, 0x10, 0x96, 0xc6, 0xcb, - 0x37, 0xaa, 0x9d, 0x68, 0x15, 0xe4, 0x6a, 0x18, 0xb2, 0x2d, 0xb8, 0x6d, 0x48, 0x67, 0x1d, 0x9c, - 0x57, 0xd0, 0xb9, 0xb0, 0xaf, 0xbb, 0x5a, 0xdb, 0x06, 0x7a, 0xbf, 0xec, 0x87, 0x83, 0xf3, 0x2d, - 0x74, 0x36, 0x91, 0xa5, 0xb6, 0x68, 0xe3, 0xbd, 0x30, 0x77, 0x9e, 0xa7, 0x5d, 0x21, 0x79, 0x87, - 0x3f, 0x91, 0x79, 0xa1, 0x83, 0xee, 0x2a, 0xea, 0x66, 0x11, 0x5c, 0xd7, 0x9c, 0x76, 0x1d, 0x87, - 0x89, 0x6d, 0xaf, 0xc7, 0x1d, 0x14, 0xd7, 0x50, 0x31, 0xae, 0xd7, 0x6b, 0x74, 0x15, 0xa6, 0x7c, - 0x51, 0x3c, 0x92, 0x03, 0xfe, 0x36, 0xe2, 0x93, 0xc4, 0xa0, 0x22, 0x11, 0x49, 0x16, 0x7a, 0xca, - 0x65, 0x07, 0xef, 0x90, 0x82, 0x18, 0x54, 0x8c, 0x50, 0xd6, 0x77, 0x49, 0x21, 0x8d, 0x7a, 0xae, - 0xc0, 0xa4, 0x88, 0xc3, 0x1d, 0x11, 0xbb, 0x6c, 0xe2, 0x3d, 0x34, 0x00, 0x22, 0x5a, 0xb0, 0x0c, - 0x0d, 0xd7, 0x46, 0xbc, 0x8f, 0xf8, 0x04, 0xa7, 0x0e, 0xb4, 0x60, 0x96, 0x86, 0x4c, 0x20, 0x62, - 0x07, 0xc5, 0x07, 0xa8, 0x98, 0x31, 0x30, 0x7c, 0x0c, 0xc5, 0xa5, 0xf2, 0xb9, 0x8b, 0xe4, 0x43, - 0x7a, 0x0c, 0x44, 0xb0, 0x94, 0x5d, 0x1e, 0xf7, 0xce, 0xb9, 0x19, 0x3e, 0xa2, 0x52, 0x12, 0xa3, - 0x15, 0x6b, 0x30, 0x1d, 0x79, 0xa9, 0x3c, 0xe7, 0x85, 0x4e, 0xed, 0xf8, 0x18, 0x1d, 0x53, 0x25, - 0x84, 0x15, 0xc9, 0xe2, 0x51, 0x34, 0x9f, 0x50, 0x45, 0x0c, 0x0c, 0x8f, 0x9e, 0x54, 0x5e, 0x37, - 0xe4, 0x9d, 0x51, 0x6c, 0x9f, 0xd2, 0xd1, 0x2b, 0xd8, 0x0d, 0xd3, 0xb8, 0x0c, 0x0d, 0x19, 0x5c, - 0x74, 0xd2, 0x7c, 0x46, 0x9d, 0xce, 0x01, 0x0d, 0x3f, 0x02, 0xb7, 0x0f, 0x1d, 0xf5, 0x0e, 0xb2, - 0xcf, 0x51, 0xb6, 0x38, 0x64, 0xdc, 0xe3, 0x48, 0x18, 0x55, 0xf9, 0x05, 0x8d, 0x04, 0x5e, 0x73, - 0xb5, 0x61, 0x21, 0x8b, 0xa5, 0xb7, 0x3d, 0x5a, 0xd5, 0xbe, 0xa4, 0xaa, 0x15, 0x6c, 0xa5, 0x6a, - 0x67, 0x60, 0x11, 0x8d, 0xa3, 0xf5, 0xf5, 0x2b, 0x1a, 0xac, 0x05, 0xbd, 0x55, 0xed, 0xee, 0xa3, - 0x70, 0xa8, 0x2c, 0xe7, 0x05, 0xc5, 0x63, 0xa9, 0x99, 0x4e, 0xe4, 0x25, 0x0e, 0xe6, 0xeb, 0x68, - 0xa6, 0x89, 0xbf, 0x5e, 0x0a, 0x36, 0xbc, 0x44, 0xcb, 0x1f, 0x86, 0x83, 0x24, 0xcf, 0xe2, 0x94, - 0xf7, 0x84, 0x1f, 0x07, 0x17, 0x79, 0xdf, 0x41, 0xfd, 0x75, 0xad, 0x55, 0x5b, 0x06, 0xae, 0xcd, - 0xa7, 0xe1, 0x96, 0xf2, 0xf7, 0x46, 0x27, 0x88, 0x12, 0x91, 0x2a, 0x8b, 0xf1, 0x1b, 0xea, 0x54, - 0xc9, 0x9d, 0xce, 0x31, 0xb6, 0x0e, 0x33, 0xf9, 0x9f, 0xae, 0x5f, 0xc9, 0x6f, 0x51, 0x34, 0x3d, - 0xa0, 0x70, 0x70, 0xf4, 0x44, 0x94, 0x78, 0xa9, 0xcb, 0xfc, 0xfb, 0x8e, 0x06, 0x07, 0x22, 0xc5, - 0xb7, 0x6f, 0xb6, 0x96, 0xc4, 0xcd, 0xbb, 0xf6, 0x49, 0x36, 0xb8, 0x94, 0x9e, 0x5f, 0x7a, 0x9e, - 0xdc, 0xc5, 0x33, 0x5b, 0x0d, 0x62, 0x76, 0x9f, 0x2e, 0x4f, 0x35, 0x2e, 0xed, 0xb2, 0x4b, 0xbb, - 0x65, 0x85, 0x2a, 0x69, 0xc9, 0x4e, 0xc1, 0x74, 0x25, 0x2a, 0xed, 0xaa, 0xa7, 0x50, 0x35, 0x65, - 0x26, 0x25, 0x3b, 0x06, 0x63, 0x3a, 0xf6, 0xec, 0xf8, 0xd3, 0x88, 0xe7, 0xcb, 0xd9, 0x09, 0x98, - 0xa0, 0xb8, 0xb3, 0xa3, 0xcf, 0x20, 0x5a, 0x22, 0x1a, 0xa7, 0xa8, 0xb3, 0xe3, 0xcf, 0x12, 0x4e, - 0x88, 0xc6, 0xdd, 0x4b, 0xf8, 0xfd, 0xf3, 0x63, 0x38, 0xae, 0xa8, 0x76, 0xcb, 0x30, 0x8e, 0x19, - 0x67, 0xa7, 0x9f, 0xc3, 0x0f, 0x27, 0x82, 0xdd, 0x03, 0x07, 0x1c, 0x0b, 0xfe, 0x02, 0xa2, 0xc5, - 0x7a, 0xb6, 0x06, 0x93, 0x46, 0xae, 0xd9, 0xf1, 0x17, 0x11, 0x37, 0x29, 0xbd, 0x75, 0xcc, 0x35, - 0xbb, 0xe0, 0x25, 0xda, 0x3a, 0x12, 0xba, 0x6c, 0x14, 0x69, 0x76, 0xfa, 0x65, 0xaa, 0x3a, 0x21, - 0x6c, 0x05, 0x1a, 0xe5, 0x98, 0xb2, 0xf3, 0xaf, 0x20, 0x3f, 0x60, 0x74, 0x05, 0x8c, 0x31, 0x69, - 0x57, 0xbc, 0x4a, 0x15, 0x30, 0x28, 0x7d, 0x8c, 0xea, 0xd1, 0x67, 0x37, 0xbd, 0x46, 0xc7, 0xa8, - 0x96, 0x7c, 0xba, 0x9b, 0xf9, 0xb4, 0xb0, 0x2b, 0x5e, 0xa7, 0x6e, 0xe6, 0xeb, 0xf5, 0x36, 0xea, - 0x59, 0x62, 0x77, 0xbc, 0x41, 0xdb, 0xa8, 0x45, 0x09, 0x6b, 0x43, 0x73, 0x7f, 0x8e, 0xd8, 0x7d, - 0x6f, 0xa2, 0x6f, 0x6e, 0x5f, 0x8c, 0xb0, 0x87, 0x60, 0x71, 0x78, 0x86, 0xd8, 0xad, 0x97, 0x77, - 0x6b, 0xbf, 0xfa, 0xcd, 0x08, 0x61, 0x67, 0x06, 0xbf, 0xfa, 0xcd, 0xfc, 0xb0, 0x6b, 0xaf, 0xec, - 0x56, 0x2f, 0x76, 0x66, 0x7c, 0xb0, 0x55, 0x80, 0xc1, 0xe8, 0xb6, 0xbb, 0xae, 0xa2, 0xcb, 0x80, - 0xf4, 0xd1, 0xc0, 0xc9, 0x6d, 0xe7, 0xaf, 0xd1, 0xd1, 0x40, 0x82, 0x2d, 0xc3, 0x44, 0x9c, 0x85, - 0xa1, 0xfe, 0x72, 0x34, 0xef, 0x1c, 0x12, 0x13, 0x3c, 0xec, 0x13, 0xfb, 0xeb, 0x1e, 0x1e, 0x0c, - 0x02, 0xd8, 0x31, 0x38, 0xc0, 0xa3, 0x2e, 0xef, 0xdb, 0xc8, 0xdf, 0xf6, 0x68, 0x20, 0xe8, 0xd5, - 0x6c, 0x05, 0xa0, 0xb8, 0x34, 0xaa, 0x9d, 0xc4, 0xfa, 0xa9, 0xbf, 0xef, 0x15, 0x77, 0x50, 0x03, - 0x19, 0x08, 0xf2, 0x5b, 0xa7, 0x45, 0x70, 0xa3, 0x2a, 0xc8, 0x2f, 0x9a, 0xc7, 0x61, 0xfc, 0x31, - 0x29, 0x62, 0xe5, 0xf9, 0x36, 0xfa, 0x0f, 0xa4, 0x69, 0xbd, 0x2e, 0x58, 0x24, 0x52, 0xae, 0x3c, - 0x5f, 0xda, 0xd8, 0x3f, 0x91, 0x2d, 0x01, 0x0d, 0xf7, 0x3c, 0xa9, 0x5c, 0x9e, 0xfb, 0x2f, 0x82, - 0x09, 0xd0, 0x9b, 0xd6, 0xaf, 0x1f, 0xe7, 0x3b, 0x36, 0xf6, 0x6f, 0xda, 0x34, 0xae, 0x67, 0x27, - 0xa0, 0xa1, 0x5f, 0xe6, 0xf7, 0x6d, 0x1b, 0xfc, 0x0f, 0xc2, 0x03, 0x42, 0x7f, 0xb2, 0x54, 0x7d, - 0x15, 0xd8, 0x8b, 0xfd, 0x2f, 0x76, 0x9a, 0xd6, 0xb3, 0x55, 0x98, 0x94, 0xaa, 0xdf, 0xcf, 0x52, - 0x2f, 0x1f, 0xfe, 0x16, 0xfc, 0xbf, 0xbd, 0xf2, 0x32, 0x57, 0x32, 0x27, 0x8f, 0xc0, 0x7c, 0x4f, - 0x44, 0x75, 0xf0, 0x24, 0xb4, 0x44, 0x4b, 0xb4, 0xf3, 0x63, 0xf0, 0x7f, 0x00, 0x00, 0x00, 0xff, - 0xff, 0x3f, 0x9b, 0x2b, 0x54, 0xfc, 0x11, 0x00, 0x00, + // 1178 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x98, 0x4b, 0x6f, 0x1c, 0x45, + 0x10, 0x80, 0x85, 0x48, 0x64, 0x6f, 0xf9, 0x85, 0xd7, 0xc6, 0x84, 0x08, 0x44, 0x72, 0xe3, 0xe4, + 0x9c, 0x22, 0x94, 0xb6, 0x22, 0xcb, 0xb1, 0x1c, 0x2b, 0x08, 0x83, 0x65, 0xe2, 0x00, 0xe2, 0xb0, + 0x1a, 0xef, 0xb6, 0x27, 0x0b, 0x33, 0xd3, 0xc3, 0x74, 0x4f, 0x14, 0xe7, 0x86, 0xc2, 0x43, 0x08, + 0xf1, 0x46, 0x82, 0x84, 0x24, 0xc0, 0x81, 0xf7, 0x33, 0x3c, 0xaf, 0x5c, 0x78, 0x5c, 0xf9, 0x0f, + 0x5c, 0x00, 0xf3, 0xf6, 0xcd, 0x17, 0x54, 0x33, 0x55, 0xb3, 0xbd, 0xeb, 0x95, 0xba, 0xf7, 0x36, + 0xbb, 0xee, 0xef, 0xdb, 0x9a, 0xaa, 0xe9, 0xaa, 0x69, 0x03, 0x84, 0x2a, 0x54, 0xb3, 0x69, 0xa6, + 0x8c, 0xaa, 0xd7, 0xf0, 0xba, 0xb8, 0x3c, 0x78, 0x28, 0x54, 0x2a, 0x8c, 0xe4, 0x91, 0xe2, 0xd3, + 0x46, 0xbe, 0x79, 0xa4, 0x25, 0x75, 0x33, 0x6b, 0xa7, 0x46, 0x65, 0xe5, 0x62, 0x71, 0x2f, 0x4c, + 0xd1, 0xe2, 0x86, 0x4c, 0xf2, 0xb8, 0x91, 0x66, 0x72, 0xb3, 0x7d, 0xbe, 0x7e, 0xdb, 0x6c, 0x49, + 0xce, 0x32, 0x39, 0xbb, 0x94, 0xe4, 0xf1, 0x7d, 0xa9, 0x69, 0xab, 0x44, 0x1f, 0xb8, 0xfe, 0xcb, + 0x8d, 0x87, 0x6e, 0xb8, 0x73, 0x78, 0x6d, 0x92, 0x50, 0xfc, 0xdb, 0x6a, 0x01, 0x8a, 0x35, 0xb8, + 0xb9, 0xcb, 0xa7, 0x4d, 0xd6, 0x4e, 0x42, 0x99, 0x39, 0x8c, 0xdf, 0x93, 0x71, 0xca, 0x32, 0xde, + 0x4f, 0xa8, 0x58, 0x84, 0xb1, 0x41, 0x5c, 0x3f, 0x90, 0x6b, 0x54, 0xda, 0x92, 0x65, 0x98, 0x28, + 0x24, 0xcd, 0x5c, 0x1b, 0x15, 0x27, 0x41, 0x2c, 0x1d, 0x9a, 0x1f, 0x0b, 0x4d, 0x6d, 0x6d, 0x1c, + 0xb1, 0xc5, 0x8a, 0x12, 0x02, 0x86, 0xf1, 0x9b, 0x96, 0x6c, 0x46, 0x0e, 0xc3, 0x4f, 0x14, 0x48, + 0xb5, 0x5e, 0x9c, 0x81, 0x69, 0xbc, 0x3e, 0x17, 0x44, 0xb9, 0xb4, 0x23, 0x39, 0xdc, 0xd7, 0x73, + 0x06, 0x97, 0xb1, 0xec, 0xe7, 0x8b, 0xfb, 0x8a, 0x70, 0xa6, 0x2a, 0x81, 0x15, 0x93, 0x55, 0xc5, + 0x50, 0x1a, 0x23, 0x33, 0xdd, 0x08, 0xa2, 0x7e, 0xe1, 0x9d, 0x6c, 0x47, 0x95, 0xf1, 0xd2, 0x76, + 0x77, 0x15, 0x97, 0x4b, 0x72, 0x21, 0x8a, 0xc4, 0x3a, 0xdc, 0xd2, 0xe7, 0xa9, 0xf0, 0x70, 0x5e, + 0x26, 0xe7, 0xf4, 0x9e, 0x27, 0x03, 0xb5, 0xab, 0xc0, 0xdf, 0x57, 0xb5, 0xf4, 0x70, 0xbe, 0x41, + 0xce, 0x3a, 0xb1, 0x5c, 0x52, 0x34, 0xde, 0x0d, 0x93, 0xe7, 0x64, 0xb6, 0xa1, 0xb4, 0x6c, 0xc8, + 0xc7, 0xf2, 0x20, 0xf2, 0xd0, 0x5d, 0x21, 0xdd, 0x04, 0x81, 0x4b, 0xc8, 0xa1, 0xeb, 0x18, 0x0c, + 0x6f, 0x06, 0x4d, 0xe9, 0xa1, 0xb8, 0x4a, 0x8a, 0x21, 0x5c, 0x8f, 0xe8, 0x02, 0x8c, 0x86, 0xaa, + 0xbc, 0x25, 0x0f, 0xfc, 0x1a, 0xe1, 0x23, 0xcc, 0x90, 0x22, 0x55, 0x69, 0x1e, 0x05, 0xc6, 0x27, + 0x82, 0x37, 0x59, 0xc1, 0x0c, 0x29, 0x06, 0x48, 0xeb, 0x5b, 0xac, 0xd0, 0x56, 0x3e, 0xe7, 0x61, + 0x44, 0x25, 0xd1, 0x96, 0x4a, 0x7c, 0x82, 0x78, 0x9b, 0x0c, 0x40, 0x08, 0x0a, 0xe6, 0xa0, 0xe6, + 0x5b, 0x88, 0x77, 0xb6, 0x79, 0x7b, 0x70, 0x05, 0x96, 0x61, 0x82, 0x1b, 0x54, 0x5b, 0x25, 0x1e, + 0x8a, 0x77, 0x49, 0x31, 0x6e, 0x61, 0x74, 0x1b, 0x46, 0x6a, 0x13, 0x4a, 0x1f, 0xc9, 0x7b, 0x7c, + 0x1b, 0x84, 0x50, 0x2a, 0x37, 0x64, 0xd2, 0x3c, 0xeb, 0x67, 0x78, 0x9f, 0x53, 0xc9, 0x0c, 0x2a, + 0x16, 0x61, 0x2c, 0x0e, 0x32, 0x7d, 0x36, 0x88, 0xbc, 0xca, 0xf1, 0x01, 0x39, 0x46, 0x2b, 0x88, + 0x32, 0x92, 0x27, 0x83, 0x68, 0x3e, 0xe4, 0x8c, 0x58, 0x18, 0x6d, 0x3d, 0x6d, 0x82, 0x8d, 0x48, + 0x36, 0x06, 0xb1, 0x7d, 0xc4, 0x5b, 0xaf, 0x64, 0x57, 0x6c, 0xe3, 0x1c, 0xd4, 0x74, 0xfb, 0x82, + 0x97, 0xe6, 0x63, 0xae, 0x74, 0x01, 0x20, 0xfc, 0x10, 0xdc, 0xda, 0x77, 0x4c, 0x78, 0xc8, 0x3e, + 0x21, 0xd9, 0x4c, 0x9f, 0x51, 0x41, 0x2d, 0x61, 0x50, 0xe5, 0xa7, 0xdc, 0x12, 0x64, 0x8f, 0x6b, + 0x15, 0xa6, 0xf3, 0x44, 0x07, 0x9b, 0x83, 0x65, 0xed, 0x33, 0xce, 0x5a, 0xc9, 0x76, 0x65, 0xed, + 0x34, 0xcc, 0x90, 0x71, 0xb0, 0xba, 0x7e, 0xce, 0x8d, 0xb5, 0xa4, 0xd7, 0xbb, 0xab, 0xfb, 0x30, + 0x1c, 0xac, 0xd2, 0x79, 0xde, 0xc8, 0x44, 0x23, 0xd3, 0x88, 0x83, 0xd4, 0xc3, 0x7c, 0x9d, 0xcc, + 0xdc, 0xf1, 0x97, 0x2a, 0xc1, 0x4a, 0x90, 0xa2, 0xfc, 0x41, 0x38, 0xc0, 0xf2, 0x3c, 0xc9, 0x64, + 0x53, 0x85, 0x49, 0xfb, 0x82, 0x6c, 0x79, 0xa8, 0xbf, 0xe8, 0x29, 0xd5, 0xba, 0x85, 0xa3, 0xf9, + 0x14, 0xdc, 0x54, 0xbd, 0xab, 0x34, 0xda, 0x71, 0xaa, 0x32, 0xe3, 0x30, 0x7e, 0xc9, 0x95, 0xaa, + 0xb8, 0x53, 0x05, 0x26, 0x96, 0x60, 0xbc, 0xf8, 0xe8, 0xfb, 0x48, 0x7e, 0x45, 0xa2, 0xb1, 0x0e, + 0x45, 0x8d, 0xa3, 0xa9, 0xe2, 0x34, 0xc8, 0x7c, 0xfa, 0xdf, 0xd7, 0xdc, 0x38, 0x08, 0xa1, 0xc6, + 0x61, 0xb6, 0x52, 0x89, 0xd3, 0xde, 0xc3, 0xf0, 0x0d, 0x37, 0x0e, 0x66, 0x48, 0xc1, 0x2f, 0x0c, + 0x1e, 0x8a, 0x6f, 0x59, 0xc1, 0x4c, 0xb9, 0x07, 0x26, 0x7a, 0xde, 0x07, 0xea, 0x77, 0xec, 0xb1, + 0xac, 0x48, 0xad, 0x83, 0xb0, 0x12, 0x3d, 0xbe, 0x43, 0x9d, 0xa3, 0xfb, 0x75, 0x40, 0xdc, 0x83, + 0x45, 0xea, 0x1e, 0xda, 0x6e, 0xd9, 0xc5, 0x9d, 0xaa, 0x4e, 0x5d, 0x33, 0x5b, 0x9c, 0x84, 0xb1, + 0xae, 0x81, 0xed, 0x56, 0x3d, 0x41, 0xaa, 0x51, 0x7b, 0x5e, 0x8b, 0xa3, 0xb0, 0x0f, 0x87, 0xaf, + 0x1b, 0x7f, 0x92, 0xf0, 0x62, 0xb9, 0x38, 0x0e, 0xc3, 0x3c, 0x74, 0xdd, 0xe8, 0x53, 0x84, 0x56, + 0x08, 0xe2, 0x3c, 0x70, 0xdd, 0xf8, 0xd3, 0x8c, 0x33, 0x82, 0xb8, 0x7f, 0x0a, 0xbf, 0x7b, 0x76, + 0x1f, 0x35, 0x4d, 0xce, 0xdd, 0x1c, 0x0c, 0xd1, 0xa4, 0x75, 0xd3, 0xcf, 0xd0, 0x8f, 0x33, 0x21, + 0xee, 0x82, 0xfd, 0x9e, 0x09, 0x7f, 0x8e, 0xd0, 0x72, 0xbd, 0x58, 0x84, 0x11, 0x6b, 0xba, 0xba, + 0xf1, 0xe7, 0x09, 0xb7, 0x29, 0x0c, 0x9d, 0xa6, 0xab, 0x5b, 0xf0, 0x02, 0x87, 0x4e, 0x04, 0xa6, + 0x8d, 0x07, 0xab, 0x9b, 0x7e, 0x91, 0xb3, 0xce, 0x88, 0x98, 0x87, 0x5a, 0xd5, 0x2c, 0xdd, 0xfc, + 0x4b, 0xc4, 0x77, 0x18, 0xcc, 0x80, 0xd5, 0xac, 0xdd, 0x8a, 0x97, 0x39, 0x03, 0x16, 0x85, 0xdb, + 0xa8, 0x77, 0x00, 0xbb, 0x4d, 0xaf, 0xf0, 0x36, 0xea, 0x99, 0xbf, 0x58, 0xcd, 0xa2, 0x67, 0xb9, + 0x15, 0xaf, 0x72, 0x35, 0x8b, 0xf5, 0x18, 0x46, 0xef, 0x44, 0x73, 0x3b, 0x5e, 0xe3, 0x30, 0x7a, + 0x06, 0x9a, 0x58, 0x85, 0xfa, 0xde, 0x69, 0xe6, 0xf6, 0xbd, 0x4e, 0xbe, 0xc9, 0x3d, 0xc3, 0x4c, + 0x3c, 0x00, 0x33, 0xfd, 0x27, 0x99, 0xdb, 0x7a, 0x69, 0xa7, 0xe7, 0xec, 0x61, 0x0f, 0x32, 0x71, + 0xba, 0x73, 0xf6, 0xb0, 0xa7, 0x98, 0x5b, 0x7b, 0x79, 0xa7, 0xfb, 0x68, 0x6a, 0x0f, 0x31, 0xb1, + 0x00, 0xd0, 0x19, 0x20, 0x6e, 0xd7, 0x15, 0x72, 0x59, 0x10, 0x6e, 0x0d, 0x9a, 0x1f, 0x6e, 0xfe, + 0x2a, 0x6f, 0x0d, 0x22, 0x70, 0x6b, 0xf0, 0xe8, 0x70, 0xd3, 0xd7, 0x78, 0x6b, 0x30, 0x22, 0xe6, + 0x60, 0x38, 0xc9, 0xa3, 0x08, 0x9f, 0xad, 0xfa, 0xed, 0x7d, 0xc6, 0x8c, 0x8c, 0x5a, 0x0c, 0xff, + 0xba, 0x4b, 0x30, 0x03, 0xe2, 0x28, 0xec, 0x97, 0xf1, 0x86, 0x6c, 0xb9, 0xc8, 0xdf, 0x76, 0xb9, + 0x9f, 0xe0, 0x6a, 0x31, 0x0f, 0x50, 0x9e, 0x7c, 0x31, 0x0a, 0x17, 0xfb, 0xfb, 0x6e, 0x79, 0x08, + 0xb7, 0x90, 0x8e, 0xa0, 0x38, 0x3a, 0x3b, 0x04, 0xdb, 0xdd, 0x82, 0xe2, 0xb4, 0x7c, 0x0c, 0x86, + 0x1e, 0xd1, 0x2a, 0x31, 0x41, 0xe8, 0xa2, 0xff, 0x20, 0x9a, 0xd7, 0x63, 0xc2, 0x62, 0x95, 0x49, + 0x13, 0x84, 0xda, 0xc5, 0xfe, 0x49, 0x6c, 0x05, 0x20, 0xdc, 0x0c, 0xb4, 0xf1, 0xb9, 0xef, 0xbf, + 0x18, 0x66, 0x00, 0x83, 0xc6, 0xeb, 0x47, 0xe5, 0x96, 0x8b, 0xfd, 0x9b, 0x83, 0xa6, 0xf5, 0xe2, + 0x38, 0xd4, 0xf0, 0xb2, 0xf8, 0xa7, 0x81, 0x0b, 0xfe, 0x87, 0xe0, 0x0e, 0x81, 0xbf, 0xac, 0x4d, + 0xcb, 0xb4, 0xdd, 0xc9, 0xfe, 0x97, 0x2a, 0xcd, 0xeb, 0xc5, 0x02, 0x8c, 0x68, 0xd3, 0x6a, 0xe5, + 0x59, 0x50, 0xcc, 0x0e, 0x07, 0xfe, 0xdf, 0x6e, 0x75, 0x22, 0xad, 0x98, 0x13, 0x87, 0x61, 0xaa, + 0xa9, 0xe2, 0x5e, 0xf0, 0x04, 0x2c, 0xab, 0x65, 0xb5, 0x5a, 0xec, 0xa2, 0xff, 0x03, 0x00, 0x00, + 0xff, 0xff, 0x06, 0x21, 0xec, 0x88, 0xfd, 0x12, 0x00, 0x00, } diff --git a/vendor/github.com/gogo/protobuf/gogoproto/gogo.proto b/vendor/github.com/gogo/protobuf/gogoproto/gogo.proto index 0da211a..1895252 100644 --- a/vendor/github.com/gogo/protobuf/gogoproto/gogo.proto +++ b/vendor/github.com/gogo/protobuf/gogoproto/gogo.proto @@ -39,6 +39,7 @@ extend google.protobuf.EnumOptions { optional bool goproto_enum_stringer = 62021; optional bool enum_stringer = 62022; optional string enum_customname = 62023; + optional bool enumdecl = 62024; } extend google.protobuf.EnumValueOptions { @@ -77,6 +78,8 @@ extend google.protobuf.FileOptions { optional bool gogoproto_import = 63027; optional bool protosizer_all = 63028; optional bool compare_all = 63029; + optional bool typedecl_all = 63030; + optional bool enumdecl_all = 63031; } extend google.protobuf.MessageOptions { @@ -107,6 +110,8 @@ extend google.protobuf.MessageOptions { optional bool protosizer = 64028; optional bool compare = 64029; + + optional bool typedecl = 64030; } extend google.protobuf.FieldOptions { diff --git a/vendor/github.com/gogo/protobuf/gogoproto/helper.go b/vendor/github.com/gogo/protobuf/gogoproto/helper.go index bb5fff4..f47708b 100644 --- a/vendor/github.com/gogo/protobuf/gogoproto/helper.go +++ b/vendor/github.com/gogo/protobuf/gogoproto/helper.go @@ -90,6 +90,14 @@ func IsCastValue(field *google_protobuf.FieldDescriptorProto) bool { return false } +func HasEnumDecl(file *google_protobuf.FileDescriptorProto, enum *google_protobuf.EnumDescriptorProto) bool { + return proto.GetBoolExtension(enum.Options, E_Enumdecl, proto.GetBoolExtension(file.Options, E_EnumdeclAll, true)) +} + +func HasTypeDecl(file *google_protobuf.FileDescriptorProto, message *google_protobuf.DescriptorProto) bool { + return proto.GetBoolExtension(message.Options, E_Typedecl, proto.GetBoolExtension(file.Options, E_TypedeclAll, true)) +} + func GetCustomType(field *google_protobuf.FieldDescriptorProto) string { if field == nil { return "" diff --git a/vendor/github.com/gogo/protobuf/plugin/gostring/gostring.go b/vendor/github.com/gogo/protobuf/plugin/gostring/gostring.go index 789cc5d..024c9f0 100644 --- a/vendor/github.com/gogo/protobuf/plugin/gostring/gostring.go +++ b/vendor/github.com/gogo/protobuf/plugin/gostring/gostring.go @@ -97,8 +97,10 @@ not print their values, while the generated GoString method will always print al package gostring import ( + "fmt" "github.com/gogo/protobuf/gogoproto" "github.com/gogo/protobuf/protoc-gen-gogo/generator" + "os" "strconv" "strings" ) @@ -145,6 +147,7 @@ func (p *gostring) Generate(file *generator.FileDescriptor) { reflectPkg := p.NewImport("reflect") sortKeysPkg := p.NewImport("github.com/gogo/protobuf/sortkeys") + extensionToGoStringUsed := false for _, message := range file.Messages() { if !p.overwrite && !gogoproto.HasGoString(file.FileDescriptorProto, message.DescriptorProto) { continue @@ -221,7 +224,7 @@ func (p *gostring) Generate(file *generator.FileDescriptor) { p.P(`s = append(s, "`, fieldname, `: " + `, mapName, `+ ",\n")`) p.Out() p.P(`}`) - } else if field.IsMessage() || p.IsGroup(field) { + } else if (field.IsMessage() && !gogoproto.IsCustomType(field) && !gogoproto.IsStdTime(field) && !gogoproto.IsStdDuration(field)) || p.IsGroup(field) { if nullable || repeated { p.P(`if this.`, fieldname, ` != nil {`) p.In() @@ -264,6 +267,7 @@ func (p *gostring) Generate(file *generator.FileDescriptor) { if message.DescriptorProto.HasExtension() { if gogoproto.HasExtensionsMap(file.FileDescriptorProto, message.DescriptorProto) { p.P(`s = append(s, "XXX_InternalExtensions: " + extensionToGoString`, p.localName, `(this) + ",\n")`) + extensionToGoStringUsed = true } else { p.P(`if this.XXX_extensions != nil {`) p.In() @@ -338,29 +342,34 @@ func (p *gostring) Generate(file *generator.FileDescriptor) { p.Out() p.P(`}`) - p.P(`func extensionToGoString`, p.localName, `(m `, protoPkg.Use(), `.Message) string {`) - p.In() - p.P(`e := `, protoPkg.Use(), `.GetUnsafeExtensionsMap(m)`) - p.P(`if e == nil { return "nil" }`) - p.P(`s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{"`) - p.P(`keys := make([]int, 0, len(e))`) - p.P(`for k := range e {`) - p.In() - p.P(`keys = append(keys, int(k))`) - p.Out() - p.P(`}`) - p.P(sortPkg.Use(), `.Ints(keys)`) - p.P(`ss := []string{}`) - p.P(`for _, k := range keys {`) - p.In() - p.P(`ss = append(ss, `, strconvPkg.Use(), `.Itoa(k) + ": " + e[int32(k)].GoString())`) - p.Out() - p.P(`}`) - p.P(`s+=`, stringsPkg.Use(), `.Join(ss, ",") + "})"`) - p.P(`return s`) - p.Out() - p.P(`}`) - + if extensionToGoStringUsed { + if !gogoproto.ImportsGoGoProto(file.FileDescriptorProto) { + fmt.Fprintf(os.Stderr, "The GoString plugin for messages with extensions requires importing gogoprotobuf. Please see file %s", file.GetName()) + os.Exit(1) + } + p.P(`func extensionToGoString`, p.localName, `(m `, protoPkg.Use(), `.Message) string {`) + p.In() + p.P(`e := `, protoPkg.Use(), `.GetUnsafeExtensionsMap(m)`) + p.P(`if e == nil { return "nil" }`) + p.P(`s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{"`) + p.P(`keys := make([]int, 0, len(e))`) + p.P(`for k := range e {`) + p.In() + p.P(`keys = append(keys, int(k))`) + p.Out() + p.P(`}`) + p.P(sortPkg.Use(), `.Ints(keys)`) + p.P(`ss := []string{}`) + p.P(`for _, k := range keys {`) + p.In() + p.P(`ss = append(ss, `, strconvPkg.Use(), `.Itoa(k) + ": " + e[int32(k)].GoString())`) + p.Out() + p.P(`}`) + p.P(`s+=`, stringsPkg.Use(), `.Join(ss, ",") + "})"`) + p.P(`return s`) + p.Out() + p.P(`}`) + } } func init() { diff --git a/vendor/github.com/gogo/protobuf/plugin/marshalto/marshalto.go b/vendor/github.com/gogo/protobuf/plugin/marshalto/marshalto.go index d8b1af0..b2631e6 100644 --- a/vendor/github.com/gogo/protobuf/plugin/marshalto/marshalto.go +++ b/vendor/github.com/gogo/protobuf/plugin/marshalto/marshalto.go @@ -420,7 +420,7 @@ func (p *marshalto) generateField(proto3 bool, numGen NumGen, file *generator.Fi p.P(`if m.`, fieldname, ` != nil {`) p.In() } - packed := field.IsPacked() || (proto3 && field.IsRepeated() && generator.IsScalar(field)) + packed := field.IsPacked() || (proto3 && field.IsPacked3()) wireType := field.WireType() fieldNumber := field.GetNumber() if packed { diff --git a/vendor/github.com/gogo/protobuf/plugin/populate/populate.go b/vendor/github.com/gogo/protobuf/plugin/populate/populate.go index e03c3e2..16aee32 100644 --- a/vendor/github.com/gogo/protobuf/plugin/populate/populate.go +++ b/vendor/github.com/gogo/protobuf/plugin/populate/populate.go @@ -194,7 +194,6 @@ func (p *plugin) getFuncName(goTypName string) string { case "time.NewPopulatedTime": funcName = p.typesPkg.Use() + ".NewPopulatedStdTime" case "time.NewPopulatedDuration": - p.typesPkg.Use() funcName = p.typesPkg.Use() + ".NewPopulatedStdDuration" } return funcName @@ -305,6 +304,23 @@ func (p *plugin) GenerateField(file *generator.FileDescriptor, message *generato } p.Out() p.P(`}`) + } else if gogoproto.IsCustomType(field) { + funcCall := p.getCustomFuncCall(goTypName) + if field.IsRepeated() { + p.P(p.varGen.Next(), ` := r.Intn(10)`) + p.P(`this.`, fieldname, ` = make(`, goTyp, `, `, p.varGen.Current(), `)`) + p.P(`for i := 0; i < `, p.varGen.Current(), `; i++ {`) + p.In() + p.P(p.varGen.Next(), `:= `, funcCall) + p.P(`this.`, fieldname, `[i] = *`, p.varGen.Current()) + p.Out() + p.P(`}`) + } else if gogoproto.IsNullable(field) { + p.P(`this.`, fieldname, ` = `, funcCall) + } else { + p.P(p.varGen.Next(), `:= `, funcCall) + p.P(`this.`, fieldname, ` = *`, p.varGen.Current()) + } } else if field.IsMessage() || p.IsGroup(field) { funcCall := p.getFuncCall(goTypName) if field.IsRepeated() { @@ -345,23 +361,6 @@ func (p *plugin) GenerateField(file *generator.FileDescriptor, message *generato p.P(p.varGen.Next(), ` := `, val) p.P(`this.`, fieldname, ` = &`, p.varGen.Current()) } - } else if gogoproto.IsCustomType(field) { - funcCall := p.getCustomFuncCall(goTypName) - if field.IsRepeated() { - p.P(p.varGen.Next(), ` := r.Intn(10)`) - p.P(`this.`, fieldname, ` = make(`, goTyp, `, `, p.varGen.Current(), `)`) - p.P(`for i := 0; i < `, p.varGen.Current(), `; i++ {`) - p.In() - p.P(p.varGen.Next(), `:= `, funcCall) - p.P(`this.`, fieldname, `[i] = *`, p.varGen.Current()) - p.Out() - p.P(`}`) - } else if gogoproto.IsNullable(field) { - p.P(`this.`, fieldname, ` = `, funcCall) - } else { - p.P(p.varGen.Next(), `:= `, funcCall) - p.P(`this.`, fieldname, ` = *`, p.varGen.Current()) - } } else if field.IsBytes() { if field.IsRepeated() { p.P(p.varGen.Next(), ` := r.Intn(10)`) diff --git a/vendor/github.com/gogo/protobuf/plugin/size/size.go b/vendor/github.com/gogo/protobuf/plugin/size/size.go index 15ab49c..584e980 100644 --- a/vendor/github.com/gogo/protobuf/plugin/size/size.go +++ b/vendor/github.com/gogo/protobuf/plugin/size/size.go @@ -230,7 +230,7 @@ func (p *size) generateField(proto3 bool, file *generator.FileDescriptor, messag p.P(`if m.`, fieldname, ` != nil {`) p.In() } - packed := field.IsPacked() || (proto3 && field.IsRepeated() && generator.IsScalar(field)) + packed := field.IsPacked() || (proto3 && field.IsPacked3()) _, wire := p.GoType(message, field) wireType := wireToType(wire) fieldNumber := field.GetNumber() diff --git a/vendor/github.com/gogo/protobuf/plugin/stringer/stringer.go b/vendor/github.com/gogo/protobuf/plugin/stringer/stringer.go index aa5f022..098a9db 100644 --- a/vendor/github.com/gogo/protobuf/plugin/stringer/stringer.go +++ b/vendor/github.com/gogo/protobuf/plugin/stringer/stringer.go @@ -203,7 +203,7 @@ func (p *stringer) Generate(file *generator.FileDescriptor) { } else if p.IsMap(field) { mapName := `mapStringFor` + fieldname p.P("`", fieldname, ":`", ` + `, mapName, " + `,", "`,") - } else if field.IsMessage() || p.IsGroup(field) { + } else if (field.IsMessage() && !gogoproto.IsCustomType(field)) || p.IsGroup(field) { desc := p.ObjectNamed(field.GetTypeName()) msgname := p.TypeName(desc) msgnames := strings.Split(msgname, ".") diff --git a/vendor/github.com/gogo/protobuf/plugin/unmarshal/unmarshal.go b/vendor/github.com/gogo/protobuf/plugin/unmarshal/unmarshal.go index c8a7049..ca3a833 100644 --- a/vendor/github.com/gogo/protobuf/plugin/unmarshal/unmarshal.go +++ b/vendor/github.com/gogo/protobuf/plugin/unmarshal/unmarshal.go @@ -806,10 +806,13 @@ func (p *unmarshal) field(file *generator.FileDescriptor, msg *generator.Descrip } else { p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, time.Duration(0))`) } - } else if nullable { + } else if nullable && !gogoproto.IsCustomType(field) { p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, &`, msgname, `{})`) } else { - p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, `, msgname, `{})`) + goType, _ := p.GoType(nil, field) + // remove the slice from the type, i.e. []*T -> *T + goType = goType[2:] + p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, `, goType, `{})`) } varName := `m.` + fieldname + `[len(m.` + fieldname + `)-1]` buf := `dAtA[iNdEx:postIndex]` @@ -840,7 +843,9 @@ func (p *unmarshal) field(file *generator.FileDescriptor, msg *generator.Descrip } else if gogoproto.IsStdDuration(field) { p.P(`m.`, fieldname, ` = new(time.Duration)`) } else { - p.P(`m.`, fieldname, ` = &`, msgname, `{}`) + goType, _ := p.GoType(nil, field) + // remove the star from the type + p.P(`m.`, fieldname, ` = &`, goType[1:], `{}`) } p.Out() p.P(`}`) @@ -869,6 +874,7 @@ func (p *unmarshal) field(file *generator.FileDescriptor, msg *generator.Descrip p.P(`}`) } p.P(`iNdEx = postIndex`) + case descriptor.FieldDescriptorProto_TYPE_BYTES: p.P(`var byteLen int`) p.decodeVarint("byteLen", "int") @@ -1164,12 +1170,16 @@ func (p *unmarshal) Generate(file *generator.FileDescriptor) { if field.OneofIndex != nil { errFieldname = p.GetOneOfFieldName(message, field) } - packed := field.IsPacked() || (proto3 && field.IsRepeated() && generator.IsScalar(field)) + possiblyPacked := field.IsScalar() && field.IsRepeated() p.P(`case `, strconv.Itoa(int(field.GetNumber())), `:`) p.In() wireType := field.WireType() - if packed { - p.P(`if wireType == `, strconv.Itoa(proto.WireBytes), `{`) + if possiblyPacked { + p.P(`if wireType == `, strconv.Itoa(wireType), `{`) + p.In() + p.field(file, message, field, fieldname, false) + p.Out() + p.P(`} else if wireType == `, strconv.Itoa(proto.WireBytes), `{`) p.In() p.P(`var packedLen int`) p.decodeVarint("packedLen", "int") @@ -1190,10 +1200,6 @@ func (p *unmarshal) Generate(file *generator.FileDescriptor) { p.Out() p.P(`}`) p.Out() - p.P(`} else if wireType == `, strconv.Itoa(wireType), `{`) - p.In() - p.field(file, message, field, fieldname, false) - p.Out() p.P(`} else {`) p.In() p.P(`return ` + fmtPkg.Use() + `.Errorf("proto: wrong wireType = %d for field ` + errFieldname + `", wireType)`) diff --git a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.go b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.go index e808a3f..a85bf19 100644 --- a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.go +++ b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.go @@ -90,3 +90,29 @@ func ForMessage(msg Message) (fd *FileDescriptorProto, md *DescriptorProto) { } return fd, md } + +// Is this field a scalar numeric type? +func (field *FieldDescriptorProto) IsScalar() bool { + if field.Type == nil { + return false + } + switch *field.Type { + case FieldDescriptorProto_TYPE_DOUBLE, + FieldDescriptorProto_TYPE_FLOAT, + FieldDescriptorProto_TYPE_INT64, + FieldDescriptorProto_TYPE_UINT64, + FieldDescriptorProto_TYPE_INT32, + FieldDescriptorProto_TYPE_FIXED64, + FieldDescriptorProto_TYPE_FIXED32, + FieldDescriptorProto_TYPE_BOOL, + FieldDescriptorProto_TYPE_UINT32, + FieldDescriptorProto_TYPE_ENUM, + FieldDescriptorProto_TYPE_SFIXED32, + FieldDescriptorProto_TYPE_SFIXED64, + FieldDescriptorProto_TYPE_SINT32, + FieldDescriptorProto_TYPE_SINT64: + return true + default: + return false + } +} diff --git a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/helper.go b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/helper.go index 861f4d0..e0846a3 100644 --- a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/helper.go +++ b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/helper.go @@ -99,6 +99,17 @@ func (field *FieldDescriptorProto) GetKeyUint64() (x uint64) { return x } +func (field *FieldDescriptorProto) GetKey3Uint64() (x uint64) { + packed := field.IsPacked3() + wireType := field.WireType() + fieldNumber := field.GetNumber() + if packed { + wireType = 2 + } + x = uint64(uint32(fieldNumber)<<3 | uint32(wireType)) + return x +} + func (field *FieldDescriptorProto) GetKey() []byte { x := field.GetKeyUint64() i := 0 @@ -111,6 +122,18 @@ func (field *FieldDescriptorProto) GetKey() []byte { return keybuf } +func (field *FieldDescriptorProto) GetKey3() []byte { + x := field.GetKey3Uint64() + i := 0 + keybuf := make([]byte, 0) + for i = 0; x > 127; i++ { + keybuf = append(keybuf, 0x80|uint8(x&0x7F)) + x >>= 7 + } + keybuf = append(keybuf, uint8(x)) + return keybuf +} + func (desc *FileDescriptorSet) GetField(packageName, messageName, fieldName string) *FieldDescriptorProto { msg := desc.GetMessage(packageName, messageName) if msg == nil { @@ -352,6 +375,16 @@ func (f *FieldDescriptorProto) IsPacked() bool { return f.Options != nil && f.GetOptions().GetPacked() } +func (f *FieldDescriptorProto) IsPacked3() bool { + if f.IsRepeated() && f.IsScalar() { + if f.Options == nil || f.GetOptions().Packed == nil { + return true + } + return f.Options != nil && f.GetOptions().GetPacked() + } + return false +} + func (m *DescriptorProto) HasExtension() bool { return len(m.ExtensionRange) > 0 } diff --git a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/generator/generator.go b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/generator/generator.go index 566454b..c030534 100644 --- a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/generator/generator.go +++ b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/generator/generator.go @@ -1465,6 +1465,12 @@ func (g *Generator) generateImports() { g.P("var _ = ", g.Pkg["proto"], ".Marshal") g.P("var _ = ", g.Pkg["fmt"], ".Errorf") g.P("var _ = ", g.Pkg["math"], ".Inf") + for _, cimport := range g.customImports { + if cimport == "time" { + g.P("var _ = time.Kitchen") + break + } + } g.P() } @@ -1506,23 +1512,27 @@ func (g *Generator) generateEnum(enum *EnumDescriptor) { if !gogoproto.EnabledGoEnumPrefix(enum.file, enum.EnumDescriptorProto) { ccPrefix = "" } - g.P("type ", ccTypeName, " int32") - g.file.addExport(enum, enumSymbol{ccTypeName, enum.proto3()}) - g.P("const (") - g.In() - for i, e := range enum.Value { - g.PrintComments(fmt.Sprintf("%s,%d,%d", enum.path, enumValuePath, i)) - name := *e.Name - if gogoproto.IsEnumValueCustomName(e) { - name = gogoproto.GetEnumValueCustomName(e) - } - name = ccPrefix + name - g.P(name, " ", ccTypeName, " = ", e.Number) - g.file.addExport(enum, constOrVarSymbol{name, "const", ccTypeName}) + if gogoproto.HasEnumDecl(enum.file, enum.EnumDescriptorProto) { + g.P("type ", ccTypeName, " int32") + g.file.addExport(enum, enumSymbol{ccTypeName, enum.proto3()}) + g.P("const (") + g.In() + for i, e := range enum.Value { + g.PrintComments(fmt.Sprintf("%s,%d,%d", enum.path, enumValuePath, i)) + name := *e.Name + if gogoproto.IsEnumValueCustomName(e) { + name = gogoproto.GetEnumValueCustomName(e) + } + name = ccPrefix + name + + g.P(name, " ", ccTypeName, " = ", e.Number) + g.file.addExport(enum, constOrVarSymbol{name, "const", ccTypeName}) + } + g.Out() + g.P(")") } - g.Out() - g.P(")") + g.P("var ", ccTypeName, "_name = map[int32]string{") g.In() generated := make(map[int32]bool) // avoid duplicate values @@ -1769,7 +1779,7 @@ func (g *Generator) goTag(message *Descriptor, field *descriptor.FieldDescriptor func needsStar(field *descriptor.FieldDescriptorProto, proto3 bool, allowOneOf bool) bool { if isRepeated(field) && - (*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) && + (*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE || gogoproto.IsCustomType(field)) && (*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) { return false } @@ -2046,10 +2056,6 @@ func (g *Generator) generateMessage(message *Descriptor) { oneofTypeName := make(map[*descriptor.FieldDescriptorProto]string) // without star oneofInsertPoints := make(map[int32]int) // oneof_index => offset of g.Buffer - g.PrintComments(message.path) - g.P("type ", ccTypeName, " struct {") - g.In() - // allocNames finds a conflict-free variation of the given strings, // consistently mutating their suffixes. // It returns the same number of strings. @@ -2071,7 +2077,7 @@ func (g *Generator) generateMessage(message *Descriptor) { } } - for i, field := range message.Field { + for _, field := range message.Field { // Allocate the getter and the field at the same time so name // collisions create field/method consistent names. // TODO: This allocation occurs based on the order of the fields @@ -2083,112 +2089,122 @@ func (g *Generator) generateMessage(message *Descriptor) { } ns := allocNames(base, "Get"+base) fieldName, fieldGetterName := ns[0], ns[1] - typename, wiretype := g.GoType(message, field) - jsonName := *field.Name - jsonTag := jsonName + ",omitempty" - repeatedNativeType := (!field.IsMessage() && !gogoproto.IsCustomType(field) && field.IsRepeated()) - if !gogoproto.IsNullable(field) && !repeatedNativeType { - jsonTag = jsonName - } - gogoJsonTag := gogoproto.GetJsonTag(field) - if gogoJsonTag != nil { - jsonTag = *gogoJsonTag - } - gogoMoreTags := gogoproto.GetMoreTags(field) - moreTags := "" - if gogoMoreTags != nil { - moreTags = " " + *gogoMoreTags - } - tag := fmt.Sprintf("protobuf:%s json:%q%s", g.goTag(message, field, wiretype), jsonTag, moreTags) fieldNames[field] = fieldName fieldGetterNames[field] = fieldGetterName - if *field.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE && gogoproto.IsEmbed(field) { - fieldName = "" - } + } - oneof := field.OneofIndex != nil && message.allowOneof() - if oneof && oneofFieldName[*field.OneofIndex] == "" { - odp := message.OneofDecl[int(*field.OneofIndex)] - fname := allocNames(CamelCase(odp.GetName()))[0] + if gogoproto.HasTypeDecl(message.file, message.DescriptorProto) { + g.PrintComments(message.path) + g.P("type ", ccTypeName, " struct {") + g.In() - // This is the first field of a oneof we haven't seen before. - // Generate the union field. - com := g.PrintComments(fmt.Sprintf("%s,%d,%d", message.path, messageOneofPath, *field.OneofIndex)) - if com { - g.P("//") + for i, field := range message.Field { + fieldName := fieldNames[field] + typename, wiretype := g.GoType(message, field) + jsonName := *field.Name + jsonTag := jsonName + ",omitempty" + repeatedNativeType := (!field.IsMessage() && !gogoproto.IsCustomType(field) && field.IsRepeated()) + if !gogoproto.IsNullable(field) && !repeatedNativeType { + jsonTag = jsonName } - g.P("// Types that are valid to be assigned to ", fname, ":") - // Generate the rest of this comment later, - // when we've computed any disambiguation. - oneofInsertPoints[*field.OneofIndex] = g.Buffer.Len() - - dname := "is" + ccTypeName + "_" + fname - oneofFieldName[*field.OneofIndex] = fname - oneofDisc[*field.OneofIndex] = dname - otag := `protobuf_oneof:"` + odp.GetName() + `"` - g.P(fname, " ", dname, " `", otag, "`") - } - - if *field.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE { - desc := g.ObjectNamed(field.GetTypeName()) - if d, ok := desc.(*Descriptor); ok && d.GetOptions().GetMapEntry() { - m := g.GoMapType(d, field) - typename = m.GoType - mapFieldTypes[field] = typename // record for the getter generation - - tag += fmt.Sprintf(" protobuf_key:%s protobuf_val:%s", m.KeyTag, m.ValueTag) + gogoJsonTag := gogoproto.GetJsonTag(field) + if gogoJsonTag != nil { + jsonTag = *gogoJsonTag + } + gogoMoreTags := gogoproto.GetMoreTags(field) + moreTags := "" + if gogoMoreTags != nil { + moreTags = " " + *gogoMoreTags + } + tag := fmt.Sprintf("protobuf:%s json:%q%s", g.goTag(message, field, wiretype), jsonTag, moreTags) + if *field.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE && gogoproto.IsEmbed(field) { + fieldName = "" } - } - fieldTypes[field] = typename + oneof := field.OneofIndex != nil && message.allowOneof() + if oneof && oneofFieldName[*field.OneofIndex] == "" { + odp := message.OneofDecl[int(*field.OneofIndex)] + fname := allocNames(CamelCase(odp.GetName()))[0] - if oneof { - tname := ccTypeName + "_" + fieldName - // It is possible for this to collide with a message or enum - // nested in this message. Check for collisions. - for { - ok := true - for _, desc := range message.nested { - if CamelCaseSlice(desc.TypeName()) == tname { - ok = false - break + // This is the first field of a oneof we haven't seen before. + // Generate the union field. + com := g.PrintComments(fmt.Sprintf("%s,%d,%d", message.path, messageOneofPath, *field.OneofIndex)) + if com { + g.P("//") + } + g.P("// Types that are valid to be assigned to ", fname, ":") + // Generate the rest of this comment later, + // when we've computed any disambiguation. + oneofInsertPoints[*field.OneofIndex] = g.Buffer.Len() + + dname := "is" + ccTypeName + "_" + fname + oneofFieldName[*field.OneofIndex] = fname + oneofDisc[*field.OneofIndex] = dname + otag := `protobuf_oneof:"` + odp.GetName() + `"` + g.P(fname, " ", dname, " `", otag, "`") + } + + if *field.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE { + desc := g.ObjectNamed(field.GetTypeName()) + if d, ok := desc.(*Descriptor); ok && d.GetOptions().GetMapEntry() { + m := g.GoMapType(d, field) + typename = m.GoType + mapFieldTypes[field] = typename // record for the getter generation + + tag += fmt.Sprintf(" protobuf_key:%s protobuf_val:%s", m.KeyTag, m.ValueTag) + } + } + + fieldTypes[field] = typename + + if oneof { + tname := ccTypeName + "_" + fieldName + // It is possible for this to collide with a message or enum + // nested in this message. Check for collisions. + for { + ok := true + for _, desc := range message.nested { + if CamelCaseSlice(desc.TypeName()) == tname { + ok = false + break + } } - } - for _, enum := range message.enums { - if CamelCaseSlice(enum.TypeName()) == tname { - ok = false - break + for _, enum := range message.enums { + if CamelCaseSlice(enum.TypeName()) == tname { + ok = false + break + } } + if !ok { + tname += "_" + continue + } + break } - if !ok { - tname += "_" - continue - } - break + + oneofTypeName[field] = tname + continue } - oneofTypeName[field] = tname - continue + g.PrintComments(fmt.Sprintf("%s,%d,%d", message.path, messageFieldPath, i)) + g.P(fieldName, "\t", typename, "\t`", tag, "`") + if !gogoproto.IsStdTime(field) && !gogoproto.IsStdDuration(field) { + g.RecordTypeUse(field.GetTypeName()) + } } - - g.PrintComments(fmt.Sprintf("%s,%d,%d", message.path, messageFieldPath, i)) - g.P(fieldName, "\t", typename, "\t`", tag, "`") - if !gogoproto.IsStdTime(field) && !gogoproto.IsStdDuration(field) { - g.RecordTypeUse(field.GetTypeName()) + if len(message.ExtensionRange) > 0 { + if gogoproto.HasExtensionsMap(g.file.FileDescriptorProto, message.DescriptorProto) { + g.P(g.Pkg["proto"], ".XXX_InternalExtensions `json:\"-\"`") + } else { + g.P("XXX_extensions\t\t[]byte `protobuf:\"bytes,0,opt\" json:\"-\"`") + } } - } - if len(message.ExtensionRange) > 0 { - if gogoproto.HasExtensionsMap(g.file.FileDescriptorProto, message.DescriptorProto) { - g.P(g.Pkg["proto"], ".XXX_InternalExtensions `json:\"-\"`") - } else { - g.P("XXX_extensions\t\t[]byte `protobuf:\"bytes,0,opt\" json:\"-\"`") + if gogoproto.HasUnrecognized(g.file.FileDescriptorProto, message.DescriptorProto) && !message.proto3() { + g.P("XXX_unrecognized\t[]byte `json:\"-\"`") } + g.Out() + g.P("}") } - if gogoproto.HasUnrecognized(g.file.FileDescriptorProto, message.DescriptorProto) && !message.proto3() { - g.P("XXX_unrecognized\t[]byte `json:\"-\"`") - } - g.Out() - g.P("}") // Update g.Buffer to list valid oneof types. // We do this down here, after we've disambiguated the oneof type names. diff --git a/vendor/github.com/gogo/protobuf/types/any.pb.go b/vendor/github.com/gogo/protobuf/types/any.pb.go index ebd976e..81a8202 100644 --- a/vendor/github.com/gogo/protobuf/types/any.pb.go +++ b/vendor/github.com/gogo/protobuf/types/any.pb.go @@ -20,9 +20,6 @@ import math "math" import bytes "bytes" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" -import strconv "strconv" import reflect "reflect" import io "io" @@ -246,24 +243,6 @@ func valueToGoStringAny(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringAny(m github_com_gogo_protobuf_proto.Message) string { - e := github_com_gogo_protobuf_proto.GetUnsafeExtensionsMap(m) - if e == nil { - return "nil" - } - s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "})" - return s -} func (m *Any) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) diff --git a/vendor/github.com/gogo/protobuf/types/duration.pb.go b/vendor/github.com/gogo/protobuf/types/duration.pb.go index dcaaa64..3b17d51 100644 --- a/vendor/github.com/gogo/protobuf/types/duration.pb.go +++ b/vendor/github.com/gogo/protobuf/types/duration.pb.go @@ -18,9 +18,6 @@ import fmt "fmt" import math "math" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" -import strconv "strconv" import reflect "reflect" import io "io" @@ -204,24 +201,6 @@ func valueToGoStringDuration(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringDuration(m github_com_gogo_protobuf_proto.Message) string { - e := github_com_gogo_protobuf_proto.GetUnsafeExtensionsMap(m) - if e == nil { - return "nil" - } - s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "})" - return s -} func (m *Duration) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) diff --git a/vendor/github.com/gogo/protobuf/types/empty.pb.go b/vendor/github.com/gogo/protobuf/types/empty.pb.go index c94f81f..58b9154 100644 --- a/vendor/github.com/gogo/protobuf/types/empty.pb.go +++ b/vendor/github.com/gogo/protobuf/types/empty.pb.go @@ -18,9 +18,6 @@ import fmt "fmt" import math "math" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" -import strconv "strconv" import reflect "reflect" import io "io" @@ -127,24 +124,6 @@ func valueToGoStringEmpty(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringEmpty(m github_com_gogo_protobuf_proto.Message) string { - e := github_com_gogo_protobuf_proto.GetUnsafeExtensionsMap(m) - if e == nil { - return "nil" - } - s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "})" - return s -} func (m *Empty) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) diff --git a/vendor/github.com/gogo/protobuf/types/field_mask.pb.go b/vendor/github.com/gogo/protobuf/types/field_mask.pb.go index d5f19e3..d22b87c 100644 --- a/vendor/github.com/gogo/protobuf/types/field_mask.pb.go +++ b/vendor/github.com/gogo/protobuf/types/field_mask.pb.go @@ -18,9 +18,6 @@ import fmt "fmt" import math "math" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" -import strconv "strconv" import reflect "reflect" import io "io" @@ -350,24 +347,6 @@ func valueToGoStringFieldMask(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringFieldMask(m github_com_gogo_protobuf_proto.Message) string { - e := github_com_gogo_protobuf_proto.GetUnsafeExtensionsMap(m) - if e == nil { - return "nil" - } - s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "})" - return s -} func (m *FieldMask) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) diff --git a/vendor/github.com/gogo/protobuf/types/struct.pb.go b/vendor/github.com/gogo/protobuf/types/struct.pb.go index 61acd4a..b1dc59f 100644 --- a/vendor/github.com/gogo/protobuf/types/struct.pb.go +++ b/vendor/github.com/gogo/protobuf/types/struct.pb.go @@ -22,8 +22,6 @@ import math "math" import strconv "strconv" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" import reflect "reflect" import github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" @@ -748,24 +746,6 @@ func valueToGoStringStruct(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringStruct(m github_com_gogo_protobuf_proto.Message) string { - e := github_com_gogo_protobuf_proto.GetUnsafeExtensionsMap(m) - if e == nil { - return "nil" - } - s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "})" - return s -} func (m *Struct) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) diff --git a/vendor/github.com/gogo/protobuf/types/timestamp.pb.go b/vendor/github.com/gogo/protobuf/types/timestamp.pb.go index 4b0c34f..cbe8a03 100644 --- a/vendor/github.com/gogo/protobuf/types/timestamp.pb.go +++ b/vendor/github.com/gogo/protobuf/types/timestamp.pb.go @@ -18,9 +18,6 @@ import fmt "fmt" import math "math" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" -import strconv "strconv" import reflect "reflect" import io "io" @@ -216,24 +213,6 @@ func valueToGoStringTimestamp(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringTimestamp(m github_com_gogo_protobuf_proto.Message) string { - e := github_com_gogo_protobuf_proto.GetUnsafeExtensionsMap(m) - if e == nil { - return "nil" - } - s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "})" - return s -} func (m *Timestamp) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) diff --git a/vendor/github.com/gogo/protobuf/types/wrappers.pb.go b/vendor/github.com/gogo/protobuf/types/wrappers.pb.go index bcd011d..6c00812 100644 --- a/vendor/github.com/gogo/protobuf/types/wrappers.pb.go +++ b/vendor/github.com/gogo/protobuf/types/wrappers.pb.go @@ -28,9 +28,6 @@ import math "math" import bytes "bytes" import strings "strings" -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -import sort "sort" -import strconv "strconv" import reflect "reflect" import io "io" @@ -899,24 +896,6 @@ func valueToGoStringWrappers(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func extensionToGoStringWrappers(m github_com_gogo_protobuf_proto.Message) string { - e := github_com_gogo_protobuf_proto.GetUnsafeExtensionsMap(m) - if e == nil { - return "nil" - } - s := "proto.NewUnsafeXXX_InternalExtensions(map[int32]proto.Extension{" - keys := make([]int, 0, len(e)) - for k := range e { - keys = append(keys, int(k)) - } - sort.Ints(keys) - ss := []string{} - for _, k := range keys { - ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) - } - s += strings.Join(ss, ",") + "})" - return s -} func (m *DoubleValue) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) From baaf7543dc07ccab7036623ffbd063bcbd0f4cce Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 13 Feb 2017 18:44:01 -0800 Subject: [PATCH 2/4] api/services/content: define the content service Bring the content service into the containerd API. This allows the content store to be coordinated in the containerd daemon with minimal effort. For the most part, this API follows the conventions and behavior of the existing content store implementation with a few caveats. Specifically, we remove the object oriented transaction mechanism in favor of a very rich `Write` call. Pains are taken to reduce race conditions around when having multiple writers to a single piece of content. Clients should be able to race towards getting a write lock on a reference, then wait on each other. For the most part, this should be generically pluggable to allow implementations of the content store to be swapped out. We'll follow this up with an implementation to validate the model. Signed-off-by: Stephen J Day --- api/services/content/content.pb.go | 2390 ++++++++++++++++++++++++++++ api/services/content/content.proto | 201 +++ 2 files changed, 2591 insertions(+) create mode 100644 api/services/content/content.pb.go create mode 100644 api/services/content/content.proto diff --git a/api/services/content/content.pb.go b/api/services/content/content.pb.go new file mode 100644 index 0000000..ddb5dc7 --- /dev/null +++ b/api/services/content/content.pb.go @@ -0,0 +1,2390 @@ +// Code generated by protoc-gen-gogo. +// source: github.com/docker/containerd/api/services/content/content.proto +// DO NOT EDIT! + +/* + Package content is a generated protocol buffer package. + + It is generated from these files: + github.com/docker/containerd/api/services/content/content.proto + + It has these top-level messages: + InfoRequest + InfoResponse + ReadRequest + ReadResponse + WriteRequest + WriteResponse + StatusRequest + StatusResponse +*/ +package content + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" +import _ "github.com/gogo/protobuf/types" + +import github_com_opencontainers_go_digest "github.com/opencontainers/go-digest" +import time "time" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" + +import strings "strings" +import reflect "reflect" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf +var _ = time.Kitchen + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +// WriteAction defines the behavior of a WriteRequest. +type WriteAction int32 + +const ( + // WriteActionStat instructs the writer to return the current status while + // holding the lock on the write. + WriteActionStat WriteAction = 0 + // WriteActionWrite sets the action for the write request to write data. + // + // Any data included will be written at the provided offset. The + // transaction will be left open for further writes. + // + // This is the default. + WriteActionWrite WriteAction = 1 + // WriteActionCommit will write any outstanding data in the message and + // commit the write, storing it under the digest. + // + // This can be used in a single message to send the data, verify it and + // commit it. + // + // This action will always terminate the write. + WriteActionCommit WriteAction = 2 + // WriteActionAbort will release any resources associated with the write + // and free up the ref for a completely new set of writes. + // + // This action will always terminate the write. + WriteActionAbort WriteAction = -1 +) + +var WriteAction_name = map[int32]string{ + 0: "STAT", + 1: "WRITE", + 2: "COMMIT", + -1: "ABORT", +} +var WriteAction_value = map[string]int32{ + "STAT": 0, + "WRITE": 1, + "COMMIT": 2, + "ABORT": -1, +} + +func (x WriteAction) String() string { + return proto.EnumName(WriteAction_name, int32(x)) +} +func (WriteAction) EnumDescriptor() ([]byte, []int) { return fileDescriptorContent, []int{0} } + +type InfoRequest struct { + Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,1,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"` +} + +func (m *InfoRequest) Reset() { *m = InfoRequest{} } +func (*InfoRequest) ProtoMessage() {} +func (*InfoRequest) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{0} } + +type InfoResponse struct { + // Digest is the hash identity of the blob. + Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,1,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"` + // Size is the total number of bytes in the blob. + Size_ int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + // CommittedAt provides the time at which the blob was committed. + CommittedAt time.Time `protobuf:"bytes,3,opt,name=committed_at,json=committedAt,stdtime" json:"committed_at"` +} + +func (m *InfoResponse) Reset() { *m = InfoResponse{} } +func (*InfoResponse) ProtoMessage() {} +func (*InfoResponse) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{1} } + +// ReadRequest defines the fields that make up a request to read a portion of +// data from a stored object. +type ReadRequest struct { + // Digest is the hash identity to read. + Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,1,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"` + // Offset specifies the number of bytes from the start at which to begin + // the read. If zero or less, the read will be from the start. This uses + // standard zero-indexed semantics. + Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` + // size is the total size of the read. If zero, the entire blob will be + // returned by the service. + Size_ int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` +} + +func (m *ReadRequest) Reset() { *m = ReadRequest{} } +func (*ReadRequest) ProtoMessage() {} +func (*ReadRequest) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{2} } + +// ReadResponse carries byte data for a read request. +type ReadResponse struct { + Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (m *ReadResponse) Reset() { *m = ReadResponse{} } +func (*ReadResponse) ProtoMessage() {} +func (*ReadResponse) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{3} } + +// WriteRequest writes data to the request ref at offset. +type WriteRequest struct { + // Action sets the behavior of the write. + // + // When this is a write and the ref is not yet allocated, the ref will be + // allocated and the data will be written at offset. + // + // If the action is write and the ref is allocated, it will accept data to + // an offset that has not yet been written. + // + // If the action is write and there is no data, the current write status + // will be returned. This works differently from status because the stream + // holds a lock. + Action WriteAction `protobuf:"varint,1,opt,name=action,proto3,enum=containerd.v1.WriteAction" json:"action,omitempty"` + // Ref identifies the pre-commit object to write to. + Ref string `protobuf:"bytes,2,opt,name=ref,proto3" json:"ref,omitempty"` + // ExpectedSize can be set to have the service validate the total size of + // the of committed content. + // + // The latest value before or with the commit action message will be use to + // validate the content. It is only required on one message for the write. + // + // If the value is zero or less, no validation of the final content will be + // performed. + ExpectedSize int64 `protobuf:"varint,3,opt,name=expected_size,json=expectedSize,proto3" json:"expected_size,omitempty"` + // ExpectedDigest can be set to have the service validate the final content + // against the provided digest. + // + // If the digest is already present in the object store, an AlreadyPresent + // error will be returned. + // + // Only the latest version will be used to check the content against the + // digest. It is only required to include it on a single message, before or + // with the commit action message. + ExpectedDigest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,4,opt,name=expected_digest,json=expectedDigest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"expected_digest"` + // Offset specifies the number of bytes from the start at which to begin + // the write. If zero or less, the write will be from the start. This uses + // standard zero-indexed semantics. + Offset int64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` + // Data is the actual bytes to be written. + // + // If this is empty and the message is not a commit, a response will be + // returned with the current write state. + Data []byte `protobuf:"bytes,6,opt,name=data,proto3" json:"data,omitempty"` +} + +func (m *WriteRequest) Reset() { *m = WriteRequest{} } +func (*WriteRequest) ProtoMessage() {} +func (*WriteRequest) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{4} } + +// WriteResponse is returned on the culmination of a write call. +type WriteResponse struct { + // Action contains the action for the final message of the stream. A writer + // should confirm that they match the intended result. + Action WriteAction `protobuf:"varint,1,opt,name=action,proto3,enum=containerd.v1.WriteAction" json:"action,omitempty"` + // Offset provides the current "committed" size for the Write. + Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` + // Digest, if present, includes the digest up to the currently committed + // bytes. If action is commit, this field will be set. It is implementation + // defined if this is set for other actions, except abort. On abort, this + // will be empty. + Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,3,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"` + // StartedAt is the time at which the write first started. + StartedAt time.Time `protobuf:"bytes,4,opt,name=started_at,json=startedAt,stdtime" json:"started_at"` + // UpdatedAt is the time the write was last updated. + UpdatedAt time.Time `protobuf:"bytes,5,opt,name=updated_at,json=updatedAt,stdtime" json:"updated_at"` +} + +func (m *WriteResponse) Reset() { *m = WriteResponse{} } +func (*WriteResponse) ProtoMessage() {} +func (*WriteResponse) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{5} } + +type StatusRequest struct { + Refs []string `protobuf:"bytes,1,rep,name=refs" json:"refs,omitempty"` + Prefix []string `protobuf:"bytes,2,rep,name=prefix" json:"prefix,omitempty"` +} + +func (m *StatusRequest) Reset() { *m = StatusRequest{} } +func (*StatusRequest) ProtoMessage() {} +func (*StatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{6} } + +type StatusResponse struct { + Ref string `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` + StartedAt time.Time `protobuf:"bytes,3,opt,name=started_at,json=startedAt,stdtime" json:"started_at"` + UpdatedAt time.Time `protobuf:"bytes,4,opt,name=updated_at,json=updatedAt,stdtime" json:"updated_at"` +} + +func (m *StatusResponse) Reset() { *m = StatusResponse{} } +func (*StatusResponse) ProtoMessage() {} +func (*StatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorContent, []int{7} } + +func init() { + proto.RegisterType((*InfoRequest)(nil), "containerd.v1.InfoRequest") + proto.RegisterType((*InfoResponse)(nil), "containerd.v1.InfoResponse") + proto.RegisterType((*ReadRequest)(nil), "containerd.v1.ReadRequest") + proto.RegisterType((*ReadResponse)(nil), "containerd.v1.ReadResponse") + proto.RegisterType((*WriteRequest)(nil), "containerd.v1.WriteRequest") + proto.RegisterType((*WriteResponse)(nil), "containerd.v1.WriteResponse") + proto.RegisterType((*StatusRequest)(nil), "containerd.v1.StatusRequest") + proto.RegisterType((*StatusResponse)(nil), "containerd.v1.StatusResponse") + proto.RegisterEnum("containerd.v1.WriteAction", WriteAction_name, WriteAction_value) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Content service + +type ContentClient interface { + // Info returns information about a committed object. + // + // This call can be used for getting the size of content and checking for + // existence. + Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) + // Read allows one to read an object based on the offset into the content. + // + // The requested data may be returned in one or more messages. + Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (Content_ReadClient, error) + // Status returns the status of ongoing object ingestions, started via + // Write. + // + // For active ingestions, the status will be streamed until the client + // closes the connection or all matched ingestions are committed. + Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (Content_StatusClient, error) + // Write begins or resumes writes to a resource identified by a unique ref. + // Only one active stream may exist at a time for each ref. + // + // Once a write stream has started, it may only write to a single ref, thus + // once a stream is started, the ref may be ommitted on subsequent writes. + // + // For any write transaction represented by a ref, only a single write may + // be made to a given offset. If overlapping writes occur, it is an error. + // Writes should be sequential and implementations may throw an error if + // this is required. + // + // If expected_digest is set and already part of the content store, the + // write will fail. + // + // When completed, the commit flag should be set to true. If expected size + // or digest is set, the content will be validated against those values. + Write(ctx context.Context, opts ...grpc.CallOption) (Content_WriteClient, error) +} + +type contentClient struct { + cc *grpc.ClientConn +} + +func NewContentClient(cc *grpc.ClientConn) ContentClient { + return &contentClient{cc} +} + +func (c *contentClient) Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) { + out := new(InfoResponse) + err := grpc.Invoke(ctx, "/containerd.v1.Content/Info", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *contentClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (Content_ReadClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Content_serviceDesc.Streams[0], c.cc, "/containerd.v1.Content/Read", opts...) + if err != nil { + return nil, err + } + x := &contentReadClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Content_ReadClient interface { + Recv() (*ReadResponse, error) + grpc.ClientStream +} + +type contentReadClient struct { + grpc.ClientStream +} + +func (x *contentReadClient) Recv() (*ReadResponse, error) { + m := new(ReadResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *contentClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (Content_StatusClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Content_serviceDesc.Streams[1], c.cc, "/containerd.v1.Content/Status", opts...) + if err != nil { + return nil, err + } + x := &contentStatusClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Content_StatusClient interface { + Recv() (*StatusResponse, error) + grpc.ClientStream +} + +type contentStatusClient struct { + grpc.ClientStream +} + +func (x *contentStatusClient) Recv() (*StatusResponse, error) { + m := new(StatusResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *contentClient) Write(ctx context.Context, opts ...grpc.CallOption) (Content_WriteClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Content_serviceDesc.Streams[2], c.cc, "/containerd.v1.Content/Write", opts...) + if err != nil { + return nil, err + } + x := &contentWriteClient{stream} + return x, nil +} + +type Content_WriteClient interface { + Send(*WriteRequest) error + Recv() (*WriteResponse, error) + grpc.ClientStream +} + +type contentWriteClient struct { + grpc.ClientStream +} + +func (x *contentWriteClient) Send(m *WriteRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *contentWriteClient) Recv() (*WriteResponse, error) { + m := new(WriteResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// Server API for Content service + +type ContentServer interface { + // Info returns information about a committed object. + // + // This call can be used for getting the size of content and checking for + // existence. + Info(context.Context, *InfoRequest) (*InfoResponse, error) + // Read allows one to read an object based on the offset into the content. + // + // The requested data may be returned in one or more messages. + Read(*ReadRequest, Content_ReadServer) error + // Status returns the status of ongoing object ingestions, started via + // Write. + // + // For active ingestions, the status will be streamed until the client + // closes the connection or all matched ingestions are committed. + Status(*StatusRequest, Content_StatusServer) error + // Write begins or resumes writes to a resource identified by a unique ref. + // Only one active stream may exist at a time for each ref. + // + // Once a write stream has started, it may only write to a single ref, thus + // once a stream is started, the ref may be ommitted on subsequent writes. + // + // For any write transaction represented by a ref, only a single write may + // be made to a given offset. If overlapping writes occur, it is an error. + // Writes should be sequential and implementations may throw an error if + // this is required. + // + // If expected_digest is set and already part of the content store, the + // write will fail. + // + // When completed, the commit flag should be set to true. If expected size + // or digest is set, the content will be validated against those values. + Write(Content_WriteServer) error +} + +func RegisterContentServer(s *grpc.Server, srv ContentServer) { + s.RegisterService(&_Content_serviceDesc, srv) +} + +func _Content_Info_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(InfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ContentServer).Info(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/containerd.v1.Content/Info", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ContentServer).Info(ctx, req.(*InfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Content_Read_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ReadRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(ContentServer).Read(m, &contentReadServer{stream}) +} + +type Content_ReadServer interface { + Send(*ReadResponse) error + grpc.ServerStream +} + +type contentReadServer struct { + grpc.ServerStream +} + +func (x *contentReadServer) Send(m *ReadResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Content_Status_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StatusRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(ContentServer).Status(m, &contentStatusServer{stream}) +} + +type Content_StatusServer interface { + Send(*StatusResponse) error + grpc.ServerStream +} + +type contentStatusServer struct { + grpc.ServerStream +} + +func (x *contentStatusServer) Send(m *StatusResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Content_Write_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(ContentServer).Write(&contentWriteServer{stream}) +} + +type Content_WriteServer interface { + Send(*WriteResponse) error + Recv() (*WriteRequest, error) + grpc.ServerStream +} + +type contentWriteServer struct { + grpc.ServerStream +} + +func (x *contentWriteServer) Send(m *WriteResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *contentWriteServer) Recv() (*WriteRequest, error) { + m := new(WriteRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _Content_serviceDesc = grpc.ServiceDesc{ + ServiceName: "containerd.v1.Content", + HandlerType: (*ContentServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Info", + Handler: _Content_Info_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Read", + Handler: _Content_Read_Handler, + ServerStreams: true, + }, + { + StreamName: "Status", + Handler: _Content_Status_Handler, + ServerStreams: true, + }, + { + StreamName: "Write", + Handler: _Content_Write_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "github.com/docker/containerd/api/services/content/content.proto", +} + +func (m *InfoRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InfoRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Digest) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Digest))) + i += copy(dAtA[i:], m.Digest) + } + return i, nil +} + +func (m *InfoResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InfoResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Digest) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Digest))) + i += copy(dAtA[i:], m.Digest) + } + if m.Size_ != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Size_)) + } + dAtA[i] = 0x1a + i++ + i = encodeVarintContent(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.CommittedAt))) + n1, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.CommittedAt, dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + return i, nil +} + +func (m *ReadRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ReadRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Digest) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Digest))) + i += copy(dAtA[i:], m.Digest) + } + if m.Offset != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Offset)) + } + if m.Size_ != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Size_)) + } + return i, nil +} + +func (m *ReadResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ReadResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Offset != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Offset)) + } + if len(m.Data) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + return i, nil +} + +func (m *WriteRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *WriteRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Action != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Action)) + } + if len(m.Ref) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Ref))) + i += copy(dAtA[i:], m.Ref) + } + if m.ExpectedSize != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.ExpectedSize)) + } + if len(m.ExpectedDigest) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.ExpectedDigest))) + i += copy(dAtA[i:], m.ExpectedDigest) + } + if m.Offset != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Offset)) + } + if len(m.Data) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + return i, nil +} + +func (m *WriteResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *WriteResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Action != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Action)) + } + if m.Offset != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Offset)) + } + if len(m.Digest) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Digest))) + i += copy(dAtA[i:], m.Digest) + } + dAtA[i] = 0x22 + i++ + i = encodeVarintContent(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.StartedAt))) + n2, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.StartedAt, dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 + dAtA[i] = 0x2a + i++ + i = encodeVarintContent(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.UpdatedAt))) + n3, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.UpdatedAt, dAtA[i:]) + if err != nil { + return 0, err + } + i += n3 + return i, nil +} + +func (m *StatusRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StatusRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Refs) > 0 { + for _, s := range m.Refs { + dAtA[i] = 0xa + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.Prefix) > 0 { + for _, s := range m.Prefix { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func (m *StatusResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StatusResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Ref) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintContent(dAtA, i, uint64(len(m.Ref))) + i += copy(dAtA[i:], m.Ref) + } + if m.Offset != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintContent(dAtA, i, uint64(m.Offset)) + } + dAtA[i] = 0x1a + i++ + i = encodeVarintContent(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.StartedAt))) + n4, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.StartedAt, dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + dAtA[i] = 0x22 + i++ + i = encodeVarintContent(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.UpdatedAt))) + n5, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.UpdatedAt, dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + return i, nil +} + +func encodeFixed64Content(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Content(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintContent(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *InfoRequest) Size() (n int) { + var l int + _ = l + l = len(m.Digest) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + return n +} + +func (m *InfoResponse) Size() (n int) { + var l int + _ = l + l = len(m.Digest) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + if m.Size_ != 0 { + n += 1 + sovContent(uint64(m.Size_)) + } + l = github_com_gogo_protobuf_types.SizeOfStdTime(m.CommittedAt) + n += 1 + l + sovContent(uint64(l)) + return n +} + +func (m *ReadRequest) Size() (n int) { + var l int + _ = l + l = len(m.Digest) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + if m.Offset != 0 { + n += 1 + sovContent(uint64(m.Offset)) + } + if m.Size_ != 0 { + n += 1 + sovContent(uint64(m.Size_)) + } + return n +} + +func (m *ReadResponse) Size() (n int) { + var l int + _ = l + if m.Offset != 0 { + n += 1 + sovContent(uint64(m.Offset)) + } + l = len(m.Data) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + return n +} + +func (m *WriteRequest) Size() (n int) { + var l int + _ = l + if m.Action != 0 { + n += 1 + sovContent(uint64(m.Action)) + } + l = len(m.Ref) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + if m.ExpectedSize != 0 { + n += 1 + sovContent(uint64(m.ExpectedSize)) + } + l = len(m.ExpectedDigest) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + if m.Offset != 0 { + n += 1 + sovContent(uint64(m.Offset)) + } + l = len(m.Data) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + return n +} + +func (m *WriteResponse) Size() (n int) { + var l int + _ = l + if m.Action != 0 { + n += 1 + sovContent(uint64(m.Action)) + } + if m.Offset != 0 { + n += 1 + sovContent(uint64(m.Offset)) + } + l = len(m.Digest) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + l = github_com_gogo_protobuf_types.SizeOfStdTime(m.StartedAt) + n += 1 + l + sovContent(uint64(l)) + l = github_com_gogo_protobuf_types.SizeOfStdTime(m.UpdatedAt) + n += 1 + l + sovContent(uint64(l)) + return n +} + +func (m *StatusRequest) Size() (n int) { + var l int + _ = l + if len(m.Refs) > 0 { + for _, s := range m.Refs { + l = len(s) + n += 1 + l + sovContent(uint64(l)) + } + } + if len(m.Prefix) > 0 { + for _, s := range m.Prefix { + l = len(s) + n += 1 + l + sovContent(uint64(l)) + } + } + return n +} + +func (m *StatusResponse) Size() (n int) { + var l int + _ = l + l = len(m.Ref) + if l > 0 { + n += 1 + l + sovContent(uint64(l)) + } + if m.Offset != 0 { + n += 1 + sovContent(uint64(m.Offset)) + } + l = github_com_gogo_protobuf_types.SizeOfStdTime(m.StartedAt) + n += 1 + l + sovContent(uint64(l)) + l = github_com_gogo_protobuf_types.SizeOfStdTime(m.UpdatedAt) + n += 1 + l + sovContent(uint64(l)) + return n +} + +func sovContent(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozContent(x uint64) (n int) { + return sovContent(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *InfoRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&InfoRequest{`, + `Digest:` + fmt.Sprintf("%v", this.Digest) + `,`, + `}`, + }, "") + return s +} +func (this *InfoResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&InfoResponse{`, + `Digest:` + fmt.Sprintf("%v", this.Digest) + `,`, + `Size_:` + fmt.Sprintf("%v", this.Size_) + `,`, + `CommittedAt:` + strings.Replace(strings.Replace(this.CommittedAt.String(), "Timestamp", "google_protobuf1.Timestamp", 1), `&`, ``, 1) + `,`, + `}`, + }, "") + return s +} +func (this *ReadRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ReadRequest{`, + `Digest:` + fmt.Sprintf("%v", this.Digest) + `,`, + `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, + `Size_:` + fmt.Sprintf("%v", this.Size_) + `,`, + `}`, + }, "") + return s +} +func (this *ReadResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ReadResponse{`, + `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, + `Data:` + fmt.Sprintf("%v", this.Data) + `,`, + `}`, + }, "") + return s +} +func (this *WriteRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&WriteRequest{`, + `Action:` + fmt.Sprintf("%v", this.Action) + `,`, + `Ref:` + fmt.Sprintf("%v", this.Ref) + `,`, + `ExpectedSize:` + fmt.Sprintf("%v", this.ExpectedSize) + `,`, + `ExpectedDigest:` + fmt.Sprintf("%v", this.ExpectedDigest) + `,`, + `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, + `Data:` + fmt.Sprintf("%v", this.Data) + `,`, + `}`, + }, "") + return s +} +func (this *WriteResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&WriteResponse{`, + `Action:` + fmt.Sprintf("%v", this.Action) + `,`, + `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, + `Digest:` + fmt.Sprintf("%v", this.Digest) + `,`, + `StartedAt:` + strings.Replace(strings.Replace(this.StartedAt.String(), "Timestamp", "google_protobuf1.Timestamp", 1), `&`, ``, 1) + `,`, + `UpdatedAt:` + strings.Replace(strings.Replace(this.UpdatedAt.String(), "Timestamp", "google_protobuf1.Timestamp", 1), `&`, ``, 1) + `,`, + `}`, + }, "") + return s +} +func (this *StatusRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StatusRequest{`, + `Refs:` + fmt.Sprintf("%v", this.Refs) + `,`, + `Prefix:` + fmt.Sprintf("%v", this.Prefix) + `,`, + `}`, + }, "") + return s +} +func (this *StatusResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StatusResponse{`, + `Ref:` + fmt.Sprintf("%v", this.Ref) + `,`, + `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, + `StartedAt:` + strings.Replace(strings.Replace(this.StartedAt.String(), "Timestamp", "google_protobuf1.Timestamp", 1), `&`, ``, 1) + `,`, + `UpdatedAt:` + strings.Replace(strings.Replace(this.UpdatedAt.String(), "Timestamp", "google_protobuf1.Timestamp", 1), `&`, ``, 1) + `,`, + `}`, + }, "") + return s +} +func valueToStringContent(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *InfoRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Digest", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Digest = github_com_opencontainers_go_digest.Digest(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InfoResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Digest", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Digest = github_com_opencontainers_go_digest.Digest(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CommittedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.CommittedAt, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ReadRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ReadRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ReadRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Digest", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Digest = github_com_opencontainers_go_digest.Digest(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ReadResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ReadResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ReadResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *WriteRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: WriteRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: WriteRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Action", wireType) + } + m.Action = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Action |= (WriteAction(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ref", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ref = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExpectedSize", wireType) + } + m.ExpectedSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ExpectedSize |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ExpectedDigest", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ExpectedDigest = github_com_opencontainers_go_digest.Digest(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *WriteResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: WriteResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: WriteResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Action", wireType) + } + m.Action = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Action |= (WriteAction(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Digest", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Digest = github_com_opencontainers_go_digest.Digest(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StartedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.StartedAt, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.UpdatedAt, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StatusRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StatusRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StatusRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Refs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Refs = append(m.Refs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Prefix", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Prefix = append(m.Prefix, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StatusResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StatusResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ref", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ref = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StartedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.StartedAt, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.UpdatedAt, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipContent(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthContent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipContent(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowContent + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowContent + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowContent + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthContent + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowContent + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipContent(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthContent = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowContent = fmt.Errorf("proto: integer overflow") +) + +func init() { + proto.RegisterFile("github.com/docker/containerd/api/services/content/content.proto", fileDescriptorContent) +} + +var fileDescriptorContent = []byte{ + // 733 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x95, 0xb1, 0x6f, 0xd3, 0x4e, + 0x14, 0xc7, 0x73, 0x89, 0x93, 0xdf, 0x2f, 0x2f, 0x49, 0x1b, 0xae, 0x05, 0x45, 0x6e, 0xeb, 0x84, + 0xb0, 0x44, 0x95, 0xb0, 0x4b, 0xd8, 0x60, 0xa8, 0x9c, 0x14, 0xaa, 0x22, 0x55, 0x95, 0xdc, 0x48, + 0x15, 0x62, 0x40, 0x4e, 0x72, 0x31, 0x16, 0xc4, 0x67, 0xec, 0x4b, 0x55, 0x31, 0x21, 0x24, 0x24, + 0xd4, 0x89, 0x7f, 0xa0, 0x2c, 0xb0, 0x23, 0x26, 0x24, 0x66, 0x86, 0x8e, 0x8c, 0x88, 0xa1, 0xd0, + 0xfc, 0x23, 0x20, 0x9f, 0xcf, 0x89, 0x9b, 0xa6, 0x43, 0x4b, 0xc9, 0xe2, 0x67, 0xbf, 0xf7, 0xbe, + 0x79, 0xf7, 0xf1, 0x37, 0x2f, 0xb0, 0x6a, 0xd9, 0xec, 0xc9, 0xa0, 0xad, 0x76, 0x68, 0x5f, 0xeb, + 0xd2, 0xce, 0x53, 0xe2, 0x69, 0x1d, 0xea, 0x30, 0xd3, 0x76, 0x88, 0xd7, 0xd5, 0x4c, 0xd7, 0xd6, + 0x7c, 0xe2, 0xed, 0xda, 0x1d, 0xe2, 0xf3, 0xe7, 0xc4, 0x61, 0xd1, 0x55, 0x75, 0x3d, 0xca, 0x28, + 0x2e, 0x8c, 0xcb, 0xd5, 0xdd, 0x5b, 0xf2, 0xbc, 0x45, 0x2d, 0xca, 0x33, 0x5a, 0x10, 0x85, 0x45, + 0x72, 0xd9, 0xa2, 0xd4, 0x7a, 0x46, 0x34, 0x7e, 0xd7, 0x1e, 0xf4, 0x34, 0x66, 0xf7, 0x89, 0xcf, + 0xcc, 0xbe, 0x1b, 0x16, 0x54, 0x1f, 0x42, 0x6e, 0xc3, 0xe9, 0x51, 0x83, 0x3c, 0x1f, 0x10, 0x9f, + 0xe1, 0x07, 0x90, 0xe9, 0xda, 0x16, 0xf1, 0x59, 0x09, 0x55, 0x50, 0x2d, 0xdb, 0xa8, 0x1f, 0x1e, + 0x95, 0x13, 0x3f, 0x8e, 0xca, 0xcb, 0xb1, 0x69, 0xa9, 0x4b, 0x9c, 0xd1, 0x77, 0xfb, 0x9a, 0x45, + 0x6f, 0x86, 0x2d, 0xea, 0x1a, 0xbf, 0x18, 0x42, 0xa1, 0xfa, 0x19, 0x41, 0x3e, 0xd4, 0xf6, 0x5d, + 0xea, 0xf8, 0xe4, 0x32, 0xc5, 0x31, 0x06, 0xc9, 0xb7, 0x5f, 0x90, 0x52, 0xb2, 0x82, 0x6a, 0x29, + 0x83, 0xc7, 0x78, 0x1d, 0xf2, 0x1d, 0xda, 0xef, 0xdb, 0x8c, 0x91, 0xee, 0x63, 0x93, 0x95, 0x52, + 0x15, 0x54, 0xcb, 0xd5, 0x65, 0x35, 0x64, 0xa0, 0x46, 0x0c, 0xd4, 0x56, 0xc4, 0xa0, 0xf1, 0x7f, + 0x30, 0xc1, 0xdb, 0x9f, 0x65, 0x64, 0xe4, 0x46, 0x9d, 0x3a, 0xab, 0xbe, 0x46, 0x90, 0x33, 0x88, + 0xd9, 0xfd, 0x07, 0x54, 0xf0, 0x35, 0xc8, 0xd0, 0x5e, 0xcf, 0x27, 0x4c, 0x8c, 0x2e, 0xee, 0x46, + 0x07, 0x4a, 0x8d, 0x0f, 0x54, 0xbd, 0x03, 0xf9, 0x70, 0x0c, 0x01, 0x70, 0xdc, 0x8b, 0x26, 0x7b, + 0xbb, 0x26, 0x33, 0xb9, 0x62, 0xde, 0xe0, 0x71, 0xf5, 0x55, 0x12, 0xf2, 0x3b, 0x9e, 0xcd, 0x48, + 0x74, 0x88, 0x3a, 0x64, 0xcc, 0x0e, 0xb3, 0xa9, 0xc3, 0x9b, 0x67, 0xea, 0xb2, 0x7a, 0xc2, 0x40, + 0x2a, 0x2f, 0xd6, 0x79, 0x85, 0x21, 0x2a, 0x71, 0x11, 0x52, 0x1e, 0xe9, 0x71, 0xdd, 0xac, 0x11, + 0x84, 0xf8, 0x06, 0x14, 0xc8, 0x9e, 0x4b, 0x3a, 0x01, 0xe2, 0xd8, 0xbc, 0xf9, 0xe8, 0xe1, 0x76, + 0xf0, 0x22, 0x1e, 0xc1, 0xec, 0xa8, 0x48, 0x80, 0x93, 0x2e, 0x0c, 0x6e, 0x26, 0x92, 0x5a, 0x9b, + 0x04, 0x98, 0x9e, 0x0a, 0x21, 0x13, 0x83, 0xf0, 0x29, 0x09, 0x05, 0x01, 0x41, 0x20, 0xbc, 0x08, + 0x85, 0xb3, 0x5e, 0xd9, 0xd8, 0x16, 0xa9, 0xbf, 0xb6, 0x45, 0x13, 0xc0, 0x67, 0xa6, 0x27, 0x9c, + 0x2b, 0x9d, 0xc3, 0xb9, 0x59, 0xd1, 0xa7, 0x73, 0x91, 0x81, 0xdb, 0x35, 0x85, 0x48, 0xfa, 0x3c, + 0x22, 0xa2, 0x4f, 0x67, 0xd5, 0xbb, 0x50, 0xd8, 0x66, 0x26, 0x1b, 0xf8, 0x91, 0x71, 0x30, 0x48, + 0x1e, 0xe9, 0xf9, 0x25, 0x54, 0x49, 0xd5, 0xb2, 0x06, 0x8f, 0x03, 0x24, 0xae, 0x47, 0x7a, 0xf6, + 0x5e, 0x29, 0xc9, 0x9f, 0x8a, 0xbb, 0xea, 0x57, 0x04, 0x33, 0x51, 0xb7, 0x20, 0x2e, 0x3c, 0x84, + 0xc6, 0x1e, 0x3a, 0x8b, 0xe7, 0x49, 0x06, 0xa9, 0xcb, 0x60, 0x20, 0x5d, 0x88, 0xc1, 0xf2, 0x47, + 0x04, 0xb9, 0x98, 0x13, 0xf0, 0x12, 0x48, 0xdb, 0x2d, 0xbd, 0x55, 0x4c, 0xc8, 0x73, 0xfb, 0x07, + 0x95, 0xd9, 0x58, 0x2a, 0x38, 0x2c, 0x2e, 0x43, 0x7a, 0xc7, 0xd8, 0x68, 0xdd, 0x2b, 0x22, 0x79, + 0x7e, 0xff, 0xa0, 0x52, 0x8c, 0xe5, 0x79, 0x88, 0xaf, 0x43, 0xa6, 0xb9, 0xb5, 0xb9, 0xb9, 0xd1, + 0x2a, 0x26, 0xe5, 0xab, 0xfb, 0x07, 0x95, 0x2b, 0xb1, 0x8a, 0x26, 0x5f, 0x3c, 0xb8, 0x06, 0x69, + 0xbd, 0xb1, 0x65, 0xb4, 0x8a, 0xbf, 0xa3, 0xcf, 0x69, 0x31, 0xbd, 0x4d, 0x3d, 0x26, 0xcf, 0xbd, + 0x79, 0xaf, 0x24, 0xbe, 0x7c, 0x50, 0xe2, 0x13, 0xd6, 0xdf, 0x25, 0xe1, 0xbf, 0x66, 0xf8, 0xff, + 0x80, 0x57, 0x41, 0x0a, 0xf6, 0x2e, 0x9e, 0xf4, 0x76, 0x6c, 0xd1, 0xcb, 0x0b, 0x53, 0x73, 0xe2, + 0x95, 0xe9, 0x20, 0x05, 0x7b, 0xe7, 0x94, 0x40, 0x6c, 0x27, 0x9e, 0x12, 0x88, 0x2f, 0xaa, 0x15, + 0x84, 0xd7, 0x21, 0x13, 0xfa, 0x00, 0x2f, 0x4e, 0x14, 0x9e, 0x30, 0x97, 0xbc, 0x74, 0x46, 0x76, + 0x24, 0x74, 0x1f, 0xd2, 0x21, 0xc3, 0x85, 0x69, 0xbf, 0xd4, 0x48, 0x66, 0x71, 0x7a, 0x32, 0x54, + 0xa9, 0xa1, 0x15, 0xd4, 0x28, 0x1d, 0x1e, 0x2b, 0x89, 0xef, 0xc7, 0x4a, 0xe2, 0xe5, 0x50, 0x41, + 0x87, 0x43, 0x05, 0x7d, 0x1b, 0x2a, 0xe8, 0xd7, 0x50, 0x41, 0xed, 0x0c, 0x77, 0xc5, 0xed, 0x3f, + 0x01, 0x00, 0x00, 0xff, 0xff, 0x6f, 0xe6, 0x59, 0x92, 0x92, 0x07, 0x00, 0x00, +} diff --git a/api/services/content/content.proto b/api/services/content/content.proto new file mode 100644 index 0000000..d6099e4 --- /dev/null +++ b/api/services/content/content.proto @@ -0,0 +1,201 @@ +syntax = "proto3"; + +package containerd.v1; + +import "gogoproto/gogo.proto"; +import "google/protobuf/timestamp.proto"; + +// Content provides access to a content addressable storage system. +service Content { + // Info returns information about a committed object. + // + // This call can be used for getting the size of content and checking for + // existence. + rpc Info(InfoRequest) returns (InfoResponse); + + // Read allows one to read an object based on the offset into the content. + // + // The requested data may be returned in one or more messages. + rpc Read(ReadRequest) returns (stream ReadResponse); + + // Status returns the status of ongoing object ingestions, started via + // Write. + // + // For active ingestions, the status will be streamed until the client + // closes the connection or all matched ingestions are committed. + rpc Status(StatusRequest) returns (stream StatusResponse); + + // Write begins or resumes writes to a resource identified by a unique ref. + // Only one active stream may exist at a time for each ref. + // + // Once a write stream has started, it may only write to a single ref, thus + // once a stream is started, the ref may be ommitted on subsequent writes. + // + // For any write transaction represented by a ref, only a single write may + // be made to a given offset. If overlapping writes occur, it is an error. + // Writes should be sequential and implementations may throw an error if + // this is required. + // + // If expected_digest is set and already part of the content store, the + // write will fail. + // + // When completed, the commit flag should be set to true. If expected size + // or digest is set, the content will be validated against those values. + rpc Write(stream WriteRequest) returns (stream WriteResponse); +} + +message InfoRequest { + string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; +} + +message InfoResponse { + // Digest is the hash identity of the blob. + string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; + + // Size is the total number of bytes in the blob. + int64 size = 2; + + // CommittedAt provides the time at which the blob was committed. + google.protobuf.Timestamp committed_at = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; +} + +// ReadRequest defines the fields that make up a request to read a portion of +// data from a stored object. +message ReadRequest { + // Digest is the hash identity to read. + string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; + + // Offset specifies the number of bytes from the start at which to begin + // the read. If zero or less, the read will be from the start. This uses + // standard zero-indexed semantics. + int64 offset = 2; + + // size is the total size of the read. If zero, the entire blob will be + // returned by the service. + int64 size = 3; +} + +// ReadResponse carries byte data for a read request. +message ReadResponse { + int64 offset = 1; // offset of the returned data + bytes data = 2; // actual data +} + + +// WriteAction defines the behavior of a WriteRequest. +enum WriteAction { + option (gogoproto.goproto_enum_prefix) = false; + option (gogoproto.enum_customname) = "WriteAction"; + + // WriteActionStat instructs the writer to return the current status while + // holding the lock on the write. + STAT = 0 [(gogoproto.enumvalue_customname)="WriteActionStat"]; + + // WriteActionWrite sets the action for the write request to write data. + // + // Any data included will be written at the provided offset. The + // transaction will be left open for further writes. + // + // This is the default. + WRITE = 1 [(gogoproto.enumvalue_customname)="WriteActionWrite"]; + + // WriteActionCommit will write any outstanding data in the message and + // commit the write, storing it under the digest. + // + // This can be used in a single message to send the data, verify it and + // commit it. + // + // This action will always terminate the write. + COMMIT = 2 [(gogoproto.enumvalue_customname)="WriteActionCommit"]; + + // WriteActionAbort will release any resources associated with the write + // and free up the ref for a completely new set of writes. + // + // This action will always terminate the write. + ABORT = -1 [(gogoproto.enumvalue_customname)="WriteActionAbort"]; +} + +// WriteRequest writes data to the request ref at offset. +message WriteRequest { + // Action sets the behavior of the write. + // + // When this is a write and the ref is not yet allocated, the ref will be + // allocated and the data will be written at offset. + // + // If the action is write and the ref is allocated, it will accept data to + // an offset that has not yet been written. + // + // If the action is write and there is no data, the current write status + // will be returned. This works differently from status because the stream + // holds a lock. + WriteAction action = 1; + + // Ref identifies the pre-commit object to write to. + string ref = 2; + + // ExpectedSize can be set to have the service validate the total size of + // the of committed content. + // + // The latest value before or with the commit action message will be use to + // validate the content. It is only required on one message for the write. + // + // If the value is zero or less, no validation of the final content will be + // performed. + int64 expected_size = 3; + + // ExpectedDigest can be set to have the service validate the final content + // against the provided digest. + // + // If the digest is already present in the object store, an AlreadyPresent + // error will be returned. + // + // Only the latest version will be used to check the content against the + // digest. It is only required to include it on a single message, before or + // with the commit action message. + string expected_digest = 4 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; + + // Offset specifies the number of bytes from the start at which to begin + // the write. If zero or less, the write will be from the start. This uses + // standard zero-indexed semantics. + int64 offset = 5; + + // Data is the actual bytes to be written. + // + // If this is empty and the message is not a commit, a response will be + // returned with the current write state. + bytes data = 6; +} + +// WriteResponse is returned on the culmination of a write call. +message WriteResponse { + // Action contains the action for the final message of the stream. A writer + // should confirm that they match the intended result. + WriteAction action = 1; + + // Offset provides the current "committed" size for the Write. + int64 offset = 2; + + // Digest, if present, includes the digest up to the currently committed + // bytes. If action is commit, this field will be set. It is implementation + // defined if this is set for other actions, except abort. On abort, this + // will be empty. + string digest = 3 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; + + // StartedAt is the time at which the write first started. + google.protobuf.Timestamp started_at = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + + // UpdatedAt is the time the write was last updated. + google.protobuf.Timestamp updated_at = 5 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; +} + +message StatusRequest { + repeated string refs = 1; + repeated string prefix = 2; +} + +message StatusResponse { + string ref = 1; + int64 offset = 2; + google.protobuf.Timestamp started_at = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + google.protobuf.Timestamp updated_at = 4 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false]; +} From 621164bc84e1481b003480fce4821a62653f5bb3 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 17 Feb 2017 00:07:02 -0800 Subject: [PATCH 3/4] content: refactor content store for API After iterating on the GRPC API, the changes required for the actual implementation are now included in the content store. The begin change is the move to a single, atomic `Ingester.Writer` method for locking content ingestion on a key. From this, comes several new interface definitions. The main benefit here is the clarification between `Status` and `Info` that came out of the GPRC API. `Status` tells the status of a write, whereas `Info` is for querying metadata about various blobs. Signed-off-by: Stephen J Day --- cmd/dist/active.go | 23 +-- cmd/dist/common.go | 34 ++++ cmd/dist/delete.go | 21 +-- cmd/dist/get.go | 39 +++++ cmd/dist/ingest.go | 52 +----- cmd/dist/list.go | 17 +- cmd/dist/main.go | 27 ++- cmd/dist/path.go | 89 ---------- content/content.go | 355 +++------------------------------------- content/content_test.go | 55 +++---- content/helpers.go | 48 +++--- content/locks.go | 12 +- content/store.go | 355 ++++++++++++++++++++++++++++++++++++++++ content/writer.go | 57 +++---- 14 files changed, 573 insertions(+), 611 deletions(-) create mode 100644 cmd/dist/common.go create mode 100644 cmd/dist/get.go delete mode 100644 cmd/dist/path.go create mode 100644 content/store.go diff --git a/cmd/dist/active.go b/cmd/dist/active.go index 12be2d3..24b39a6 100644 --- a/cmd/dist/active.go +++ b/cmd/dist/active.go @@ -3,11 +3,9 @@ package main import ( "fmt" "os" - "path/filepath" "text/tabwriter" "time" - "github.com/docker/containerd/content" units "github.com/docker/go-units" "github.com/urfave/cli" ) @@ -26,24 +24,11 @@ var activeCommand = cli.Command{ cli.StringFlag{ Name: "root", Usage: "path to content store root", - Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content + Value: "/tmp/content", // TODO(stevvooe): for now, just use the PWD/.content }, }, Action: func(context *cli.Context) error { - var ( - // ctx = contextpkg.Background() - root = context.String("root") - ) - - if !filepath.IsAbs(root) { - var err error - root, err = filepath.Abs(root) - if err != nil { - return err - } - } - - cs, err := content.Open(root) + cs, err := resolveContentStore(context) if err != nil { return err } @@ -58,8 +43,8 @@ var activeCommand = cli.Command{ for _, active := range active { fmt.Fprintf(tw, "%s\t%s\t%s\n", active.Ref, - units.HumanSize(float64(active.Size)), - units.HumanDuration(time.Since(active.ModTime))) + units.HumanSize(float64(active.Offset)), + units.HumanDuration(time.Since(active.StartedAt))) } tw.Flush() diff --git a/cmd/dist/common.go b/cmd/dist/common.go new file mode 100644 index 0000000..404aa76 --- /dev/null +++ b/cmd/dist/common.go @@ -0,0 +1,34 @@ +package main + +import ( + "net" + "path/filepath" + "time" + + "github.com/docker/containerd/content" + "github.com/urfave/cli" + "google.golang.org/grpc" +) + +func resolveContentStore(context *cli.Context) (*content.Store, error) { + root := context.GlobalString("root") + if !filepath.IsAbs(root) { + var err error + root, err = filepath.Abs(root) + if err != nil { + return nil, err + } + } + return content.NewStore(root) +} + +func connectGRPC(context *cli.Context) (*grpc.ClientConn, error) { + socket := context.GlobalString("socket") + return grpc.Dial(socket, + grpc.WithBlock(), + grpc.WithInsecure(), + grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { + return net.DialTimeout("unix", socket, timeout) + }), + ) +} diff --git a/cmd/dist/delete.go b/cmd/dist/delete.go index 9cce3b5..6657286 100644 --- a/cmd/dist/delete.go +++ b/cmd/dist/delete.go @@ -3,9 +3,7 @@ package main import ( contextpkg "context" "fmt" - "path/filepath" - "github.com/docker/containerd/content" "github.com/docker/containerd/log" digest "github.com/opencontainers/go-digest" "github.com/urfave/cli" @@ -18,30 +16,15 @@ var deleteCommand = cli.Command{ ArgsUsage: "[flags] [, ...]", Description: `Delete one or more blobs permanently. Successfully deleted blobs are printed to stdout.`, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "root", - Usage: "path to content store root", - Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content - }, - }, + Flags: []cli.Flag{}, Action: func(context *cli.Context) error { var ( ctx = contextpkg.Background() - root = context.String("root") args = []string(context.Args()) exitError error ) - if !filepath.IsAbs(root) { - var err error - root, err = filepath.Abs(root) - if err != nil { - return err - } - } - - cs, err := content.Open(root) + cs, err := resolveContentStore(context) if err != nil { return err } diff --git a/cmd/dist/get.go b/cmd/dist/get.go new file mode 100644 index 0000000..bf35933 --- /dev/null +++ b/cmd/dist/get.go @@ -0,0 +1,39 @@ +package main + +import ( + "io" + "os" + + digest "github.com/opencontainers/go-digest" + "github.com/urfave/cli" +) + +var getCommand = cli.Command{ + Name: "get", + Usage: "get the data for an object", + ArgsUsage: "[flags] [, ...]", + Description: `Display the paths to one or more blobs. + +Output paths can be used to directly access blobs on disk.`, + Flags: []cli.Flag{}, + Action: func(context *cli.Context) error { + cs, err := resolveContentStore(context) + if err != nil { + return err + } + + dgst, err := digest.Parse(context.Args().First()) + if err != nil { + return err + } + + rc, err := cs.Open(dgst) + if err != nil { + return err + } + defer rc.Close() + + _, err = io.Copy(os.Stdout, rc) + return err + }, +} diff --git a/cmd/dist/ingest.go b/cmd/dist/ingest.go index 206c186..e3353cc 100644 --- a/cmd/dist/ingest.go +++ b/cmd/dist/ingest.go @@ -4,8 +4,6 @@ import ( contextpkg "context" "fmt" "os" - "path/filepath" - "strings" "github.com/docker/containerd/content" "github.com/opencontainers/go-digest" @@ -18,17 +16,6 @@ var ingestCommand = cli.Command{ ArgsUsage: "[flags] ", Description: `Ingest objects into the local content store.`, Flags: []cli.Flag{ - cli.DurationFlag{ - Name: "timeout", - Usage: "total timeout for fetch", - EnvVar: "CONTAINERD_FETCH_TIMEOUT", - }, - cli.StringFlag{ - Name: "path, p", - Usage: "path to content store", - Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content - EnvVar: "CONTAINERD_DIST_CONTENT_STORE", - }, cli.Int64Flag{ Name: "expected-size", Usage: "validate against provided size", @@ -40,57 +27,32 @@ var ingestCommand = cli.Command{ }, Action: func(context *cli.Context) error { var ( - ctx = contextpkg.Background() - timeout = context.Duration("timeout") - root = context.String("path") + ctx = background + cancel func() ref = context.Args().First() expectedSize = context.Int64("expected-size") expectedDigest = digest.Digest(context.String("expected-digest")) ) - if timeout > 0 { - var cancel func() - ctx, cancel = contextpkg.WithTimeout(ctx, timeout) - defer cancel() - } + ctx, cancel = contextpkg.WithCancel(ctx) + defer cancel() if err := expectedDigest.Validate(); expectedDigest != "" && err != nil { return err } - if !filepath.IsAbs(root) { - var err error - root, err = filepath.Abs(root) - if err != nil { - return err - } - } - - cs, err := content.Open(root) + cs, err := resolveContentStore(context) if err != nil { return err } - if expectedDigest != "" { - if ok, err := cs.Exists(expectedDigest); err != nil { - return err - } else if ok { - fmt.Fprintf(os.Stderr, "content with digest %v already exists\n", expectedDigest) - return nil - } - } - if ref == "" { - if expectedDigest == "" { - return fmt.Errorf("must specify a transaction reference or expected digest") - } - - ref = strings.Replace(expectedDigest.String(), ":", "-", -1) + return fmt.Errorf("must specify a transaction reference") } // TODO(stevvooe): Allow ingest to be reentrant. Currently, we expect // all data to be written in a single invocation. Allow multiple writes // to the same transaction key followed by a commit. - return content.WriteBlob(cs, os.Stdin, ref, expectedSize, expectedDigest) + return content.WriteBlob(ctx, cs, os.Stdin, ref, expectedSize, expectedDigest) }, } diff --git a/cmd/dist/list.go b/cmd/dist/list.go index 11b5894..2ba1d88 100644 --- a/cmd/dist/list.go +++ b/cmd/dist/list.go @@ -4,7 +4,6 @@ import ( contextpkg "context" "fmt" "os" - "path/filepath" "text/tabwriter" "time" @@ -22,11 +21,6 @@ var listCommand = cli.Command{ ArgsUsage: "[flags] [, ...]", Description: `List blobs in the content store.`, Flags: []cli.Flag{ - cli.StringFlag{ - Name: "root", - Usage: "path to content store root", - Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content - }, cli.BoolFlag{ Name: "quiet, q", Usage: "print only the blob digest", @@ -35,20 +29,11 @@ var listCommand = cli.Command{ Action: func(context *cli.Context) error { var ( ctx = contextpkg.Background() - root = context.String("root") quiet = context.Bool("quiet") args = []string(context.Args()) ) - if !filepath.IsAbs(root) { - var err error - root, err = filepath.Abs(root) - if err != nil { - return err - } - } - - cs, err := content.Open(root) + cs, err := resolveContentStore(context) if err != nil { return err } diff --git a/cmd/dist/main.go b/cmd/dist/main.go index 5a19608..de4cf22 100644 --- a/cmd/dist/main.go +++ b/cmd/dist/main.go @@ -1,6 +1,7 @@ package main import ( + contextpkg "context" "fmt" "os" @@ -9,6 +10,10 @@ import ( "github.com/urfave/cli" ) +var ( + background = contextpkg.Background() +) + func main() { app := cli.NewApp() app.Name = "dist" @@ -27,20 +32,38 @@ distribution tool Name: "debug", Usage: "enable debug output in logs", }, + cli.DurationFlag{ + Name: "timeout", + Usage: "total timeout for fetch", + EnvVar: "CONTAINERD_FETCH_TIMEOUT", + }, + cli.StringFlag{ + Name: "root", + Usage: "path to content store root", + Value: "/tmp/content", // TODO(stevvooe): for now, just use the PWD/.content + }, } app.Commands = []cli.Command{ fetchCommand, ingestCommand, activeCommand, - pathCommand, + getCommand, deleteCommand, listCommand, applyCommand, } app.Before = func(context *cli.Context) error { - if context.GlobalBool("debug") { + var ( + debug = context.GlobalBool("debug") + timeout = context.GlobalDuration("timeout") + ) + if debug { logrus.SetLevel(logrus.DebugLevel) } + + if timeout > 0 { + background, _ = contextpkg.WithTimeout(background, timeout) + } return nil } if err := app.Run(os.Args); err != nil { diff --git a/cmd/dist/path.go b/cmd/dist/path.go deleted file mode 100644 index 02215a8..0000000 --- a/cmd/dist/path.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - contextpkg "context" - "fmt" - "path/filepath" - - "github.com/docker/containerd/content" - "github.com/docker/containerd/log" - digest "github.com/opencontainers/go-digest" - "github.com/urfave/cli" -) - -var pathCommand = cli.Command{ - Name: "path", - Usage: "print the path to one or more blobs", - ArgsUsage: "[flags] [, ...]", - Description: `Display the paths to one or more blobs. - -Output paths can be used to directly access blobs on disk.`, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "root", - Usage: "path to content store root", - Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content - EnvVar: "CONTAINERD_DIST_CONTENT_STORE", - }, - cli.BoolFlag{ - Name: "quiet, q", - Usage: "elide digests in output", - }, - }, - Action: func(context *cli.Context) error { - var ( - ctx = contextpkg.Background() - root = context.String("root") - args = []string(context.Args()) - quiet = context.Bool("quiet") - exitError error - ) - - if !filepath.IsAbs(root) { - var err error - root, err = filepath.Abs(root) - if err != nil { - return err - } - } - - cs, err := content.Open(root) - if err != nil { - return err - } - - // TODO(stevvooe): Take the set of paths from stdin. - - if len(args) < 1 { - return fmt.Errorf("please specify a blob digest") - } - - for _, arg := range args { - dgst, err := digest.Parse(arg) - if err != nil { - log.G(ctx).WithError(err).Errorf("parsing %q as digest failed", arg) - if exitError == nil { - exitError = err - } - continue - } - - p, err := cs.GetPath(dgst) - if err != nil { - log.G(ctx).WithError(err).Errorf("getting path for %q failed", dgst) - if exitError == nil { - exitError = err - } - continue - } - - if !quiet { - fmt.Println(dgst, p) - } else { - fmt.Println(p) - } - } - - return exitError - }, -} diff --git a/content/content.go b/content/content.go index d7fa750..b63ed1d 100644 --- a/content/content.go +++ b/content/content.go @@ -1,364 +1,53 @@ package content import ( + "context" "io" - "io/ioutil" - "os" - "path/filepath" "sync" "time" - "github.com/docker/containerd/log" - "github.com/nightlyone/lockfile" "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) var ( - ErrBlobNotFound = errors.New("blob not found") + errNotFound = errors.New("content: not found") bufPool = sync.Pool{ New: func() interface{} { - return make([]byte, 32<<10) + return make([]byte, 1<<20) }, } ) -// Store is digest-keyed store for content. All data written into the store is -// stored under a verifiable digest. -// -// Store can generally support multi-reader, single-writer ingest of data, -// including resumable ingest. -type Store struct { - root string +type Info struct { + Digest digest.Digest + Size int64 + CommittedAt time.Time } -func Open(root string) (*Store, error) { - if err := os.MkdirAll(filepath.Join(root, "ingest"), 0777); err != nil && !os.IsExist(err) { - return nil, err - } - - return &Store{ - root: root, - }, nil +type Provider interface { + Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) } type Status struct { - Ref string - Size int64 - ModTime time.Time - Meta interface{} + Ref string + Offset int64 + StartedAt time.Time + UpdatedAt time.Time } -func (cs *Store) Exists(dgst digest.Digest) (bool, error) { - if _, err := os.Stat(cs.blobPath(dgst)); err != nil { - if !os.IsNotExist(err) { - return false, err - } - - return false, nil - } - - return true, nil +type Writer interface { + io.WriteCloser + Status() (Status, error) + Digest() digest.Digest + Commit(size int64, expected digest.Digest) error } -func (cs *Store) GetPath(dgst digest.Digest) (string, error) { - p := cs.blobPath(dgst) - if _, err := os.Stat(p); err != nil { - if os.IsNotExist(err) { - return "", ErrBlobNotFound - } - - return "", err - } - - return p, nil +type Ingester interface { + Writer(ctx context.Context, ref string) (Writer, error) } -// Delete removes a blob by its digest. -// -// While this is safe to do concurrently, safe exist-removal logic must hold -// some global lock on the store. -func (cs *Store) Delete(dgst digest.Digest) error { - if err := os.RemoveAll(cs.blobPath(dgst)); err != nil { - if !os.IsNotExist(err) { - return err - } - - return nil - } - - return nil -} - -func (cs *Store) blobPath(dgst digest.Digest) string { - return filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex()) -} - -// Stat returns the current status of a blob by the ingest ref. -func (cs *Store) Stat(ref string) (Status, error) { - dp := filepath.Join(cs.ingestRoot(ref), "data") - return cs.stat(dp) -} - -// stat works like stat above except uses the path to the ingest. -func (cs *Store) stat(ingestPath string) (Status, error) { - dp := filepath.Join(ingestPath, "data") - dfi, err := os.Stat(dp) - if err != nil { - return Status{}, err - } - - ref, err := readFileString(filepath.Join(ingestPath, "ref")) - if err != nil { - return Status{}, err - } - - return Status{ - Ref: ref, - Size: dfi.Size(), - ModTime: dfi.ModTime(), - }, nil -} - -func (cs *Store) Active() ([]Status, error) { - ip := filepath.Join(cs.root, "ingest") - - fp, err := os.Open(ip) - if err != nil { - return nil, err - } - - fis, err := fp.Readdir(-1) - if err != nil { - return nil, err - } - - var active []Status - for _, fi := range fis { - p := filepath.Join(ip, fi.Name()) - stat, err := cs.stat(p) - if err != nil { - if !os.IsNotExist(err) { - return nil, err - } - - // TODO(stevvooe): This is a common error if uploads are being - // completed while making this listing. Need to consider taking a - // lock on the whole store to coordinate this aspect. - // - // Another option is to cleanup downloads asynchronously and - // coordinate this method with the cleanup process. - // - // For now, we just skip them, as they really don't exist. - continue - } - - active = append(active, stat) - } - - return active, nil -} - -// TODO(stevvooe): Allow querying the set of blobs in the blob store. - -// WalkFunc defines the callback for a blob walk. -// -// TODO(stevvooe): Remove the file info. Just need size and modtime. Perhaps, -// not a huge deal, considering we have a path, but let's not just let this one -// go without scrutiny. -type WalkFunc func(path string, fi os.FileInfo, dgst digest.Digest) error - -func (cs *Store) Walk(fn WalkFunc) error { - root := filepath.Join(cs.root, "blobs") - var alg digest.Algorithm - return filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if !fi.IsDir() && !alg.Available() { - return nil - } - - // TODO(stevvooe): There are few more cases with subdirs that should be - // handled in case the layout gets corrupted. This isn't strict enough - // an may spew bad data. - - if path == root { - return nil - } - if filepath.Dir(path) == root { - alg = digest.Algorithm(filepath.Base(path)) - - if !alg.Available() { - alg = "" - return filepath.SkipDir - } - - // descending into a hash directory - return nil - } - - dgst := digest.NewDigestFromHex(alg.String(), filepath.Base(path)) - if err := dgst.Validate(); err != nil { - // log error but don't report - log.L.WithError(err).WithField("path", path).Error("invalid digest for blob path") - // if we see this, it could mean some sort of corruption of the - // store or extra paths not expected previously. - } - - return fn(path, fi, dgst) - }) -} - -// Begin starts a new write transaction against the blob store. -// -// The argument `ref` is used to identify the transaction. It must be a valid -// path component, meaning it has no `/` characters and no `:` (we'll ban -// others fs characters, as needed). -func (cs *Store) Begin(ref string) (*Writer, error) { - path, refp, data, lock, err := cs.ingestPaths(ref) - if err != nil { - return nil, err - } - - // use single path mkdir for this to ensure ref is only base path, in - // addition to validation above. - if err := os.Mkdir(path, 0755); err != nil { - return nil, err - } - - if err := tryLock(lock); err != nil { - return nil, err - } - - // write the ref to a file for later use - if err := ioutil.WriteFile(refp, []byte(ref), 0666); err != nil { - return nil, err - } - - fp, err := os.OpenFile(data, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) - if err != nil { - return nil, errors.Wrap(err, "failed to open data file") - } - defer fp.Close() - - // re-open the file in append mode - fp, err = os.OpenFile(data, os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - return nil, errors.Wrap(err, "error opening for append") - } - - return &Writer{ - cs: cs, - fp: fp, - lock: lock, - path: path, - digester: digest.Canonical.Digester(), - }, nil -} - -func (cs *Store) Resume(ref string) (*Writer, error) { - path, refp, data, lock, err := cs.ingestPaths(ref) - if err != nil { - return nil, err - } - - if err := tryLock(lock); err != nil { - return nil, err - } - - refraw, err := readFileString(refp) - if err != nil { - return nil, errors.Wrap(err, "could not read ref") - } - - if ref != refraw { - // NOTE(stevvooe): This is fairly catastrophic. Either we have some - // layout corruption or a hash collision for the ref key. - return nil, errors.Wrapf(err, "ref key does not match: %v != %v", ref, refraw) - } - - digester := digest.Canonical.Digester() - - // slow slow slow!!, send to goroutine or use resumable hashes - fp, err := os.Open(data) - if err != nil { - return nil, err - } - defer fp.Close() - - p := bufPool.Get().([]byte) - defer bufPool.Put(p) - - offset, err := io.CopyBuffer(digester.Hash(), fp, p) - if err != nil { - return nil, err - } - - fp1, err := os.OpenFile(data, os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - if os.IsNotExist(err) { - return nil, errors.Wrap(err, "ingest does not exist") - } - - return nil, errors.Wrap(err, "error opening for append") - } - - return &Writer{ - cs: cs, - fp: fp1, - lock: lock, - ref: ref, - path: path, - offset: offset, - digester: digester, - }, nil -} - -// Remove an active transaction keyed by ref. -func (cs *Store) Remove(ref string) error { - root := cs.ingestRoot(ref) - if err := os.RemoveAll(root); err != nil { - if os.IsNotExist(err) { - return nil - } - - return err - } - - return nil -} - -func (cs *Store) ingestRoot(ref string) string { - dgst := digest.FromString(ref) - return filepath.Join(cs.root, "ingest", dgst.Hex()) -} - -// ingestPaths are returned, including the lockfile. The paths are the following: -// -// - root: entire ingest directory -// - ref: name of the starting ref, must be unique -// - data: file where data is written -// - lock: lock file location -// -func (cs *Store) ingestPaths(ref string) (string, string, string, lockfile.Lockfile, error) { - var ( - fp = cs.ingestRoot(ref) - rp = filepath.Join(fp, "ref") - lp = filepath.Join(fp, "lock") - dp = filepath.Join(fp, "data") - ) - - lock, err := lockfile.New(lp) - if err != nil { - return "", "", "", "", errors.Wrapf(err, "error creating lockfile %v", lp) - } - - return fp, rp, dp, lock, nil -} - -func readFileString(path string) (string, error) { - p, err := ioutil.ReadFile(path) - return string(p), err +func IsNotFound(err error) bool { + return errors.Cause(err) == errNotFound } diff --git a/content/content_test.go b/content/content_test.go index a469630..b831a54 100644 --- a/content/content_test.go +++ b/content/content_test.go @@ -3,6 +3,7 @@ package content import ( "bufio" "bytes" + "context" "crypto/rand" _ "crypto/sha256" // required for digest package "fmt" @@ -21,7 +22,7 @@ import ( ) func TestContentWriter(t *testing.T) { - tmpdir, cs, cleanup := contentStoreEnv(t) + ctx, tmpdir, cs, cleanup := contentStoreEnv(t) defer cleanup() defer testutil.DumpDir(t, tmpdir) @@ -29,7 +30,7 @@ func TestContentWriter(t *testing.T) { t.Fatal("ingest dir should be created", err) } - cw, err := cs.Begin("myref") + cw, err := cs.Writer(ctx, "myref") if err != nil { t.Fatal(err) } @@ -37,20 +38,14 @@ func TestContentWriter(t *testing.T) { t.Fatal(err) } - // try to begin again with same ref, should fail - cw, err = cs.Begin("myref") - if err == nil { - t.Fatal("expected error on repeated begin") - } - // reopen, so we can test things - cw, err = cs.Resume("myref") + cw, err = cs.Writer(ctx, "myref") if err != nil { t.Fatal(err) } // make sure that second resume also fails - if _, err = cs.Resume("myref"); err == nil { + if _, err = cs.Writer(ctx, "myref"); err == nil { // TODO(stevvooe): This also works across processes. Need to find a way // to test that, as well. t.Fatal("no error on second resume") @@ -64,14 +59,14 @@ func TestContentWriter(t *testing.T) { // clear out the time and meta cause we don't care for this test for i := range ingestions { - ingestions[i].Meta = nil - ingestions[i].ModTime = time.Time{} + ingestions[i].UpdatedAt = time.Time{} + ingestions[i].StartedAt = time.Time{} } if !reflect.DeepEqual(ingestions, []Status{ { - Ref: "myref", - Size: 0, + Ref: "myref", + Offset: 0, }, }) { t.Fatalf("unexpected ingestion set: %v", ingestions) @@ -93,7 +88,7 @@ func TestContentWriter(t *testing.T) { t.Fatal(err) } - cw, err = cs.Begin("aref") + cw, err = cs.Writer(ctx, "aref") if err != nil { t.Fatal(err) } @@ -119,7 +114,7 @@ func TestContentWriter(t *testing.T) { } func TestWalkBlobs(t *testing.T) { - _, cs, cleanup := contentStoreEnv(t) + ctx, _, cs, cleanup := contentStoreEnv(t) defer cleanup() const ( @@ -128,7 +123,7 @@ func TestWalkBlobs(t *testing.T) { ) var ( - blobs = populateBlobStore(t, cs, nblobs, maxsize) + blobs = populateBlobStore(t, ctx, cs, nblobs, maxsize) expected = map[digest.Digest]struct{}{} found = map[digest.Digest]struct{}{} ) @@ -158,7 +153,7 @@ func TestWalkBlobs(t *testing.T) { // for blobs. This seems to be due to the number of syscalls and file io we do // coordinating the ingestion. func BenchmarkIngests(b *testing.B) { - _, cs, cleanup := contentStoreEnv(b) + ctx, _, cs, cleanup := contentStoreEnv(b) defer cleanup() for _, size := range []int64{ @@ -181,7 +176,7 @@ func BenchmarkIngests(b *testing.B) { b.StartTimer() for dgst, p := range blobs { - checkWrite(b, cs, dgst, p) + checkWrite(b, ctx, cs, dgst, p) } }) } @@ -208,17 +203,17 @@ func generateBlobs(t checker, nblobs, maxsize int64) map[digest.Digest][]byte { return blobs } -func populateBlobStore(t checker, cs *Store, nblobs, maxsize int64) map[digest.Digest][]byte { +func populateBlobStore(t checker, ctx context.Context, cs *Store, nblobs, maxsize int64) map[digest.Digest][]byte { blobs := generateBlobs(t, nblobs, maxsize) for dgst, p := range blobs { - checkWrite(t, cs, dgst, p) + checkWrite(t, ctx, cs, dgst, p) } return blobs } -func contentStoreEnv(t checker) (string, *Store, func()) { +func contentStoreEnv(t checker) (context.Context, string, *Store, func()) { pc, _, _, ok := runtime.Caller(1) if !ok { t.Fatal("failed to resolve caller") @@ -230,13 +225,15 @@ func contentStoreEnv(t checker) (string, *Store, func()) { t.Fatal(err) } - cs, err := Open(tmpdir) + cs, err := NewStore(tmpdir) if err != nil { os.RemoveAll(tmpdir) t.Fatal(err) } - return tmpdir, cs, func() { + ctx, cancel := context.WithCancel(context.Background()) + return ctx, tmpdir, cs, func() { + cancel() os.RemoveAll(tmpdir) } } @@ -253,10 +250,8 @@ func checkCopy(t checker, size int64, dst io.Writer, src io.Reader) { } func checkBlobPath(t *testing.T, cs *Store, dgst digest.Digest) string { - path, err := cs.GetPath(dgst) - if err != nil { - t.Fatal(err, dgst) - } + path := cs.blobPath(dgst) + if path != filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex()) { t.Fatalf("unexpected path: %q", path) } @@ -273,8 +268,8 @@ func checkBlobPath(t *testing.T, cs *Store, dgst digest.Digest) string { return path } -func checkWrite(t checker, cs *Store, dgst digest.Digest, p []byte) digest.Digest { - if err := WriteBlob(cs, bytes.NewReader(p), dgst.String(), int64(len(p)), dgst); err != nil { +func checkWrite(t checker, ctx context.Context, cs *Store, dgst digest.Digest, p []byte) digest.Digest { + if err := WriteBlob(ctx, cs, bytes.NewReader(p), dgst.String(), int64(len(p)), dgst); err != nil { t.Fatal(err) } diff --git a/content/helpers.go b/content/helpers.go index 37cb71e..4209350 100644 --- a/content/helpers.go +++ b/content/helpers.go @@ -1,37 +1,14 @@ package content import ( + "context" "io" - "os" + "io/ioutil" "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) -// Provider gives access to blob content by paths. -// -// Typically, this is implemented by `*Store`. -type Provider interface { - GetPath(dgst digest.Digest) (string, error) -} - -// OpenBlob opens the blob for reading identified by dgst. -// -// The opened blob may also implement seek. Callers can detect with io.Seeker. -func OpenBlob(provider Provider, dgst digest.Digest) (io.ReadCloser, error) { - path, err := provider.GetPath(dgst) - if err != nil { - return nil, err - } - - fp, err := os.Open(path) - return fp, err -} - -type Ingester interface { - Begin(key string) (*Writer, error) -} - // WriteBlob writes data with the expected digest into the content store. If // expected already exists, the method returns immediately and the reader will // not be consumed. @@ -39,11 +16,23 @@ type Ingester interface { // This is useful when the digest and size are known beforehand. // // Copy is buffered, so no need to wrap reader in buffered io. -func WriteBlob(cs Ingester, r io.Reader, ref string, size int64, expected digest.Digest) error { - cw, err := cs.Begin(ref) +func WriteBlob(ctx context.Context, cs Ingester, r io.Reader, ref string, size int64, expected digest.Digest) error { + cw, err := cs.Writer(ctx, ref) if err != nil { return err } + + ws, err := cw.Status() + if err != nil { + return err + } + + if ws.Offset > 0 { + // Arbitrary limitation for now. We can detect io.Seeker on r and + // resume. + return errors.Errorf("cannot resume already started write") + } + buf := bufPool.Get().([]byte) defer bufPool.Put(buf) @@ -62,3 +51,8 @@ func WriteBlob(cs Ingester, r io.Reader, ref string, size int64, expected digest return nil } + +func readFileString(path string) (string, error) { + p, err := ioutil.ReadFile(path) + return string(p), err +} diff --git a/content/locks.go b/content/locks.go index dd2cae9..400793d 100644 --- a/content/locks.go +++ b/content/locks.go @@ -1,10 +1,10 @@ package content import ( - "errors" "sync" "github.com/nightlyone/lockfile" + "github.com/pkg/errors" ) // In addition to providing inter-process locks for content ingest, we also @@ -16,6 +16,8 @@ import ( // error reporting. var ( + errLocked = errors.New("key is locked") + // locks lets us lock in process, as well as output of process. locks = map[lockfile.Lockfile]struct{}{} locksMu sync.Mutex @@ -26,11 +28,15 @@ func tryLock(lock lockfile.Lockfile) error { defer locksMu.Unlock() if _, ok := locks[lock]; ok { - return errors.New("file in use") + return errLocked } if err := lock.TryLock(); err != nil { - return err + if errors.Cause(err) == lockfile.ErrBusy { + return errLocked + } + + return errors.Wrapf(err, "lock.TryLock() encountered an error") } locks[lock] = struct{}{} diff --git a/content/store.go b/content/store.go new file mode 100644 index 0000000..dfe1993 --- /dev/null +++ b/content/store.go @@ -0,0 +1,355 @@ +package content + +import ( + "context" + "io" + "io/ioutil" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/docker/containerd/log" + "github.com/nightlyone/lockfile" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// Store is digest-keyed store for content. All data written into the store is +// stored under a verifiable digest. +// +// Store can generally support multi-reader, single-writer ingest of data, +// including resumable ingest. +type Store struct { + root string +} + +func NewStore(root string) (*Store, error) { + if err := os.MkdirAll(filepath.Join(root, "ingest"), 0777); err != nil && !os.IsExist(err) { + return nil, err + } + + return &Store{ + root: root, + }, nil +} + +func (s *Store) Info(dgst digest.Digest) (Info, error) { + p := s.blobPath(dgst) + fi, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + err = errNotFound + } + + return Info{}, err + } + + return Info{ + Digest: dgst, + Size: fi.Size(), + CommittedAt: fi.ModTime(), + }, nil +} + +// Open returns an io.ReadCloser for the blob. +// +// TODO(stevvooe): This would work much better as an io.ReaderAt in practice. +// Right now, we are doing type assertion to tease that out, but it won't scale +// well. +func (s *Store) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) { + fp, err := os.Open(s.blobPath(dgst)) + if err != nil { + if os.IsNotExist(err) { + err = errNotFound + } + return nil, err + } + + return fp, nil +} + +// Delete removes a blob by its digest. +// +// While this is safe to do concurrently, safe exist-removal logic must hold +// some global lock on the store. +func (cs *Store) Delete(dgst digest.Digest) error { + if err := os.RemoveAll(cs.blobPath(dgst)); err != nil { + if !os.IsNotExist(err) { + return err + } + + return nil + } + + return nil +} + +// TODO(stevvooe): Allow querying the set of blobs in the blob store. + +// WalkFunc defines the callback for a blob walk. +// +// TODO(stevvooe): Remove the file info. Just need size and modtime. Perhaps, +// not a huge deal, considering we have a path, but let's not just let this one +// go without scrutiny. +type WalkFunc func(path string, fi os.FileInfo, dgst digest.Digest) error + +func (cs *Store) Walk(fn WalkFunc) error { + root := filepath.Join(cs.root, "blobs") + var alg digest.Algorithm + return filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if !fi.IsDir() && !alg.Available() { + return nil + } + + // TODO(stevvooe): There are few more cases with subdirs that should be + // handled in case the layout gets corrupted. This isn't strict enough + // an may spew bad data. + + if path == root { + return nil + } + if filepath.Dir(path) == root { + alg = digest.Algorithm(filepath.Base(path)) + + if !alg.Available() { + alg = "" + return filepath.SkipDir + } + + // descending into a hash directory + return nil + } + + dgst := digest.NewDigestFromHex(alg.String(), filepath.Base(path)) + if err := dgst.Validate(); err != nil { + // log error but don't report + log.L.WithError(err).WithField("path", path).Error("invalid digest for blob path") + // if we see this, it could mean some sort of corruption of the + // store or extra paths not expected previously. + } + + return fn(path, fi, dgst) + }) +} + +// Stat returns the current status of a blob by the ingest ref. +func (s *Store) Status(ref string) (Status, error) { + dp := filepath.Join(s.ingestRoot(ref), "data") + return s.status(dp) +} + +// stat works like stat above except uses the path to the ingest. +func (s *Store) status(ingestPath string) (Status, error) { + dp := filepath.Join(ingestPath, "data") + fi, err := os.Stat(dp) + if err != nil { + return Status{}, err + } + + ref, err := readFileString(filepath.Join(ingestPath, "ref")) + if err != nil { + return Status{}, err + } + + var startedAt time.Time + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + startedAt = time.Unix(st.Ctim.Sec, st.Ctim.Nsec) + } else { + startedAt = fi.ModTime() + } + + return Status{ + Ref: ref, + Offset: fi.Size(), + UpdatedAt: fi.ModTime(), + StartedAt: startedAt, + }, nil +} + +// Writer begins or resumes the active writer identified by ref. If the writer +// is already in use, an error is returned. Only one writer may be in use per +// ref at a time. +// +// The argument `ref` is used to uniquely identify a long-lived writer transaction. +func (s *Store) Writer(ctx context.Context, ref string) (Writer, error) { + path, refp, data, lock, err := s.ingestPaths(ref) + if err != nil { + return nil, err + } + + if err := tryLock(lock); err != nil { + if !os.IsNotExist(errors.Cause(err)) { + return nil, errors.Wrapf(err, "locking %v failed", ref) + } + + // if it doesn't exist, we'll make it so below! + } + + var ( + digester = digest.Canonical.Digester() + offset int64 + startedAt time.Time + updatedAt time.Time + ) + + // ensure that the ingest path has been created. + if err := os.Mkdir(path, 0755); err != nil { + if !os.IsExist(err) { + return nil, err + } + + // validate that we have no collision for the ref. + refraw, err := readFileString(refp) + if err != nil { + return nil, errors.Wrap(err, "could not read ref") + } + + if ref != refraw { + // NOTE(stevvooe): This is fairly catastrophic. Either we have some + // layout corruption or a hash collision for the ref key. + return nil, errors.Wrapf(err, "ref key does not match: %v != %v", ref, refraw) + } + + // slow slow slow!!, send to goroutine or use resumable hashes + fp, err := os.Open(data) + if err != nil { + return nil, err + } + defer fp.Close() + + p := bufPool.Get().([]byte) + defer bufPool.Put(p) + + offset, err = io.CopyBuffer(digester.Hash(), fp, p) + if err != nil { + return nil, err + } + + fi, err := os.Stat(data) + if err != nil { + return nil, err + } + + updatedAt = fi.ModTime() + + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + startedAt = time.Unix(st.Ctim.Sec, st.Ctim.Nsec) + } else { + startedAt = updatedAt + } + } else { + // the ingest is new, we need to setup the target location. + // write the ref to a file for later use + if err := ioutil.WriteFile(refp, []byte(ref), 0666); err != nil { + return nil, err + } + + startedAt = time.Now() + updatedAt = startedAt + } + + fp, err := os.OpenFile(data, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return nil, errors.Wrap(err, "failed to open data file") + } + + return &writer{ + s: s, + fp: fp, + lock: lock, + ref: ref, + path: path, + offset: offset, + digester: digester, + startedAt: startedAt, + updatedAt: updatedAt, + }, nil +} + +// Abort an active transaction keyed by ref. If the ingest is active, it will +// be cancelled. Any resoures associated with the ingest will be cleaned. +func (s *Store) Abort(ref string) error { + root := s.ingestRoot(ref) + if err := os.RemoveAll(root); err != nil { + if os.IsNotExist(err) { + return nil + } + + return err + } + + return nil +} + +func (s *Store) Active() ([]Status, error) { + fp, err := os.Open(filepath.Join(s.root, "ingest")) + if err != nil { + return nil, err + } + + fis, err := fp.Readdir(-1) + if err != nil { + return nil, err + } + + var active []Status + for _, fi := range fis { + p := filepath.Join(s.root, "ingest", fi.Name()) + stat, err := s.status(p) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + + // TODO(stevvooe): This is a common error if uploads are being + // completed while making this listing. Need to consider taking a + // lock on the whole store to coordinate this aspect. + // + // Another option is to cleanup downloads asynchronously and + // coordinate this method with the cleanup process. + // + // For now, we just skip them, as they really don't exist. + continue + } + + active = append(active, stat) + } + + return active, nil +} + +func (cs *Store) blobPath(dgst digest.Digest) string { + return filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex()) +} + +func (s *Store) ingestRoot(ref string) string { + dgst := digest.FromString(ref) + return filepath.Join(s.root, "ingest", dgst.Hex()) +} + +// ingestPaths are returned, including the lockfile. The paths are the following: +// +// - root: entire ingest directory +// - ref: name of the starting ref, must be unique +// - data: file where data is written +// - lock: lock file location +// +func (s *Store) ingestPaths(ref string) (string, string, string, lockfile.Lockfile, error) { + var ( + fp = s.ingestRoot(ref) + rp = filepath.Join(fp, "ref") + lp = filepath.Join(fp, "lock") + dp = filepath.Join(fp, "data") + ) + + lock, err := lockfile.New(lp) + if err != nil { + return "", "", "", "", errors.Wrapf(err, "error creating lockfile %v", lp) + } + + return fp, rp, dp, lock, nil +} diff --git a/content/writer.go b/content/writer.go index a2ff020..df33234 100644 --- a/content/writer.go +++ b/content/writer.go @@ -4,54 +4,55 @@ import ( "log" "os" "path/filepath" + "time" "github.com/nightlyone/lockfile" "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) -// Writer represents a write transaction against the blob store. -type Writer struct { - cs *Store - fp *os.File // opened data file - lock lockfile.Lockfile - path string // path to writer dir - ref string // ref key - offset int64 - digester digest.Digester +// writer represents a write transaction against the blob store. +type writer struct { + s *Store + fp *os.File // opened data file + lock lockfile.Lockfile + path string // path to writer dir + ref string // ref key + offset int64 + digester digest.Digester + startedAt time.Time + updatedAt time.Time } -func (cw *Writer) Ref() string { - return cw.ref -} - -// Size returns the current size written. -// -// Cannot be called concurrently with `Write`. If you need need concurrent -// status, query it with `Store.Stat`. -func (cw *Writer) Size() int64 { - return cw.offset +func (w *writer) Status() (Status, error) { + return Status{ + Ref: w.ref, + Offset: w.offset, + StartedAt: w.startedAt, + UpdatedAt: w.updatedAt, + }, nil } // Digest returns the current digest of the content, up to the current write. // // Cannot be called concurrently with `Write`. -func (cw *Writer) Digest() digest.Digest { - return cw.digester.Digest() +func (w *writer) Digest() digest.Digest { + return w.digester.Digest() } // Write p to the transaction. // // Note that writes are unbuffered to the backing file. When writing, it is // recommended to wrap in a bufio.Writer or, preferably, use io.CopyBuffer. -func (cw *Writer) Write(p []byte) (n int, err error) { - n, err = cw.fp.Write(p) - cw.digester.Hash().Write(p[:n]) - cw.offset += int64(len(p)) +func (w *writer) Write(p []byte) (n int, err error) { + n, err = w.fp.Write(p) + w.digester.Hash().Write(p[:n]) + w.offset += int64(len(p)) + w.updatedAt = time.Now() return n, err } -func (cw *Writer) Commit(size int64, expected digest.Digest) error { +func (cw *writer) Commit(size int64, expected digest.Digest) error { if err := cw.fp.Sync(); err != nil { return errors.Wrap(err, "sync failed") } @@ -85,7 +86,7 @@ func (cw *Writer) Commit(size int64, expected digest.Digest) error { var ( ingest = filepath.Join(cw.path, "data") - target = cw.cs.blobPath(dgst) + target = cw.s.blobPath(dgst) ) // make sure parent directories of blob exist @@ -118,7 +119,7 @@ func (cw *Writer) Commit(size int64, expected digest.Digest) error { // // To abandon a transaction completely, first call close then `Store.Remove` to // clean up the associated resources. -func (cw *Writer) Close() (err error) { +func (cw *writer) Close() (err error) { if err := unlock(cw.lock); err != nil { log.Printf("unlock failed: %v", err) } From e6efb397cfb16c6cd69a026b6bf6e71513c9717b Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 17 Feb 2017 16:49:59 -0800 Subject: [PATCH 4/4] cmd/dist: port commands over to use GRPC content store Following from the rest of the work in this branch, we now are porting the dist command to work directly against the containerd content API. Signed-off-by: Stephen J Day --- cmd/containerd/main.go | 32 ++++-- cmd/dist/get.go | 18 +++- cmd/dist/ingest.go | 7 +- cmd/dist/main.go | 5 + content/client.go | 216 ++++++++++++++++++++++++++++++++++++++ content/service.go | 233 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 496 insertions(+), 15 deletions(-) create mode 100644 content/client.go create mode 100644 content/service.go diff --git a/cmd/containerd/main.go b/cmd/containerd/main.go index 5cdb8c0..9002b5d 100644 --- a/cmd/containerd/main.go +++ b/cmd/containerd/main.go @@ -7,6 +7,7 @@ import ( _ "net/http/pprof" "os" "os/signal" + "path/filepath" "syscall" "time" @@ -15,7 +16,9 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/containerd" + contentapi "github.com/docker/containerd/api/services/content" api "github.com/docker/containerd/api/services/execution" + "github.com/docker/containerd/content" _ "github.com/docker/containerd/linux" "github.com/docker/containerd/log" "github.com/docker/containerd/services/execution" @@ -55,10 +58,6 @@ func main() { Name: "log-level,l", Usage: "set the logging level [debug, info, warn, error, fatal, panic]", }, - cli.StringFlag{ - Name: "root,r", - Usage: "containerd root directory", - }, cli.StringFlag{ Name: "state", Usage: "containerd state directory", @@ -90,14 +89,27 @@ func main() { return err } serveMetricsAPI() + + contentStore, err := resolveContentStore(context) + if err != nil { + return err + } + contentService := content.NewService(contentStore) + // start the GRPC api with the execution service registered - server := newGRPCServer(execution.New(supervisor)) + server := newGRPCServer() + + api.RegisterContainerServiceServer(server, execution.New(supervisor)) + contentapi.RegisterContentServer(server, contentService) + + // start the GRPC api with registered services if err := serveGRPC(server); err != nil { return err } log.G(global).Infof("containerd successfully booted in %fs", time.Now().Sub(start).Seconds()) return handleSignals(signals, server) } + if err := app.Run(os.Args); err != nil { fmt.Fprintf(os.Stderr, "containerd: %s\n", err) os.Exit(1) @@ -192,8 +204,13 @@ func serveDebugAPI() error { return nil } +func resolveContentStore(context *cli.Context) (*content.Store, error) { + cp := filepath.Join(conf.Root, "content") + return content.NewStore(cp) +} + func loadRuntimes() (map[string]containerd.Runtime, error) { - o := make(map[string]containerd.Runtime) + o := map[string]containerd.Runtime{} for _, name := range containerd.Runtimes() { r, err := containerd.NewRuntime(name, conf.State) if err != nil { @@ -205,9 +222,8 @@ func loadRuntimes() (map[string]containerd.Runtime, error) { return o, nil } -func newGRPCServer(service api.ContainerServiceServer) *grpc.Server { +func newGRPCServer() *grpc.Server { s := grpc.NewServer(grpc.UnaryInterceptor(interceptor)) - api.RegisterContainerServiceServer(s, service) return s } diff --git a/cmd/dist/get.go b/cmd/dist/get.go index bf35933..f4bfbb2 100644 --- a/cmd/dist/get.go +++ b/cmd/dist/get.go @@ -4,6 +4,8 @@ import ( "io" "os" + contentapi "github.com/docker/containerd/api/services/content" + "github.com/docker/containerd/content" digest "github.com/opencontainers/go-digest" "github.com/urfave/cli" ) @@ -17,17 +19,23 @@ var getCommand = cli.Command{ Output paths can be used to directly access blobs on disk.`, Flags: []cli.Flag{}, Action: func(context *cli.Context) error { - cs, err := resolveContentStore(context) - if err != nil { - return err - } + var ( + ctx = background + ) dgst, err := digest.Parse(context.Args().First()) if err != nil { return err } - rc, err := cs.Open(dgst) + conn, err := connectGRPC(context) + if err != nil { + return err + } + + cs := content.NewProviderFromClient(contentapi.NewContentClient(conn)) + + rc, err := cs.Reader(ctx, dgst) if err != nil { return err } diff --git a/cmd/dist/ingest.go b/cmd/dist/ingest.go index e3353cc..8d8a7cc 100644 --- a/cmd/dist/ingest.go +++ b/cmd/dist/ingest.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + contentapi "github.com/docker/containerd/api/services/content" "github.com/docker/containerd/content" "github.com/opencontainers/go-digest" "github.com/urfave/cli" @@ -41,7 +42,7 @@ var ingestCommand = cli.Command{ return err } - cs, err := resolveContentStore(context) + conn, err := connectGRPC(context) if err != nil { return err } @@ -50,9 +51,11 @@ var ingestCommand = cli.Command{ return fmt.Errorf("must specify a transaction reference") } + ingester := content.NewIngesterFromClient(contentapi.NewContentClient(conn)) + // TODO(stevvooe): Allow ingest to be reentrant. Currently, we expect // all data to be written in a single invocation. Allow multiple writes // to the same transaction key followed by a commit. - return content.WriteBlob(ctx, cs, os.Stdin, ref, expectedSize, expectedDigest) + return content.WriteBlob(ctx, ingester, os.Stdin, ref, expectedSize, expectedDigest) }, } diff --git a/cmd/dist/main.go b/cmd/dist/main.go index de4cf22..bfe6b24 100644 --- a/cmd/dist/main.go +++ b/cmd/dist/main.go @@ -42,6 +42,11 @@ distribution tool Usage: "path to content store root", Value: "/tmp/content", // TODO(stevvooe): for now, just use the PWD/.content }, + cli.StringFlag{ + Name: "socket, s", + Usage: "socket path for containerd's GRPC server", + Value: "/run/containerd/containerd.sock", + }, } app.Commands = []cli.Command{ fetchCommand, diff --git a/content/client.go b/content/client.go new file mode 100644 index 0000000..92bea31 --- /dev/null +++ b/content/client.go @@ -0,0 +1,216 @@ +package content + +import ( + "context" + "io" + + contentapi "github.com/docker/containerd/api/services/content" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +func NewProviderFromClient(client contentapi.ContentClient) Provider { + return &remoteProvider{ + client: client, + } +} + +type remoteProvider struct { + client contentapi.ContentClient +} + +func (rp *remoteProvider) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) { + client, err := rp.client.Read(ctx, &contentapi.ReadRequest{Digest: dgst}) + if err != nil { + return nil, err + } + + return &remoteReader{ + client: client, + }, nil +} + +type remoteReader struct { + client contentapi.Content_ReadClient + extra []byte +} + +func (rr *remoteReader) Read(p []byte) (n int, err error) { + n += copy(p, rr.extra) + if n >= len(p) { + if n <= len(rr.extra) { + rr.extra = rr.extra[n:] + } else { + rr.extra = rr.extra[:0] + } + return + } + + p = p[n:] + for len(p) > 0 { + var resp *contentapi.ReadResponse + // fill our buffer up until we can fill p. + resp, err = rr.client.Recv() + if err != nil { + return + } + + copied := copy(p, resp.Data) + n += copied + p = p[copied:] + + if copied < len(p) { + continue + } + + rr.extra = append(rr.extra, resp.Data[copied:]...) + } + + return +} + +func (rr *remoteReader) Close() error { + return rr.client.CloseSend() +} + +func NewIngesterFromClient(client contentapi.ContentClient) Ingester { + return &remoteIngester{ + client: client, + } +} + +type remoteIngester struct { + client contentapi.ContentClient +} + +func (ri *remoteIngester) Writer(ctx context.Context, ref string) (Writer, error) { + wrclient, offset, err := ri.negotiate(ctx, ref) + if err != nil { + return nil, err + } + + return &remoteWriter{ + client: wrclient, + offset: offset, + }, nil +} + +func (ri *remoteIngester) negotiate(ctx context.Context, ref string) (contentapi.Content_WriteClient, int64, error) { + wrclient, err := ri.client.Write(ctx) + if err != nil { + return nil, 0, err + } + + if err := wrclient.Send(&contentapi.WriteRequest{ + Action: contentapi.WriteActionStat, + Ref: ref, + }); err != nil { + return nil, 0, err + } + + resp, err := wrclient.Recv() + if err != nil { + return nil, 0, err + } + + return wrclient, resp.Offset, nil +} + +type remoteWriter struct { + ref string + client contentapi.Content_WriteClient + offset int64 + digest digest.Digest +} + +func newRemoteWriter(client contentapi.Content_WriteClient, ref string, offset int64) (*remoteWriter, error) { + return &remoteWriter{ + ref: ref, + client: client, + offset: offset, + }, nil +} + +// send performs a synchronous req-resp cycle on the client. +func (rw *remoteWriter) send(req *contentapi.WriteRequest) (*contentapi.WriteResponse, error) { + if err := rw.client.Send(req); err != nil { + return nil, err + } + + resp, err := rw.client.Recv() + + if err == nil { + // try to keep these in sync + if resp.Digest != "" { + rw.digest = resp.Digest + } + } + + return resp, err +} + +func (rw *remoteWriter) Status() (Status, error) { + resp, err := rw.send(&contentapi.WriteRequest{ + Action: contentapi.WriteActionStat, + }) + if err != nil { + return Status{}, err + } + + return Status{ + Ref: rw.ref, + Offset: resp.Offset, + StartedAt: resp.StartedAt, + UpdatedAt: resp.UpdatedAt, + }, nil +} + +func (rw *remoteWriter) Digest() digest.Digest { + return rw.digest +} + +func (rw *remoteWriter) Write(p []byte) (n int, err error) { + offset := rw.offset + + resp, err := rw.send(&contentapi.WriteRequest{ + Action: contentapi.WriteActionWrite, + Offset: offset, + Data: p, + }) + if err != nil { + return 0, err + } + + n = int(resp.Offset - offset) + if n < len(p) { + err = io.ErrShortWrite + } + + rw.offset += int64(n) + return +} + +func (rw *remoteWriter) Commit(size int64, expected digest.Digest) error { + resp, err := rw.send(&contentapi.WriteRequest{ + Action: contentapi.WriteActionCommit, + ExpectedSize: size, + ExpectedDigest: expected, + }) + if err != nil { + return err + } + + if size != 0 && resp.Offset != size { + return errors.Errorf("unexpected size: %v != %v", resp.Offset, size) + } + + if expected != "" && resp.Digest != expected { + return errors.New("unexpected digest") + } + + return nil +} + +func (rw *remoteWriter) Close() error { + return rw.client.CloseSend() +} diff --git a/content/service.go b/content/service.go new file mode 100644 index 0000000..a3c8184 --- /dev/null +++ b/content/service.go @@ -0,0 +1,233 @@ +package content + +import ( + "errors" + "io" + + contentapi "github.com/docker/containerd/api/services/content" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +type Service struct { + store *Store +} + +var _ contentapi.ContentServer = &Service{} + +func NewService(store *Store) contentapi.ContentServer { + return &Service{store: store} +} + +func (s *Service) Info(ctx context.Context, req *contentapi.InfoRequest) (*contentapi.InfoResponse, error) { + if err := req.Digest.Validate(); err != nil { + return nil, grpc.Errorf(codes.InvalidArgument, "%q failed validation", req.Digest) + } + + bi, err := s.store.Info(req.Digest) + if err != nil { + return nil, maybeNotFoundGRPC(err, req.Digest.String()) + } + + return &contentapi.InfoResponse{ + Digest: req.Digest, + Size_: bi.Size, + CommittedAt: bi.CommittedAt, + }, nil +} + +func (s *Service) Read(req *contentapi.ReadRequest, session contentapi.Content_ReadServer) error { + if err := req.Digest.Validate(); err != nil { + return grpc.Errorf(codes.InvalidArgument, "%v: %v", req.Digest, err) + } + + oi, err := s.store.Info(req.Digest) + if err != nil { + return maybeNotFoundGRPC(err, req.Digest.String()) + } + + rc, err := s.store.Reader(session.Context(), req.Digest) + if err != nil { + return maybeNotFoundGRPC(err, req.Digest.String()) + } + defer rc.Close() // TODO(stevvooe): Cache these file descriptors for performance. + + ra, ok := rc.(io.ReaderAt) + if !ok { + // TODO(stevvooe): Need to set this up to get correct behavior across + // board. May change interface to store to just return ReaderAtCloser. + // Possibly, we could just return io.ReaderAt and handle file + // descriptors internally. + return errors.New("content service only supports content stores that return ReaderAt") + } + + var ( + offset = req.Offset + size = req.Size_ + + // TODO(stevvooe): Using the global buffer pool. At 32KB, it is probably + // little inefficient for work over a fast network. We can tune this later. + p = bufPool.Get().([]byte) + ) + defer bufPool.Put(p) + + if offset < 0 { + offset = 0 + } + + if size <= 0 { + size = oi.Size - offset + } + + if offset+size > oi.Size { + return grpc.Errorf(codes.OutOfRange, "read past object length %v bytes", oi.Size) + } + + if _, err := io.CopyBuffer( + &readResponseWriter{session: session}, + io.NewSectionReader(ra, offset, size), p); err != nil { + return err + } + + return nil +} + +type readResponseWriter struct { + offset int64 + session contentapi.Content_ReadServer +} + +func (rw *readResponseWriter) Write(p []byte) (n int, err error) { + if err := rw.session.Send(&contentapi.ReadResponse{ + Offset: rw.offset, + Data: p, + }); err != nil { + return 0, err + } + + rw.offset += int64(len(p)) + return len(p), nil +} + +func (s *Service) Write(session contentapi.Content_WriteServer) (err error) { + var ( + ref string + msg contentapi.WriteResponse + req *contentapi.WriteRequest + ) + + defer func(msg *contentapi.WriteResponse) { + // pump through the last message if no error was encountered + if err != nil { + return + } + + err = session.Send(msg) + }(&msg) + + // handle the very first request! + req, err = session.Recv() + if err != nil { + return err + } + + ref = req.Ref + if ref == "" { + return grpc.Errorf(codes.InvalidArgument, "first message must have a reference") + } + + // this action locks the writer for the session. + wr, err := s.store.Writer(session.Context(), ref) + if err != nil { + return err + } + defer wr.Close() + + for { + // TODO(stevvooe): We need to study this behavior in containerd a + // little better to decide where to put this. We may be able to make + // this determination elsewhere and avoid even creating the writer. + // + // Ideally, we just use the expected digest on commit to abandon the + // cost of the move when they collide. + if req.ExpectedDigest != "" { + if _, err := s.store.Info(req.ExpectedDigest); err != nil { + if !IsNotFound(err) { + return err + } + + return grpc.Errorf(codes.AlreadyExists, "blob with expected digest %v exists", req.ExpectedDigest) + } + } + + msg.Action = req.Action + ws, err := wr.Status() + if err != nil { + return err + } + + msg.Offset = ws.Offset + msg.StartedAt = ws.StartedAt + msg.UpdatedAt = ws.UpdatedAt + + switch req.Action { + case contentapi.WriteActionStat: + msg.Digest = wr.Digest() + case contentapi.WriteActionWrite, contentapi.WriteActionCommit: + if req.Offset > 0 { + // validate the offset if provided + if req.Offset != ws.Offset { + return grpc.Errorf(codes.OutOfRange, "write @%v must occur at current offset %v", req.Offset, ws.Offset) + } + } + + // issue the write if we actually have data. + if len(req.Data) > 0 { + // While this looks like we could use io.WriterAt here, because we + // maintain the offset as append only, we just issue the write. + n, err := wr.Write(req.Data) + if err != nil { + return err + } + + if n != len(req.Data) { + // TODO(stevvooe): Perhaps, we can recover this by including it + // in the offset on the write return. + return grpc.Errorf(codes.DataLoss, "wrote %v of %v bytes", n, len(req.Data)) + } + + msg.Offset += int64(n) + } + + if req.Action == contentapi.WriteActionCommit { + return wr.Commit(req.ExpectedSize, req.ExpectedDigest) + } + case contentapi.WriteActionAbort: + return s.store.Abort(ref) + } + + if err := session.Send(&msg); err != nil { + return err + } + + req, err = session.Recv() + if err != nil { + return err + } + } + + return nil +} + +func (s *Service) Status(*contentapi.StatusRequest, contentapi.Content_StatusServer) error { + return grpc.Errorf(codes.Unimplemented, "not implemented") +} + +func maybeNotFoundGRPC(err error, id string) error { + if IsNotFound(err) { + return grpc.Errorf(codes.NotFound, "%v: not found", id) + } + + return err +}