package storage import ( "bufio" "encoding/json" "errors" "io" "path" ) var ( ErrDuplicatePath = errors.New("duplicates of file paths not supported") ) // Packer describes the methods to pack Entries to a storage destination type Packer interface { // AddEntry packs the Entry and returns its position AddEntry(e Entry) (int, error) } // Unpacker describes the methods to read Entries from a source type Unpacker interface { // Next returns the next Entry being unpacked, or error, until io.EOF Next() (*Entry, error) } /* TODO(vbatts) figure out a good model for this type PackUnpacker interface { Packer Unpacker } */ type jsonUnpacker struct { r io.Reader b *bufio.Reader isEOF bool seen seenNames } func (jup *jsonUnpacker) Next() (*Entry, error) { var e Entry if jup.isEOF { // since ReadBytes() will return read bytes AND an EOF, we handle it this // round-a-bout way so we can Unmarshal the tail with relevant errors, but // still get an io.EOF when the stream is ended. return nil, io.EOF } line, err := jup.b.ReadBytes('\n') if err != nil && err != io.EOF { return nil, err } else if err == io.EOF { jup.isEOF = true } err = json.Unmarshal(line, &e) if err != nil && jup.isEOF { // if the remainder actually _wasn't_ a remaining json structure, then just EOF return nil, io.EOF } // check for dup name if e.Type == FileType { cName := path.Clean(e.Name) if _, ok := jup.seen[cName]; ok { return nil, ErrDuplicatePath } jup.seen[cName] = emptyByte } return &e, err } // NewJsonUnpacker provides an Unpacker that reads Entries (SegmentType and // FileType) as a json document. // // Each Entry read are expected to be delimited by new line. func NewJsonUnpacker(r io.Reader) Unpacker { return &jsonUnpacker{ r: r, b: bufio.NewReader(r), seen: seenNames{}, } } type jsonPacker struct { w io.Writer e *json.Encoder pos int seen seenNames } type seenNames map[string]byte // used in the seenNames map. byte is a uint8, and we'll re-use the same one // for minimalism. const emptyByte byte = 0 func (jp *jsonPacker) AddEntry(e Entry) (int, error) { // check early for dup name if e.Type == FileType { cName := path.Clean(e.Name) if _, ok := jp.seen[cName]; ok { return -1, ErrDuplicatePath } jp.seen[cName] = emptyByte } e.Position = jp.pos err := jp.e.Encode(e) if err != nil { return -1, err } // made it this far, increment now jp.pos++ return e.Position, nil } // NewJsonPacker provides an Packer that writes each Entry (SegmentType and // FileType) as a json document. // // The Entries are delimited by new line. func NewJsonPacker(w io.Writer) Packer { return &jsonPacker{ w: w, e: json.NewEncoder(w), seen: seenNames{}, } } /* TODO(vbatts) perhaps have a more compact packer/unpacker, maybe using msgapck (https://github.com/ugorji/go) Even though, since our jsonUnpacker and jsonPacker just take io.Reader/io.Writer, then we can get away with passing them a gzip.Reader/gzip.Writer */