vbatts/tar-split
1
0
Fork 0
forked from mirrors/tar-split
Code Releases Activity

WIP

Browse source
This commit is contained in:
Vincent Batts 2015-08-11 21:59:23 -04:00
parent e2a62d6b0d
commit 5c8d5cacba
13 changed files with 265 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tar/asm/README.md
View file

@ -10,7 +10,7 @@ Concerns
For completely safe assembly/disassembly, there will need to be a Content
Addressable Storage (CAS) directory, that maps to a checksum in the
`storage.Entity` of `storage.FileType`.
`storage.Entity` of `storage.FileCheckEntry`.
This is due to the fact that tar archives _can_ allow multiple records for the
same path, but the last one effectively wins. Even if the prior records had a

4
tar/asm/assemble.go
View file

@ -19,6 +19,10 @@ import (
// metadata. With the combination of these two items, a precise assembled Tar
// archive is possible.
func NewOutputTarStream(fg storage.FileGetter, up storage.Unpacker) io.ReadCloser {
return newOutputTarStreamWithOptions(fg, up, DefaultOutputOptions)
}
func newOutputTarStreamWithOptions(fg storage.FileGetter, up storage.Unpacker, opts Options) io.ReadCloser {
// ... Since these are interfaces, this is possible, so let's not have a nil pointer
if fg == nil || up == nil {
return nil

16
tar/asm/assemble_test.go
View file

@ -20,7 +20,7 @@ var entries = []struct {
}{
{
Entry: storage.Entry{
Type: storage.FileType,
Type: storage.FileCheckEntry,
Name: "./hurr.txt",
Payload: []byte{2, 116, 164, 177, 171, 236, 107, 78},
Size: 20,
@ -29,7 +29,7 @@ var entries = []struct {
},
{
Entry: storage.Entry{
Type: storage.FileType,
Type: storage.FileCheckEntry,
Name: "./ermahgerd.txt",
Payload: []byte{126, 72, 89, 239, 230, 252, 160, 187},
Size: 26,
@ -38,7 +38,7 @@ var entries = []struct {
},
{
Entry: storage.Entry{
Type: storage.FileType,
Type: storage.FileCheckEntry,
NameRaw: []byte{0x66, 0x69, 0x6c, 0x65, 0x2d, 0xe4}, // this is invalid UTF-8. Just checking the round trip.
Payload: []byte{126, 72, 89, 239, 230, 252, 160, 187},
Size: 26,
@ -52,7 +52,7 @@ var entriesMangled = []struct {
}{
{
Entry: storage.Entry{
Type: storage.FileType,
Type: storage.FileCheckEntry,
Name: "./hurr.txt",
Payload: []byte{3, 116, 164, 177, 171, 236, 107, 78},
Size: 20,
@ -62,7 +62,7 @@ var entriesMangled = []struct {
},
{
Entry: storage.Entry{
Type: storage.FileType,
Type: storage.FileCheckEntry,
Name: "./ermahgerd.txt",
Payload: []byte{127, 72, 89, 239, 230, 252, 160, 187},
Size: 26,
@ -72,7 +72,7 @@ var entriesMangled = []struct {
},
{
Entry: storage.Entry{
Type: storage.FileType,
Type: storage.FileCheckEntry,
NameRaw: []byte{0x66, 0x69, 0x6c, 0x65, 0x2d, 0xe4},
Payload: []byte{127, 72, 89, 239, 230, 252, 160, 187},
Size: 26,
@ -86,7 +86,7 @@ func TestTarStreamMangledGetterPutter(t *testing.T) {
// first lets prep a GetPutter and Packer
for i := range entries {
if entries[i].Entry.Type == storage.FileType {
if entries[i].Entry.Type == storage.FileCheckEntry {
j, csum, err := fgp.Put(entries[i].Entry.GetName(), bytes.NewBuffer(entries[i].Body))
if err != nil {
t.Error(err)
@ -107,7 +107,7 @@ func TestTarStreamMangledGetterPutter(t *testing.T) {
}
for _, e := range entriesMangled {
if e.Entry.Type == storage.FileType {
if e.Entry.Type == storage.FileCheckEntry {
rdr, err := fgp.Get(e.Entry.GetName())
if err != nil {
t.Error(err)

14
tar/asm/disassemble.go
View file

@ -19,6 +19,10 @@ import (
// storage.FilePutter. Since the checksumming is still needed, then a default
// of NewDiscardFilePutter will be used internally
func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io.Reader, error) {
return newInputTarStreamWithOptions(r, p, fp, DefaultInputOptions)
}
func newInputTarStreamWithOptions(r io.Reader, p storage.Packer, fp storage.FilePutter, opts Options) (io.Reader, error) {
// What to do here... folks will want their own access to the Reader that is
// their tar archive stream, but we'll need that same stream to use our
// forked 'archive/tar'.
@ -57,7 +61,7 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
// the end of an archive. Collect them too.
if b := tr.RawBytes(); len(b) > 0 {
_, err := p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Type: storage.SegmentEntry,
Payload: b,
})
if err != nil {
@ -73,7 +77,7 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
if b := tr.RawBytes(); len(b) > 0 {
_, err := p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Type: storage.SegmentEntry,
Payload: b,
})
if err != nil {
@ -93,7 +97,7 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
}
entry := storage.Entry{
Type: storage.FileType,
Type: storage.FileCheckEntry,
Size: hdr.Size,
Payload: csum,
}
@ -109,7 +113,7 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
if b := tr.RawBytes(); len(b) > 0 {
_, err = p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Type: storage.SegmentEntry,
Payload: b,
})
if err != nil {
@ -127,7 +131,7 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
return
}
_, err = p.AddEntry(storage.Entry{
Type: storage.SegmentType,
Type: storage.SegmentEntry,
Payload: remainder,
})
if err != nil {

18
tar/asm/options.go Normal file
View file

@ -0,0 +1,18 @@
package asm
// Defaults that matched existing behavior
var (
DefaultOutputOptions = OptFileCheck | OptSegment
DefaultInputOptions = OptFileCheck | OptSegment
)
// Options for processing the tar stream with additional options. Like
// including entries for on-disk verification.
type Options int
// The options include the FileCheckEntry, SegmentEntry, and for VerficationEntry
const (
OptFileCheck Options = 1 << iota
OptSegment
OptVerify
)