From aded075f80e97afe32bde3864ab6990b3b5112a8 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Thu, 28 Jul 2016 09:59:28 -0400 Subject: [PATCH] Rework some of the build machinery - Run tests in VMs with sufficiently-new gccgo - Assume that all binary builds build dynamic binaries Signed-off-by: Nalin Dahyabhai --- ioutils/bytespipe.go | 186 -------------------------------------- ioutils/bytespipe_test.go | 159 -------------------------------- 2 files changed, 345 deletions(-) delete mode 100644 ioutils/bytespipe.go delete mode 100644 ioutils/bytespipe_test.go diff --git a/ioutils/bytespipe.go b/ioutils/bytespipe.go deleted file mode 100644 index 72a04f3..0000000 --- a/ioutils/bytespipe.go +++ /dev/null @@ -1,186 +0,0 @@ -package ioutils - -import ( - "errors" - "io" - "sync" -) - -// maxCap is the highest capacity to use in byte slices that buffer data. -const maxCap = 1e6 - -// minCap is the lowest capacity to use in byte slices that buffer data -const minCap = 64 - -// blockThreshold is the minimum number of bytes in the buffer which will cause -// a write to BytesPipe to block when allocating a new slice. -const blockThreshold = 1e6 - -var ( - // ErrClosed is returned when Write is called on a closed BytesPipe. - ErrClosed = errors.New("write to closed BytesPipe") - - bufPools = make(map[int]*sync.Pool) - bufPoolsLock sync.Mutex -) - -// BytesPipe is io.ReadWriteCloser which works similarly to pipe(queue). -// All written data may be read at most once. Also, BytesPipe allocates -// and releases new byte slices to adjust to current needs, so the buffer -// won't be overgrown after peak loads. -type BytesPipe struct { - mu sync.Mutex - wait *sync.Cond - buf []*fixedBuffer - bufLen int - closeErr error // error to return from next Read. set to nil if not closed. -} - -// NewBytesPipe creates new BytesPipe, initialized by specified slice. -// If buf is nil, then it will be initialized with slice which cap is 64. -// buf will be adjusted in a way that len(buf) == 0, cap(buf) == cap(buf). -func NewBytesPipe() *BytesPipe { - bp := &BytesPipe{} - bp.buf = append(bp.buf, getBuffer(minCap)) - bp.wait = sync.NewCond(&bp.mu) - return bp -} - -// Write writes p to BytesPipe. -// It can allocate new []byte slices in a process of writing. -func (bp *BytesPipe) Write(p []byte) (int, error) { - bp.mu.Lock() - - written := 0 -loop0: - for { - if bp.closeErr != nil { - bp.mu.Unlock() - return written, ErrClosed - } - - if len(bp.buf) == 0 { - bp.buf = append(bp.buf, getBuffer(64)) - } - // get the last buffer - b := bp.buf[len(bp.buf)-1] - - n, err := b.Write(p) - written += n - bp.bufLen += n - - // errBufferFull is an error we expect to get if the buffer is full - if err != nil && err != errBufferFull { - bp.wait.Broadcast() - bp.mu.Unlock() - return written, err - } - - // if there was enough room to write all then break - if len(p) == n { - break - } - - // more data: write to the next slice - p = p[n:] - - // make sure the buffer doesn't grow too big from this write - for bp.bufLen >= blockThreshold { - bp.wait.Wait() - if bp.closeErr != nil { - continue loop0 - } - } - - // add new byte slice to the buffers slice and continue writing - nextCap := b.Cap() * 2 - if nextCap > maxCap { - nextCap = maxCap - } - bp.buf = append(bp.buf, getBuffer(nextCap)) - } - bp.wait.Broadcast() - bp.mu.Unlock() - return written, nil -} - -// CloseWithError causes further reads from a BytesPipe to return immediately. -func (bp *BytesPipe) CloseWithError(err error) error { - bp.mu.Lock() - if err != nil { - bp.closeErr = err - } else { - bp.closeErr = io.EOF - } - bp.wait.Broadcast() - bp.mu.Unlock() - return nil -} - -// Close causes further reads from a BytesPipe to return immediately. -func (bp *BytesPipe) Close() error { - return bp.CloseWithError(nil) -} - -// Read reads bytes from BytesPipe. -// Data could be read only once. -func (bp *BytesPipe) Read(p []byte) (n int, err error) { - bp.mu.Lock() - if bp.bufLen == 0 { - if bp.closeErr != nil { - bp.mu.Unlock() - return 0, bp.closeErr - } - bp.wait.Wait() - if bp.bufLen == 0 && bp.closeErr != nil { - err := bp.closeErr - bp.mu.Unlock() - return 0, err - } - } - - for bp.bufLen > 0 { - b := bp.buf[0] - read, _ := b.Read(p) // ignore error since fixedBuffer doesn't really return an error - n += read - bp.bufLen -= read - - if b.Len() == 0 { - // it's empty so return it to the pool and move to the next one - returnBuffer(b) - bp.buf[0] = nil - bp.buf = bp.buf[1:] - } - - if len(p) == read { - break - } - - p = p[read:] - } - - bp.wait.Broadcast() - bp.mu.Unlock() - return -} - -func returnBuffer(b *fixedBuffer) { - b.Reset() - bufPoolsLock.Lock() - pool := bufPools[b.Cap()] - bufPoolsLock.Unlock() - if pool != nil { - pool.Put(b) - } -} - -func getBuffer(size int) *fixedBuffer { - bufPoolsLock.Lock() - pool, ok := bufPools[size] - if !ok { - pool = &sync.Pool{New: func() interface{} { return &fixedBuffer{buf: make([]byte, 0, size)} }} - bufPools[size] = pool - } - bufPoolsLock.Unlock() - return pool.Get().(*fixedBuffer) -} diff --git a/ioutils/bytespipe_test.go b/ioutils/bytespipe_test.go deleted file mode 100644 index 300fb5f..0000000 --- a/ioutils/bytespipe_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package ioutils - -import ( - "crypto/sha1" - "encoding/hex" - "math/rand" - "testing" - "time" -) - -func TestBytesPipeRead(t *testing.T) { - buf := NewBytesPipe() - buf.Write([]byte("12")) - buf.Write([]byte("34")) - buf.Write([]byte("56")) - buf.Write([]byte("78")) - buf.Write([]byte("90")) - rd := make([]byte, 4) - n, err := buf.Read(rd) - if err != nil { - t.Fatal(err) - } - if n != 4 { - t.Fatalf("Wrong number of bytes read: %d, should be %d", n, 4) - } - if string(rd) != "1234" { - t.Fatalf("Read %s, but must be %s", rd, "1234") - } - n, err = buf.Read(rd) - if err != nil { - t.Fatal(err) - } - if n != 4 { - t.Fatalf("Wrong number of bytes read: %d, should be %d", n, 4) - } - if string(rd) != "5678" { - t.Fatalf("Read %s, but must be %s", rd, "5679") - } - n, err = buf.Read(rd) - if err != nil { - t.Fatal(err) - } - if n != 2 { - t.Fatalf("Wrong number of bytes read: %d, should be %d", n, 2) - } - if string(rd[:n]) != "90" { - t.Fatalf("Read %s, but must be %s", rd, "90") - } -} - -func TestBytesPipeWrite(t *testing.T) { - buf := NewBytesPipe() - buf.Write([]byte("12")) - buf.Write([]byte("34")) - buf.Write([]byte("56")) - buf.Write([]byte("78")) - buf.Write([]byte("90")) - if buf.buf[0].String() != "1234567890" { - t.Fatalf("Buffer %q, must be %q", buf.buf[0].String(), "1234567890") - } -} - -// Write and read in different speeds/chunk sizes and check valid data is read. -func TestBytesPipeWriteRandomChunks(t *testing.T) { - cases := []struct{ iterations, writesPerLoop, readsPerLoop int }{ - {100, 10, 1}, - {1000, 10, 5}, - {1000, 100, 0}, - {1000, 5, 6}, - {10000, 50, 25}, - } - - testMessage := []byte("this is a random string for testing") - // random slice sizes to read and write - writeChunks := []int{25, 35, 15, 20} - readChunks := []int{5, 45, 20, 25} - - for _, c := range cases { - // first pass: write directly to hash - hash := sha1.New() - for i := 0; i < c.iterations*c.writesPerLoop; i++ { - if _, err := hash.Write(testMessage[:writeChunks[i%len(writeChunks)]]); err != nil { - t.Fatal(err) - } - } - expected := hex.EncodeToString(hash.Sum(nil)) - - // write/read through buffer - buf := NewBytesPipe() - hash.Reset() - - done := make(chan struct{}) - - go func() { - // random delay before read starts - <-time.After(time.Duration(rand.Intn(10)) * time.Millisecond) - for i := 0; ; i++ { - p := make([]byte, readChunks[(c.iterations*c.readsPerLoop+i)%len(readChunks)]) - n, _ := buf.Read(p) - if n == 0 { - break - } - hash.Write(p[:n]) - } - - close(done) - }() - - for i := 0; i < c.iterations; i++ { - for w := 0; w < c.writesPerLoop; w++ { - buf.Write(testMessage[:writeChunks[(i*c.writesPerLoop+w)%len(writeChunks)]]) - } - } - buf.Close() - <-done - - actual := hex.EncodeToString(hash.Sum(nil)) - - if expected != actual { - t.Fatalf("BytesPipe returned invalid data. Expected checksum %v, got %v", expected, actual) - } - - } -} - -func BenchmarkBytesPipeWrite(b *testing.B) { - testData := []byte("pretty short line, because why not?") - for i := 0; i < b.N; i++ { - readBuf := make([]byte, 1024) - buf := NewBytesPipe() - go func() { - var err error - for err == nil { - _, err = buf.Read(readBuf) - } - }() - for j := 0; j < 1000; j++ { - buf.Write(testData) - } - buf.Close() - } -} - -func BenchmarkBytesPipeRead(b *testing.B) { - rd := make([]byte, 512) - for i := 0; i < b.N; i++ { - b.StopTimer() - buf := NewBytesPipe() - for j := 0; j < 500; j++ { - buf.Write(make([]byte, 1024)) - } - b.StartTimer() - for j := 0; j < 1000; j++ { - if n, _ := buf.Read(rd); n != 512 { - b.Fatalf("Wrong number of bytes: %d", n) - } - } - } -}