Merge pull request #5822 from creack/update_vendors

Update archive/tar vendored

archive/archive_test.go

@@ -137,3 +137,32 @@ func TestTypeXGlobalHeaderDoesNotFail(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+// Some tar have both GNU specific (huge uid) and Ustar specific (long name) things.
+// Not supposed to happen (should use PAX instead of Ustar for long name) but it does and it should still work.
+func TestUntarUstarGnuConflict(t *testing.T) {
+	f, err := os.Open("testdata/broken.tar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := false
+	tr := tar.NewReader(f)
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if hdr.Name == "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm" {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatal("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm")
+	}
+}

hack/vendor.sh

@@ -53,7 +53,7 @@ clone hg code.google.com/p/gosqlite 74691fb6f837
 
 # get Go tip's archive/tar, for xattr support
 # TODO after Go 1.3 drops, bump our minimum supported version and drop this vendored dep
-clone hg code.google.com/p/go a15f344a9efa
+clone hg code.google.com/p/go 3458ba248590
 mv src/code.google.com/p/go/src/pkg/archive/tar tmp-tar
 rm -rf src/code.google.com/p/go
 mkdir -p src/code.google.com/p/go/src/pkg/archive

vendor/src/code.google.com/p/go/src/pkg/archive/tar/common.go

@@ -38,6 +38,7 @@ const (
 	TypeXGlobalHeader = 'g'    // global extended header
 	TypeGNULongName   = 'L'    // Next file has a long name
 	TypeGNULongLink   = 'K'    // Next file symlinks to a file w/ a long name
+	TypeGNUSparse     = 'S'    // sparse file
 )
 
 // A Header represents a single header in a tar archive.

vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go

@@ -29,12 +29,57 @@ const maxNanoSecondIntSize = 9
 // The Next method advances to the next file in the archive (including the first),
 // and then it can be treated as an io.Reader to access the file's data.
 type Reader struct {
-	r   io.Reader
+	r    io.Reader
-	err error
+	err  error
-	nb  int64 // number of unread bytes for current file entry
+	pad  int64          // amount of padding (ignored) after current file entry
-	pad int64 // amount of padding (ignored) after current file entry
+	curr numBytesReader // reader for current file entry
 }
 
+// A numBytesReader is an io.Reader with a numBytes method, returning the number
+// of bytes remaining in the underlying encoded data.
+type numBytesReader interface {
+	io.Reader
+	numBytes() int64
+}
+
+// A regFileReader is a numBytesReader for reading file data from a tar archive.
+type regFileReader struct {
+	r  io.Reader // underlying reader
+	nb int64     // number of unread bytes for current file entry
+}
+
+// A sparseFileReader is a numBytesReader for reading sparse file data from a tar archive.
+type sparseFileReader struct {
+	rfr *regFileReader // reads the sparse-encoded file data
+	sp  []sparseEntry  // the sparse map for the file
+	pos int64          // keeps track of file position
+	tot int64          // total size of the file
+}
+
+// Keywords for GNU sparse files in a PAX extended header
+const (
+	paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
+	paxGNUSparseOffset    = "GNU.sparse.offset"
+	paxGNUSparseNumBytes  = "GNU.sparse.numbytes"
+	paxGNUSparseMap       = "GNU.sparse.map"
+	paxGNUSparseName      = "GNU.sparse.name"
+	paxGNUSparseMajor     = "GNU.sparse.major"
+	paxGNUSparseMinor     = "GNU.sparse.minor"
+	paxGNUSparseSize      = "GNU.sparse.size"
+	paxGNUSparseRealSize  = "GNU.sparse.realsize"
+)
+
+// Keywords for old GNU sparse headers
+const (
+	oldGNUSparseMainHeaderOffset               = 386
+	oldGNUSparseMainHeaderIsExtendedOffset     = 482
+	oldGNUSparseMainHeaderNumEntries           = 4
+	oldGNUSparseExtendedHeaderIsExtendedOffset = 504
+	oldGNUSparseExtendedHeaderNumEntries       = 21
+	oldGNUSparseOffsetSize                     = 12
+	oldGNUSparseNumBytesSize                   = 12
+)
+
 // NewReader creates a new Reader reading from r.
 func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
 
@@ -64,6 +109,18 @@ func (tr *Reader) Next() (*Header, error) {
 		tr.skipUnread()
 		hdr = tr.readHeader()
 		mergePAX(hdr, headers)
+
+		// Check for a PAX format sparse file
+		sp, err := tr.checkForGNUSparsePAXHeaders(hdr, headers)
+		if err != nil {
+			tr.err = err
+			return nil, err
+		}
+		if sp != nil {
+			// Current file is a PAX format GNU sparse file.
+			// Set the current file reader to a sparse file reader.
+			tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
+		}
 		return hdr, nil
 	case TypeGNULongName:
 		// We have a GNU long name header. Its contents are the real file name.
@@ -87,6 +144,67 @@ func (tr *Reader) Next() (*Header, error) {
 	return hdr, tr.err
 }
 
+// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then
+// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to
+// be treated as a regular file.
+func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) {
+	var sparseFormat string
+
+	// Check for sparse format indicators
+	major, majorOk := headers[paxGNUSparseMajor]
+	minor, minorOk := headers[paxGNUSparseMinor]
+	sparseName, sparseNameOk := headers[paxGNUSparseName]
+	_, sparseMapOk := headers[paxGNUSparseMap]
+	sparseSize, sparseSizeOk := headers[paxGNUSparseSize]
+	sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize]
+
+	// Identify which, if any, sparse format applies from which PAX headers are set
+	if majorOk && minorOk {
+		sparseFormat = major + "." + minor
+	} else if sparseNameOk && sparseMapOk {
+		sparseFormat = "0.1"
+	} else if sparseSizeOk {
+		sparseFormat = "0.0"
+	} else {
+		// Not a PAX format GNU sparse file.
+		return nil, nil
+	}
+
+	// Check for unknown sparse format
+	if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" {
+		return nil, nil
+	}
+
+	// Update hdr from GNU sparse PAX headers
+	if sparseNameOk {
+		hdr.Name = sparseName
+	}
+	if sparseSizeOk {
+		realSize, err := strconv.ParseInt(sparseSize, 10, 0)
+		if err != nil {
+			return nil, ErrHeader
+		}
+		hdr.Size = realSize
+	} else if sparseRealSizeOk {
+		realSize, err := strconv.ParseInt(sparseRealSize, 10, 0)
+		if err != nil {
+			return nil, ErrHeader
+		}
+		hdr.Size = realSize
+	}
+
+	// Set up the sparse map, according to the particular sparse format in use
+	var sp []sparseEntry
+	var err error
+	switch sparseFormat {
+	case "0.0", "0.1":
+		sp, err = readGNUSparseMap0x1(headers)
+	case "1.0":
+		sp, err = readGNUSparseMap1x0(tr.curr)
+	}
+	return sp, err
+}
+
 // mergePAX merges well known headers according to PAX standard.
 // In general headers with the same name as those found
 // in the header struct overwrite those found in the header
@@ -194,6 +312,11 @@ func parsePAX(r io.Reader) (map[string]string, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// For GNU PAX sparse format 0.0 support.
+	// This function transforms the sparse format 0.0 headers into sparse format 0.1 headers.
+	var sparseMap bytes.Buffer
+
 	headers := make(map[string]string)
 	// Each record is constructed as
 	//     "%d %s=%s\n", length, keyword, value
@@ -211,7 +334,7 @@ func parsePAX(r io.Reader) (map[string]string, error) {
 			return nil, ErrHeader
 		}
 		// Extract everything between the decimal and the n -1 on the
-		// beginning to to eat the ' ', -1 on the end to skip the newline.
+		// beginning to eat the ' ', -1 on the end to skip the newline.
 		var record []byte
 		record, buf = buf[sp+1:n-1], buf[n:]
 		// The first equals is guaranteed to mark the end of the key.
@@ -221,7 +344,21 @@ func parsePAX(r io.Reader) (map[string]string, error) {
 			return nil, ErrHeader
 		}
 		key, value := record[:eq], record[eq+1:]
-		headers[string(key)] = string(value)
+
+		keyStr := string(key)
+		if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes {
+			// GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map.
+			sparseMap.Write(value)
+			sparseMap.Write([]byte{','})
+		} else {
+			// Normal key. Set the value in the headers map.
+			headers[keyStr] = string(value)
+		}
+	}
+	if sparseMap.Len() != 0 {
+		// Add sparse info to headers, chopping off the extra comma
+		sparseMap.Truncate(sparseMap.Len() - 1)
+		headers[paxGNUSparseMap] = sparseMap.String()
 	}
 	return headers, nil
 }
@@ -268,8 +405,8 @@ func (tr *Reader) octal(b []byte) int64 {
 
 // skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding.
 func (tr *Reader) skipUnread() {
-	nr := tr.nb + tr.pad // number of bytes to skip
+	nr := tr.numBytes() + tr.pad // number of bytes to skip
-	tr.nb, tr.pad = 0, 0
+	tr.curr, tr.pad = nil, 0
 	if sr, ok := tr.r.(io.Seeker); ok {
 		if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil {
 			return
@@ -331,14 +468,14 @@ func (tr *Reader) readHeader() *Header {
 	// so its magic bytes, like the rest of the block, are NULs.
 	magic := string(s.next(8)) // contains version field as well.
 	var format string
-	switch magic {
+	switch {
-	case "ustar\x0000": // POSIX tar (1003.1-1988)
+	case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988)
 		if string(header[508:512]) == "tar\x00" {
 			format = "star"
 		} else {
 			format = "posix"
 		}
-	case "ustar  \x00": // old GNU tar
+	case magic == "ustar  \x00": // old GNU tar
 		format = "gnu"
 	}
 
@@ -373,30 +510,308 @@ func (tr *Reader) readHeader() *Header {
 
 	// Maximum value of hdr.Size is 64 GB (12 octal digits),
 	// so there's no risk of int64 overflowing.
-	tr.nb = int64(hdr.Size)
+	nb := int64(hdr.Size)
-	tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two
+	tr.pad = -nb & (blockSize - 1) // blockSize is a power of two
+
+	// Set the current file reader.
+	tr.curr = &regFileReader{r: tr.r, nb: nb}
+
+	// Check for old GNU sparse format entry.
+	if hdr.Typeflag == TypeGNUSparse {
+		// Get the real size of the file.
+		hdr.Size = tr.octal(header[483:495])
+
+		// Read the sparse map.
+		sp := tr.readOldGNUSparseMap(header)
+		if tr.err != nil {
+			return nil
+		}
+		// Current file is a GNU sparse file. Update the current file reader.
+		tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
+	}
 
 	return hdr
 }
 
+// A sparseEntry holds a single entry in a sparse file's sparse map.
+// A sparse entry indicates the offset and size in a sparse file of a
+// block of data.
+type sparseEntry struct {
+	offset   int64
+	numBytes int64
+}
+
+// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format.
+// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries,
+// then one or more extension headers are used to store the rest of the sparse map.
+func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
+	isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0
+	spCap := oldGNUSparseMainHeaderNumEntries
+	if isExtended {
+		spCap += oldGNUSparseExtendedHeaderNumEntries
+	}
+	sp := make([]sparseEntry, 0, spCap)
+	s := slicer(header[oldGNUSparseMainHeaderOffset:])
+
+	// Read the four entries from the main tar header
+	for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ {
+		offset := tr.octal(s.next(oldGNUSparseOffsetSize))
+		numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
+		if tr.err != nil {
+			tr.err = ErrHeader
+			return nil
+		}
+		if offset == 0 && numBytes == 0 {
+			break
+		}
+		sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+	}
+
+	for isExtended {
+		// There are more entries. Read an extension header and parse its entries.
+		sparseHeader := make([]byte, blockSize)
+		if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil {
+			return nil
+		}
+		isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0
+		s = slicer(sparseHeader)
+		for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ {
+			offset := tr.octal(s.next(oldGNUSparseOffsetSize))
+			numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
+			if tr.err != nil {
+				tr.err = ErrHeader
+				return nil
+			}
+			if offset == 0 && numBytes == 0 {
+				break
+			}
+			sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+		}
+	}
+	return sp
+}
+
+// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format version 1.0.
+// The sparse map is stored just before the file data and padded out to the nearest block boundary.
+func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) {
+	buf := make([]byte, 2*blockSize)
+	sparseHeader := buf[:blockSize]
+
+	// readDecimal is a helper function to read a decimal integer from the sparse map
+	// while making sure to read from the file in blocks of size blockSize
+	readDecimal := func() (int64, error) {
+		// Look for newline
+		nl := bytes.IndexByte(sparseHeader, '\n')
+		if nl == -1 {
+			if len(sparseHeader) >= blockSize {
+				// This is an error
+				return 0, ErrHeader
+			}
+			oldLen := len(sparseHeader)
+			newLen := oldLen + blockSize
+			if cap(sparseHeader) < newLen {
+				// There's more header, but we need to make room for the next block
+				copy(buf, sparseHeader)
+				sparseHeader = buf[:newLen]
+			} else {
+				// There's more header, and we can just reslice
+				sparseHeader = sparseHeader[:newLen]
+			}
+
+			// Now that sparseHeader is large enough, read next block
+			if _, err := io.ReadFull(r, sparseHeader[oldLen:newLen]); err != nil {
+				return 0, err
+			}
+
+			// Look for a newline in the new data
+			nl = bytes.IndexByte(sparseHeader[oldLen:newLen], '\n')
+			if nl == -1 {
+				// This is an error
+				return 0, ErrHeader
+			}
+			nl += oldLen // We want the position from the beginning
+		}
+		// Now that we've found a newline, read a number
+		n, err := strconv.ParseInt(string(sparseHeader[:nl]), 10, 0)
+		if err != nil {
+			return 0, ErrHeader
+		}
+
+		// Update sparseHeader to consume this number
+		sparseHeader = sparseHeader[nl+1:]
+		return n, nil
+	}
+
+	// Read the first block
+	if _, err := io.ReadFull(r, sparseHeader); err != nil {
+		return nil, err
+	}
+
+	// The first line contains the number of entries
+	numEntries, err := readDecimal()
+	if err != nil {
+		return nil, err
+	}
+
+	// Read all the entries
+	sp := make([]sparseEntry, 0, numEntries)
+	for i := int64(0); i < numEntries; i++ {
+		// Read the offset
+		offset, err := readDecimal()
+		if err != nil {
+			return nil, err
+		}
+		// Read numBytes
+		numBytes, err := readDecimal()
+		if err != nil {
+			return nil, err
+		}
+
+		sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+	}
+
+	return sp, nil
+}
+
+// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format version 0.1.
+// The sparse map is stored in the PAX headers.
+func readGNUSparseMap0x1(headers map[string]string) ([]sparseEntry, error) {
+	// Get number of entries
+	numEntriesStr, ok := headers[paxGNUSparseNumBlocks]
+	if !ok {
+		return nil, ErrHeader
+	}
+	numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0)
+	if err != nil {
+		return nil, ErrHeader
+	}
+
+	sparseMap := strings.Split(headers[paxGNUSparseMap], ",")
+
+	// There should be two numbers in sparseMap for each entry
+	if int64(len(sparseMap)) != 2*numEntries {
+		return nil, ErrHeader
+	}
+
+	// Loop through the entries in the sparse map
+	sp := make([]sparseEntry, 0, numEntries)
+	for i := int64(0); i < numEntries; i++ {
+		offset, err := strconv.ParseInt(sparseMap[2*i], 10, 0)
+		if err != nil {
+			return nil, ErrHeader
+		}
+		numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 0)
+		if err != nil {
+			return nil, ErrHeader
+		}
+		sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+	}
+
+	return sp, nil
+}
+
+// numBytes returns the number of bytes left to read in the current file's entry
+// in the tar archive, or 0 if there is no current file.
+func (tr *Reader) numBytes() int64 {
+	if tr.curr == nil {
+		// No current file, so no bytes
+		return 0
+	}
+	return tr.curr.numBytes()
+}
+
 // Read reads from the current entry in the tar archive.
 // It returns 0, io.EOF when it reaches the end of that entry,
 // until Next is called to advance to the next entry.
 func (tr *Reader) Read(b []byte) (n int, err error) {
-	if tr.nb == 0 {
+	if tr.curr == nil {
+		return 0, io.EOF
+	}
+	n, err = tr.curr.Read(b)
+	if err != nil && err != io.EOF {
+		tr.err = err
+	}
+	return
+}
+
+func (rfr *regFileReader) Read(b []byte) (n int, err error) {
+	if rfr.nb == 0 {
 		// file consumed
 		return 0, io.EOF
 	}
-
+	if int64(len(b)) > rfr.nb {
-	if int64(len(b)) > tr.nb {
+		b = b[0:rfr.nb]
-		b = b[0:tr.nb]
 	}
-	n, err = tr.r.Read(b)
+	n, err = rfr.r.Read(b)
-	tr.nb -= int64(n)
+	rfr.nb -= int64(n)
 
-	if err == io.EOF && tr.nb > 0 {
+	if err == io.EOF && rfr.nb > 0 {
 		err = io.ErrUnexpectedEOF
 	}
-	tr.err = err
 	return
 }
+
+// numBytes returns the number of bytes left to read in the file's data in the tar archive.
+func (rfr *regFileReader) numBytes() int64 {
+	return rfr.nb
+}
+
+// readHole reads a sparse file hole ending at offset toOffset
+func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int {
+	n64 := toOffset - sfr.pos
+	if n64 > int64(len(b)) {
+		n64 = int64(len(b))
+	}
+	n := int(n64)
+	for i := 0; i < n; i++ {
+		b[i] = 0
+	}
+	sfr.pos += n64
+	return n
+}
+
+// Read reads the sparse file data in expanded form.
+func (sfr *sparseFileReader) Read(b []byte) (n int, err error) {
+	if len(sfr.sp) == 0 {
+		// No more data fragments to read from.
+		if sfr.pos < sfr.tot {
+			// We're in the last hole
+			n = sfr.readHole(b, sfr.tot)
+			return
+		}
+		// Otherwise, we're at the end of the file
+		return 0, io.EOF
+	}
+	if sfr.pos < sfr.sp[0].offset {
+		// We're in a hole
+		n = sfr.readHole(b, sfr.sp[0].offset)
+		return
+	}
+
+	// We're not in a hole, so we'll read from the next data fragment
+	posInFragment := sfr.pos - sfr.sp[0].offset
+	bytesLeft := sfr.sp[0].numBytes - posInFragment
+	if int64(len(b)) > bytesLeft {
+		b = b[0:bytesLeft]
+	}
+
+	n, err = sfr.rfr.Read(b)
+	sfr.pos += int64(n)
+
+	if int64(n) == bytesLeft {
+		// We're done with this fragment
+		sfr.sp = sfr.sp[1:]
+	}
+
+	if err == io.EOF && sfr.pos < sfr.tot {
+		// We reached the end of the last fragment's data, but there's a final hole
+		err = nil
+	}
+	return
+}
+
+// numBytes returns the number of bytes left to read in the sparse file's
+// sparse-encoded data in the tar archive.
+func (sfr *sparseFileReader) numBytes() int64 {
+	return sfr.rfr.nb
+}

vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader_test.go

@@ -9,6 +9,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"reflect"
 	"strings"
@@ -54,8 +55,92 @@ var gnuTarTest = &untarTest{
 	},
 }
 
+var sparseTarTest = &untarTest{
+	file: "testdata/sparse-formats.tar",
+	headers: []*Header{
+		{
+			Name:     "sparse-gnu",
+			Mode:     420,
+			Uid:      1000,
+			Gid:      1000,
+			Size:     200,
+			ModTime:  time.Unix(1392395740, 0),
+			Typeflag: 0x53,
+			Linkname: "",
+			Uname:    "david",
+			Gname:    "david",
+			Devmajor: 0,
+			Devminor: 0,
+		},
+		{
+			Name:     "sparse-posix-0.0",
+			Mode:     420,
+			Uid:      1000,
+			Gid:      1000,
+			Size:     200,
+			ModTime:  time.Unix(1392342187, 0),
+			Typeflag: 0x30,
+			Linkname: "",
+			Uname:    "david",
+			Gname:    "david",
+			Devmajor: 0,
+			Devminor: 0,
+		},
+		{
+			Name:     "sparse-posix-0.1",
+			Mode:     420,
+			Uid:      1000,
+			Gid:      1000,
+			Size:     200,
+			ModTime:  time.Unix(1392340456, 0),
+			Typeflag: 0x30,
+			Linkname: "",
+			Uname:    "david",
+			Gname:    "david",
+			Devmajor: 0,
+			Devminor: 0,
+		},
+		{
+			Name:     "sparse-posix-1.0",
+			Mode:     420,
+			Uid:      1000,
+			Gid:      1000,
+			Size:     200,
+			ModTime:  time.Unix(1392337404, 0),
+			Typeflag: 0x30,
+			Linkname: "",
+			Uname:    "david",
+			Gname:    "david",
+			Devmajor: 0,
+			Devminor: 0,
+		},
+		{
+			Name:     "end",
+			Mode:     420,
+			Uid:      1000,
+			Gid:      1000,
+			Size:     4,
+			ModTime:  time.Unix(1392398319, 0),
+			Typeflag: 0x30,
+			Linkname: "",
+			Uname:    "david",
+			Gname:    "david",
+			Devmajor: 0,
+			Devminor: 0,
+		},
+	},
+	cksums: []string{
+		"6f53234398c2449fe67c1812d993012f",
+		"6f53234398c2449fe67c1812d993012f",
+		"6f53234398c2449fe67c1812d993012f",
+		"6f53234398c2449fe67c1812d993012f",
+		"b0061974914468de549a2af8ced10316",
+	},
+}
+
 var untarTests = []*untarTest{
 	gnuTarTest,
+	sparseTarTest,
 	{
 		file: "testdata/star.tar",
 		headers: []*Header{
@@ -386,7 +471,7 @@ func TestParsePAXHeader(t *testing.T) {
 func TestParsePAXTime(t *testing.T) {
 	// Some valid PAX time values
 	timestamps := map[string]time.Time{
-		"1350244992.023960108":  time.Unix(1350244992, 23960108), // The commoon case
+		"1350244992.023960108":  time.Unix(1350244992, 23960108), // The common case
 		"1350244992.02396010":   time.Unix(1350244992, 23960100), // Lower precision value
 		"1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value
 		"1350244992":            time.Unix(1350244992, 0),        // Low precision value
@@ -423,3 +508,236 @@ func TestMergePAX(t *testing.T) {
 		t.Errorf("incorrect merge: got %+v, want %+v", hdr, want)
 	}
 }
+
+func TestSparseEndToEnd(t *testing.T) {
+	test := sparseTarTest
+	f, err := os.Open(test.file)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer f.Close()
+
+	tr := NewReader(f)
+
+	headers := test.headers
+	cksums := test.cksums
+	nread := 0
+
+	// loop over all files
+	for ; ; nread++ {
+		hdr, err := tr.Next()
+		if hdr == nil || err == io.EOF {
+			break
+		}
+
+		// check the header
+		if !reflect.DeepEqual(*hdr, *headers[nread]) {
+			t.Errorf("Incorrect header:\nhave %+v\nwant %+v",
+				*hdr, headers[nread])
+		}
+
+		// read and checksum the file data
+		h := md5.New()
+		_, err = io.Copy(h, tr)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+
+		// verify checksum
+		have := fmt.Sprintf("%x", h.Sum(nil))
+		want := cksums[nread]
+		if want != have {
+			t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want)
+		}
+	}
+	if nread != len(headers) {
+		t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread)
+	}
+}
+
+type sparseFileReadTest struct {
+	sparseData []byte
+	sparseMap  []sparseEntry
+	realSize   int64
+	expected   []byte
+}
+
+var sparseFileReadTests = []sparseFileReadTest{
+	{
+		sparseData: []byte("abcde"),
+		sparseMap: []sparseEntry{
+			{offset: 0, numBytes: 2},
+			{offset: 5, numBytes: 3},
+		},
+		realSize: 8,
+		expected: []byte("ab\x00\x00\x00cde"),
+	},
+	{
+		sparseData: []byte("abcde"),
+		sparseMap: []sparseEntry{
+			{offset: 0, numBytes: 2},
+			{offset: 5, numBytes: 3},
+		},
+		realSize: 10,
+		expected: []byte("ab\x00\x00\x00cde\x00\x00"),
+	},
+	{
+		sparseData: []byte("abcde"),
+		sparseMap: []sparseEntry{
+			{offset: 1, numBytes: 3},
+			{offset: 6, numBytes: 2},
+		},
+		realSize: 8,
+		expected: []byte("\x00abc\x00\x00de"),
+	},
+	{
+		sparseData: []byte("abcde"),
+		sparseMap: []sparseEntry{
+			{offset: 1, numBytes: 3},
+			{offset: 6, numBytes: 2},
+		},
+		realSize: 10,
+		expected: []byte("\x00abc\x00\x00de\x00\x00"),
+	},
+	{
+		sparseData: []byte(""),
+		sparseMap:  nil,
+		realSize:   2,
+		expected:   []byte("\x00\x00"),
+	},
+}
+
+func TestSparseFileReader(t *testing.T) {
+	for i, test := range sparseFileReadTests {
+		r := bytes.NewReader(test.sparseData)
+		nb := int64(r.Len())
+		sfr := &sparseFileReader{
+			rfr: &regFileReader{r: r, nb: nb},
+			sp:  test.sparseMap,
+			pos: 0,
+			tot: test.realSize,
+		}
+		if sfr.numBytes() != nb {
+			t.Errorf("test %d: Before reading, sfr.numBytes() = %d, want %d", i, sfr.numBytes(), nb)
+		}
+		buf, err := ioutil.ReadAll(sfr)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+		}
+		if e := test.expected; !bytes.Equal(buf, e) {
+			t.Errorf("test %d: Contents = %v, want %v", i, buf, e)
+		}
+		if sfr.numBytes() != 0 {
+			t.Errorf("test %d: After draining the reader, numBytes() was nonzero", i)
+		}
+	}
+}
+
+func TestSparseIncrementalRead(t *testing.T) {
+	sparseMap := []sparseEntry{{10, 2}}
+	sparseData := []byte("Go")
+	expected := "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Go\x00\x00\x00\x00\x00\x00\x00\x00"
+
+	r := bytes.NewReader(sparseData)
+	nb := int64(r.Len())
+	sfr := &sparseFileReader{
+		rfr: &regFileReader{r: r, nb: nb},
+		sp:  sparseMap,
+		pos: 0,
+		tot: int64(len(expected)),
+	}
+
+	// We'll read the data 6 bytes at a time, with a hole of size 10 at
+	// the beginning and one of size 8 at the end.
+	var outputBuf bytes.Buffer
+	buf := make([]byte, 6)
+	for {
+		n, err := sfr.Read(buf)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Errorf("Read: unexpected error %v\n", err)
+		}
+		if n > 0 {
+			_, err := outputBuf.Write(buf[:n])
+			if err != nil {
+				t.Errorf("Write: unexpected error %v\n", err)
+			}
+		}
+	}
+	got := outputBuf.String()
+	if got != expected {
+		t.Errorf("Contents = %v, want %v", got, expected)
+	}
+}
+
+func TestReadGNUSparseMap0x1(t *testing.T) {
+	headers := map[string]string{
+		paxGNUSparseNumBlocks: "4",
+		paxGNUSparseMap:       "0,5,10,5,20,5,30,5",
+	}
+	expected := []sparseEntry{
+		{offset: 0, numBytes: 5},
+		{offset: 10, numBytes: 5},
+		{offset: 20, numBytes: 5},
+		{offset: 30, numBytes: 5},
+	}
+
+	sp, err := readGNUSparseMap0x1(headers)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(sp, expected) {
+		t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected)
+	}
+}
+
+func TestReadGNUSparseMap1x0(t *testing.T) {
+	// This test uses lots of holes so the sparse header takes up more than two blocks
+	numEntries := 100
+	expected := make([]sparseEntry, 0, numEntries)
+	sparseMap := new(bytes.Buffer)
+
+	fmt.Fprintf(sparseMap, "%d\n", numEntries)
+	for i := 0; i < numEntries; i++ {
+		offset := int64(2048 * i)
+		numBytes := int64(1024)
+		expected = append(expected, sparseEntry{offset: offset, numBytes: numBytes})
+		fmt.Fprintf(sparseMap, "%d\n%d\n", offset, numBytes)
+	}
+
+	// Make the header the smallest multiple of blockSize that fits the sparseMap
+	headerBlocks := (sparseMap.Len() + blockSize - 1) / blockSize
+	bufLen := blockSize * headerBlocks
+	buf := make([]byte, bufLen)
+	copy(buf, sparseMap.Bytes())
+
+	// Get an reader to read the sparse map
+	r := bytes.NewReader(buf)
+
+	// Read the sparse map
+	sp, err := readGNUSparseMap1x0(r)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(sp, expected) {
+		t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected)
+	}
+}
+
+func TestUninitializedRead(t *testing.T) {
+	test := gnuTarTest
+	f, err := os.Open(test.file)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer f.Close()
+
+	tr := NewReader(f)
+	_, err = tr.Read([]byte{})
+	if err == nil || err != io.EOF {
+		t.Errorf("Unexpected error: %v, wanted %v", err, io.EOF)
+	}
+
+}

vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go

@@ -218,8 +218,8 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
 				tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil)
 
 				// Use the ustar magic if we used ustar long names.
-				if len(prefix) > 0 {
+				if len(prefix) > 0 && !tw.usedBinary {
-					copy(header[257:265], []byte("ustar\000"))
+					copy(header[257:265], []byte("ustar\x00"))
 				}
 			}
 		}

vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer_test.go

@@ -103,6 +103,29 @@ var writerTests = []*writerTest{
 			},
 		},
 	},
+	// The truncated test file was produced using these commands:
+	//   dd if=/dev/zero bs=1048576 count=16384 > (longname/)*15 /16gig.txt
+	//   tar -b 1 -c -f- (longname/)*15 /16gig.txt | dd bs=512 count=8 > writer-big-long.tar
+	{
+		file: "testdata/writer-big-long.tar",
+		entries: []*writerTestEntry{
+			{
+				header: &Header{
+					Name:     strings.Repeat("longname/", 15) + "16gig.txt",
+					Mode:     0644,
+					Uid:      1000,
+					Gid:      1000,
+					Size:     16 << 30,
+					ModTime:  time.Unix(1399583047, 0),
+					Typeflag: '0',
+					Uname:    "guillaume",
+					Gname:    "guillaume",
+				},
+				// fake contents
+				contents: strings.Repeat("\x00", 4<<10),
+			},
+		},
+	},
 	// This file was produced using gnu tar 1.17
 	// gnutar  -b 4 --format=ustar (longname/)*15 + file.txt
 	{