InTheForest/moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2026-01-11 10:41:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move pkg/tarsum to daemon/builder/remotecontext/internal/tarsum

Browse Source 
Signed-off-by: Derek McGowan <derek@mcg.dev>
This commit is contained in:
Derek McGowan
2025-07-24 12:12:23 -07:00
parent ca5d2348d0
commit 9566272f30
22 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
pkg/tarsum/builder_context.go
View File

@@ -1,21 +0,0 @@
package tarsum
// BuilderContext is an interface extending TarSum by adding the Remove method.
// In general there was concern about adding this method to TarSum itself
// so instead it is being added just to "BuilderContext" which will then
// only be used during the .dockerignore file processing
// - see builder/evaluator.go
type BuilderContext interface {
TarSum
Remove(string)
}
func (ts *tarSum) Remove(filename string) {
for i, fis := range ts.sums {
if fis.Name() == filename {
ts.sums = append(ts.sums[:i], ts.sums[i+1:]...)
// Note, we don't just return because there could be
// more than one with this name
}
}
}

66
pkg/tarsum/builder_context_test.go
View File

@@ -1,66 +0,0 @@
package tarsum
import (
"io"
"os"
"testing"
)
// Try to remove tarsum (in the BuilderContext) that do not exists, won't change a thing
func TestTarSumRemoveNonExistent(t *testing.T) {
filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar"
reader, err := os.Open(filename)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
ts, err := NewTarSum(reader, false, Version0)
if err != nil {
t.Fatal(err)
}
// Read and discard bytes so that it populates sums
_, err = io.Copy(io.Discard, ts)
if err != nil {
t.Errorf("failed to read from %s: %s", filename, err)
}
expected := len(ts.GetSums())
ts.(BuilderContext).Remove("")
ts.(BuilderContext).Remove("Anything")
if len(ts.GetSums()) != expected {
t.Fatalf("Expected %v sums, go %v.", expected, ts.GetSums())
}
}
// Remove a tarsum (in the BuilderContext)
func TestTarSumRemove(t *testing.T) {
filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar"
reader, err := os.Open(filename)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
ts, err := NewTarSum(reader, false, Version0)
if err != nil {
t.Fatal(err)
}
// Read and discard bytes so that it populates sums
_, err = io.Copy(io.Discard, ts)
if err != nil {
t.Errorf("failed to read from %s: %s", filename, err)
}
expected := len(ts.GetSums()) - 1
ts.(BuilderContext).Remove("etc/sudoers")
if len(ts.GetSums()) != expected {
t.Fatalf("Expected %v sums, go %v.", expected, len(ts.GetSums()))
}
}

135
pkg/tarsum/fileinfosums.go
View File

@@ -1,135 +0,0 @@
package tarsum
import (
"runtime"
"sort"
"strings"
)
// FileInfoSumInterface provides an interface for accessing file checksum
// information within a tar file. This info is accessed through interface
// so the actual name and sum cannot be melded with.
type FileInfoSumInterface interface {
// File name
Name() string
// Checksum of this particular file and its headers
Sum() string
// Position of file in the tar
Pos() int64
}
type fileInfoSum struct {
name string
sum string
pos int64
}
func (fis fileInfoSum) Name() string {
return fis.name
}
func (fis fileInfoSum) Sum() string {
return fis.sum
}
func (fis fileInfoSum) Pos() int64 {
return fis.pos
}
// FileInfoSums provides a list of FileInfoSumInterfaces.
type FileInfoSums []FileInfoSumInterface
// GetFile returns the first FileInfoSumInterface with a matching name.
func (fis FileInfoSums) GetFile(name string) FileInfoSumInterface {
// We do case insensitive matching on Windows as c:\APP and c:\app are
// the same. See issue #33107.
for i := range fis {
if (runtime.GOOS == "windows" && strings.EqualFold(fis[i].Name(), name)) ||
(runtime.GOOS != "windows" && fis[i].Name() == name) {
return fis[i]
}
}
return nil
}
// GetAllFile returns a FileInfoSums with all matching names.
func (fis FileInfoSums) GetAllFile(name string) FileInfoSums {
f := FileInfoSums{}
for i := range fis {
if fis[i].Name() == name {
f = append(f, fis[i])
}
}
return f
}
// GetDuplicatePaths returns a FileInfoSums with all duplicated paths.
func (fis FileInfoSums) GetDuplicatePaths() (dups FileInfoSums) {
seen := make(map[string]int, len(fis)) // allocate earl. no need to grow this map.
for i := range fis {
f := fis[i]
if _, ok := seen[f.Name()]; ok {
dups = append(dups, f)
} else {
seen[f.Name()] = 0
}
}
return dups
}
// Len returns the size of the FileInfoSums.
func (fis FileInfoSums) Len() int { return len(fis) }
// Swap swaps two FileInfoSum values if a FileInfoSums list.
func (fis FileInfoSums) Swap(i, j int) { fis[i], fis[j] = fis[j], fis[i] }
// SortByPos sorts FileInfoSums content by position.
func (fis FileInfoSums) SortByPos() {
sort.Sort(byPos{fis})
}
// SortByNames sorts FileInfoSums content by name.
func (fis FileInfoSums) SortByNames() {
sort.Sort(byName{fis})
}
// SortBySums sorts FileInfoSums content by sums.
func (fis FileInfoSums) SortBySums() {
dups := fis.GetDuplicatePaths()
if len(dups) > 0 {
sort.Sort(bySum{fis, dups})
} else {
sort.Sort(bySum{fis, nil})
}
}
// byName is a sort.Sort helper for sorting by file names.
// If names are the same, order them by their appearance in the tar archive
type byName struct{ FileInfoSums }
func (bn byName) Less(i, j int) bool {
if bn.FileInfoSums[i].Name() == bn.FileInfoSums[j].Name() {
return bn.FileInfoSums[i].Pos() < bn.FileInfoSums[j].Pos()
}
return bn.FileInfoSums[i].Name() < bn.FileInfoSums[j].Name()
}
// bySum is a sort.Sort helper for sorting by the sums of all the fileinfos in the tar archive
type bySum struct {
FileInfoSums
dups FileInfoSums
}
func (bs bySum) Less(i, j int) bool {
if bs.dups != nil && bs.FileInfoSums[i].Name() == bs.FileInfoSums[j].Name() {
return bs.FileInfoSums[i].Pos() < bs.FileInfoSums[j].Pos()
}
return bs.FileInfoSums[i].Sum() < bs.FileInfoSums[j].Sum()
}
// byPos is a sort.Sort helper for sorting by the sums of all the fileinfos by their original order
type byPos struct{ FileInfoSums }
func (bp byPos) Less(i, j int) bool {
return bp.FileInfoSums[i].Pos() < bp.FileInfoSums[j].Pos()
}

61
pkg/tarsum/fileinfosums_test.go
View File

@@ -1,61 +0,0 @@
package tarsum
import "testing"
func newFileInfoSums() FileInfoSums {
return FileInfoSums{
fileInfoSum{name: "file3", sum: "2abcdef1234567890", pos: 2},
fileInfoSum{name: "dup1", sum: "deadbeef1", pos: 5},
fileInfoSum{name: "file1", sum: "0abcdef1234567890", pos: 0},
fileInfoSum{name: "file4", sum: "3abcdef1234567890", pos: 3},
fileInfoSum{name: "dup1", sum: "deadbeef0", pos: 4},
fileInfoSum{name: "file2", sum: "1abcdef1234567890", pos: 1},
}
}
func TestSortFileInfoSums(t *testing.T) {
dups := newFileInfoSums().GetAllFile("dup1")
if len(dups) != 2 {
t.Errorf("expected length 2, got %d", len(dups))
}
dups.SortByNames()
if dups[0].Pos() != 4 {
t.Errorf("sorted dups should be ordered by position. Expected 4, got %d", dups[0].Pos())
}
fis := newFileInfoSums()
expected := "0abcdef1234567890"
fis.SortBySums()
got := fis[0].Sum()
if got != expected {
t.Errorf("Expected %q, got %q", expected, got)
}
fis = newFileInfoSums()
expected = "dup1"
fis.SortByNames()
gotFis := fis[0]
if gotFis.Name() != expected {
t.Errorf("Expected %q, got %q", expected, gotFis.Name())
}
// since a duplicate is first, ensure it is ordered first by position too
if gotFis.Pos() != 4 {
t.Errorf("Expected %d, got %d", 4, gotFis.Pos())
}
fis = newFileInfoSums()
fis.SortByPos()
if fis[0].Pos() != 0 {
t.Error("sorted fileInfoSums by Pos should order them by position.")
}
fis = newFileInfoSums()
expected = "deadbeef1"
gotFileInfoSum := fis.GetFile("dup1")
if gotFileInfoSum.Sum() != expected {
t.Errorf("Expected %q, got %q", expected, gotFileInfoSum)
}
if fis.GetFile("noPresent") != nil {
t.Error("Should have return nil if name not found.")
}
}

298
pkg/tarsum/tarsum.go
View File

@@ -1,298 +0,0 @@
// Package tarsum provides algorithms to perform checksum calculation on
// filesystem layers.
//
// The transportation of filesystems, regarding Docker, is done with tar(1)
// archives. There are a variety of tar serialization formats [2], and a key
// concern here is ensuring a repeatable checksum given a set of inputs from a
// generic tar archive. Types of transportation include distribution to and from a
// registry endpoint, saving and loading through commands or Docker daemon APIs,
// transferring the build context from client to Docker daemon, and committing the
// filesystem of a container to become an image.
//
// As tar archives are used for transit, but not preserved in many situations, the
// focus of the algorithm is to ensure the integrity of the preserved filesystem,
// while maintaining a deterministic accountability. This includes neither
// constraining the ordering or manipulation of the files during the creation or
// unpacking of the archive, nor include additional metadata state about the file
// system attributes.
package tarsum
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"path"
"strings"
)
const (
buf8K = 8 * 1024
buf16K = 16 * 1024
buf32K = 32 * 1024
)
// NewTarSum creates a new interface for calculating a fixed time checksum of a
// tar archive.
//
// This is used for calculating checksums of layers of an image, in some cases
// including the byte payload of the image's json metadata as well, and for
// calculating the checksums for buildcache.
func NewTarSum(r io.Reader, dc bool, v Version) (TarSum, error) {
return NewTarSumHash(r, dc, v, DefaultTHash)
}
// NewTarSumHash creates a new TarSum, providing a THash to use rather than
// the DefaultTHash.
func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) {
headerSelector, err := getTarHeaderSelector(v)
if err != nil {
return nil, err
}
ts := &tarSum{Reader: r, DisableCompression: dc, tarSumVersion: v, headerSelector: headerSelector, tHash: tHash}
err = ts.initTarSum()
return ts, err
}
// NewTarSumForLabel creates a new TarSum using the provided TarSum version+hash label.
func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) {
versionName, hashName, ok := strings.Cut(label, "+")
if !ok {
return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}")
}
version, ok := tarSumVersionsByName[versionName]
if !ok {
return nil, fmt.Errorf("unknown TarSum version name: %q", versionName)
}
hashConfig, ok := standardHashConfigs[hashName]
if !ok {
return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName)
}
tHash := NewTHash(hashConfig.name, hashConfig.hash.New)
return NewTarSumHash(r, disableCompression, version, tHash)
}
// TarSum is the generic interface for calculating fixed time
// checksums of a tar archive.
type TarSum interface {
io.Reader
GetSums() FileInfoSums
Sum([]byte) string
Version() Version
Hash() THash
}
// tarSum struct is the structure for a Version0 checksum calculation.
type tarSum struct {
io.Reader
tarR *tar.Reader
tarW *tar.Writer
writer writeCloseFlusher
bufTar *bytes.Buffer
bufWriter *bytes.Buffer
bufData []byte
h hash.Hash
tHash THash
sums FileInfoSums
fileCounter int64
currentFile string
finished bool
first bool
DisableCompression bool // false by default. When false, the output gzip compressed.
tarSumVersion Version // this field is not exported so it can not be mutated during use
headerSelector tarHeaderSelector // handles selecting and ordering headers for files in the archive
}
func (ts tarSum) Hash() THash {
return ts.tHash
}
func (ts tarSum) Version() Version {
return ts.tarSumVersion
}
// THash provides a hash.Hash type generator and its name.
type THash interface {
Hash() hash.Hash
Name() string
}
// NewTHash is a convenience method for creating a THash.
func NewTHash(name string, h func() hash.Hash) THash {
return simpleTHash{n: name, h: h}
}
type tHashConfig struct {
name string
hash crypto.Hash
}
// NOTE: DO NOT include MD5 or SHA1, which are considered insecure.
var standardHashConfigs = map[string]tHashConfig{
"sha256": {name: "sha256", hash: crypto.SHA256},
"sha512": {name: "sha512", hash: crypto.SHA512},
}
// DefaultTHash is default TarSum hashing algorithm - "sha256".
var DefaultTHash = NewTHash("sha256", sha256.New)
type simpleTHash struct {
n string
h func() hash.Hash
}
func (sth simpleTHash) Name() string { return sth.n }
func (sth simpleTHash) Hash() hash.Hash { return sth.h() }
func (ts *tarSum) encodeHeader(h *tar.Header) error {
for _, elem := range ts.headerSelector.selectHeaders(h) {
// Ignore these headers to be compatible with versions
// before go 1.10
if elem[0] == "gname" || elem[0] == "uname" {
elem[1] = ""
}
if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil {
return err
}
}
return nil
}
func (ts *tarSum) initTarSum() error {
ts.bufTar = bytes.NewBuffer([]byte{})
ts.bufWriter = bytes.NewBuffer([]byte{})
ts.tarR = tar.NewReader(ts.Reader)
ts.tarW = tar.NewWriter(ts.bufTar)
if !ts.DisableCompression {
ts.writer = gzip.NewWriter(ts.bufWriter)
} else {
ts.writer = &nopCloseFlusher{Writer: ts.bufWriter}
}
if ts.tHash == nil {
ts.tHash = DefaultTHash
}
ts.h = ts.tHash.Hash()
ts.h.Reset()
ts.first = true
ts.sums = FileInfoSums{}
return nil
}
func (ts *tarSum) Read(buf []byte) (int, error) {
if ts.finished {
return ts.bufWriter.Read(buf)
}
if len(ts.bufData) < len(buf) {
switch {
case len(buf) <= buf8K:
ts.bufData = make([]byte, buf8K)
case len(buf) <= buf16K:
ts.bufData = make([]byte, buf16K)
case len(buf) <= buf32K:
ts.bufData = make([]byte, buf32K)
default:
ts.bufData = make([]byte, len(buf))
}
}
buf2 := ts.bufData[:len(buf)]
n, err := ts.tarR.Read(buf2)
if err != nil {
if errors.Is(err, io.EOF) {
if _, err := ts.h.Write(buf2[:n]); err != nil {
return 0, err
}
if !ts.first {
ts.sums = append(ts.sums, fileInfoSum{name: ts.currentFile, sum: hex.EncodeToString(ts.h.Sum(nil)), pos: ts.fileCounter})
ts.fileCounter++
ts.h.Reset()
} else {
ts.first = false
}
if _, err := ts.tarW.Write(buf2[:n]); err != nil {
return 0, err
}
currentHeader, err := ts.tarR.Next()
if err != nil {
if err == io.EOF {
if err := ts.tarW.Close(); err != nil {
return 0, err
}
if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
return 0, err
}
if err := ts.writer.Close(); err != nil {
return 0, err
}
ts.finished = true
return ts.bufWriter.Read(buf)
}
return 0, err
}
// #nosec G305 -- The joined path is not passed to any filesystem APIs.
ts.currentFile = path.Join(".", path.Join("/", currentHeader.Name))
if err := ts.encodeHeader(currentHeader); err != nil {
return 0, err
}
if err := ts.tarW.WriteHeader(currentHeader); err != nil {
return 0, err
}
if _, err := io.Copy(ts.writer, ts.bufTar); err != nil {
return 0, err
}
ts.writer.Flush()
return ts.bufWriter.Read(buf)
}
return 0, err
}
// Filling the hash buffer
if _, err = ts.h.Write(buf2[:n]); err != nil {
return 0, err
}
// Filling the tar writer
if _, err = ts.tarW.Write(buf2[:n]); err != nil {
return 0, err
}
// Filling the output writer
if _, err = io.Copy(ts.writer, ts.bufTar); err != nil {
return 0, err
}
ts.writer.Flush()
return ts.bufWriter.Read(buf)
}
func (ts *tarSum) Sum(extra []byte) string {
ts.sums.SortBySums()
h := ts.tHash.Hash()
if extra != nil {
h.Write(extra)
}
for _, fis := range ts.sums {
h.Write([]byte(fis.Sum()))
}
checksum := ts.Version().String() + "+" + ts.tHash.Name() + ":" + hex.EncodeToString(h.Sum(nil))
return checksum
}
func (ts *tarSum) GetSums() FileInfoSums {
return ts.sums
}

230
pkg/tarsum/tarsum_spec.md
View File

@@ -1,230 +0,0 @@
page_title: TarSum checksum specification
page_description: Documentation for algorithms used in the TarSum checksum calculation
page_keywords: docker, checksum, validation, tarsum
# TarSum Checksum Specification
## Abstract
This document describes the algorithms used in performing the TarSum checksum
calculation on filesystem layers, the need for this method over existing
methods, and the versioning of this calculation.
## Warning
This checksum algorithm is for best-effort comparison of file trees with fuzzy logic.
This is _not_ a cryptographic attestation, and should not be considered secure.
## Introduction
The transportation of filesystems, regarding Docker, is done with tar(1)
archives. There are a variety of tar serialization formats [2], and a key
concern here is ensuring a repeatable checksum given a set of inputs from a
generic tar archive. Types of transportation include distribution to and from a
registry endpoint, saving and loading through commands or Docker daemon APIs,
transferring the build context from client to Docker daemon, and committing the
filesystem of a container to become an image.
As tar archives are used for transit, but not preserved in many situations, the
focus of the algorithm is to ensure the integrity of the preserved filesystem,
while maintaining a deterministic accountability. This includes neither
constraining the ordering or manipulation of the files during the creation or
unpacking of the archive, nor include additional metadata state about the file
system attributes.
## Intended Audience
This document is outlining the methods used for consistent checksum calculation
for filesystems transported via tar archives.
Auditing these methodologies is an open and iterative process. This document
should accommodate the review of source code. Ultimately, this document should
be the starting point of further refinements to the algorithm and its future
versions.
## Concept
The checksum mechanism must ensure the integrity and assurance of the
filesystem payload.
## Checksum Algorithm Profile
A checksum mechanism must define the following operations and attributes:
* Associated hashing cipher - used to checksum each file payload and attribute
information.
* Checksum list - each file of the filesystem archive has its checksum
calculated from the payload and attributes of the file. The final checksum is
calculated from this list, with specific ordering.
* Version - as the algorithm adapts to requirements, there are behaviors of the
algorithm to manage by versioning.
* Archive being calculated - the tar archive having its checksum calculated
## Elements of TarSum checksum
The calculated sum output is a text string. The elements included in the output
of the calculated sum comprise the information needed for validation of the sum
(TarSum version and hashing cipher used) and the expected checksum in hexadecimal
form.
There are two delimiters used:
* '+' separates TarSum version from hashing cipher
* ':' separates calculation mechanics from expected hash
Example:
```
"tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e"
| | \ |
| | \ |
|_version_|_cipher__|__ |
| \ |
|_calculation_mechanics_|______________________expected_sum_______________________|
```
## Versioning
Versioning was introduced [0] to accommodate differences in calculation needed,
and ability to maintain reverse compatibility.
The general algorithm will be describe further in the 'Calculation'.
### Version0
This is the initial version of TarSum.
Its element in the TarSum checksum string is `tarsum`.
### Version1
Its element in the TarSum checksum is `tarsum.v1`.
The notable changes in this version:
* Exclusion of file `mtime` from the file information headers, in each file
checksum calculation
* Inclusion of extended attributes (`xattrs`. Also seen as `SCHILY.xattr.` prefixed Pax
tar file info headers) keys and values in each file checksum calculation
### VersionDev
*Do not use unless validating refinements to the checksum algorithm*
Its element in the TarSum checksum is `tarsum.dev`.
This is a floating place holder for a next version and grounds for testing
changes. The methods used for calculation are subject to change without notice,
and this version is for testing and not for production use.
## Ciphers
The official default and standard hashing cipher used in the calculation mechanic
is `sha256`. This refers to SHA256 hash algorithm as defined in FIPS 180-4.
Though the TarSum algorithm itself is not exclusively bound to the single
hashing cipher `sha256`, support for alternate hashing ciphers was later added
[1]. Use cases for alternate cipher could include future-proofing TarSum
checksum format and using faster cipher hashes for tar filesystem checksums.
## Calculation
### Requirement
As mentioned earlier, the calculation is such that it takes into consideration
the lifecycle of the tar archive. In that the tar archive is not an immutable,
permanent artifact. Otherwise options like relying on a known hashing cipher
checksum of the archive itself would be reliable enough. The tar archive of the
filesystem is used as a transportation medium for Docker images, and the
archive is discarded once its contents are extracted. Therefore, for consistent
validation items such as order of files in the tar archive and time stamps are
subject to change once an image is received.
### Process
The method is typically iterative due to reading tar info headers from the
archive stream, though this is not a strict requirement.
#### Files
Each file in the tar archive have their contents (headers and body) checksummed
individually using the designated associated hashing cipher. The ordered
headers of the file are written to the checksum calculation first, and then the
payload of the file body.
The resulting checksum of the file is appended to the list of file sums. The
sum is encoded as a string of the hexadecimal digest. Additionally, the file
name and position in the archive is kept as reference for special ordering.
#### Headers
The following headers are read, in this
order ( and the corresponding representation of its value):
* 'name' - string
* 'mode' - string of the base10 integer
* 'uid' - string of the integer
* 'gid' - string of the integer
* 'size' - string of the integer
* 'mtime' (_Version0 only_) - string of integer of the seconds since 1970-01-01 00:00:00 UTC
* 'typeflag' - string of the char
* 'linkname' - string
* 'uname' - string
* 'gname' - string
* 'devmajor' - string of the integer
* 'devminor' - string of the integer
For >= Version1, the extended attribute headers ("SCHILY.xattr." prefixed pax
headers) included after the above list. These xattrs key/values are first
sorted by the keys.
#### Header Format
The ordered headers are written to the hash in the format of
"{.key}{.value}"
with no newline.
#### Body
After the order headers of the file have been added to the checksum for the
file, the body of the file is written to the hash.
#### List of file sums
The list of file sums is sorted by the string of the hexadecimal digest.
If there are two files in the tar with matching paths, the order of occurrence
for that path is reflected for the sums of the corresponding file header and
body.
#### Final Checksum
Begin with a fresh or initial state of the associated hash cipher. If there is
additional payload to include in the TarSum calculation for the archive, it is
written first. Then each checksum from the ordered list of file sums is written
to the hash.
The resulting digest is formatted per the Elements of TarSum checksum,
including the TarSum version, the associated hash cipher and the hexadecimal
encoded checksum digest.
## Security Considerations
The initial version of TarSum has undergone one update that could invalidate
handcrafted tar archives. The tar archive format supports appending of files
with same names as prior files in the archive. The latter file will clobber the
prior file of the same path. Due to this the algorithm now accounts for files
with matching paths, and orders the list of file sums accordingly [3].
## Footnotes
* [0] Versioning https://github.com/docker/docker/commit/747f89cd327db9d50251b17797c4d825162226d0
* [1] Alternate ciphers https://github.com/docker/docker/commit/4e9925d780665149b8bc940d5ba242ada1973c4e
* [2] Tar http://en.wikipedia.org/wiki/Tar_%28computing%29
* [3] Name collision https://github.com/docker/docker/commit/c5e6362c53cbbc09ddbabd5a7323e04438b57d31
## Acknowledgments
Joffrey F (shin-) and Guillaume J. Charmes (creack) on the initial work of the
TarSum calculation.

668
pkg/tarsum/tarsum_test.go
View File

@@ -1,668 +0,0 @@
package tarsum
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/md5" // #nosec G501
"crypto/rand"
"crypto/sha1" // #nosec G505
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"strings"
"testing"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
type testLayer struct {
filename string
options *sizedOptions
jsonfile string
gzip bool
tarsum string
version Version
hash THash
}
var testLayers = []testLayer{
{
filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
version: Version0,
tarsum: "tarsum+sha256:4095cc12fa5fdb1ab2760377e1cd0c4ecdd3e61b4f9b82319d96fcea6c9a41c6",
},
{
filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
version: VersionDev,
tarsum: "tarsum.dev+sha256:db56e35eec6ce65ba1588c20ba6b1ea23743b59e81fb6b7f358ccbde5580345c",
},
{
filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
gzip: true,
tarsum: "tarsum+sha256:4095cc12fa5fdb1ab2760377e1cd0c4ecdd3e61b4f9b82319d96fcea6c9a41c6",
},
{
// Tests existing version of TarSum when xattrs are present
filename: "testdata/xattr/layer.tar",
jsonfile: "testdata/xattr/json",
version: Version0,
tarsum: "tarsum+sha256:07e304a8dbcb215b37649fde1a699f8aeea47e60815707f1cdf4d55d25ff6ab4",
},
{
// Tests next version of TarSum when xattrs are present
filename: "testdata/xattr/layer.tar",
jsonfile: "testdata/xattr/json",
version: VersionDev,
tarsum: "tarsum.dev+sha256:6c58917892d77b3b357b0f9ad1e28e1f4ae4de3a8006bd3beb8beda214d8fd16",
},
{
filename: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar",
jsonfile: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json",
tarsum: "tarsum+sha256:c66bd5ec9f87b8f4c6135ca37684618f486a3dd1d113b138d0a177bfa39c2571",
},
{
options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
tarsum: "tarsum+sha256:75258b2c5dcd9adfe24ce71eeca5fc5019c7e669912f15703ede92b1a60cb11f",
},
{
// this tar has two files with the same path
filename: "testdata/collision/collision-0.tar",
tarsum: "tarsum+sha256:7cabb5e9128bb4a93ff867b9464d7c66a644ae51ea2e90e6ef313f3bef93f077",
},
{
// this tar has the same two files (with the same path), but reversed order. ensuring is has different hash than above
filename: "testdata/collision/collision-1.tar",
tarsum: "tarsum+sha256:805fd393cfd58900b10c5636cf9bab48b2406d9b66523122f2352620c85dc7f9",
},
{
// this tar has newer of collider-0.tar, ensuring is has different hash
filename: "testdata/collision/collision-2.tar",
tarsum: "tarsum+sha256:85d2b8389f077659d78aca898f9e632ed9161f553f144aef100648eac540147b",
},
{
// this tar has newer of collider-1.tar, ensuring is has different hash
filename: "testdata/collision/collision-3.tar",
tarsum: "tarsum+sha256:cbe4dee79fe979d69c16c2bccd032e3205716a562f4a3c1ca1cbeed7b256eb19",
},
{
options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
tarsum: "tarsum+md5:3a6cdb475d90459ac0d3280703d17be2",
hash: md5THash,
},
{
options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
tarsum: "tarsum+sha1:14b5e0d12a0c50a4281e86e92153fa06d55d00c6",
hash: sha1Hash,
},
{
options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
tarsum: "tarsum+sha224:dd8925b7a4c71b13f3a68a0f9428a757c76b93752c398f272a9062d5",
hash: sha224Hash,
},
{
options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
tarsum: "tarsum+sha384:e39e82f40005134bed13fb632d1a5f2aa4675c9ddb4a136fbcec202797e68d2f635e1200dee2e3a8d7f69d54d3f2fd27",
hash: sha384Hash,
},
{
options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
tarsum: "tarsum+sha512:7c56de40b2d1ed3863ff25d83b59cdc8f53e67d1c01c3ee8f201f8e4dec3107da976d0c0ec9109c962a152b32699fe329b2dab13966020e400c32878a0761a7e",
hash: sha512Hash,
},
}
type sizedOptions struct {
num int64
size int64
isRand bool
realFile bool
}
// make a tar:
// * num is the number of files the tar should have
// * size is the bytes per file
// * isRand is whether the contents of the files should be a random chunk (otherwise it's all zeros)
// * realFile will write to a TempFile, instead of an in memory buffer
func sizedTar(opts sizedOptions) io.Reader {
var (
fh io.ReadWriter
err error
)
if opts.realFile {
fh, err = os.CreateTemp("", "tarsum")
if err != nil {
return nil
}
} else {
fh = bytes.NewBuffer([]byte{})
}
tarW := tar.NewWriter(fh)
defer tarW.Close()
for i := int64(0); i < opts.num; i++ {
err := tarW.WriteHeader(&tar.Header{
Name: fmt.Sprintf("/testdata%d", i),
Mode: 0o755,
Uid: 0,
Gid: 0,
Size: opts.size,
Typeflag: tar.TypeReg,
})
if err != nil {
return nil
}
var rBuf []byte
if opts.isRand {
rBuf = make([]byte, 8)
_, err = rand.Read(rBuf)
if err != nil {
return nil
}
} else {
rBuf = []byte{0, 0, 0, 0, 0, 0, 0, 0}
}
for i := int64(0); i < opts.size/int64(8); i++ {
tarW.Write(rBuf)
}
}
return fh
}
func emptyTarSum(gzip bool) (TarSum, error) {
reader, writer := io.Pipe()
tarWriter := tar.NewWriter(writer)
// Immediately close tarWriter and write-end of the
// Pipe in a separate goroutine so we don't block.
go func() {
tarWriter.Close()
writer.Close()
}()
return NewTarSum(reader, !gzip, Version0)
}
// Test errors on NewTarsumForLabel
func TestNewTarSumForLabelInvalid(t *testing.T) {
reader := strings.NewReader("")
if _, err := NewTarSumForLabel(reader, true, "invalidlabel"); err == nil {
t.Fatalf("Expected an error, got nothing.")
}
if _, err := NewTarSumForLabel(reader, true, "invalid+sha256"); err == nil {
t.Fatalf("Expected an error, got nothing.")
}
if _, err := NewTarSumForLabel(reader, true, "tarsum.v1+invalid"); err == nil {
t.Fatalf("Expected an error, got nothing.")
}
}
func TestNewTarSumForLabel(t *testing.T) {
layer := testLayers[0]
reader, err := os.Open(layer.filename)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
label := strings.Split(layer.tarsum, ":")[0]
ts, err := NewTarSumForLabel(reader, false, label)
if err != nil {
t.Fatal(err)
}
// Make sure it actually worked by reading a little bit of it
nbByteToRead := 8 * 1024
dBuf := make([]byte, nbByteToRead)
_, err = ts.Read(dBuf)
if err != nil {
t.Errorf("failed to read %vKB from %s: %s", nbByteToRead, layer.filename, err)
}
}
// TestEmptyTar tests that tarsum does not fail to read an empty tar
// and correctly returns the hex digest of an empty hash.
func TestEmptyTar(t *testing.T) {
// Test without gzip.
ts, err := emptyTarSum(false)
assert.NilError(t, err)
zeroBlock := make([]byte, 1024)
buf := new(bytes.Buffer)
n, err := io.Copy(buf, ts)
assert.NilError(t, err)
if n != int64(len(zeroBlock)) || !bytes.Equal(buf.Bytes(), zeroBlock) {
t.Fatalf("tarSum did not write the correct number of zeroed bytes: %d", n)
}
expectedSum := ts.Version().String() + "+sha256:" + hex.EncodeToString(sha256.New().Sum(nil))
resultSum := ts.Sum(nil)
if resultSum != expectedSum {
t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum)
}
// Test with gzip.
ts, err = emptyTarSum(true)
assert.NilError(t, err)
buf.Reset()
_, err = io.Copy(buf, ts)
assert.NilError(t, err)
bufgz := new(bytes.Buffer)
gz := gzip.NewWriter(bufgz)
n, err = io.Copy(gz, bytes.NewBuffer(zeroBlock))
assert.NilError(t, err)
gz.Close()
gzBytes := bufgz.Bytes()
if n != int64(len(zeroBlock)) || !bytes.Equal(buf.Bytes(), gzBytes) {
t.Fatalf("tarSum did not write the correct number of gzipped-zeroed bytes: %d", n)
}
resultSum = ts.Sum(nil)
if resultSum != expectedSum {
t.Fatalf("expected [%s] but got [%s]", expectedSum, resultSum)
}
// Test without ever actually writing anything.
if ts, err = NewTarSum(bytes.NewReader([]byte{}), true, Version0); err != nil {
t.Fatal(err)
}
resultSum = ts.Sum(nil)
assert.Check(t, is.Equal(expectedSum, resultSum))
}
var (
md5THash = NewTHash("md5", md5.New)
sha1Hash = NewTHash("sha1", sha1.New)
sha224Hash = NewTHash("sha224", sha256.New224)
sha384Hash = NewTHash("sha384", sha512.New384)
sha512Hash = NewTHash("sha512", sha512.New)
)
// Test all the build-in read size : buf8K, buf16K, buf32K and more
func TestTarSumsReadSize(t *testing.T) {
// Test always on the same layer (that is big enough)
layer := testLayers[0]
for i := 0; i < 5; i++ {
reader, err := os.Open(layer.filename)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
ts, err := NewTarSum(reader, false, layer.version)
if err != nil {
t.Fatal(err)
}
// Read and discard bytes so that it populates sums
nbByteToRead := (i + 1) * 8 * 1024
dBuf := make([]byte, nbByteToRead)
_, err = ts.Read(dBuf)
if err != nil {
t.Errorf("failed to read %vKB from %s: %s", nbByteToRead, layer.filename, err)
continue
}
}
}
func TestTarSums(t *testing.T) {
for _, layer := range testLayers {
var (
fh io.Reader
err error
)
if layer.filename != "" {
fh, err = os.Open(layer.filename)
if err != nil {
t.Errorf("failed to open %s: %s", layer.filename, err)
continue
}
} else if layer.options != nil {
fh = sizedTar(*layer.options)
} else {
// What else is there to test?
t.Errorf("what to do with %#v", layer)
continue
}
if file, ok := fh.(*os.File); ok {
defer file.Close()
}
var ts TarSum
if layer.hash == nil {
// double negatives!
ts, err = NewTarSum(fh, !layer.gzip, layer.version)
} else {
ts, err = NewTarSumHash(fh, !layer.gzip, layer.version, layer.hash)
}
if err != nil {
t.Errorf("%q :: %q", err, layer.filename)
continue
}
// Read variable number of bytes to test dynamic buffer
dBuf := make([]byte, 1)
_, err = ts.Read(dBuf)
if err != nil {
t.Errorf("failed to read 1B from %s: %s", layer.filename, err)
continue
}
dBuf = make([]byte, 16*1024)
_, err = ts.Read(dBuf)
if err != nil {
t.Errorf("failed to read 16KB from %s: %s", layer.filename, err)
continue
}
// Read and discard remaining bytes
_, err = io.Copy(io.Discard, ts)
if err != nil {
t.Errorf("failed to copy from %s: %s", layer.filename, err)
continue
}
var gotSum string
if layer.jsonfile != "" {
jfh, err := os.Open(layer.jsonfile)
if err != nil {
t.Errorf("failed to open %s: %s", layer.jsonfile, err)
continue
}
defer jfh.Close()
buf, err := io.ReadAll(jfh)
if err != nil {
t.Errorf("failed to readAll %s: %s", layer.jsonfile, err)
continue
}
gotSum = ts.Sum(buf)
} else {
gotSum = ts.Sum(nil)
}
if layer.tarsum != gotSum {
t.Errorf("expecting [%s], but got [%s]", layer.tarsum, gotSum)
}
var expectedHashName string
if layer.hash != nil {
expectedHashName = layer.hash.Name()
} else {
expectedHashName = DefaultTHash.Name()
}
if expectedHashName != ts.Hash().Name() {
t.Errorf("expecting hash [%v], but got [%s]", expectedHashName, ts.Hash().Name())
}
}
}
func TestIteration(t *testing.T) {
headerTests := []struct {
expectedSum string // TODO(vbatts) it would be nice to get individual sums of each
version Version
hdr *tar.Header
data []byte
}{
{
"tarsum+sha256:626c4a2e9a467d65c33ae81f7f3dedd4de8ccaee72af73223c4bc4718cbc7bbd",
Version0,
&tar.Header{
Name: "file.txt",
Size: 0,
Typeflag: tar.TypeReg,
Devminor: 0,
Devmajor: 0,
},
[]byte(""),
},
{
"tarsum.dev+sha256:6ffd43a1573a9913325b4918e124ee982a99c0f3cba90fc032a65f5e20bdd465",
VersionDev,
&tar.Header{
Name: "file.txt",
Size: 0,
Typeflag: tar.TypeReg,
Devminor: 0,
Devmajor: 0,
},
[]byte(""),
},
{
"tarsum.dev+sha256:862964db95e0fa7e42836ae4caab3576ab1df8d275720a45bdd01a5a3730cc63",
VersionDev,
&tar.Header{
Name: "another.txt",
Uid: 1000,
Gid: 1000,
Uname: "slartibartfast",
Gname: "users",
Size: 4,
Typeflag: tar.TypeReg,
Devminor: 0,
Devmajor: 0,
},
[]byte("test"),
},
{
"tarsum.dev+sha256:4b1ba03544b49d96a32bacc77f8113220bd2f6a77e7e6d1e7b33cd87117d88e7",
VersionDev,
&tar.Header{
Name: "xattrs.txt",
Uid: 1000,
Gid: 1000,
Uname: "slartibartfast",
Gname: "users",
Size: 4,
Typeflag: tar.TypeReg,
Xattrs: map[string]string{
"user.key1": "value1",
"user.key2": "value2",
},
},
[]byte("test"),
},
{
"tarsum.dev+sha256:410b602c898bd4e82e800050f89848fc2cf20fd52aa59c1ce29df76b878b84a6",
VersionDev,
&tar.Header{
Name: "xattrs.txt",
Uid: 1000,
Gid: 1000,
Uname: "slartibartfast",
Gname: "users",
Size: 4,
Typeflag: tar.TypeReg,
Xattrs: map[string]string{
"user.KEY1": "value1", // adding different case to ensure different sum
"user.key2": "value2",
},
},
[]byte("test"),
},
{
"tarsum+sha256:b1f97eab73abd7593c245e51070f9fbdb1824c6b00a0b7a3d7f0015cd05e9e86",
Version0,
&tar.Header{
Name: "xattrs.txt",
Uid: 1000,
Gid: 1000,
Uname: "slartibartfast",
Gname: "users",
Size: 4,
Typeflag: tar.TypeReg,
Xattrs: map[string]string{
"user.NOT": "CALCULATED",
},
},
[]byte("test"),
},
}
for _, htest := range headerTests {
s, err := renderSumForHeader(htest.version, htest.hdr, htest.data)
if err != nil {
t.Fatal(err)
}
if s != htest.expectedSum {
t.Errorf("expected sum: %q, got: %q", htest.expectedSum, s)
}
}
}
func renderSumForHeader(v Version, h *tar.Header, data []byte) (string, error) {
buf := bytes.NewBuffer(nil)
// first build our test tar
tw := tar.NewWriter(buf)
if err := tw.WriteHeader(h); err != nil {
return "", err
}
if _, err := tw.Write(data); err != nil {
return "", err
}
tw.Close()
ts, err := NewTarSum(buf, true, v)
if err != nil {
return "", err
}
tr := tar.NewReader(ts)
for {
hdr, err := tr.Next()
if hdr == nil || errors.Is(err, io.EOF) {
// Signals the end of the archive.
break
}
if err != nil {
return "", err
}
// #nosec G110 -- ignore "potential DoS vulnerability via decompression bomb"
if _, err = io.Copy(io.Discard, tr); err != nil {
return "", err
}
}
return ts.Sum(nil), nil
}
func Benchmark9kTar(b *testing.B) {
buf := bytes.NewBuffer([]byte{})
fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar")
if err != nil {
b.Error(err)
return
}
defer fh.Close()
n, err := io.Copy(buf, fh)
if err != nil {
b.Error(err)
return
}
reader := bytes.NewReader(buf.Bytes())
b.SetBytes(n)
b.ResetTimer()
for i := 0; i < b.N; i++ {
reader.Seek(0, 0)
ts, err := NewTarSum(reader, true, Version0)
if err != nil {
b.Error(err)
return
}
io.Copy(io.Discard, ts)
ts.Sum(nil)
}
}
func Benchmark9kTarGzip(b *testing.B) {
buf := bytes.NewBuffer([]byte{})
fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar")
if err != nil {
b.Error(err)
return
}
defer fh.Close()
n, err := io.Copy(buf, fh)
if err != nil {
b.Error(err)
return
}
reader := bytes.NewReader(buf.Bytes())
b.SetBytes(n)
b.ResetTimer()
for i := 0; i < b.N; i++ {
reader.Seek(0, 0)
ts, err := NewTarSum(reader, false, Version0)
if err != nil {
b.Error(err)
return
}
io.Copy(io.Discard, ts)
ts.Sum(nil)
}
}
// this is a single big file in the tar archive
func Benchmark1mbSingleFileTar(b *testing.B) {
benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, false)
}
// this is a single big file in the tar archive
func Benchmark1mbSingleFileTarGzip(b *testing.B) {
benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, true)
}
// this is 1024 1k files in the tar archive
func Benchmark1kFilesTar(b *testing.B) {
benchmarkTar(b, sizedOptions{1024, 1024, true, true}, false)
}
// this is 1024 1k files in the tar archive
func Benchmark1kFilesTarGzip(b *testing.B) {
benchmarkTar(b, sizedOptions{1024, 1024, true, true}, true)
}
func benchmarkTar(b *testing.B, opts sizedOptions, isGzip bool) {
var fh *os.File
tarReader := sizedTar(opts)
if br, ok := tarReader.(*os.File); ok {
fh = br
}
defer os.Remove(fh.Name())
defer fh.Close()
b.SetBytes(opts.size * opts.num)
b.ResetTimer()
for i := 0; i < b.N; i++ {
ts, err := NewTarSum(fh, !isGzip, Version0)
if err != nil {
b.Error(err)
return
}
io.Copy(io.Discard, ts)
ts.Sum(nil)
fh.Seek(0, 0)
}
}

1
pkg/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json vendored
View File

@@ -1 +0,0 @@
{"id":"46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457","parent":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","created":"2014-04-07T02:45:52.610504484Z","container":"e0f07f8d72cae171a3dcc35859960e7e956e0628bce6fedc4122bf55b2c287c7","container_config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","sed -ri 's/^(%wheel.*)(ALL)$/\\1NOPASSWD: \\2/' /etc/sudoers"],"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.9.1-dev","config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":3425}

BIN
pkg/tarsum/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar vendored
View File

Binary file not shown.

1
pkg/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json vendored
View File

@@ -1 +0,0 @@
{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}

BIN
pkg/tarsum/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar vendored
View File

Binary file not shown.

BIN
pkg/tarsum/testdata/collision/collision-0.tar vendored
View File

Binary file not shown.

BIN
pkg/tarsum/testdata/collision/collision-1.tar vendored
View File

Binary file not shown.

BIN
pkg/tarsum/testdata/collision/collision-2.tar vendored
View File

Binary file not shown.

BIN
pkg/tarsum/testdata/collision/collision-3.tar vendored
View File

Binary file not shown.

1
pkg/tarsum/testdata/xattr/json vendored
View File

@@ -1 +0,0 @@
{"id":"4439c3c7f847954100b42b267e7e5529cac1d6934db082f65795c5ca2e594d93","parent":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","created":"2014-05-16T17:19:44.091534414Z","container":"5f92fb06cc58f357f0cde41394e2bbbb664e663974b2ac1693ab07b7a306749b","container_config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","setcap 'cap_setgid,cap_setuid+ep' ./file \u0026\u0026 getcap ./file"],"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.11.1-dev","config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":0}

BIN
pkg/tarsum/testdata/xattr/layer.tar vendored
View File

Binary file not shown.

166
pkg/tarsum/versioning.go
View File

@@ -1,166 +0,0 @@
package tarsum
import (
"archive/tar"
"errors"
"io"
"sort"
"strconv"
"strings"
)
// Version is used for versioning of the TarSum algorithm
// based on the prefix of the hash used
// i.e. "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b"
type Version int
// Prefix of "tarsum"
const (
Version0 Version = iota
Version1
// VersionDev this constant will be either the latest or an unsettled next-version of the TarSum calculation
VersionDev
)
// WriteV1Header writes a tar header to a writer in V1 tarsum format.
func WriteV1Header(h *tar.Header, w io.Writer) {
for _, elem := range v1TarHeaderSelect(h) {
w.Write([]byte(elem[0] + elem[1]))
}
}
// VersionLabelForChecksum returns the label for the given tarsum
// checksum, i.e., everything before the first `+` character in
// the string or an empty string if no label separator is found.
func VersionLabelForChecksum(checksum string) string {
// Checksums are in the form: {versionLabel}+{hashID}:{hex}
sepIndex := strings.Index(checksum, "+")
if sepIndex < 0 {
return ""
}
return checksum[:sepIndex]
}
// GetVersions gets a list of all known tarsum versions.
func GetVersions() []Version {
v := []Version{}
for k := range tarSumVersions {
v = append(v, k)
}
return v
}
var (
tarSumVersions = map[Version]string{
Version0: "tarsum",
Version1: "tarsum.v1",
VersionDev: "tarsum.dev",
}
tarSumVersionsByName = map[string]Version{
"tarsum": Version0,
"tarsum.v1": Version1,
"tarsum.dev": VersionDev,
}
)
func (tsv Version) String() string {
return tarSumVersions[tsv]
}
// GetVersionFromTarsum returns the Version from the provided string.
func GetVersionFromTarsum(tarsum string) (Version, error) {
versionName, _, _ := strings.Cut(tarsum, "+")
version, ok := tarSumVersionsByName[versionName]
if !ok {
return -1, ErrNotVersion
}
return version, nil
}
// Errors that may be returned by functions in this package
var (
ErrNotVersion = errors.New("string does not include a TarSum Version")
ErrVersionNotImplemented = errors.New("TarSum Version is not yet implemented")
)
// tarHeaderSelector is the interface which different versions
// of tarsum should use for selecting and ordering tar headers
// for each item in the archive.
type tarHeaderSelector interface {
selectHeaders(h *tar.Header) (orderedHeaders [][2]string)
}
type tarHeaderSelectFunc func(h *tar.Header) (orderedHeaders [][2]string)
func (f tarHeaderSelectFunc) selectHeaders(h *tar.Header) (orderedHeaders [][2]string) {
return f(h)
}
func v0TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
return [][2]string{
{"name", h.Name},
{"mode", strconv.FormatInt(h.Mode, 10)},
{"uid", strconv.Itoa(h.Uid)},
{"gid", strconv.Itoa(h.Gid)},
{"size", strconv.FormatInt(h.Size, 10)},
{"mtime", strconv.FormatInt(h.ModTime.UTC().Unix(), 10)},
{"typeflag", string([]byte{h.Typeflag})},
{"linkname", h.Linkname},
{"uname", h.Uname},
{"gname", h.Gname},
{"devmajor", strconv.FormatInt(h.Devmajor, 10)},
{"devminor", strconv.FormatInt(h.Devminor, 10)},
}
}
func v1TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
// Get extended attributes.
const paxSchilyXattr = "SCHILY.xattr."
var xattrs [][2]string
for k, v := range h.PAXRecords {
if xattr, ok := strings.CutPrefix(k, paxSchilyXattr); ok {
// h.Xattrs keys take precedence over h.PAXRecords keys, like
// archive/tar does when writing.
if vv, ok := h.Xattrs[xattr]; ok { //nolint:staticcheck // field deprecated in stdlib
v = vv
}
xattrs = append(xattrs, [2]string{xattr, v})
}
}
// Get extended attributes which are not in PAXRecords.
for k, v := range h.Xattrs { //nolint:staticcheck // field deprecated in stdlib
if _, ok := h.PAXRecords[paxSchilyXattr+k]; !ok {
xattrs = append(xattrs, [2]string{k, v})
}
}
sort.Slice(xattrs, func(i, j int) bool { return xattrs[i][0] < xattrs[j][0] })
// Make the slice with enough capacity to hold the 11 basic headers
// we want from the v0 selector plus however many xattrs we have.
orderedHeaders = make([][2]string, 0, 11+len(xattrs))
// Copy all headers from v0 excluding the 'mtime' header (the 5th element).
v0headers := v0TarHeaderSelect(h)
orderedHeaders = append(orderedHeaders, v0headers[0:5]...)
orderedHeaders = append(orderedHeaders, v0headers[6:]...)
// Finally, append the sorted xattrs.
orderedHeaders = append(orderedHeaders, xattrs...)
return orderedHeaders
}
var registeredHeaderSelectors = map[Version]tarHeaderSelectFunc{
Version0: v0TarHeaderSelect,
Version1: v1TarHeaderSelect,
VersionDev: v1TarHeaderSelect,
}
func getTarHeaderSelector(v Version) (tarHeaderSelector, error) {
headerSelector, ok := registeredHeaderSelectors[v]
if !ok {
return nil, ErrVersionNotImplemented
}
return headerSelector, nil
}

131
pkg/tarsum/versioning_test.go
View File

@@ -1,131 +0,0 @@
package tarsum
import (
"archive/tar"
"errors"
"fmt"
"strings"
"testing"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
func TestVersionLabelForChecksum(t *testing.T) {
version := VersionLabelForChecksum("tarsum+sha256:deadbeef")
if version != "tarsum" {
t.Fatalf("Version should have been 'tarsum', was %v", version)
}
version = VersionLabelForChecksum("tarsum.v1+sha256:deadbeef")
if version != "tarsum.v1" {
t.Fatalf("Version should have been 'tarsum.v1', was %v", version)
}
version = VersionLabelForChecksum("something+somethingelse")
if version != "something" {
t.Fatalf("Version should have been 'something', was %v", version)
}
version = VersionLabelForChecksum("invalidChecksum")
if version != "" {
t.Fatalf("Version should have been empty, was %v", version)
}
}
func TestVersion(t *testing.T) {
expected := "tarsum"
var v Version
if v.String() != expected {
t.Errorf("expected %q, got %q", expected, v.String())
}
expected = "tarsum.v1"
v = 1
if v.String() != expected {
t.Errorf("expected %q, got %q", expected, v.String())
}
expected = "tarsum.dev"
v = 2
if v.String() != expected {
t.Errorf("expected %q, got %q", expected, v.String())
}
}
func TestGetVersion(t *testing.T) {
testSet := []struct {
Str string
Expected Version
}{
{"tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", Version0},
{"tarsum+sha256", Version0},
{"tarsum", Version0},
{"tarsum.dev", VersionDev},
{"tarsum.dev+sha256:deadbeef", VersionDev},
}
for _, ts := range testSet {
v, err := GetVersionFromTarsum(ts.Str)
if err != nil {
t.Fatalf("%q : %s", err, ts.Str)
}
if v != ts.Expected {
t.Errorf("expected %d (%q), got %d (%q)", ts.Expected, ts.Expected, v, v)
}
}
// test one that does not exist, to ensure it errors
str := "weak+md5:abcdeabcde"
_, err := GetVersionFromTarsum(str)
if !errors.Is(err, ErrNotVersion) {
t.Fatalf("%q : %s", err, str)
}
}
func TestGetVersions(t *testing.T) {
expected := []Version{
Version0,
Version1,
VersionDev,
}
versions := GetVersions()
if len(versions) != len(expected) {
t.Fatalf("Expected %v versions, got %v", len(expected), len(versions))
}
if !containsVersion(versions, expected[0]) || !containsVersion(versions, expected[1]) || !containsVersion(versions, expected[2]) {
t.Fatalf("Expected [%v], got [%v]", expected, versions)
}
}
func containsVersion(versions []Version, version Version) bool {
for _, v := range versions {
if v == version {
return true
}
}
return false
}
func TestSelectXattrsV1(t *testing.T) {
hdr := &tar.Header{
Xattrs: map[string]string{ //nolint:staticcheck
"user.xattronly": "x",
"user.foo": "xattr",
},
PAXRecords: map[string]string{
"SCHILY.xattr.user.paxonly": "p",
"SCHILY.xattr.user.foo": "paxrecord",
},
}
selected := v1TarHeaderSelect(hdr)
var s strings.Builder
for _, elem := range selected {
fmt.Fprintf(&s, "%s=%s\n", elem[0], elem[1])
}
t.Logf("Selected headers:\n%s", s.String())
assert.Check(t, is.DeepEqual(selected[len(selected)-3:], [][2]string{
{"user.foo", "xattr"},
{"user.paxonly", "p"},
{"user.xattronly", "x"},
}))
}

22
pkg/tarsum/writercloser.go
View File

@@ -1,22 +0,0 @@
package tarsum
import (
"io"
)
type writeCloseFlusher interface {
io.WriteCloser
Flush() error
}
type nopCloseFlusher struct {
io.Writer
}
func (n *nopCloseFlusher) Close() error {
return nil
}
func (n *nopCloseFlusher) Flush() error {
return nil
}