Merge pull request #5053 from containers/dependabot/go_modules/github.com/containers/storage-1.15.8

build(deps): bump github.com/containers/storage from 1.15.7 to 1.15.8
This commit is contained in:
Daniel J Walsh
2020-02-03 09:27:58 -05:00
committed by GitHub
38 changed files with 681 additions and 2410 deletions

2
go.mod
View File

@ -14,7 +14,7 @@ require (
github.com/containers/conmon v2.0.10+incompatible
github.com/containers/image/v5 v5.1.0
github.com/containers/psgo v1.4.0
github.com/containers/storage v1.15.7
github.com/containers/storage v1.15.8
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/cri-o/ocicni v0.1.1-0.20190920040751-deac903fd99b

6
go.sum
View File

@ -109,6 +109,8 @@ github.com/containers/storage v1.15.5 h1:dBZx9yRFHod9c8FVaXlVtRqr2cmlAhpl+9rt87c
github.com/containers/storage v1.15.5/go.mod h1:v0lq/3f+cXH3Y/HiDaFYRR0zilwDve7I4W7U5xQxvF8=
github.com/containers/storage v1.15.7 h1:ecPmv2y/qpxeSTHZ147jQLO6to8wDn8yUPtDCZlz0H4=
github.com/containers/storage v1.15.7/go.mod h1:gLZIp+/hP8nFn9tLS0uJlnk4h1tSoDu3oS2eFiaIqkE=
github.com/containers/storage v1.15.8 h1:ef7OfUMTpyq0PIVAhV7qfufEI92gAldk25nItrip+6Q=
github.com/containers/storage v1.15.8/go.mod h1:zhvjIIl/fR6wt/lgqQAC+xanHQ+8gUQ0GBVeXYN81qI=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-iptables v0.4.2 h1:KH0EwId05JwWIfb96gWvkiT2cbuOu8ygqUaB+yPAwIg=
@ -311,6 +313,8 @@ github.com/klauspost/compress v1.9.4 h1:xhvAeUPQ2drNUhKtrGdTGNvV9nNafHMUkRyLkzxJ
github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.7 h1:hYW1gP94JUmAhBtJ+LNz5My+gBobDxPR1iVuKug26aA=
github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM=
@ -341,6 +345,8 @@ github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3Zk
github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-shellwords v1.0.7 h1:KqhVjVZomx2puPACkj9vrGFqnp42Htvo9SEAWePHKOs=
github.com/mattn/go-shellwords v1.0.7/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-shellwords v1.0.9 h1:eaB5JspOwiKKcHdqcjbfe5lA9cNn/4NRRtddXJCimqk=
github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mistifyio/go-zfs v2.1.1+incompatible h1:gAMO1HM9xBRONLHHYnu5iFsOJUiJdNZo6oqSENd4eW8=

View File

@ -3,37 +3,35 @@ run:
concurrency: 6
deadline: 5m
linters:
disable-all: true
enable:
- bodyclose
- depguard
- gofmt
- interfacer
- typecheck
# - deadcode
# - dupl
# - errcheck
# - gochecknoglobals
# - gochecknoinits
# - goconst
# - gocritic
# - gocyclo
# - goimports
# - golint
# - gosec
# - gosimple
# - govet
# - ineffassign
# - lll
# - maligned
# - misspell
# - nakedret
# - prealloc
# - scopelint
# - staticcheck
# - structcheck
# - stylecheck
# - unconvert
# - unparam
# - unused
# - varcheck
enable-all: true
disable:
- dogsled
- dupl
- errcheck
- funlen
- gochecknoglobals
- gochecknoinits
- gocognit
- gocritic
- gocyclo
- godox
- gomnd
- gosec
- gosimple
- govet
- ineffassign
- lll
- maligned
- misspell
- nakedret
- prealloc
- scopelint
- staticcheck
- structcheck
- stylecheck
- unconvert
- unparam
- unused
- varcheck
- whitespace
- wsl

View File

@ -1 +1 @@
1.15.8-dev
1.15.8

View File

@ -35,7 +35,7 @@ import (
"sync"
"time"
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/chrootarchive"
"github.com/containers/storage/pkg/directory"

View File

@ -26,7 +26,7 @@ import (
"sync"
"unsafe"
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/mount"
"github.com/containers/storage/pkg/parsers"

View File

@ -25,14 +25,14 @@ func platformLChown(path string, info os.FileInfo, toHost, toContainer *idtools.
UID: uid,
GID: gid,
}
mappedUid, mappedGid, err := toContainer.ToContainer(pair)
mappedUID, mappedGID, err := toContainer.ToContainer(pair)
if err != nil {
if (uid != 0) || (gid != 0) {
return fmt.Errorf("error mapping host ID pair %#v for %q to container: %v", pair, path, err)
}
mappedUid, mappedGid = uid, gid
mappedUID, mappedGID = uid, gid
}
uid, gid = mappedUid, mappedGid
uid, gid = mappedUID, mappedGID
}
if toHost != nil {
pair := idtools.IDPair{

View File

@ -18,7 +18,7 @@ import (
"sync"
"time"
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/devicemapper"
"github.com/containers/storage/pkg/dmesg"
"github.com/containers/storage/pkg/idtools"
@ -49,8 +49,13 @@ var (
lvmSetupConfigForce bool
)
const deviceSetMetaFile string = "deviceset-metadata"
const transactionMetaFile string = "transaction-metadata"
const (
deviceSetMetaFile = "deviceset-metadata"
transactionMetaFile = "transaction-metadata"
xfs = "xfs"
ext4 = "ext4"
base = "base"
)
type transaction struct {
OpenTransactionID uint64 `json:"open_transaction_id"`
@ -199,7 +204,7 @@ func getDevName(name string) string {
func (info *devInfo) Name() string {
hash := info.Hash
if hash == "" {
hash = "base"
hash = base
}
return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
}
@ -219,7 +224,7 @@ func (devices *DeviceSet) metadataDir() string {
func (devices *DeviceSet) metadataFile(info *devInfo) string {
file := info.Hash
if file == "" {
file = "base"
file = base
}
return path.Join(devices.metadataDir(), file)
}
@ -440,7 +445,7 @@ func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo)
logrus.Debugf("devmapper: Loading data for file %s", path)
hash := finfo.Name()
if hash == "base" {
if hash == base {
hash = ""
}
@ -542,7 +547,7 @@ func xfsSupported() error {
}
// Check if kernel supports xfs filesystem or not.
exec.Command("modprobe", "xfs").Run()
exec.Command("modprobe", xfs).Run()
f, err := os.Open("/proc/filesystems")
if err != nil {
@ -567,16 +572,16 @@ func xfsSupported() error {
func determineDefaultFS() string {
err := xfsSupported()
if err == nil {
return "xfs"
return xfs
}
logrus.Warnf("devmapper: XFS is not supported in your system (%v). Defaulting to ext4 filesystem", err)
return "ext4"
logrus.Warnf("devmapper: XFS is not supported in your system (%v). Defaulting to %s filesystem", ext4, err)
return ext4
}
// mkfsOptions tries to figure out whether some additional mkfs options are required
func mkfsOptions(fs string) []string {
if fs == "xfs" && !kernel.CheckKernelVersion(3, 16, 0) {
if fs == xfs && !kernel.CheckKernelVersion(3, 16, 0) {
// For kernels earlier than 3.16 (and newer xfsutils),
// some xfs features need to be explicitly disabled.
return []string{"-m", "crc=0,finobt=0"}
@ -609,9 +614,9 @@ func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) {
}()
switch devices.filesystem {
case "xfs":
case xfs:
err = exec.Command("mkfs.xfs", args...).Run()
case "ext4":
case ext4:
err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
if err != nil {
err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
@ -1197,7 +1202,7 @@ func (devices *DeviceSet) growFS(info *devInfo) error {
}
options := ""
if devices.BaseDeviceFilesystem == "xfs" {
if devices.BaseDeviceFilesystem == xfs {
// XFS needs nouuid or it can't mount filesystems with the same fs
options = joinMountOptions(options, "nouuid")
}
@ -1210,11 +1215,11 @@ func (devices *DeviceSet) growFS(info *devInfo) error {
defer unix.Unmount(fsMountPoint, unix.MNT_DETACH)
switch devices.BaseDeviceFilesystem {
case "ext4":
case ext4:
if out, err := exec.Command("resize2fs", info.DevName()).CombinedOutput(); err != nil {
return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
}
case "xfs":
case xfs:
if out, err := exec.Command("xfs_growfs", info.DevName()).CombinedOutput(); err != nil {
return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
}
@ -2391,7 +2396,7 @@ func (devices *DeviceSet) MountDevice(hash, path string, moptions graphdriver.Mo
options := ""
if fstype == "xfs" {
if fstype == xfs {
// XFS needs nouuid or it can't mount filesystems with the same fs
options = joinMountOptions(options, "nouuid")
}
@ -2412,7 +2417,7 @@ func (devices *DeviceSet) MountDevice(hash, path string, moptions graphdriver.Mo
return fmt.Errorf("devmapper: Error mounting '%s' on '%s': %s\n%v", info.DevName(), path, err, string(dmesg.Dmesg(256)))
}
if fstype == "xfs" && devices.xfsNospaceRetries != "" {
if fstype == xfs && devices.xfsNospaceRetries != "" {
if err := devices.xfsSetNospaceRetries(info); err != nil {
unix.Unmount(path, unix.MNT_DETACH)
devices.deactivateDevice(info)
@ -2693,7 +2698,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
}
devices.metaDataLoopbackSize = size
case "dm.fs":
if val != "ext4" && val != "xfs" {
if val != ext4 && val != xfs {
return nil, fmt.Errorf("devmapper: Unsupported filesystem %s", val)
}
devices.filesystem = val

View File

@ -9,7 +9,7 @@ import (
"path"
"strconv"
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/devicemapper"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/locker"

View File

@ -49,8 +49,8 @@ type MountOpts struct {
// Mount label is the MAC Labels to assign to mount point (SELINUX)
MountLabel string
// UidMaps & GidMaps are the User Namespace mappings to be assigned to content in the mount point
UidMaps []idtools.IDMap
GidMaps []idtools.IDMap
UidMaps []idtools.IDMap // nolint: golint
GidMaps []idtools.IDMap // nolint: golint
Options []string
}

View File

@ -401,9 +401,8 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI
if err == nil {
logrus.Debugf("overlay test mount with multiple lowers succeeded")
return supportsDType, nil
} else {
logrus.Debugf("overlay test mount with multiple lowers failed %v", err)
}
logrus.Debugf("overlay test mount with multiple lowers failed %v", err)
}
flags = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower1Dir, upperDir, workDir)
if len(flags) < unix.Getpagesize() {
@ -411,9 +410,8 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI
if err == nil {
logrus.Errorf("overlay test mount with multiple lowers failed, but succeeded with a single lower")
return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay")
} else {
logrus.Debugf("overlay test mount with a single lower failed %v", err)
}
logrus.Debugf("overlay test mount with a single lower failed %v", err)
}
logrus.Errorf("'overlay' is not supported over %s at %q", backingFs, home)
return supportsDType, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s at %q", backingFs, home)
@ -810,15 +808,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
return "", err
}
readWrite := true
// fuse-overlayfs doesn't support working without an upperdir.
if d.options.mountProgram == "" {
for _, o := range options.Options {
if o == "ro" {
readWrite = false
break
}
}
}
lowers, err := ioutil.ReadFile(path.Join(dir, lowerFile))
if err != nil && !os.IsNotExist(err) {

View File

@ -5,7 +5,7 @@ package overlayutils
import (
"fmt"
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/pkg/errors"
)

View File

@ -8,7 +8,7 @@ import (
"strconv"
"strings"
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/parsers"

View File

@ -12,7 +12,7 @@ import (
"sync"
"time"
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/mount"
"github.com/containers/storage/pkg/parsers"

View File

@ -1,7 +1,7 @@
package zfs
import (
"github.com/containers/storage/drivers"
graphdriver "github.com/containers/storage/drivers"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

View File

@ -7,14 +7,14 @@ require (
github.com/Microsoft/hcsshim v0.8.7
github.com/docker/docker v0.0.0-20171019062838-86f080cff091 // indirect
github.com/docker/go-units v0.4.0
github.com/klauspost/compress v1.9.7
github.com/klauspost/compress v1.9.8
github.com/klauspost/cpuid v1.2.1 // indirect
github.com/klauspost/pgzip v1.2.1
github.com/mattn/go-shellwords v1.0.7
github.com/mattn/go-shellwords v1.0.9
github.com/mistifyio/go-zfs v2.1.1+incompatible
github.com/opencontainers/go-digest v1.0.0-rc1
github.com/opencontainers/runc v1.0.0-rc9
github.com/opencontainers/selinux v1.3.0
github.com/opencontainers/selinux v1.3.1
github.com/pkg/errors v0.9.1
github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7
github.com/sirupsen/logrus v1.4.2
@ -24,7 +24,7 @@ require (
github.com/tchap/go-patricia v2.3.0+incompatible
github.com/vbatts/tar-split v0.11.1
golang.org/x/net v0.0.0-20190628185345-da137c7871d7
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
golang.org/x/sys v0.0.0-20191115151921-52ab43148777
gotest.tools v2.2.0+incompatible
)

View File

@ -77,6 +77,8 @@ github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzV
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.7 h1:hYW1gP94JUmAhBtJ+LNz5My+gBobDxPR1iVuKug26aA=
github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM=
@ -89,6 +91,8 @@ github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3Zk
github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-shellwords v1.0.7 h1:KqhVjVZomx2puPACkj9vrGFqnp42Htvo9SEAWePHKOs=
github.com/mattn/go-shellwords v1.0.7/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-shellwords v1.0.9 h1:eaB5JspOwiKKcHdqcjbfe5lA9cNn/4NRRtddXJCimqk=
github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mistifyio/go-zfs v2.1.1+incompatible h1:gAMO1HM9xBRONLHHYnu5iFsOJUiJdNZo6oqSENd4eW8=
github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618 h1:7InQ7/zrOh6SlFjaXFubv0xX0HsuC9qJsdqm7bNQpYM=
@ -111,6 +115,8 @@ github.com/opencontainers/selinux v1.2.2 h1:Kx9J6eDG5/24A6DtUquGSpJQ+m2MUTahn4Ft
github.com/opencontainers/selinux v1.2.2/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs=
github.com/opencontainers/selinux v1.3.0 h1:xsI95WzPZu5exzA6JzkLSfdr/DilzOhCJOqGe5TgR0g=
github.com/opencontainers/selinux v1.3.0/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs=
github.com/opencontainers/selinux v1.3.1 h1:dn2Rc3wTEvTB6iVqoFrKKeMb0uZ38ZheeyMu2h5C1TI=
github.com/opencontainers/selinux v1.3.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -194,6 +200,8 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339 h1:zSqWKgm/o7HAnlAzBQ+aetp9fpuyytsXnKA8eiLHYQM=
golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191115151921-52ab43148777 h1:wejkGHRTr38uaKRqECZlsCsJ1/TGxIyFbH32x5zUdu4=
golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2 h1:/J2nHFg1MTqaRLFO7M+J78ASNsJoz3r0cvHBPQ77fsE=
golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

View File

@ -214,17 +214,17 @@ func bigDataNameIsManifest(name string) bool {
// recomputeDigests takes a fixed digest and a name-to-digest map and builds a
// list of the unique values that would identify the image.
func (image *Image) recomputeDigests() error {
validDigests := make([]digest.Digest, 0, len(image.BigDataDigests)+1)
func (i *Image) recomputeDigests() error {
validDigests := make([]digest.Digest, 0, len(i.BigDataDigests)+1)
digests := make(map[digest.Digest]struct{})
if image.Digest != "" {
if err := image.Digest.Validate(); err != nil {
return errors.Wrapf(err, "error validating image digest %q", string(image.Digest))
if i.Digest != "" {
if err := i.Digest.Validate(); err != nil {
return errors.Wrapf(err, "error validating image digest %q", string(i.Digest))
}
digests[image.Digest] = struct{}{}
validDigests = append(validDigests, image.Digest)
digests[i.Digest] = struct{}{}
validDigests = append(validDigests, i.Digest)
}
for name, digest := range image.BigDataDigests {
for name, digest := range i.BigDataDigests {
if !bigDataNameIsManifest(name) {
continue
}
@ -237,10 +237,10 @@ func (image *Image) recomputeDigests() error {
validDigests = append(validDigests, digest)
}
}
if image.Digest == "" && len(validDigests) > 0 {
image.Digest = validDigests[0]
if i.Digest == "" && len(validDigests) > 0 {
i.Digest = validDigests[0]
}
image.Digests = validDigests
i.Digests = validDigests
return nil
}

View File

@ -239,6 +239,10 @@ type LayerStore interface {
// ApplyDiff reads a tarstream which was created by a previous call to Diff and
// applies its changes to a specified layer.
ApplyDiff(to string, diff io.Reader) (int64, error)
// LoadLocked wraps Load in a locked state. This means it loads the store
// and cleans-up invalid layers if needed.
LoadLocked() error
}
type layerStore struct {
@ -346,6 +350,7 @@ func (r *layerStore) Load() error {
r.byname = names
r.bycompressedsum = compressedsums
r.byuncompressedsum = uncompressedsums
// Load and merge information about which layers are mounted, and where.
if r.IsReadWrite() {
r.mountsLockfile.RLock()
@ -353,22 +358,23 @@ func (r *layerStore) Load() error {
if err = r.loadMounts(); err != nil {
return err
}
}
// Last step: if we're writable, try to remove anything that a previous
// user of this storage area marked for deletion but didn't manage to
// actually delete.
if r.IsReadWrite() && r.Locked() {
for _, layer := range r.layers {
if layer.Flags == nil {
layer.Flags = make(map[string]interface{})
}
if cleanup, ok := layer.Flags[incompleteFlag]; ok {
if b, ok := cleanup.(bool); ok && b {
err = r.deleteInternal(layer.ID)
if err != nil {
break
// Last step: as were writable, try to remove anything that a previous
// user of this storage area marked for deletion but didn't manage to
// actually delete.
if r.Locked() {
for _, layer := range r.layers {
if layer.Flags == nil {
layer.Flags = make(map[string]interface{})
}
if cleanup, ok := layer.Flags[incompleteFlag]; ok {
if b, ok := cleanup.(bool); ok && b {
err = r.deleteInternal(layer.ID)
if err != nil {
break
}
shouldSave = true
}
shouldSave = true
}
}
}
@ -376,9 +382,16 @@ func (r *layerStore) Load() error {
return r.saveLayers()
}
}
return err
}
func (r *layerStore) LoadLocked() error {
r.lockfile.Lock()
defer r.lockfile.Unlock()
return r.Load()
}
func (r *layerStore) loadMounts() error {
mounts := make(map[string]*Layer)
mpath := r.mountspath()
@ -487,8 +500,6 @@ func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Dri
if err != nil {
return nil, err
}
lockfile.Lock()
defer lockfile.Unlock()
mountsLockfile, err := GetLockfile(filepath.Join(rundir, "mountpoints.lock"))
if err != nil {
return nil, err
@ -516,8 +527,6 @@ func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (ROL
if err != nil {
return nil, err
}
lockfile.RLock()
defer lockfile.Unlock()
rlstore := layerStore{
lockfile: lockfile,
mountsLockfile: nil,

File diff suppressed because it is too large Load Diff

View File

@ -68,6 +68,12 @@ type (
}
)
const (
tarExt = "tar"
solaris = "solaris"
windows = "windows"
)
// Archiver allows the reuse of most utility functions of this package with a
// pluggable Untar function. To facilitate the passing of specific id mappings
// for untar, an archiver can be created with maps which will then be passed to
@ -325,15 +331,15 @@ func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModi
func (compression *Compression) Extension() string {
switch *compression {
case Uncompressed:
return "tar"
return tarExt
case Bzip2:
return "tar.bz2"
return tarExt + ".bz2"
case Gzip:
return "tar.gz"
return tarExt + ".gz"
case Xz:
return "tar.xz"
return tarExt + ".xz"
case Zstd:
return "tar.zst"
return tarExt + ".zst"
}
return ""
}
@ -670,7 +676,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L
}
// Lchown is not supported on Windows.
if Lchown && runtime.GOOS != "windows" {
if Lchown && runtime.GOOS != windows {
if chownOpts == nil {
chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}
}

View File

@ -13,17 +13,17 @@ import (
func statDifferent(oldStat *system.StatT, oldInfo *FileInfo, newStat *system.StatT, newInfo *FileInfo) bool {
// Don't look at size for dirs, its not a good measure of change
oldUid, oldGid := oldStat.UID(), oldStat.GID()
oldUID, oldGID := oldStat.UID(), oldStat.GID()
uid, gid := newStat.UID(), newStat.GID()
if cuid, cgid, err := newInfo.idMappings.ToContainer(idtools.IDPair{UID: int(uid), GID: int(gid)}); err == nil {
uid = uint32(cuid)
gid = uint32(cgid)
if oldcuid, oldcgid, err := oldInfo.idMappings.ToContainer(idtools.IDPair{UID: int(oldUid), GID: int(oldGid)}); err == nil {
oldUid = uint32(oldcuid)
oldGid = uint32(oldcgid)
if oldcuid, oldcgid, err := oldInfo.idMappings.ToContainer(idtools.IDPair{UID: int(oldUID), GID: int(oldGID)}); err == nil {
oldUID = uint32(oldcuid)
oldGID = uint32(oldcgid)
}
}
ownerChanged := uid != oldUid || gid != oldGid
ownerChanged := uid != oldUID || gid != oldGID
if oldStat.Mode() != newStat.Mode() ||
ownerChanged ||
oldStat.Rdev() != newStat.Rdev() ||

View File

@ -68,7 +68,7 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
// specific or Linux-specific, this warning should be changed to an error
// to cater for the situation where someone does manage to upload a Linux
// image but have it tagged as Windows inadvertently.
if runtime.GOOS == "windows" {
if runtime.GOOS == windows {
if strings.Contains(hdr.Name, ":") {
logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name)
continue

View File

@ -226,8 +226,9 @@ func (p *Pattern) compile() error {
sl := string(os.PathSeparator)
escSL := sl
if sl == `\` {
escSL += `\`
const bs = `\`
if sl == bs {
escSL += bs
}
for scan.Peek() != scanner.EOF {
@ -262,11 +263,11 @@ func (p *Pattern) compile() error {
} else if ch == '.' || ch == '$' {
// Escape some regexp special chars that have no meaning
// in golang's filepath.Match
regStr += `\` + string(ch)
regStr += bs + string(ch)
} else if ch == '\\' {
// escape next char. Note that a trailing \ in the pattern
// will be left alone (but need to escape it)
if sl == `\` {
if sl == bs {
// On windows map "\" to "\\", meaning an escaped backslash,
// and then just continue because filepath.Match on
// Windows doesn't allow escaping at all
@ -274,9 +275,9 @@ func (p *Pattern) compile() error {
continue
}
if scan.Peek() != scanner.EOF {
regStr += `\` + string(scan.Next())
regStr += bs + string(scan.Next())
} else {
regStr += `\`
regStr += bs
}
} else {
regStr += string(ch)

View File

@ -77,14 +77,14 @@ func createLockerForPath(path string, ro bool) (Locker, error) {
// lock locks the lockfile via FCTNL(2) based on the specified type and
// command.
func (l *lockfile) lock(l_type int16, recursive bool) {
func (l *lockfile) lock(lType int16, recursive bool) {
lk := unix.Flock_t{
Type: l_type,
Type: lType,
Whence: int16(os.SEEK_SET),
Start: 0,
Len: 0,
}
switch l_type {
switch lType {
case unix.F_RDLCK:
l.rwMutex.RLock()
case unix.F_WRLCK:
@ -96,7 +96,7 @@ func (l *lockfile) lock(l_type int16, recursive bool) {
l.rwMutex.Lock()
}
default:
panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", l_type))
panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", lType))
}
l.stateMutex.Lock()
defer l.stateMutex.Unlock()
@ -116,7 +116,7 @@ func (l *lockfile) lock(l_type int16, recursive bool) {
time.Sleep(10 * time.Millisecond)
}
}
l.locktype = l_type
l.locktype = lType
l.locked = true
l.recursive = recursive
l.counter++

View File

@ -82,6 +82,4 @@ const (
// it possible for the kernel to default to relatime or noatime but still
// allow userspace to override it.
STRICTATIME = unix.MS_STRICTATIME
mntDetach = unix.MNT_DETACH
)

View File

@ -13,6 +13,8 @@ const (
// broflags is the combination of bind and read only
broflags = unix.MS_BIND | unix.MS_RDONLY
none = "none"
)
// isremount returns true if either device name or flags identify a remount request, false otherwise.
@ -20,7 +22,7 @@ func isremount(device string, flags uintptr) bool {
switch {
// We treat device "" and "none" as a remount request to provide compatibility with
// requests that don't explicitly set MS_REMOUNT such as those manipulating bind mounts.
case flags&unix.MS_REMOUNT != 0, device == "", device == "none":
case flags&unix.MS_REMOUNT != 0, device == "", device == none:
return true
default:
return false

View File

@ -2783,18 +2783,24 @@ func (s *store) ContainerParentOwners(id string) ([]int, []int, error) {
}
func (s *store) Layers() ([]Layer, error) {
var layers []Layer
lstore, err := s.LayerStore()
if err != nil {
return nil, err
}
if err := lstore.LoadLocked(); err != nil {
return nil, err
}
layers, err := lstore.Layers()
if err != nil {
return nil, err
}
lstores, err := s.ROLayerStores()
if err != nil {
return nil, err
}
for _, s := range append([]ROLayerStore{lstore}, lstores...) {
for _, s := range lstores {
store := s
store.RLock()
defer store.Unlock()

View File

@ -69,8 +69,8 @@ func ParseIDMapping(UIDMapSlice, GIDMapSlice []string, subUIDMap, subGIDMap stri
}
// GetRootlessRuntimeDir returns the runtime directory when running as non root
func GetRootlessRuntimeDir(rootlessUid int) (string, error) {
path, err := getRootlessRuntimeDir(rootlessUid)
func GetRootlessRuntimeDir(rootlessUID int) (string, error) {
path, err := getRootlessRuntimeDir(rootlessUID)
if err != nil {
return "", err
}
@ -81,18 +81,18 @@ func GetRootlessRuntimeDir(rootlessUid int) (string, error) {
return path, nil
}
func getRootlessRuntimeDir(rootlessUid int) (string, error) {
func getRootlessRuntimeDir(rootlessUID int) (string, error) {
runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
if runtimeDir != "" {
return runtimeDir, nil
}
tmpDir := fmt.Sprintf("/run/user/%d", rootlessUid)
tmpDir := fmt.Sprintf("/run/user/%d", rootlessUID)
st, err := system.Stat(tmpDir)
if err == nil && int(st.UID()) == os.Getuid() && st.Mode()&0700 == 0700 && st.Mode()&0066 == 0000 {
return tmpDir, nil
}
tmpDir = fmt.Sprintf("%s/%d", os.TempDir(), rootlessUid)
tmpDir = fmt.Sprintf("%s/%d", os.TempDir(), rootlessUID)
if err := os.MkdirAll(tmpDir, 0700); err != nil {
logrus.Errorf("failed to create %s: %v", tmpDir, err)
} else {
@ -111,8 +111,8 @@ func getRootlessRuntimeDir(rootlessUid int) (string, error) {
// getRootlessDirInfo returns the parent path of where the storage for containers and
// volumes will be in rootless mode
func getRootlessDirInfo(rootlessUid int) (string, string, error) {
rootlessRuntime, err := GetRootlessRuntimeDir(rootlessUid)
func getRootlessDirInfo(rootlessUID int) (string, string, error) {
rootlessRuntime, err := GetRootlessRuntimeDir(rootlessUID)
if err != nil {
return "", "", err
}
@ -135,10 +135,10 @@ func getRootlessDirInfo(rootlessUid int) (string, string, error) {
}
// getRootlessStorageOpts returns the storage opts for containers running as non root
func getRootlessStorageOpts(rootlessUid int) (StoreOptions, error) {
func getRootlessStorageOpts(rootlessUID int) (StoreOptions, error) {
var opts StoreOptions
dataDir, rootlessRuntime, err := getRootlessDirInfo(rootlessUid)
dataDir, rootlessRuntime, err := getRootlessDirInfo(rootlessUID)
if err != nil {
return opts, err
}
@ -153,10 +153,6 @@ func getRootlessStorageOpts(rootlessUid int) (StoreOptions, error) {
return opts, nil
}
type tomlOptionsConfig struct {
MountProgram string `toml:"mount_program"`
}
func getTomlStorage(storeOptions *StoreOptions) *tomlConfig {
config := new(tomlConfig)
@ -189,21 +185,21 @@ func DefaultStoreOptionsAutoDetectUID() (StoreOptions, error) {
}
// DefaultStoreOptions returns the default storage ops for containers
func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) {
func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) {
var (
defaultRootlessRunRoot string
defaultRootlessGraphRoot string
err error
)
storageOpts := defaultStoreOptions
if rootless && rootlessUid != 0 {
storageOpts, err = getRootlessStorageOpts(rootlessUid)
if rootless && rootlessUID != 0 {
storageOpts, err = getRootlessStorageOpts(rootlessUID)
if err != nil {
return storageOpts, err
}
}
storageConf, err := DefaultConfigFile(rootless && rootlessUid != 0)
storageConf, err := DefaultConfigFile(rootless && rootlessUID != 0)
if err != nil {
return storageOpts, err
}
@ -218,7 +214,7 @@ func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) {
ReloadConfigurationFile(storageConf, &storageOpts)
}
if rootless && rootlessUid != 0 {
if rootless && rootlessUID != 0 {
if err == nil {
// If the file did not specify a graphroot or runroot,
// set sane defaults so we don't try and use root-owned

View File

@ -644,7 +644,7 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
d.fill = (*compressor).fillBlock
d.step = (*compressor).store
case level == ConstantCompression:
d.w.logReusePenalty = uint(4)
d.w.logNewTablePenalty = 4
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillBlock
d.step = (*compressor).storeHuff
@ -652,13 +652,13 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
level = 5
fallthrough
case level >= 1 && level <= 6:
d.w.logReusePenalty = uint(level + 1)
d.w.logNewTablePenalty = 6
d.fast = newFastEnc(level)
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillBlock
d.step = (*compressor).storeFast
case 7 <= level && level <= 9:
d.w.logReusePenalty = uint(level)
d.w.logNewTablePenalty = 10
d.state = &advancedState{}
d.compressionLevel = levels[level]
d.initDeflate()

View File

@ -93,12 +93,12 @@ type huffmanBitWriter struct {
err error
lastHeader int
// Set between 0 (reused block can be up to 2x the size)
logReusePenalty uint
lastHuffMan bool
bytes [256]byte
literalFreq [lengthCodesStart + 32]uint16
offsetFreq [32]uint16
codegenFreq [codegenCodeCount]uint16
logNewTablePenalty uint
lastHuffMan bool
bytes [256]byte
literalFreq [lengthCodesStart + 32]uint16
offsetFreq [32]uint16
codegenFreq [codegenCodeCount]uint16
// codegen must have an extra space for the final symbol.
codegen [literalCount + offsetCodeCount + 1]uint8
@ -119,7 +119,7 @@ type huffmanBitWriter struct {
// If lastHuffMan is set, a table for outputting literals has been generated and offsets are invalid.
//
// An incoming block estimates the output size of a new table using a 'fresh' by calculating the
// optimal size and adding a penalty in 'logReusePenalty'.
// optimal size and adding a penalty in 'logNewTablePenalty'.
// A Huffman table is not optimal, which is why we add a penalty, and generating a new table
// is slower both for compression and decompression.
@ -349,6 +349,13 @@ func (w *huffmanBitWriter) headerSize() (size, numCodegens int) {
int(w.codegenFreq[18])*7, numCodegens
}
// dynamicSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) dynamicReuseSize(litEnc, offEnc *huffmanEncoder) (size int) {
size = litEnc.bitLength(w.literalFreq[:]) +
offEnc.bitLength(w.offsetFreq[:])
return size
}
// dynamicSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
header, numCodegens := w.headerSize()
@ -451,12 +458,12 @@ func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, n
i := 0
for {
var codeWord int = int(w.codegen[i])
var codeWord = uint32(w.codegen[i])
i++
if codeWord == badCode {
break
}
w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
w.writeCode(w.codegenEncoding.codes[codeWord])
switch codeWord {
case 16:
@ -602,14 +609,14 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
var size int
// Check if we should reuse.
if w.lastHeader > 0 {
// Estimate size for using a new table
// Estimate size for using a new table.
// Use the previous header size as the best estimate.
newSize := w.lastHeader + tokens.EstimatedBits()
newSize += newSize >> w.logNewTablePenalty
// The estimated size is calculated as an optimal table.
// We add a penalty to make it more realistic and re-use a bit more.
newSize += newSize >> (w.logReusePenalty & 31)
extra := w.extraBitSize()
reuseSize, _ := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extra)
reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + w.extraBitSize()
// Check if a new table is better.
if newSize < reuseSize {
@ -801,21 +808,30 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
}
// Add everything as literals
estBits := histogramSize(input, w.literalFreq[:], !eof && !sync) + 15
// We have to estimate the header size.
// Assume header is around 70 bytes:
// https://stackoverflow.com/a/25454430
const guessHeaderSizeBits = 70 * 8
estBits, estExtra := histogramSize(input, w.literalFreq[:], !eof && !sync)
estBits += w.lastHeader + 15
if w.lastHeader == 0 {
estBits += guessHeaderSizeBits
}
estBits += estBits >> w.logNewTablePenalty
// Store bytes, if we don't get a reasonable improvement.
ssize, storable := w.storedSize(input)
if storable && ssize < (estBits+estBits>>4) {
if storable && ssize < estBits {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
if w.lastHeader > 0 {
size, _ := w.dynamicSize(w.literalEncoding, huffOffset, w.lastHeader)
estBits += estBits >> (w.logReusePenalty)
reuseSize := w.literalEncoding.bitLength(w.literalFreq[:256])
estBits += estExtra
if estBits < size {
if estBits < reuseSize {
// We owe an EOB
w.writeCode(w.literalEncoding.codes[endBlockMarker])
w.lastHeader = 0

View File

@ -7,7 +7,6 @@ package flate
import (
"math"
"math/bits"
"sort"
)
const (
@ -25,8 +24,6 @@ type huffmanEncoder struct {
codes []hcode
freqcache []literalNode
bitCount [17]int32
lns byLiteral // stored to avoid repeated allocation in generate
lfs byFreq // stored to avoid repeated allocation in generate
}
type literalNode struct {
@ -270,7 +267,7 @@ func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalN
// assigned in literal order (not frequency order).
chunk := list[len(list)-int(bits):]
h.lns.sort(chunk)
sortByLiteral(chunk)
for _, node := range chunk {
h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
code++
@ -315,7 +312,7 @@ func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
}
return
}
h.lfs.sort(list)
sortByFreq(list)
// Get the number of literals for each bit count
bitCount := h.bitCounts(list, maxBits)
@ -323,59 +320,44 @@ func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
h.assignEncodingAndSize(bitCount, list)
}
type byLiteral []literalNode
func (s *byLiteral) sort(a []literalNode) {
*s = byLiteral(a)
sort.Sort(s)
}
func (s byLiteral) Len() int { return len(s) }
func (s byLiteral) Less(i, j int) bool {
return s[i].literal < s[j].literal
}
func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
type byFreq []literalNode
func (s *byFreq) sort(a []literalNode) {
*s = byFreq(a)
sort.Sort(s)
}
func (s byFreq) Len() int { return len(s) }
func (s byFreq) Less(i, j int) bool {
if s[i].freq == s[j].freq {
return s[i].literal < s[j].literal
func atLeastOne(v float32) float32 {
if v < 1 {
return 1
}
return s[i].freq < s[j].freq
return v
}
func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// histogramSize accumulates a histogram of b in h.
// An estimated size in bits is returned.
// Unassigned values are assigned '1' in the histogram.
// len(h) must be >= 256, and h's elements must be all zeroes.
func histogramSize(b []byte, h []uint16, fill bool) int {
func histogramSize(b []byte, h []uint16, fill bool) (int, int) {
h = h[:256]
for _, t := range b {
h[t]++
}
invTotal := 1.0 / float64(len(b))
shannon := 0.0
single := math.Ceil(-math.Log2(invTotal))
for i, v := range h[:] {
if v > 0 {
n := float64(v)
shannon += math.Ceil(-math.Log2(n*invTotal) * n)
} else if fill {
shannon += single
h[i] = 1
invTotal := 1.0 / float32(len(b))
shannon := float32(0.0)
var extra float32
if fill {
oneBits := atLeastOne(-mFastLog2(invTotal))
for i, v := range h[:] {
if v > 0 {
n := float32(v)
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
} else {
h[i] = 1
extra += oneBits
}
}
} else {
for _, v := range h[:] {
if v > 0 {
n := float32(v)
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
}
}
}
return int(shannon + 0.99)
return int(shannon + 0.99), int(extra + 0.99)
}

View File

@ -0,0 +1,178 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// Sort sorts data.
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
// data.Less and data.Swap. The sort is not guaranteed to be stable.
func sortByFreq(data []literalNode) {
n := len(data)
quickSortByFreq(data, 0, n, maxDepth(n))
}
func quickSortByFreq(data []literalNode, a, b, maxDepth int) {
for b-a > 12 { // Use ShellSort for slices <= 12 elements
if maxDepth == 0 {
heapSort(data, a, b)
return
}
maxDepth--
mlo, mhi := doPivotByFreq(data, a, b)
// Avoiding recursion on the larger subproblem guarantees
// a stack depth of at most lg(b-a).
if mlo-a < b-mhi {
quickSortByFreq(data, a, mlo, maxDepth)
a = mhi // i.e., quickSortByFreq(data, mhi, b)
} else {
quickSortByFreq(data, mhi, b, maxDepth)
b = mlo // i.e., quickSortByFreq(data, a, mlo)
}
}
if b-a > 1 {
// Do ShellSort pass with gap 6
// It could be written in this simplified form cause b-a <= 12
for i := a + 6; i < b; i++ {
if data[i].freq == data[i-6].freq && data[i].literal < data[i-6].literal || data[i].freq < data[i-6].freq {
data[i], data[i-6] = data[i-6], data[i]
}
}
insertionSortByFreq(data, a, b)
}
}
// siftDownByFreq implements the heap property on data[lo, hi).
// first is an offset into the array where the root of the heap lies.
func siftDownByFreq(data []literalNode, lo, hi, first int) {
root := lo
for {
child := 2*root + 1
if child >= hi {
break
}
if child+1 < hi && (data[first+child].freq == data[first+child+1].freq && data[first+child].literal < data[first+child+1].literal || data[first+child].freq < data[first+child+1].freq) {
child++
}
if data[first+root].freq == data[first+child].freq && data[first+root].literal > data[first+child].literal || data[first+root].freq > data[first+child].freq {
return
}
data[first+root], data[first+child] = data[first+child], data[first+root]
root = child
}
}
func doPivotByFreq(data []literalNode, lo, hi int) (midlo, midhi int) {
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
if hi-lo > 40 {
// Tukey's ``Ninther,'' median of three medians of three.
s := (hi - lo) / 8
medianOfThreeSortByFreq(data, lo, lo+s, lo+2*s)
medianOfThreeSortByFreq(data, m, m-s, m+s)
medianOfThreeSortByFreq(data, hi-1, hi-1-s, hi-1-2*s)
}
medianOfThreeSortByFreq(data, lo, m, hi-1)
// Invariants are:
// data[lo] = pivot (set up by ChoosePivot)
// data[lo < i < a] < pivot
// data[a <= i < b] <= pivot
// data[b <= i < c] unexamined
// data[c <= i < hi-1] > pivot
// data[hi-1] >= pivot
pivot := lo
a, c := lo+1, hi-1
for ; a < c && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ {
}
b := a
for {
for ; b < c && (data[pivot].freq == data[b].freq && data[pivot].literal > data[b].literal || data[pivot].freq > data[b].freq); b++ { // data[b] <= pivot
}
for ; b < c && (data[pivot].freq == data[c-1].freq && data[pivot].literal < data[c-1].literal || data[pivot].freq < data[c-1].freq); c-- { // data[c-1] > pivot
}
if b >= c {
break
}
// data[b] > pivot; data[c-1] <= pivot
data[b], data[c-1] = data[c-1], data[b]
b++
c--
}
// If hi-c<3 then there are duplicates (by property of median of nine).
// Let's be a bit more conservative, and set border to 5.
protect := hi-c < 5
if !protect && hi-c < (hi-lo)/4 {
// Lets test some points for equality to pivot
dups := 0
if data[pivot].freq == data[hi-1].freq && data[pivot].literal > data[hi-1].literal || data[pivot].freq > data[hi-1].freq { // data[hi-1] = pivot
data[c], data[hi-1] = data[hi-1], data[c]
c++
dups++
}
if data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq { // data[b-1] = pivot
b--
dups++
}
// m-lo = (hi-lo)/2 > 6
// b-lo > (hi-lo)*3/4-1 > 8
// ==> m < b ==> data[m] <= pivot
if data[m].freq == data[pivot].freq && data[m].literal > data[pivot].literal || data[m].freq > data[pivot].freq { // data[m] = pivot
data[m], data[b-1] = data[b-1], data[m]
b--
dups++
}
// if at least 2 points are equal to pivot, assume skewed distribution
protect = dups > 1
}
if protect {
// Protect against a lot of duplicates
// Add invariant:
// data[a <= i < b] unexamined
// data[b <= i < c] = pivot
for {
for ; a < b && (data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq); b-- { // data[b] == pivot
}
for ; a < b && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { // data[a] < pivot
}
if a >= b {
break
}
// data[a] == pivot; data[b-1] < pivot
data[a], data[b-1] = data[b-1], data[a]
a++
b--
}
}
// Swap pivot into middle
data[pivot], data[b-1] = data[b-1], data[pivot]
return b - 1, c
}
// Insertion sort
func insertionSortByFreq(data []literalNode, a, b int) {
for i := a + 1; i < b; i++ {
for j := i; j > a && (data[j].freq == data[j-1].freq && data[j].literal < data[j-1].literal || data[j].freq < data[j-1].freq); j-- {
data[j], data[j-1] = data[j-1], data[j]
}
}
}
// quickSortByFreq, loosely following Bentley and McIlroy,
// ``Engineering a Sort Function,'' SP&E November 1993.
// medianOfThreeSortByFreq moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
func medianOfThreeSortByFreq(data []literalNode, m1, m0, m2 int) {
// sort 3 elements
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
data[m1], data[m0] = data[m0], data[m1]
}
// data[m0] <= data[m1]
if data[m2].freq == data[m1].freq && data[m2].literal < data[m1].literal || data[m2].freq < data[m1].freq {
data[m2], data[m1] = data[m1], data[m2]
// data[m0] <= data[m2] && data[m1] < data[m2]
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
data[m1], data[m0] = data[m0], data[m1]
}
}
// now data[m0] <= data[m1] <= data[m2]
}

View File

@ -0,0 +1,201 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// Sort sorts data.
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
// data.Less and data.Swap. The sort is not guaranteed to be stable.
func sortByLiteral(data []literalNode) {
n := len(data)
quickSort(data, 0, n, maxDepth(n))
}
func quickSort(data []literalNode, a, b, maxDepth int) {
for b-a > 12 { // Use ShellSort for slices <= 12 elements
if maxDepth == 0 {
heapSort(data, a, b)
return
}
maxDepth--
mlo, mhi := doPivot(data, a, b)
// Avoiding recursion on the larger subproblem guarantees
// a stack depth of at most lg(b-a).
if mlo-a < b-mhi {
quickSort(data, a, mlo, maxDepth)
a = mhi // i.e., quickSort(data, mhi, b)
} else {
quickSort(data, mhi, b, maxDepth)
b = mlo // i.e., quickSort(data, a, mlo)
}
}
if b-a > 1 {
// Do ShellSort pass with gap 6
// It could be written in this simplified form cause b-a <= 12
for i := a + 6; i < b; i++ {
if data[i].literal < data[i-6].literal {
data[i], data[i-6] = data[i-6], data[i]
}
}
insertionSort(data, a, b)
}
}
func heapSort(data []literalNode, a, b int) {
first := a
lo := 0
hi := b - a
// Build heap with greatest element at top.
for i := (hi - 1) / 2; i >= 0; i-- {
siftDown(data, i, hi, first)
}
// Pop elements, largest first, into end of data.
for i := hi - 1; i >= 0; i-- {
data[first], data[first+i] = data[first+i], data[first]
siftDown(data, lo, i, first)
}
}
// siftDown implements the heap property on data[lo, hi).
// first is an offset into the array where the root of the heap lies.
func siftDown(data []literalNode, lo, hi, first int) {
root := lo
for {
child := 2*root + 1
if child >= hi {
break
}
if child+1 < hi && data[first+child].literal < data[first+child+1].literal {
child++
}
if data[first+root].literal > data[first+child].literal {
return
}
data[first+root], data[first+child] = data[first+child], data[first+root]
root = child
}
}
func doPivot(data []literalNode, lo, hi int) (midlo, midhi int) {
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
if hi-lo > 40 {
// Tukey's ``Ninther,'' median of three medians of three.
s := (hi - lo) / 8
medianOfThree(data, lo, lo+s, lo+2*s)
medianOfThree(data, m, m-s, m+s)
medianOfThree(data, hi-1, hi-1-s, hi-1-2*s)
}
medianOfThree(data, lo, m, hi-1)
// Invariants are:
// data[lo] = pivot (set up by ChoosePivot)
// data[lo < i < a] < pivot
// data[a <= i < b] <= pivot
// data[b <= i < c] unexamined
// data[c <= i < hi-1] > pivot
// data[hi-1] >= pivot
pivot := lo
a, c := lo+1, hi-1
for ; a < c && data[a].literal < data[pivot].literal; a++ {
}
b := a
for {
for ; b < c && data[pivot].literal > data[b].literal; b++ { // data[b] <= pivot
}
for ; b < c && data[pivot].literal < data[c-1].literal; c-- { // data[c-1] > pivot
}
if b >= c {
break
}
// data[b] > pivot; data[c-1] <= pivot
data[b], data[c-1] = data[c-1], data[b]
b++
c--
}
// If hi-c<3 then there are duplicates (by property of median of nine).
// Let's be a bit more conservative, and set border to 5.
protect := hi-c < 5
if !protect && hi-c < (hi-lo)/4 {
// Lets test some points for equality to pivot
dups := 0
if data[pivot].literal > data[hi-1].literal { // data[hi-1] = pivot
data[c], data[hi-1] = data[hi-1], data[c]
c++
dups++
}
if data[b-1].literal > data[pivot].literal { // data[b-1] = pivot
b--
dups++
}
// m-lo = (hi-lo)/2 > 6
// b-lo > (hi-lo)*3/4-1 > 8
// ==> m < b ==> data[m] <= pivot
if data[m].literal > data[pivot].literal { // data[m] = pivot
data[m], data[b-1] = data[b-1], data[m]
b--
dups++
}
// if at least 2 points are equal to pivot, assume skewed distribution
protect = dups > 1
}
if protect {
// Protect against a lot of duplicates
// Add invariant:
// data[a <= i < b] unexamined
// data[b <= i < c] = pivot
for {
for ; a < b && data[b-1].literal > data[pivot].literal; b-- { // data[b] == pivot
}
for ; a < b && data[a].literal < data[pivot].literal; a++ { // data[a] < pivot
}
if a >= b {
break
}
// data[a] == pivot; data[b-1] < pivot
data[a], data[b-1] = data[b-1], data[a]
a++
b--
}
}
// Swap pivot into middle
data[pivot], data[b-1] = data[b-1], data[pivot]
return b - 1, c
}
// Insertion sort
func insertionSort(data []literalNode, a, b int) {
for i := a + 1; i < b; i++ {
for j := i; j > a && data[j].literal < data[j-1].literal; j-- {
data[j], data[j-1] = data[j-1], data[j]
}
}
}
// maxDepth returns a threshold at which quicksort should switch
// to heapsort. It returns 2*ceil(lg(n+1)).
func maxDepth(n int) int {
var depth int
for i := n; i > 0; i >>= 1 {
depth++
}
return depth * 2
}
// medianOfThree moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
func medianOfThree(data []literalNode, m1, m0, m2 int) {
// sort 3 elements
if data[m1].literal < data[m0].literal {
data[m1], data[m0] = data[m0], data[m1]
}
// data[m0] <= data[m1]
if data[m2].literal < data[m1].literal {
data[m2], data[m1] = data[m1], data[m2]
// data[m0] <= data[m2] && data[m1] < data[m2]
if data[m1].literal < data[m0].literal {
data[m1], data[m0] = data[m0], data[m1]
}
}
// now data[m0] <= data[m1] <= data[m2]
}

View File

@ -184,9 +184,7 @@ func (t *tokens) indexTokens(in []token) {
t.Reset()
for _, tok := range in {
if tok < matchType {
t.tokens[t.n] = tok
t.litHist[tok]++
t.n++
t.AddLiteral(tok.literal())
continue
}
t.AddMatch(uint32(tok.length()), tok.offset())
@ -211,43 +209,53 @@ func (t *tokens) AddLiteral(lit byte) {
t.nLits++
}
// from https://stackoverflow.com/a/28730362
func mFastLog2(val float32) float32 {
ux := int32(math.Float32bits(val))
log2 := (float32)(((ux >> 23) & 255) - 128)
ux &= -0x7f800001
ux += 127 << 23
uval := math.Float32frombits(uint32(ux))
log2 += ((-0.34484843)*uval+2.02466578)*uval - 0.67487759
return log2
}
// EstimatedBits will return an minimum size estimated by an *optimal*
// compression of the block.
// The size of the block
func (t *tokens) EstimatedBits() int {
shannon := float64(0)
shannon := float32(0)
bits := int(0)
nMatches := 0
if t.nLits > 0 {
invTotal := 1.0 / float64(t.nLits)
invTotal := 1.0 / float32(t.nLits)
for _, v := range t.litHist[:] {
if v > 0 {
n := float64(v)
shannon += math.Ceil(-math.Log2(n*invTotal) * n)
n := float32(v)
shannon += -mFastLog2(n*invTotal) * n
}
}
// Just add 15 for EOB
shannon += 15
for _, v := range t.extraHist[1 : literalCount-256] {
for i, v := range t.extraHist[1 : literalCount-256] {
if v > 0 {
n := float64(v)
shannon += math.Ceil(-math.Log2(n*invTotal) * n)
bits += int(lengthExtraBits[v&31]) * int(v)
n := float32(v)
shannon += -mFastLog2(n*invTotal) * n
bits += int(lengthExtraBits[i&31]) * int(v)
nMatches += int(v)
}
}
}
if nMatches > 0 {
invTotal := 1.0 / float64(nMatches)
for _, v := range t.offHist[:offsetCodeCount] {
invTotal := 1.0 / float32(nMatches)
for i, v := range t.offHist[:offsetCodeCount] {
if v > 0 {
n := float64(v)
shannon += math.Ceil(-math.Log2(n*invTotal) * n)
bits += int(offsetExtraBits[v&31]) * int(n)
n := float32(v)
shannon += -mFastLog2(n*invTotal) * n
bits += int(offsetExtraBits[i&31]) * int(v)
}
}
}
return int(shannon) + bits
}

View File

@ -1 +1,3 @@
module github.com/mattn/go-shellwords
go 1.13

View File

@ -88,9 +88,17 @@ loop:
backtick += string(r)
} else if got {
if p.ParseEnv {
buf = replaceEnv(p.Getenv, buf)
parser := &Parser{ParseEnv: false, ParseBacktick: false, Position: 0, Dir: p.Dir}
strs, err := parser.Parse(replaceEnv(p.Getenv, buf))
if err != nil {
return nil, err
}
for _, str := range strs {
args = append(args, str)
}
} else {
args = append(args, buf)
}
args = append(args, buf)
buf = ""
got = false
}
@ -144,7 +152,7 @@ loop:
}
case '"':
if !singleQuoted && !dollarQuote {
if doubleQuoted && buf == "" {
if doubleQuoted {
got = true
}
doubleQuoted = !doubleQuoted
@ -152,7 +160,7 @@ loop:
}
case '\'':
if !doubleQuoted && !dollarQuote {
if singleQuoted && buf == "" {
if singleQuoted {
got = true
}
singleQuoted = !singleQuoted
@ -180,9 +188,17 @@ loop:
if got {
if p.ParseEnv {
buf = replaceEnv(p.Getenv, buf)
parser := &Parser{ParseEnv: false, ParseBacktick: false, Position: 0, Dir: p.Dir}
strs, err := parser.Parse(replaceEnv(p.Getenv, buf))
if err != nil {
return nil, err
}
for _, str := range strs {
args = append(args, str)
}
} else {
args = append(args, buf)
}
args = append(args, buf)
}
if escaped || singleQuoted || doubleQuoted || backQuote || dollarQuote {

6
vendor/modules.txt vendored
View File

@ -138,7 +138,7 @@ github.com/containers/psgo/internal/dev
github.com/containers/psgo/internal/host
github.com/containers/psgo/internal/proc
github.com/containers/psgo/internal/process
# github.com/containers/storage v1.15.7
# github.com/containers/storage v1.15.8
github.com/containers/storage
github.com/containers/storage/drivers
github.com/containers/storage/drivers/aufs
@ -312,7 +312,7 @@ github.com/inconshreveable/mousetrap
github.com/ishidawataru/sctp
# github.com/json-iterator/go v1.1.9
github.com/json-iterator/go
# github.com/klauspost/compress v1.9.7
# github.com/klauspost/compress v1.9.8
github.com/klauspost/compress/flate
github.com/klauspost/compress/fse
github.com/klauspost/compress/huff0
@ -323,7 +323,7 @@ github.com/klauspost/compress/zstd/internal/xxhash
github.com/klauspost/pgzip
# github.com/konsorten/go-windows-terminal-sequences v1.0.2
github.com/konsorten/go-windows-terminal-sequences
# github.com/mattn/go-shellwords v1.0.7
# github.com/mattn/go-shellwords v1.0.9
github.com/mattn/go-shellwords
# github.com/matttproud/golang_protobuf_extensions v1.0.1
github.com/matttproud/golang_protobuf_extensions/pbutil