support multi-image (docker) archives

Support loading and saving tarballs with more than one image.
Add a new `/libpod/images/export` endpoint to the rest API to
allow for exporting/saving multiple images into an archive.

Note that a non-release version of containers/image is vendored.
A release version must be vendored before cutting a new Podman
release.  We force the containers/image version via a replace in
the go.mod file; this way go won't try to match the versions.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

cmd/podman/images/save.go

@@ -16,7 +16,10 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-var validFormats = []string{define.OCIManifestDir, define.OCIArchive, define.V2s2ManifestDir, define.V2s2Archive}
+var (
+	validFormats    = []string{define.OCIManifestDir, define.OCIArchive, define.V2s2ManifestDir, define.V2s2Archive}
+	containerConfig = registry.PodmanConfig()
+)
 
 var (
 	saveDescription = `Save an image to docker-archive or oci-archive on the local machine. Default is docker-archive.`
@@ -79,7 +82,7 @@ func saveFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&saveOpts.Format, "format", define.V2s2Archive, "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-archive, docker-dir (directory with v2s2 manifest type)")
 	flags.StringVarP(&saveOpts.Output, "output", "o", "", "Write to a specified file (default: stdout, which must be redirected)")
 	flags.BoolVarP(&saveOpts.Quiet, "quiet", "q", false, "Suppress the output")
-
+	flags.BoolVarP(&saveOpts.MultiImageArchive, "multi-image-archive", "m", containerConfig.Engine.MultiImageArchive, "Interpret additional arguments as images not tags and create a multi-image-archive (only for docker-archive)")
 }
 
 func save(cmd *cobra.Command, args []string) (finalErr error) {
@@ -118,6 +121,13 @@ func save(cmd *cobra.Command, args []string) (finalErr error) {
 	if len(args) > 1 {
 		tags = args[1:]
 	}
+
+	// Decide whether c/image's progress bars should use stderr or stdout.
+	// If the output is set of stdout, any log message there would corrupt
+	// the tarfile.
+	if saveOpts.Output == os.Stdout.Name() {
+		saveOpts.Quiet = true
+	}
 	err := registry.ImageEngine().Save(context.Background(), args[0], tags, saveOpts)
 	if err == nil {
 		succeeded = true

docs/source/markdown/podman-save.1.md

@@ -40,6 +40,10 @@ Save image to **oci-archive, oci-dir** (directory with oci manifest type), or **
 --format docker-dir
 ```
 
+**--multi-image-archive**, **-m**
+
+Allow for creating archives with more than one image.  Additional names will be interpreted as images instead of tags.  Only supported for **docker-archive**.
+
 **--quiet**, **-q**
 
 Suppress the output

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/containernetworking/cni v0.8.0
 	github.com/containernetworking/plugins v0.8.7
 	github.com/containers/buildah v1.15.1-0.20200813183340-0a8dc1f8064c
-	github.com/containers/common v0.20.3-0.20200827091701-a550d6a98aa3
+	github.com/containers/common v0.21.0
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.5.2
 	github.com/containers/psgo v1.5.1
@@ -60,8 +60,10 @@ require (
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1
+	golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed
 	k8s.io/api v0.0.0-20190620084959-7cf5895f2711
 	k8s.io/apimachinery v0.19.0
 	k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
 )
+
+replace github.com/containers/image/v5 => github.com/containers/image/v5 v5.5.2-0.20200902171422-1c313b2d23e0

go.sum

@@ -52,7 +52,6 @@ github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmY
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
-github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA=
@@ -72,24 +71,20 @@ github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CY
 github.com/containers/buildah v1.15.1-0.20200813183340-0a8dc1f8064c h1:elGbJcB3UjBdk7fBxfAzUNS3IT288U1Dzm0gmhgsnB8=
 github.com/containers/buildah v1.15.1-0.20200813183340-0a8dc1f8064c/go.mod h1:+IklBLPix5wxPEWn26aDay5f5q4A5VtmNjkdyK5YVsI=
 github.com/containers/common v0.19.0/go.mod h1:+NUHV8V5Kmo260ja9Dxtr8ialrDnK4RNzyeEbSgmLac=
-github.com/containers/common v0.20.3-0.20200827091701-a550d6a98aa3 h1:rTSiIMOH3fbCBN+2L8Xr9BJ19AejEIaBQvzkAXZCz/k=
+github.com/containers/common v0.21.0 h1:v2U9MrGw0vMgefQf0/uJYBsSnengxLbSORYqhCVEBs0=
-github.com/containers/common v0.20.3-0.20200827091701-a550d6a98aa3/go.mod h1:z5HJtHWU8sopAHO0Od5s9EpVkXPrLIcNszVvN1Fc3fQ=
+github.com/containers/common v0.21.0/go.mod h1:8w8SVwc+P2p1MOnRMbSKNWXt1Iwd2bKFu2LLZx55DTM=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
-github.com/containers/image/v5 v5.5.1/go.mod h1:4PyNYR0nwlGq/ybVJD9hWlhmIsNra4Q8uOQX2s6E2uM=
+github.com/containers/image/v5 v5.5.2-0.20200902171422-1c313b2d23e0 h1:MJ0bKRn2I5I2NJlVzMU7/eP/9yfMCeWaUskl6zgY/nc=
-github.com/containers/image/v5 v5.5.2 h1:fv7FArz0zUnjH0W0l8t90CqWFlFcQrPP6Pug+9dUtVI=
+github.com/containers/image/v5 v5.5.2-0.20200902171422-1c313b2d23e0/go.mod h1:pBnp9KTyDqM84XTHwmk2lXRvTL6jayAQS47GC6PaPGM=
-github.com/containers/image/v5 v5.5.2/go.mod h1:4PyNYR0nwlGq/ybVJD9hWlhmIsNra4Q8uOQX2s6E2uM=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
-github.com/containers/ocicrypt v1.0.2/go.mod h1:nsOhbP19flrX6rE7ieGFvBlr7modwmNjsqWarIUce4M=
 github.com/containers/ocicrypt v1.0.3 h1:vYgl+RZ9Q3DPMuTfxmN+qp0X2Bj52uuY2vnt6GzVe1c=
 github.com/containers/ocicrypt v1.0.3/go.mod h1:CUBa+8MRNL/VkpxYIpaMtgn1WgXGyvPQj8jcy0EVG6g=
 github.com/containers/psgo v1.5.1 h1:MQNb7FLbXqBdqz6u4lI2QWizVz4RSTzs1+Nk9XT1iVA=
 github.com/containers/psgo v1.5.1/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzPUWfawVU=
-github.com/containers/storage v1.20.2/go.mod h1:oOB9Ie8OVPojvoaKWEGSEtHbXUAs+tSyr7RO7ZGteMc=
 github.com/containers/storage v1.23.0/go.mod h1:I1EIAA7B4OwWRSA0b4yq2AW1wjvvfcY0zLWQuwTa4zw=
-github.com/containers/storage v1.23.2 h1:GPZ8PXYezML1gmZ/uFaXQpyps7AH645lmdvvOJwJYNc=
+github.com/containers/storage v1.23.3/go.mod h1:0azTMiuBhArp/VUmH1o4DJAGaaH+qLtEu17pJ/iKJCg=
-github.com/containers/storage v1.23.2/go.mod h1:AyTMMiE5ANvZJiqvatQgSZ85wAl5yHucY3NDN/kemr4=
 github.com/containers/storage v1.23.4 h1:1raHKGNs2C52tEq2ydHqZ+wu2u1d79BHMO6O5JO20xQ=
 github.com/containers/storage v1.23.4/go.mod h1:KzpVgmUucelPYHq2YsseUTiTuucdVh3xfpPNmxmPZRU=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -154,7 +149,6 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsouza/go-dockerclient v1.6.5 h1:vuFDnPcds3LvTWGYb9h0Rty14FLgkjHZdwLDROCdgsw=
 github.com/fsouza/go-dockerclient v1.6.5/go.mod h1:GOdftxWLWIbIWKbIMDroKFJzPdg6Iw7r+jX1DDZdVsA=
-github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -233,7 +227,6 @@ github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FK
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -243,8 +236,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
+github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
-github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/insomniacslk/dhcp v0.0.0-20200420235442-ed3125c2efe7/go.mod h1:CfMdguCK66I5DAUJgGKyNz8aB6vO5dZzkm9Xep6WGvw=
@@ -262,12 +255,9 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=
 github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
 github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -282,8 +272,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
@@ -325,7 +313,6 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
 github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
@@ -347,7 +334,6 @@ github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod
 github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v0.0.0-20190425234816-dae70e8efea4/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc91/go.mod h1:3Sm6Dt7OT8z88EbdQqqcRN2oCT54jbi72tT/HqgflT8=
 github.com/opencontainers/runc v1.0.0-rc91.0.20200708210054-ce54a9d4d79b h1:wjSgG2Z5xWv1wpAI7JbwKR9aJH0p4HJ+ROZ7ViKh9qU=
 github.com/opencontainers/runc v1.0.0-rc91.0.20200708210054-ce54a9d4d79b/go.mod h1:ZuXhqlr4EiRYgDrBDNfSbE4+n9JX4+V107NwAmF7sZA=
@@ -360,7 +346,6 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo
 github.com/opencontainers/runtime-tools v0.9.0 h1:FYgwVsKRI/H9hU32MJ/4MLOzXWodKK5zsQavY8NPMkU=
 github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
-github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
 github.com/opencontainers/selinux v1.6.0 h1:+bIAS/Za3q5FTwWym4fTB0vObnfCf3G/NC7K6Jx62mY=
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/openshift/imagebuilder v1.1.6 h1:1+YzRxIIefY4QqtCImx6rg+75QrKNfBoPAKxgMo/khM=
@@ -446,7 +431,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -461,8 +445,8 @@ github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
 github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4=
+github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
-github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
@@ -470,8 +454,8 @@ github.com/varlink/go v0.0.0-20190502142041-0f1d566d194b h1:hdDRrn9OP/roL8a/e/5Z
 github.com/varlink/go v0.0.0-20190502142041-0f1d566d194b/go.mod h1:YHaw8N660ESgMgLOZfLQqT1htFItynAUxMesFBho52s=
 github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02dE=
 github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
-github.com/vbauerster/mpb/v5 v5.2.2 h1:zIICVOm+XD+uV6crpSORaL6I0Q1WqOdvxZTp+r3L9cw=
+github.com/vbauerster/mpb/v5 v5.3.0 h1:vgrEJjUzHaSZKDRRxul5Oh4C72Yy/5VEMb0em+9M0mQ=
-github.com/vbauerster/mpb/v5 v5.2.2/go.mod h1:W5Fvgw4dm3/0NhqzV8j6EacfuTe5SvnzBRwiXxDR9ww=
+github.com/vbauerster/mpb/v5 v5.3.0/go.mod h1:4yTkvAb8Cm4eylAp6t0JRq6pXDkFJ4krUlDqWYkakAs=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
@@ -491,7 +475,6 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M=
@@ -566,7 +549,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -575,8 +557,9 @@ golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 h1:sIky/MyNRSHTrdxfsiUSS4WIAMvInbeXljJz+jDjeYE=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed h1:WBkVNH1zd9jg/dK4HCM4lNANnmd12EHC9z+LmcCG4ns=
+golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -611,7 +594,6 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=

libpod/image/image.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/common/pkg/retry"
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
+	"github.com/containers/image/v5/docker/archive"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/image"
@@ -173,13 +174,182 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile
 	return newImage, nil
 }
 
+// SaveImages stores one more images in a multi-image archive.
+// Note that only `docker-archive` supports storing multiple
+// image.
+func (ir *Runtime) SaveImages(ctx context.Context, namesOrIDs []string, format string, outputFile string, quiet bool) (finalErr error) {
+	if format != DockerArchive {
+		return errors.Errorf("multi-image archives are only supported in in the %q format", DockerArchive)
+	}
+
+	sys := GetSystemContext("", "", false)
+
+	archWriter, err := archive.NewWriter(sys, outputFile)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := archWriter.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+			return
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
+
+	// Decide whether c/image's progress bars should use stderr or stdout.
+	// Use stderr in case we need to be quiet or if the output is set to
+	// stdout.  If the output is set of stdout, any log message there would
+	// corrupt the tarfile.
+	writer := os.Stdout
+	if quiet {
+		writer = os.Stderr
+	}
+
+	// extend an image with additional tags
+	type imageData struct {
+		*Image
+		tags []reference.NamedTagged
+	}
+
+	// Look up the images (and their tags) in the local storage.
+	imageMap := make(map[string]*imageData) // to group tags for an image
+	imageQueue := []string{}                // to preserve relative image order
+	for _, nameOrID := range namesOrIDs {
+		// Look up the name or ID in the local image storage.
+		localImage, err := ir.NewFromLocal(nameOrID)
+		if err != nil {
+			return err
+		}
+		id := localImage.ID()
+
+		iData, exists := imageMap[id]
+		if !exists {
+			imageQueue = append(imageQueue, id)
+			iData = &imageData{Image: localImage}
+			imageMap[id] = iData
+		}
+
+		// Unless we referred to an ID, add the input as a tag.
+		if !strings.HasPrefix(id, nameOrID) {
+			tag, err := NormalizedTag(nameOrID)
+			if err != nil {
+				return err
+			}
+			refTagged, isTagged := tag.(reference.NamedTagged)
+			if isTagged {
+				iData.tags = append(iData.tags, refTagged)
+			}
+		}
+	}
+
+	policyContext, err := getPolicyContext(sys)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := policyContext.Destroy(); err != nil {
+			logrus.Errorf("failed to destroy policy context: %q", err)
+		}
+	}()
+
+	// Now copy the images one-by-one.
+	for _, id := range imageQueue {
+		dest, err := archWriter.NewReference(nil)
+		if err != nil {
+			return err
+		}
+
+		img := imageMap[id]
+		copyOptions := getCopyOptions(sys, writer, nil, nil, SigningOptions{}, "", img.tags)
+		copyOptions.DestinationCtx.SystemRegistriesConfPath = registries.SystemRegistriesConfPath()
+
+		// For copying, we need a source reference that we can create
+		// from the image.
+		src, err := is.Transport.NewStoreReference(img.imageruntime.store, nil, id)
+		if err != nil {
+			return errors.Wrapf(err, "error getting source imageReference for %q", img.InputName)
+		}
+		_, err = cp.Image(ctx, policyContext, dest, src, copyOptions)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// LoadAllImagesFromDockerArchive loads all images from the docker archive that
+// fileName points to.
+func (ir *Runtime) LoadAllImagesFromDockerArchive(ctx context.Context, fileName string, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
+	if signaturePolicyPath == "" {
+		signaturePolicyPath = ir.SignaturePolicyPath
+	}
+
+	sc := GetSystemContext(signaturePolicyPath, "", false)
+	reader, err := archive.NewReader(sc, fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err := reader.Close(); err != nil {
+			logrus.Errorf(err.Error())
+		}
+	}()
+
+	refLists, err := reader.List()
+	if err != nil {
+		return nil, err
+	}
+
+	refPairs := []pullRefPair{}
+	for _, refList := range refLists {
+		for _, ref := range refList {
+			pairs, err := ir.getPullRefPairsFromDockerArchiveReference(ctx, reader, ref, sc)
+			if err != nil {
+				return nil, err
+			}
+			refPairs = append(refPairs, pairs...)
+		}
+	}
+
+	goal := pullGoal{
+		pullAllPairs:         true,
+		usedSearchRegistries: false,
+		refPairs:             refPairs,
+		searchedRegistries:   nil,
+	}
+
+	defer goal.cleanUp()
+	imageNames, err := ir.doPullImage(ctx, sc, goal, writer, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{}, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	newImages := make([]*Image, 0, len(imageNames))
+	for _, name := range imageNames {
+		newImage, err := ir.NewFromLocal(name)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error retrieving local image after pulling %s", name)
+		}
+		newImages = append(newImages, newImage)
+	}
+	ir.newImageEvent(events.LoadFromArchive, "")
+	return newImages, nil
+}
+
 // LoadFromArchiveReference creates a new image object for images pulled from a tar archive and the like (podman load)
 // This function is needed because it is possible for a tar archive to have multiple tags for one image
 func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.ImageReference, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
 	if signaturePolicyPath == "" {
 		signaturePolicyPath = ir.SignaturePolicyPath
 	}
-	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{MaxRetry: maxRetry})
+
+	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{})
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
 	}

libpod/image/pull.go

@@ -11,8 +11,8 @@ import (
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
 	"github.com/containers/image/v5/docker"
+	"github.com/containers/image/v5/docker/archive"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
-	"github.com/containers/image/v5/docker/tarfile"
 	ociarchive "github.com/containers/image/v5/oci/archive"
 	oci "github.com/containers/image/v5/oci/layout"
 	is "github.com/containers/image/v5/storage"
@@ -61,12 +61,26 @@ type pullRefPair struct {
 	dstRef types.ImageReference
 }
 
+// cleanUpFunc is a function prototype for clean-up functions.
+type cleanUpFunc func() error
+
 // pullGoal represents the prepared image references and decided behavior to be executed by imagePull
 type pullGoal struct {
 	refPairs             []pullRefPair
-	pullAllPairs         bool     // Pull all refPairs instead of stopping on first success.
+	pullAllPairs         bool          // Pull all refPairs instead of stopping on first success.
-	usedSearchRegistries bool     // refPairs construction has depended on registries.GetRegistries()
+	usedSearchRegistries bool          // refPairs construction has depended on registries.GetRegistries()
-	searchedRegistries   []string // The list of search registries used; set only if usedSearchRegistries
+	searchedRegistries   []string      // The list of search registries used; set only if usedSearchRegistries
+	cleanUpFuncs         []cleanUpFunc // Mainly used to close long-lived objects (e.g., an archive.Reader)
+}
+
+// cleanUp invokes all cleanUpFuncs.  Certain resources may not be available
+// anymore.  Errors are logged.
+func (p *pullGoal) cleanUp() {
+	for _, f := range p.cleanUpFuncs {
+		if err := f(); err != nil {
+			logrus.Error(err.Error())
+		}
+	}
 }
 
 // singlePullRefPairGoal returns a no-frills pull goal for the specified reference pair.
@@ -114,7 +128,49 @@ func (ir *Runtime) getSinglePullRefPairGoal(srcRef types.ImageReference, destNam
 	return singlePullRefPairGoal(rp), nil
 }
 
+// getPullRefPairsFromDockerArchiveReference returns a slice of pullRefPairs
+// for the specified docker reference and the corresponding archive.Reader.
+func (ir *Runtime) getPullRefPairsFromDockerArchiveReference(ctx context.Context, reader *archive.Reader, ref types.ImageReference, sc *types.SystemContext) ([]pullRefPair, error) {
+	destNames, err := reader.ManifestTagsForReference(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(destNames) == 0 {
+		destName, err := getImageDigest(ctx, ref, sc)
+		if err != nil {
+			return nil, err
+		}
+		destNames = append(destNames, destName)
+	} else {
+		for i := range destNames {
+			ref, err := NormalizedTag(destNames[i])
+			if err != nil {
+				return nil, err
+			}
+			destNames[i] = ref.String()
+		}
+	}
+
+	refPairs := []pullRefPair{}
+	for _, destName := range destNames {
+		destRef, err := is.Transport.ParseStoreReference(ir.store, destName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error parsing dest reference name %#v", destName)
+		}
+		pair := pullRefPair{
+			image:  destName,
+			srcRef: ref,
+			dstRef: destRef,
+		}
+		refPairs = append(refPairs, pair)
+	}
+
+	return refPairs, nil
+}
+
 // pullGoalFromImageReference returns a pull goal for a single ImageReference, depending on the used transport.
+// Note that callers are responsible for invoking (*pullGoal).cleanUp() to clean up possibly open resources.
 func (ir *Runtime) pullGoalFromImageReference(ctx context.Context, srcRef types.ImageReference, imgName string, sc *types.SystemContext) (*pullGoal, error) {
 	span, _ := opentracing.StartSpanFromContext(ctx, "pullGoalFromImageReference")
 	defer span.Finish()
@@ -122,57 +178,26 @@ func (ir *Runtime) pullGoalFromImageReference(ctx context.Context, srcRef types.
 	// supports pulling from docker-archive, oci, and registries
 	switch srcRef.Transport().Name() {
 	case DockerArchive:
-		archivePath := srcRef.StringWithinTransport()
+		reader, readerRef, err := archive.NewReaderForReference(sc, srcRef)
-		tarSource, err := tarfile.NewSourceFromFile(archivePath)
 		if err != nil {
 			return nil, err
 		}
-		defer tarSource.Close()
-		manifest, err := tarSource.LoadTarManifest()
 
+		pairs, err := ir.getPullRefPairsFromDockerArchiveReference(ctx, reader, readerRef, sc)
 		if err != nil {
-			return nil, errors.Wrapf(err, "error retrieving manifest.json")
+			// No need to defer for a single error path.
-		}
+			if err := reader.Close(); err != nil {
-		// to pull the first image stored in the tar file
+				logrus.Error(err.Error())
-		if len(manifest) == 0 {
-			// use the hex of the digest if no manifest is found
-			reference, err := getImageDigest(ctx, srcRef, sc)
-			if err != nil {
-				return nil, err
 			}
-			return ir.getSinglePullRefPairGoal(srcRef, reference)
+			return nil, err
 		}
 
-		if len(manifest[0].RepoTags) == 0 {
-			// If the input image has no repotags, we need to feed it a dest anyways
-			digest, err := getImageDigest(ctx, srcRef, sc)
-			if err != nil {
-				return nil, err
-			}
-			return ir.getSinglePullRefPairGoal(srcRef, digest)
-		}
-
-		// Need to load in all the repo tags from the manifest
-		res := []pullRefPair{}
-		for _, dst := range manifest[0].RepoTags {
-			//check if image exists and gives a warning of untagging
-			localImage, err := ir.NewFromLocal(dst)
-			imageID := strings.TrimSuffix(manifest[0].Config, ".json")
-			if err == nil && imageID != localImage.ID() {
-				logrus.Errorf("the image %s already exists, renaming the old one with ID %s to empty string", dst, localImage.ID())
-			}
-
-			pullInfo, err := ir.getPullRefPair(srcRef, dst)
-			if err != nil {
-				return nil, err
-			}
-			res = append(res, pullInfo)
-		}
 		return &pullGoal{
-			refPairs:             res,
 			pullAllPairs:         true,
 			usedSearchRegistries: false,
+			refPairs:             pairs,
 			searchedRegistries:   nil,
+			cleanUpFuncs:         []cleanUpFunc{reader.Close},
 		}, nil
 
 	case OCIArchive:
@@ -249,6 +274,7 @@ func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName s
 			return nil, errors.Wrapf(err, "error determining pull goal for image %q", inputName)
 		}
 	}
+	defer goal.cleanUp()
 	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, label)
 }
 
@@ -267,6 +293,7 @@ func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.Imag
 	if err != nil {
 		return nil, errors.Wrapf(err, "error determining pull goal for image %q", transports.ImageName(srcRef))
 	}
+	defer goal.cleanUp()
 	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, nil)
 }
 

libpod/image/pull_test.go

@@ -150,7 +150,7 @@ func TestPullGoalFromImageReference(t *testing.T) {
 		{ // RepoTags is empty
 			"docker-archive:testdata/docker-unnamed.tar.xz",
 			[]expected{{"@ec9293436c2e66da44edb9efb8d41f6b13baf62283ebe846468bc992d76d7951", "@ec9293436c2e66da44edb9efb8d41f6b13baf62283ebe846468bc992d76d7951"}},
-			false,
+			true,
 		},
 		{ // RepoTags is a [docker.io/library/]name:latest, normalized to the short format.
 			"docker-archive:testdata/docker-name-only.tar.xz",
@@ -170,11 +170,37 @@ func TestPullGoalFromImageReference(t *testing.T) {
 			},
 			true,
 		},
-		{ // FIXME: Two images in a single archive - only the "first" one (whichever it is) is returned
+		{ // Reference image by name in multi-image archive
-			// (and docker-archive: then refuses to read anything when the manifest has more than 1 item)
+			"docker-archive:testdata/docker-two-images.tar.xz:example.com/empty:latest",
+			[]expected{
+				{"example.com/empty:latest", "example.com/empty:latest"},
+			},
+			true,
+		},
+		{ // Reference image by name in multi-image archive
+			"docker-archive:testdata/docker-two-images.tar.xz:example.com/empty/but:different",
+			[]expected{
+				{"example.com/empty/but:different", "example.com/empty/but:different"},
+			},
+			true,
+		},
+		{ // Reference image by index in multi-image archive
+			"docker-archive:testdata/docker-two-images.tar.xz:@0",
+			[]expected{
+				{"example.com/empty:latest", "example.com/empty:latest"},
+			},
+			true,
+		},
+		{ // Reference image by index in multi-image archive
+			"docker-archive:testdata/docker-two-images.tar.xz:@1",
+			[]expected{
+				{"example.com/empty/but:different", "example.com/empty/but:different"},
+			},
+			true,
+		},
+		{ // Reference entire multi-image archive must fail (more than one manifest)
 			"docker-archive:testdata/docker-two-images.tar.xz",
-			[]expected{{"example.com/empty:latest", "example.com/empty:latest"}},
+			[]expected{},
-			// "example.com/empty/but:different" exists but is ignored
 			true,
 		},
 
@@ -248,7 +274,7 @@ func TestPullGoalFromImageReference(t *testing.T) {
 			for i, e := range c.expected {
 				testDescription := fmt.Sprintf("%s #%d", c.srcName, i)
 				assert.Equal(t, e.image, res.refPairs[i].image, testDescription)
-				assert.Equal(t, srcRef, res.refPairs[i].srcRef, testDescription)
+				assert.Equal(t, transports.ImageName(srcRef), transports.ImageName(res.refPairs[i].srcRef), testDescription)
 				assert.Equal(t, e.dstName, storageReferenceWithoutLocation(res.refPairs[i].dstRef), testDescription)
 			}
 			assert.Equal(t, c.expectedPullAllPairs, res.pullAllPairs, c.srcName)

libpod/runtime_img.go

@@ -282,9 +282,16 @@ func (r *Runtime) LoadImage(ctx context.Context, name, inputFile string, writer
 		src       types.ImageReference
 	)
 
+	if name == "" {
+		newImages, err = r.ImageRuntime().LoadAllImagesFromDockerArchive(ctx, inputFile, signaturePolicy, writer)
+		if err == nil {
+			return getImageNames(newImages), nil
+		}
+	}
+
 	for _, referenceFn := range []func() (types.ImageReference, error){
 		func() (types.ImageReference, error) {
-			return dockerarchive.ParseReference(inputFile) // FIXME? We should add dockerarchive.NewReference()
+			return dockerarchive.ParseReference(inputFile)
 		},
 		func() (types.ImageReference, error) {
 			return ociarchive.NewReference(inputFile, name) // name may be ""

pkg/api/handlers/compat/images.go

@@ -365,7 +365,6 @@ func LoadImages(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	id, err := runtime.LoadImage(r.Context(), "", f.Name(), writer, "")
-	//id, err := runtime.Import(r.Context())
 	if err != nil {
 		utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "failed to load image"))
 		return

pkg/api/handlers/libpod/images.go

@@ -234,6 +234,76 @@ func ExportImage(w http.ResponseWriter, r *http.Request) {
 	utils.WriteResponse(w, http.StatusOK, rdr)
 }
 
+func ExportImages(w http.ResponseWriter, r *http.Request) {
+	var (
+		output string
+	)
+	runtime := r.Context().Value("runtime").(*libpod.Runtime)
+	decoder := r.Context().Value("decoder").(*schema.Decoder)
+	query := struct {
+		Compress   bool     `schema:"compress"`
+		Format     string   `schema:"format"`
+		References []string `schema:"references"`
+	}{
+		Format: define.OCIArchive,
+	}
+
+	if err := decoder.Decode(&query, r.URL.Query()); err != nil {
+		utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest,
+			errors.Wrapf(err, "Failed to parse parameters for %s", r.URL.String()))
+		return
+	}
+
+	// References are mandatory!
+	if len(query.References) == 0 {
+		utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest,
+			errors.New("No references"))
+		return
+	}
+
+	// Format is mandatory! Currently, we only support multi-image docker
+	// archives.
+	switch query.Format {
+	case define.V2s2Archive:
+		tmpfile, err := ioutil.TempFile("", "api.tar")
+		if err != nil {
+			utils.Error(w, "unable to create tmpfile", http.StatusInternalServerError, errors.Wrap(err, "unable to create tempfile"))
+			return
+		}
+		output = tmpfile.Name()
+		if err := tmpfile.Close(); err != nil {
+			utils.Error(w, "unable to close tmpfile", http.StatusInternalServerError, errors.Wrap(err, "unable to close tempfile"))
+			return
+		}
+	default:
+		utils.Error(w, "unsupported format", http.StatusInternalServerError, errors.Errorf("unsupported format %q", query.Format))
+		return
+	}
+	defer os.RemoveAll(output)
+
+	// Use the ABI image engine to share as much code as possible.
+	opts := entities.ImageSaveOptions{
+		Compress:          query.Compress,
+		Format:            query.Format,
+		MultiImageArchive: true,
+		Output:            output,
+	}
+
+	imageEngine := abi.ImageEngine{Libpod: runtime}
+	if err := imageEngine.Save(r.Context(), query.References[0], query.References[1:], opts); err != nil {
+		utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, err)
+		return
+	}
+
+	rdr, err := os.Open(output)
+	if err != nil {
+		utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "failed to read the exported tarfile"))
+		return
+	}
+	defer rdr.Close()
+	utils.WriteResponse(w, http.StatusOK, rdr)
+}
+
 func ImagesLoad(w http.ResponseWriter, r *http.Request) {
 	runtime := r.Context().Value("runtime").(*libpod.Runtime)
 	decoder := r.Context().Value("decoder").(*schema.Decoder)

pkg/api/server/register_images.go

@@ -1028,6 +1028,40 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
 	//   500:
 	//     $ref: '#/responses/InternalError'
 	r.Handle(VersionedPath("/libpod/images/{name:.*}/get"), s.APIHandler(libpod.ExportImage)).Methods(http.MethodGet)
+	// swagger:operation GET /libpod/images/export libpod libpodExportImages
+	// ---
+	// tags:
+	//  - images
+	// summary: Export multiple images
+	// description: Export multiple images into a single object. Only `docker-archive` is currently supported.
+	// parameters:
+	//  - in: query
+	//    name: format
+	//    type: string
+	//    description: format for exported image (only docker-archive is supported)
+	//  - in: query
+	//    name: references
+	//    description: references to images to export
+	//    type: array
+	//    items:
+	//      type: string
+	//  - in: query
+	//    name: compress
+	//    type: boolean
+	//    description: use compression on image
+	// produces:
+	// - application/json
+	// responses:
+	//   200:
+	//     description: no error
+	//     schema:
+	//      type: string
+	//      format: binary
+	//   404:
+	//     $ref: '#/responses/NoSuchImage'
+	//   500:
+	//     $ref: '#/responses/InternalError'
+	r.Handle(VersionedPath("/libpod/images/export"), s.APIHandler(libpod.ExportImages)).Methods(http.MethodGet)
 	// swagger:operation GET /libpod/images/{name:.*}/json libpod libpodInspectImage
 	// ---
 	// tags:

pkg/bindings/images/images.go

@@ -128,6 +128,34 @@ func Load(ctx context.Context, r io.Reader, name *string) (*entities.ImageLoadRe
 	return &report, response.Process(&report)
 }
 
+func MultiExport(ctx context.Context, namesOrIds []string, w io.Writer, format *string, compress *bool) error {
+	conn, err := bindings.GetClient(ctx)
+	if err != nil {
+		return err
+	}
+	params := url.Values{}
+	if format != nil {
+		params.Set("format", *format)
+	}
+	if compress != nil {
+		params.Set("compress", strconv.FormatBool(*compress))
+	}
+	for _, ref := range namesOrIds {
+		params.Add("references", ref)
+	}
+	response, err := conn.DoRequest(nil, http.MethodGet, "/images/export", params, nil)
+	if err != nil {
+		return err
+	}
+
+	if response.StatusCode/100 == 2 || response.StatusCode/100 == 3 {
+		_, err = io.Copy(w, response.Body)
+		return err
+	}
+	return response.Process(nil)
+
+}
+
 // Export saves an image from local storage as a tarball or image archive.  The optional format
 // parameter is used to change the format of the output.
 func Export(ctx context.Context, nameOrID string, w io.Writer, format *string, compress *bool) error {

pkg/domain/entities/images.go

@@ -271,11 +271,22 @@ type ImageImportReport struct {
 	Id string //nolint
 }
 
+// ImageSaveOptions provide options for saving images.
 type ImageSaveOptions struct {
+	// Compress layers when saving to a directory.
 	Compress bool
-	Format   string
+	// Format of saving the image: oci-archive, oci-dir (directory with oci
-	Output   string
+	// manifest type), docker-archive, docker-dir (directory with v2s2
-	Quiet    bool
+	// manifest type).
+	Format string
+	// MultiImageArchive denotes if the created archive shall include more
+	// than one image.  Additional tags will be interpreted as references
+	// to images which are added to the archive.
+	MultiImageArchive bool
+	// Output - write image to the specified path.
+	Output string
+	// Quiet - suppress output when copying images
+	Quiet bool
 }
 
 // ImageTreeOptions provides options for ImageEngine.Tree()

pkg/domain/infra/abi/images.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/containers/common/pkg/config"
 	"github.com/containers/image/v5/docker"
-	dockerarchive "github.com/containers/image/v5/docker/archive"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/signature"
@@ -230,15 +229,6 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entiti
 		}
 	}
 
-	// Special-case for docker-archive which allows multiple tags.
-	if imageRef.Transport().Name() == dockerarchive.Transport.Name() {
-		newImage, err := ir.Libpod.ImageRuntime().LoadFromArchiveReference(ctx, imageRef, options.SignaturePolicy, writer)
-		if err != nil {
-			return nil, err
-		}
-		return &entities.ImagePullReport{Images: []string{newImage[0].ID()}}, nil
-	}
-
 	var registryCreds *types.DockerAuthConfig
 	if len(options.Username) > 0 && len(options.Password) > 0 {
 		registryCreds = &types.DockerAuthConfig{
@@ -481,6 +471,10 @@ func (ir *ImageEngine) Import(ctx context.Context, opts entities.ImageImportOpti
 }
 
 func (ir *ImageEngine) Save(ctx context.Context, nameOrID string, tags []string, options entities.ImageSaveOptions) error {
+	if options.MultiImageArchive {
+		nameOrIDs := append([]string{nameOrID}, tags...)
+		return ir.Libpod.ImageRuntime().SaveImages(ctx, nameOrIDs, options.Format, options.Output, options.Quiet)
+	}
 	newImage, err := ir.Libpod.ImageRuntime().NewFromLocal(nameOrID)
 	if err != nil {
 		return err

pkg/domain/infra/tunnel/images.go

@@ -251,12 +251,23 @@ func (ir *ImageEngine) Save(ctx context.Context, nameOrID string, tags []string,
 		return err
 	}
 
-	exErr := images.Export(ir.ClientCxt, nameOrID, f, &options.Format, &options.Compress)
+	if options.MultiImageArchive {
-	if err := f.Close(); err != nil {
+		exErr := images.MultiExport(ir.ClientCxt, append([]string{nameOrID}, tags...), f, &options.Format, &options.Compress)
-		return err
+		if err := f.Close(); err != nil {
-	}
+			return err
-	if exErr != nil {
+		}
-		return exErr
+		if exErr != nil {
+			return exErr
+		}
+	} else {
+		// FIXME: tags are entirely ignored here but shouldn't.
+		exErr := images.Export(ir.ClientCxt, nameOrID, f, &options.Format, &options.Compress)
+		if err := f.Close(); err != nil {
+			return err
+		}
+		if exErr != nil {
+			return exErr
+		}
 	}
 
 	if options.Format != "oci-dir" && options.Format != "docker-dir" {

test/e2e/load_test.go

@@ -269,4 +269,12 @@ var _ = Describe("Podman load", func() {
 		result.WaitWithDefaultTimeout()
 		Expect(result.ExitCode()).To(Equal(0))
 	})
+
+	It("podman load multi-image archive", func() {
+		result := podmanTest.PodmanNoCache([]string{"load", "-i", "./testdata/image/docker-two-images.tar.xz"})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(result.LineInOutputContains("example.com/empty:latest")).To(BeTrue())
+		Expect(result.LineInOutputContains("example.com/empty/but:different")).To(BeTrue())
+	})
 })

test/e2e/pull_test.go

@@ -251,6 +251,49 @@ var _ = Describe("Podman pull", func() {
 		session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
+
+		// Pulling a multi-image archive without further specifying
+		// which image _must_ error out. Pulling is restricted to one
+		// image.
+		session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:./testdata/image/docker-two-images.tar.xz")})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(125))
+		expectedError := "Unexpected tar manifest.json: expected 1 item, got 2"
+		found, _ := session.ErrorGrepString(expectedError)
+		Expect(found).To(Equal(true))
+
+		// Now pull _one_ image from a multi-image archive via the name
+		// and index syntax.
+		session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:./testdata/image/docker-two-images.tar.xz:@0")})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:./testdata/image/docker-two-images.tar.xz:example.com/empty:latest")})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:./testdata/image/docker-two-images.tar.xz:@1")})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:./testdata/image/docker-two-images.tar.xz:example.com/empty/but:different")})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		// Now check for some errors.
+		session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:./testdata/image/docker-two-images.tar.xz:foo.com/does/not/exist:latest")})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(125))
+		expectedError = "Tag \"foo.com/does/not/exist:latest\" not found"
+		found, _ = session.ErrorGrepString(expectedError)
+		Expect(found).To(Equal(true))
+
+		session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:./testdata/image/docker-two-images.tar.xz:@2")})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(125))
+		expectedError = "Invalid source index @2, only 2 manifest items available"
+		found, _ = session.ErrorGrepString(expectedError)
+		Expect(found).To(Equal(true))
 	})
 
 	It("podman pull from oci-archive", func() {

test/e2e/save_test.go

@@ -128,4 +128,51 @@ var _ = Describe("Podman save", func() {
 		save.WaitWithDefaultTimeout()
 		Expect(save.ExitCode()).To(Equal(0))
 	})
+
+	It("podman save --multi-image-archive (tagged images)", func() {
+		multiImageSave(podmanTest, RESTORE_IMAGES)
+	})
+
+	It("podman save --multi-image-archive (untagged images)", func() {
+		// Refer to images via ID instead of tag.
+		session := podmanTest.PodmanNoCache([]string{"images", "--format", "{{.ID}}"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		ids := session.OutputToStringArray()
+
+		Expect(len(RESTORE_IMAGES), len(ids))
+		multiImageSave(podmanTest, ids)
+	})
 })
+
+// Create a multi-image archive, remove all images, load it and
+// make sure that all images are (again) present.
+func multiImageSave(podmanTest *PodmanTestIntegration, images []string) {
+	// Create the archive.
+	outfile := filepath.Join(podmanTest.TempDir, "temp.tar")
+	session := podmanTest.PodmanNoCache(append([]string{"save", "-o", outfile, "--multi-image-archive"}, images...))
+	session.WaitWithDefaultTimeout()
+	Expect(session.ExitCode()).To(Equal(0))
+
+	// Remove all images.
+	session = podmanTest.PodmanNoCache([]string{"rmi", "-af"})
+	session.WaitWithDefaultTimeout()
+	Expect(session.ExitCode()).To(Equal(0))
+
+	// Now load the archive.
+	session = podmanTest.PodmanNoCache([]string{"load", "-i", outfile})
+	session.WaitWithDefaultTimeout()
+	Expect(session.ExitCode()).To(Equal(0))
+	// Grep for each image in the `podman load` output.
+	for _, image := range images {
+		found, _ := session.GrepString(image)
+		Expect(found).Should(BeTrue())
+	}
+
+	// Make sure that each image has really been loaded.
+	for _, image := range images {
+		session = podmanTest.PodmanNoCache([]string{"image", "exists", image})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+	}
+}

test/e2e/testdata/image

@@ -0,0 +1 @@
+../../../libpod/image/testdata/

vendor/github.com/containers/common/pkg/seccomp/conversion.go

@@ -1,25 +1,92 @@
+// NOTE: this package has originally been copied from
+// github.com/opencontainers/runc and modified to work for other use cases
+
 package seccomp
 
-import "fmt"
+import (
+	"fmt"
 
-var goArchToSeccompArchMap = map[string]Arch{
+	"github.com/opencontainers/runtime-spec/specs-go"
-	"386":         ArchX86,
+	"github.com/pkg/errors"
-	"amd64":       ArchX86_64,
+)
-	"amd64p32":    ArchX32,
+
-	"arm":         ArchARM,
+var (
-	"arm64":       ArchAARCH64,
+	goArchToSeccompArchMap = map[string]Arch{
-	"mips":        ArchMIPS,
+		"386":         ArchX86,
-	"mips64":      ArchMIPS64,
+		"amd64":       ArchX86_64,
-	"mips64le":    ArchMIPSEL64,
+		"amd64p32":    ArchX32,
-	"mips64p32":   ArchMIPS64N32,
+		"arm":         ArchARM,
-	"mips64p32le": ArchMIPSEL64N32,
+		"arm64":       ArchAARCH64,
-	"mipsle":      ArchMIPSEL,
+		"mips":        ArchMIPS,
-	"ppc":         ArchPPC,
+		"mips64":      ArchMIPS64,
-	"ppc64":       ArchPPC64,
+		"mips64le":    ArchMIPSEL64,
-	"ppc64le":     ArchPPC64LE,
+		"mips64p32":   ArchMIPS64N32,
-	"s390":        ArchS390,
+		"mips64p32le": ArchMIPSEL64N32,
-	"s390x":       ArchS390X,
+		"mipsle":      ArchMIPSEL,
-}
+		"ppc":         ArchPPC,
+		"ppc64":       ArchPPC64,
+		"ppc64le":     ArchPPC64LE,
+		"s390":        ArchS390,
+		"s390x":       ArchS390X,
+	}
+	specArchToLibseccompArchMap = map[specs.Arch]string{
+		specs.ArchX86:         "x86",
+		specs.ArchX86_64:      "amd64",
+		specs.ArchX32:         "x32",
+		specs.ArchARM:         "arm",
+		specs.ArchAARCH64:     "arm64",
+		specs.ArchMIPS:        "mips",
+		specs.ArchMIPS64:      "mips64",
+		specs.ArchMIPS64N32:   "mips64n32",
+		specs.ArchMIPSEL:      "mipsel",
+		specs.ArchMIPSEL64:    "mipsel64",
+		specs.ArchMIPSEL64N32: "mipsel64n32",
+		specs.ArchPPC:         "ppc",
+		specs.ArchPPC64:       "ppc64",
+		specs.ArchPPC64LE:     "ppc64le",
+		specs.ArchS390:        "s390",
+		specs.ArchS390X:       "s390x",
+	}
+	specArchToSeccompArchMap = map[specs.Arch]Arch{
+		specs.ArchX86:         ArchX86,
+		specs.ArchX86_64:      ArchX86_64,
+		specs.ArchX32:         ArchX32,
+		specs.ArchARM:         ArchARM,
+		specs.ArchAARCH64:     ArchAARCH64,
+		specs.ArchMIPS:        ArchMIPS,
+		specs.ArchMIPS64:      ArchMIPS64,
+		specs.ArchMIPS64N32:   ArchMIPS64N32,
+		specs.ArchMIPSEL:      ArchMIPSEL,
+		specs.ArchMIPSEL64:    ArchMIPSEL64,
+		specs.ArchMIPSEL64N32: ArchMIPSEL64N32,
+		specs.ArchPPC:         ArchPPC,
+		specs.ArchPPC64:       ArchPPC64,
+		specs.ArchPPC64LE:     ArchPPC64LE,
+		specs.ArchS390:        ArchS390,
+		specs.ArchS390X:       ArchS390X,
+	}
+	specActionToSeccompActionMap = map[specs.LinuxSeccompAction]Action{
+		specs.ActKill: ActKill,
+		// TODO: wait for this PR to get merged:
+		// https://github.com/opencontainers/runtime-spec/pull/1064
+		// specs.ActKillProcess   ActKillProcess,
+		// specs.ActKillThread   ActKillThread,
+		specs.ActErrno: ActErrno,
+		specs.ActTrap:  ActTrap,
+		specs.ActAllow: ActAllow,
+		specs.ActTrace: ActTrace,
+		specs.ActLog:   ActLog,
+	}
+	specOperatorToSeccompOperatorMap = map[specs.LinuxSeccompOperator]Operator{
+		specs.OpNotEqual:     OpNotEqual,
+		specs.OpLessThan:     OpLessThan,
+		specs.OpLessEqual:    OpLessEqual,
+		specs.OpEqualTo:      OpEqualTo,
+		specs.OpGreaterEqual: OpGreaterEqual,
+		specs.OpGreaterThan:  OpGreaterThan,
+		specs.OpMaskedEqual:  OpMaskedEqual,
+	}
+)
 
 // GoArchToSeccompArch converts a runtime.GOARCH to a seccomp `Arch`. The
 // function returns an error if the architecture conversion is not supported.
@@ -30,3 +97,100 @@ func GoArchToSeccompArch(goArch string) (Arch, error) {
 	}
 	return arch, nil
 }
+
+// specToSeccomp converts a `LinuxSeccomp` spec into a `Seccomp` struct.
+func specToSeccomp(spec *specs.LinuxSeccomp) (*Seccomp, error) {
+	res := &Seccomp{
+		Syscalls: []*Syscall{},
+	}
+
+	for _, arch := range spec.Architectures {
+		newArch, err := specArchToSeccompArch(arch)
+		if err != nil {
+			return nil, errors.Wrap(err, "convert spec arch")
+		}
+		res.Architectures = append(res.Architectures, newArch)
+	}
+
+	// Convert default action
+	newDefaultAction, err := specActionToSeccompAction(spec.DefaultAction)
+	if err != nil {
+		return nil, errors.Wrap(err, "convert default action")
+	}
+	res.DefaultAction = newDefaultAction
+
+	// Loop through all syscall blocks and convert them to the internal format
+	for _, call := range spec.Syscalls {
+		newAction, err := specActionToSeccompAction(call.Action)
+		if err != nil {
+			return nil, errors.Wrap(err, "convert action")
+		}
+
+		for _, name := range call.Names {
+			newCall := Syscall{
+				Name:     name,
+				Action:   newAction,
+				ErrnoRet: call.ErrnoRet,
+				Args:     []*Arg{},
+			}
+
+			// Loop through all the arguments of the syscall and convert them
+			for _, arg := range call.Args {
+				newOp, err := specOperatorToSeccompOperator(arg.Op)
+				if err != nil {
+					return nil, errors.Wrap(err, "convert operator")
+				}
+
+				newArg := Arg{
+					Index:    arg.Index,
+					Value:    arg.Value,
+					ValueTwo: arg.ValueTwo,
+					Op:       newOp,
+				}
+
+				newCall.Args = append(newCall.Args, &newArg)
+			}
+			res.Syscalls = append(res.Syscalls, &newCall)
+		}
+	}
+
+	return res, nil
+}
+
+// specArchToLibseccompArch converts a spec arch into a libseccomp one.
+func specArchToLibseccompArch(arch specs.Arch) (string, error) {
+	if res, ok := specArchToLibseccompArchMap[arch]; ok {
+		return res, nil
+	}
+	return "", errors.Errorf(
+		"architecture %q is not valid for libseccomp", arch,
+	)
+}
+
+// specArchToSeccompArch converts a spec arch into an internal one.
+func specArchToSeccompArch(arch specs.Arch) (Arch, error) {
+	if res, ok := specArchToSeccompArchMap[arch]; ok {
+		return res, nil
+	}
+	return "", errors.Errorf("architecture %q is not valid", arch)
+}
+
+// specActionToSeccompAction converts a spec action into a seccomp one.
+func specActionToSeccompAction(action specs.LinuxSeccompAction) (Action, error) {
+	if res, ok := specActionToSeccompActionMap[action]; ok {
+		return res, nil
+	}
+	return "", errors.Errorf(
+		"spec action %q is not valid internal action", action,
+	)
+}
+
+// specOperatorToSeccompOperator converts a spec operator into a seccomp one.
+func specOperatorToSeccompOperator(operator specs.LinuxSeccompOperator) (Operator, error) {
+	if op, ok := specOperatorToSeccompOperatorMap[operator]; ok {
+		return op, nil
+	}
+	return "", errors.Errorf(
+		"spec operator %q is not a valid internal operator", operator,
+	)
+}

vendor/github.com/containers/common/pkg/seccomp/filter.go

@@ -0,0 +1,237 @@
+// +build seccomp
+
+// NOTE: this package has originally been copied from
+// github.com/opencontainers/runc and modified to work for other use cases
+
+package seccomp
+
+import (
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	libseccomp "github.com/seccomp/libseccomp-golang"
+	"golang.org/x/sys/unix"
+)
+
+// NOTE: this package has originally been copied from
+// github.com/opencontainers/runc and modified to work for other use cases
+
+var (
+	// ErrSpecNil is a possible return error from BuildFilter() and occurs if
+	// the provided spec is nil.
+	ErrSpecNil = errors.New("spec is nil")
+
+	// ErrSpecEmpty is a possible return error from BuildFilter() and occurs if
+	// the provided spec has neither a DefaultAction nor any syscalls.
+	ErrSpecEmpty = errors.New("spec contains neither a default action nor any syscalls")
+)
+
+// BuildFilter does a basic validation for the provided seccomp profile
+// string and returns a filter for it.
+func BuildFilter(spec *specs.LinuxSeccomp) (*libseccomp.ScmpFilter, error) {
+	// Sanity checking to allow consumers to act accordingly
+	if spec == nil {
+		return nil, ErrSpecNil
+	}
+	if spec.DefaultAction == "" && len(spec.Syscalls) == 0 {
+		return nil, ErrSpecEmpty
+	}
+
+	profile, err := specToSeccomp(spec)
+	if err != nil {
+		return nil, errors.Wrap(err, "convert spec to seccomp profile")
+	}
+
+	defaultAction, err := toAction(profile.DefaultAction, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "convert default action %s", profile.DefaultAction)
+	}
+
+	filter, err := libseccomp.NewFilter(defaultAction)
+	if err != nil {
+		return nil, errors.Wrapf(err, "create filter for default action %s", defaultAction)
+	}
+
+	// Add extra architectures
+	for _, arch := range spec.Architectures {
+		libseccompArch, err := specArchToLibseccompArch(arch)
+		if err != nil {
+			return nil, errors.Wrap(err, "convert spec arch")
+		}
+
+		scmpArch, err := libseccomp.GetArchFromString(libseccompArch)
+		if err != nil {
+			return nil, errors.Wrapf(err, "validate Seccomp architecture %s", arch)
+		}
+
+		if err := filter.AddArch(scmpArch); err != nil {
+			return nil, errors.Wrap(err, "add architecture to seccomp filter")
+		}
+	}
+
+	// Unset no new privs bit
+	if err := filter.SetNoNewPrivsBit(false); err != nil {
+		return nil, errors.Wrap(err, "set no new privileges flag")
+	}
+
+	// Add a rule for each syscall
+	for _, call := range profile.Syscalls {
+		if call == nil {
+			return nil, errors.New("encountered nil syscall while initializing seccomp")
+		}
+
+		if err = matchSyscall(filter, call); err != nil {
+			return nil, errors.Wrap(err, "filter matches syscall")
+		}
+	}
+
+	return filter, nil
+}
+
+func matchSyscall(filter *libseccomp.ScmpFilter, call *Syscall) error {
+	if call == nil || filter == nil {
+		return errors.New("cannot use nil as syscall to block")
+	}
+
+	if call.Name == "" {
+		return errors.New("empty string is not a valid syscall")
+	}
+
+	// If we can't resolve the syscall, assume it's not supported on this kernel
+	// Ignore it, don't error out
+	callNum, err := libseccomp.GetSyscallFromName(call.Name)
+	if err != nil {
+		return nil
+	}
+
+	// Convert the call's action to the libseccomp equivalent
+	callAct, err := toAction(call.Action, call.ErrnoRet)
+	if err != nil {
+		return errors.Wrapf(err, "convert action %s", call.Action)
+	}
+
+	// Unconditional match - just add the rule
+	if len(call.Args) == 0 {
+		if err = filter.AddRule(callNum, callAct); err != nil {
+			return errors.Wrapf(err, "add seccomp filter rule for syscall %s", call.Name)
+		}
+	} else {
+		// Linux system calls can have at most 6 arguments
+		const syscallMaxArguments int = 6
+
+		// If two or more arguments have the same condition,
+		// Revert to old behavior, adding each condition as a separate rule
+		argCounts := make([]uint, syscallMaxArguments)
+		conditions := []libseccomp.ScmpCondition{}
+
+		for _, cond := range call.Args {
+			newCond, err := toCondition(cond)
+			if err != nil {
+				return errors.Wrapf(err, "create seccomp syscall condition for syscall %s", call.Name)
+			}
+
+			argCounts[cond.Index] += 1
+
+			conditions = append(conditions, newCond)
+		}
+
+		hasMultipleArgs := false
+		for _, count := range argCounts {
+			if count > 1 {
+				hasMultipleArgs = true
+				break
+			}
+		}
+
+		if hasMultipleArgs {
+			// Revert to old behavior
+			// Add each condition attached to a separate rule
+			for _, cond := range conditions {
+				condArr := []libseccomp.ScmpCondition{cond}
+
+				if err = filter.AddRuleConditional(callNum, callAct, condArr); err != nil {
+					return errors.Wrapf(err, "add seccomp rule for syscall %s", call.Name)
+				}
+			}
+		} else if err = filter.AddRuleConditional(callNum, callAct, conditions); err != nil {
+			// No conditions share same argument
+			// Use new, proper behavior
+			return errors.Wrapf(err, "add seccomp rule for syscall %s", call.Name)
+		}
+	}
+
+	return nil
+}
+
+// toAction converts an internal `Action` type to a `libseccomp.ScmpAction`
+// type.
+func toAction(act Action, errnoRet *uint) (libseccomp.ScmpAction, error) {
+	switch act {
+	case ActKill:
+		return libseccomp.ActKill, nil
+	case ActKillProcess:
+		return libseccomp.ActKillProcess, nil
+	case ActErrno:
+		if errnoRet != nil {
+			return libseccomp.ActErrno.SetReturnCode(int16(*errnoRet)), nil
+		}
+		return libseccomp.ActErrno.SetReturnCode(int16(unix.EPERM)), nil
+	case ActTrap:
+		return libseccomp.ActTrap, nil
+	case ActAllow:
+		return libseccomp.ActAllow, nil
+	case ActTrace:
+		if errnoRet != nil {
+			return libseccomp.ActTrace.SetReturnCode(int16(*errnoRet)), nil
+		}
+		return libseccomp.ActTrace.SetReturnCode(int16(unix.EPERM)), nil
+	case ActLog:
+		return libseccomp.ActLog, nil
+	default:
+		return libseccomp.ActInvalid, errors.Errorf("invalid action %s", act)
+	}
+}
+
+// toCondition converts an internal `Arg` type to a `libseccomp.ScmpCondition`
+// type.
+func toCondition(arg *Arg) (cond libseccomp.ScmpCondition, err error) {
+	if arg == nil {
+		return cond, errors.New("cannot convert nil to syscall condition")
+	}
+
+	op, err := toCompareOp(arg.Op)
+	if err != nil {
+		return cond, errors.Wrap(err, "convert compare operator")
+	}
+
+	condition, err := libseccomp.MakeCondition(
+		arg.Index, op, arg.Value, arg.ValueTwo,
+	)
+	if err != nil {
+		return cond, errors.Wrap(err, "make condition")
+	}
+
+	return condition, nil
+}
+
+// toCompareOp converts an internal `Operator` type to a
+// `libseccomp.ScmpCompareOp`.
+func toCompareOp(op Operator) (libseccomp.ScmpCompareOp, error) {
+	switch op {
+	case OpEqualTo:
+		return libseccomp.CompareEqual, nil
+	case OpNotEqual:
+		return libseccomp.CompareNotEqual, nil
+	case OpGreaterThan:
+		return libseccomp.CompareGreater, nil
+	case OpGreaterEqual:
+		return libseccomp.CompareGreaterEqual, nil
+	case OpLessThan:
+		return libseccomp.CompareLess, nil
+	case OpLessEqual:
+		return libseccomp.CompareLessOrEqual, nil
+	case OpMaskedEqual:
+		return libseccomp.CompareMaskedEqual, nil
+	default:
+		return libseccomp.CompareInvalid, errors.Errorf("invalid operator %s", op)
+	}
+}

vendor/github.com/containers/common/pkg/seccomp/seccomp_linux.go

@@ -122,7 +122,7 @@ Loop:
 		}
 		if len(call.Excludes.Caps) > 0 {
 			for _, c := range call.Excludes.Caps {
-				if inSlice(rs.Process.Capabilities.Bounding, c) {
+				if rs != nil && rs.Process != nil && rs.Process.Capabilities != nil && inSlice(rs.Process.Capabilities.Bounding, c) {
 					continue Loop
 				}
 			}
@@ -134,7 +134,7 @@ Loop:
 		}
 		if len(call.Includes.Caps) > 0 {
 			for _, c := range call.Includes.Caps {
-				if !inSlice(rs.Process.Capabilities.Bounding, c) {
+				if rs != nil && rs.Process != nil && rs.Process.Capabilities != nil && !inSlice(rs.Process.Capabilities.Bounding, c) {
 					continue Loop
 				}
 			}

vendor/github.com/containers/common/pkg/seccomp/validate.go

@@ -0,0 +1,29 @@
+// +build seccomp
+
+package seccomp
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
+// ValidateProfile does a basic validation for the provided seccomp profile
+// string.
+func ValidateProfile(content string) error {
+	profile := &Seccomp{}
+	if err := json.Unmarshal([]byte(content), &profile); err != nil {
+		return errors.Wrap(err, "decoding seccomp profile")
+	}
+
+	spec, err := setupSeccomp(profile, nil)
+	if err != nil {
+		return errors.Wrap(err, "create seccomp spec")
+	}
+
+	if _, err := BuildFilter(spec); err != nil {
+		return errors.Wrap(err, "build seccomp filter")
+	}
+
+	return nil
+}

vendor/github.com/containers/common/version/version.go

@@ -1,4 +1,4 @@
 package version
 
 // Version is the version of the build.
-const Version = "0.20.4-dev"
+const Version = "0.21.0"

vendor/github.com/containers/image/v5/copy/copy.go

@@ -377,7 +377,7 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
 	if len(sigs) != 0 {
 		c.Printf("Checking if image list destination supports signatures\n")
 		if err := c.dest.SupportsSignatures(ctx); err != nil {
-			return nil, "", errors.Wrap(err, "Can not copy signatures")
+			return nil, "", errors.Wrapf(err, "Can not copy signatures to %s", transports.ImageName(c.dest.Reference()))
 		}
 	}
 	canModifyManifestList := (len(sigs) == 0)
@@ -595,7 +595,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
 	if len(sigs) != 0 {
 		c.Printf("Checking if image destination supports signatures\n")
 		if err := c.dest.SupportsSignatures(ctx); err != nil {
-			return nil, "", "", errors.Wrap(err, "Can not copy signatures")
+			return nil, "", "", errors.Wrapf(err, "Can not copy signatures to %s", transports.ImageName(c.dest.Reference()))
 		}
 	}
 

vendor/github.com/containers/image/v5/docker/archive/dest.go

@@ -3,9 +3,8 @@ package archive
 import (
 	"context"
 	"io"
-	"os"
 
-	"github.com/containers/image/v5/docker/tarfile"
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/types"
 	"github.com/pkg/errors"
 )
@@ -13,37 +12,38 @@ import (
 type archiveImageDestination struct {
 	*tarfile.Destination // Implements most of types.ImageDestination
 	ref                  archiveReference
-	writer               io.Closer
+	archive              *tarfile.Writer // Should only be closed if writer != nil
+	writer               io.Closer       // May be nil if the archive is shared
 }
 
 func newImageDestination(sys *types.SystemContext, ref archiveReference) (types.ImageDestination, error) {
-	// ref.path can be either a pipe or a regular file
+	if ref.sourceIndex != -1 {
-	// in the case of a pipe, we require that we can open it for write
+		return nil, errors.Errorf("Destination reference must not contain a manifest index @%d", ref.sourceIndex)
-	// in the case of a regular file, we don't want to overwrite any pre-existing file
-	// so we check for Size() == 0 below (This is racy, but using O_EXCL would also be racy,
-	// only in a different way. Either way, it’s up to the user to not have two writers to the same path.)
-	fh, err := os.OpenFile(ref.path, os.O_WRONLY|os.O_CREATE, 0644)
-	if err != nil {
-		return nil, errors.Wrapf(err, "error opening file %q", ref.path)
 	}
 
-	fhStat, err := fh.Stat()
+	var archive *tarfile.Writer
-	if err != nil {
+	var writer io.Closer
-		return nil, errors.Wrapf(err, "error statting file %q", ref.path)
+	if ref.archiveWriter != nil {
-	}
+		archive = ref.archiveWriter
+		writer = nil
+	} else {
+		fh, err := openArchiveForWriting(ref.path)
+		if err != nil {
+			return nil, err
+		}
 
-	if fhStat.Mode().IsRegular() && fhStat.Size() != 0 {
+		archive = tarfile.NewWriter(fh)
-		return nil, errors.New("docker-archive doesn't support modifying existing images")
+		writer = fh
 	}
-
+	tarDest := tarfile.NewDestination(sys, archive, ref.ref)
-	tarDest := tarfile.NewDestinationWithContext(sys, fh, ref.destinationRef)
 	if sys != nil && sys.DockerArchiveAdditionalTags != nil {
 		tarDest.AddRepoTags(sys.DockerArchiveAdditionalTags)
 	}
 	return &archiveImageDestination{
 		Destination: tarDest,
 		ref:         ref,
-		writer:      fh,
+		archive:     archive,
+		writer:      writer,
 	}, nil
 }
 
@@ -60,7 +60,10 @@ func (d *archiveImageDestination) Reference() types.ImageReference {
 
 // Close removes resources associated with an initialized ImageDestination, if any.
 func (d *archiveImageDestination) Close() error {
-	return d.writer.Close()
+	if d.writer != nil {
+		return d.writer.Close()
+	}
+	return nil
 }
 
 // Commit marks the process of storing the image as successful and asks for the image to be persisted.
@@ -68,5 +71,8 @@ func (d *archiveImageDestination) Close() error {
 // - Uploaded data MAY be visible to others before Commit() is called
 // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
 func (d *archiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
-	return d.Destination.Commit(ctx)
+	if d.writer != nil {
+		return d.archive.Close()
+	}
+	return nil
 }

vendor/github.com/containers/image/v5/docker/archive/reader.go

@@ -0,0 +1,120 @@
+package archive
+
+import (
+	"github.com/containers/image/v5/docker/internal/tarfile"
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/transports"
+	"github.com/containers/image/v5/types"
+	"github.com/pkg/errors"
+)
+
+// Reader manages a single Docker archive, allows listing its contents and accessing
+// individual images with less overhead than creating image references individually
+// (because the archive is, if necessary, copied or decompressed only once).
+type Reader struct {
+	path    string // The original, user-specified path; not the maintained temporary file, if any
+	archive *tarfile.Reader
+}
+
+// NewReader returns a Reader for path.
+// The caller should call .Close() on the returned object.
+func NewReader(sys *types.SystemContext, path string) (*Reader, error) {
+	archive, err := tarfile.NewReaderFromFile(sys, path)
+	if err != nil {
+		return nil, err
+	}
+	return &Reader{
+		path:    path,
+		archive: archive,
+	}, nil
+}
+
+// Close deletes temporary files associated with the Reader, if any.
+func (r *Reader) Close() error {
+	return r.archive.Close()
+}
+
+// NewReaderForReference creates a Reader from a Reader-independent imageReference, which must be from docker/archive.Transport,
+// and a variant of imageReference that points at the same image within the reader.
+// The caller should call .Close() on the returned Reader.
+func NewReaderForReference(sys *types.SystemContext, ref types.ImageReference) (*Reader, types.ImageReference, error) {
+	standalone, ok := ref.(archiveReference)
+	if !ok {
+		return nil, nil, errors.Errorf("Internal error: NewReaderForReference called for a non-docker/archive ImageReference %s", transports.ImageName(ref))
+	}
+	if standalone.archiveReader != nil {
+		return nil, nil, errors.Errorf("Internal error: NewReaderForReference called for a reader-bound reference %s", standalone.StringWithinTransport())
+	}
+	reader, err := NewReader(sys, standalone.path)
+	if err != nil {
+		return nil, nil, err
+	}
+	succeeded := false
+	defer func() {
+		if !succeeded {
+			reader.Close()
+		}
+	}()
+	readerRef, err := newReference(standalone.path, standalone.ref, standalone.sourceIndex, reader.archive, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	succeeded = true
+	return reader, readerRef, nil
+}
+
+// List returns the a set of references for images in the Reader,
+// grouped by the image the references point to.
+// The references are valid only until the Reader is closed.
+func (r *Reader) List() ([][]types.ImageReference, error) {
+	res := [][]types.ImageReference{}
+	for imageIndex, image := range r.archive.Manifest {
+		refs := []types.ImageReference{}
+		for _, tag := range image.RepoTags {
+			parsedTag, err := reference.ParseNormalizedNamed(tag)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Invalid tag %#v in manifest item @%d", tag, imageIndex)
+			}
+			nt, ok := parsedTag.(reference.NamedTagged)
+			if !ok {
+				return nil, errors.Errorf("Invalid tag %s (%s): does not contain a tag", tag, parsedTag.String())
+			}
+			ref, err := newReference(r.path, nt, -1, r.archive, nil)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Error creating a reference for tag %#v in manifest item @%d", tag, imageIndex)
+			}
+			refs = append(refs, ref)
+		}
+		if len(refs) == 0 {
+			ref, err := newReference(r.path, nil, imageIndex, r.archive, nil)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Error creating a reference for manifest item @%d", imageIndex)
+			}
+			refs = append(refs, ref)
+		}
+		res = append(res, refs)
+	}
+	return res, nil
+}
+
+// ManifestTagsForReference returns the set of tags “matching” ref in reader, as strings
+// (i.e. exposing the short names before normalization).
+// The function reports an error if ref does not identify a single image.
+// If ref contains a NamedTagged reference, only a single tag “matching” ref is returned;
+// If ref contains a source index, or neither a NamedTagged nor a source index, all tags
+// matching the image are returned.
+// Almost all users should use List() or ImageReference.DockerReference() instead.
+func (r *Reader) ManifestTagsForReference(ref types.ImageReference) ([]string, error) {
+	archiveRef, ok := ref.(archiveReference)
+	if !ok {
+		return nil, errors.Errorf("Internal error: ManifestTagsForReference called for a non-docker/archive ImageReference %s", transports.ImageName(ref))
+	}
+	manifestItem, tagIndex, err := r.archive.ChooseManifestItem(archiveRef.ref, archiveRef.sourceIndex)
+	if err != nil {
+		return nil, err
+	}
+	if tagIndex != -1 {
+		return []string{manifestItem.RepoTags[tagIndex]}, nil
+	}
+	return manifestItem.RepoTags, nil
+}

vendor/github.com/containers/image/v5/docker/archive/src.go

@@ -3,9 +3,8 @@ package archive
 import (
 	"context"
 
-	"github.com/containers/image/v5/docker/tarfile"
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/types"
-	"github.com/sirupsen/logrus"
 )
 
 type archiveImageSource struct {
@@ -16,13 +15,20 @@ type archiveImageSource struct {
 // newImageSource returns a types.ImageSource for the specified image reference.
 // The caller must call .Close() on the returned ImageSource.
 func newImageSource(ctx context.Context, sys *types.SystemContext, ref archiveReference) (types.ImageSource, error) {
-	if ref.destinationRef != nil {
+	var archive *tarfile.Reader
-		logrus.Warnf("docker-archive: references are not supported for sources (ignoring)")
+	var closeArchive bool
-	}
+	if ref.archiveReader != nil {
-	src, err := tarfile.NewSourceFromFileWithContext(sys, ref.path)
+		archive = ref.archiveReader
-	if err != nil {
+		closeArchive = false
-		return nil, err
+	} else {
+		a, err := tarfile.NewReaderFromFile(sys, ref.path)
+		if err != nil {
+			return nil, err
+		}
+		archive = a
+		closeArchive = true
 	}
+	src := tarfile.NewSource(archive, closeArchive, ref.ref, ref.sourceIndex)
 	return &archiveImageSource{
 		Source: src,
 		ref:    ref,

vendor/github.com/containers/image/v5/docker/archive/transport.go

@@ -3,8 +3,10 @@ package archive
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/docker/reference"
 	ctrImage "github.com/containers/image/v5/image"
 	"github.com/containers/image/v5/transports"
@@ -42,9 +44,16 @@ func (t archiveTransport) ValidatePolicyConfigurationScope(scope string) error {
 // archiveReference is an ImageReference for Docker images.
 type archiveReference struct {
 	path string
-	// only used for destinations,
+	// May be nil to read the only image in an archive, or to create an untagged image.
-	// archiveReference.destinationRef is optional and can be nil for destinations as well.
+	ref reference.NamedTagged
-	destinationRef reference.NamedTagged
+	// If not -1, a zero-based index of the image in the manifest. Valid only for sources.
+	// Must not be set if ref is set.
+	sourceIndex int
+	// If not nil, must have been created from path (but archiveReader.path may point at a temporary
+	// file, not necesarily path precisely).
+	archiveReader *tarfile.Reader
+	// If not nil, must have been created for path
+	archiveWriter *tarfile.Writer
 }
 
 // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an Docker ImageReference.
@@ -55,37 +64,69 @@ func ParseReference(refString string) (types.ImageReference, error) {
 
 	parts := strings.SplitN(refString, ":", 2)
 	path := parts[0]
-	var destinationRef reference.NamedTagged
+	var nt reference.NamedTagged
+	sourceIndex := -1
 
-	// A :tag was specified, which is only necessary for destinations.
 	if len(parts) == 2 {
-		ref, err := reference.ParseNormalizedNamed(parts[1])
+		// A :tag or :@index was specified.
-		if err != nil {
+		if len(parts[1]) > 0 && parts[1][0] == '@' {
-			return nil, errors.Wrapf(err, "docker-archive parsing reference")
+			i, err := strconv.Atoi(parts[1][1:])
+			if err != nil {
+				return nil, errors.Wrapf(err, "Invalid source index %s", parts[1])
+			}
+			if i < 0 {
+				return nil, errors.Errorf("Invalid source index @%d: must not be negative", i)
+			}
+			sourceIndex = i
+		} else {
+			ref, err := reference.ParseNormalizedNamed(parts[1])
+			if err != nil {
+				return nil, errors.Wrapf(err, "docker-archive parsing reference")
+			}
+			ref = reference.TagNameOnly(ref)
+			refTagged, isTagged := ref.(reference.NamedTagged)
+			if !isTagged { // If ref contains a digest, TagNameOnly does not change it
+				return nil, errors.Errorf("reference does not include a tag: %s", ref.String())
+			}
+			nt = refTagged
 		}
-		ref = reference.TagNameOnly(ref)
-		refTagged, isTagged := ref.(reference.NamedTagged)
-		if !isTagged {
-			// Really shouldn't be hit...
-			return nil, errors.Errorf("internal error: reference is not tagged even after reference.TagNameOnly: %s", refString)
-		}
-		destinationRef = refTagged
 	}
 
-	return NewReference(path, destinationRef)
+	return newReference(path, nt, sourceIndex, nil, nil)
 }
 
-// NewReference rethrns a Docker archive reference for a path and an optional destination reference.
+// NewReference returns a Docker archive reference for a path and an optional reference.
-func NewReference(path string, destinationRef reference.NamedTagged) (types.ImageReference, error) {
+func NewReference(path string, ref reference.NamedTagged) (types.ImageReference, error) {
+	return newReference(path, ref, -1, nil, nil)
+}
+
+// NewIndexReference returns a Docker archive reference for a path and a zero-based source manifest index.
+func NewIndexReference(path string, sourceIndex int) (types.ImageReference, error) {
+	return newReference(path, nil, sourceIndex, nil, nil)
+}
+
+// newReference returns a docker archive reference for a path, an optional reference or sourceIndex,
+// and optionally a tarfile.Reader and/or a tarfile.Writer matching path.
+func newReference(path string, ref reference.NamedTagged, sourceIndex int,
+	archiveReader *tarfile.Reader, archiveWriter *tarfile.Writer) (types.ImageReference, error) {
 	if strings.Contains(path, ":") {
 		return nil, errors.Errorf("Invalid docker-archive: reference: colon in path %q is not supported", path)
 	}
-	if _, isDigest := destinationRef.(reference.Canonical); isDigest {
+	if ref != nil && sourceIndex != -1 {
-		return nil, errors.Errorf("docker-archive doesn't support digest references: %s", destinationRef.String())
+		return nil, errors.Errorf("Invalid docker-archive: reference: cannot use both a tag and a source index")
+	}
+	if _, isDigest := ref.(reference.Canonical); isDigest {
+		return nil, errors.Errorf("docker-archive doesn't support digest references: %s", ref.String())
+	}
+	if sourceIndex != -1 && sourceIndex < 0 {
+		return nil, errors.Errorf("Invalid docker-archive: reference: index @%d must not be negative", sourceIndex)
 	}
 	return archiveReference{
-		path:           path,
+		path:          path,
-		destinationRef: destinationRef,
+		ref:           ref,
+		sourceIndex:   sourceIndex,
+		archiveReader: archiveReader,
+		archiveWriter: archiveWriter,
 	}, nil
 }
 
@@ -99,17 +140,21 @@ func (ref archiveReference) Transport() types.ImageTransport {
 // e.g. default attribute values omitted by the user may be filled in in the return value, or vice versa.
 // WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix.
 func (ref archiveReference) StringWithinTransport() string {
-	if ref.destinationRef == nil {
+	switch {
+	case ref.ref != nil:
+		return fmt.Sprintf("%s:%s", ref.path, ref.ref.String())
+	case ref.sourceIndex != -1:
+		return fmt.Sprintf("%s:@%d", ref.path, ref.sourceIndex)
+	default:
 		return ref.path
 	}
-	return fmt.Sprintf("%s:%s", ref.path, ref.destinationRef.String())
 }
 
 // DockerReference returns a Docker reference associated with this reference
 // (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent,
 // not e.g. after redirect or alias processing), or nil if unknown/not applicable.
 func (ref archiveReference) DockerReference() reference.Named {
-	return ref.destinationRef
+	return ref.ref
 }
 
 // PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.

vendor/github.com/containers/image/v5/docker/archive/writer.go

@@ -0,0 +1,82 @@
+package archive
+
+import (
+	"io"
+	"os"
+
+	"github.com/containers/image/v5/docker/internal/tarfile"
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/types"
+	"github.com/pkg/errors"
+)
+
+// Writer manages a single in-progress Docker archive and allows adding images to it.
+type Writer struct {
+	path    string // The original, user-specified path; not the maintained temporary file, if any
+	archive *tarfile.Writer
+	writer  io.Closer
+}
+
+// NewWriter returns a Writer for path.
+// The caller should call .Close() on the returned object.
+func NewWriter(sys *types.SystemContext, path string) (*Writer, error) {
+	fh, err := openArchiveForWriting(path)
+	if err != nil {
+		return nil, err
+	}
+	archive := tarfile.NewWriter(fh)
+
+	return &Writer{
+		path:    path,
+		archive: archive,
+		writer:  fh,
+	}, nil
+}
+
+// Close writes all outstanding data about images to the archive, and
+// releases state associated with the Writer, if any.
+// No more images can be added after this is called.
+func (w *Writer) Close() error {
+	err := w.archive.Close()
+	if err2 := w.writer.Close(); err2 != nil && err == nil {
+		err = err2
+	}
+	return err
+}
+
+// NewReference returns an ImageReference that allows adding an image to Writer,
+// with an optional reference.
+func (w *Writer) NewReference(destinationRef reference.NamedTagged) (types.ImageReference, error) {
+	return newReference(w.path, destinationRef, -1, nil, w.archive)
+}
+
+// openArchiveForWriting opens path for writing a tar archive,
+// making a few sanity checks.
+func openArchiveForWriting(path string) (*os.File, error) {
+	// path can be either a pipe or a regular file
+	// in the case of a pipe, we require that we can open it for write
+	// in the case of a regular file, we don't want to overwrite any pre-existing file
+	// so we check for Size() == 0 below (This is racy, but using O_EXCL would also be racy,
+	// only in a different way. Either way, it’s up to the user to not have two writers to the same path.)
+	fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error opening file %q", path)
+	}
+	succeeded := false
+	defer func() {
+		if !succeeded {
+			fh.Close()
+		}
+	}()
+	fhStat, err := fh.Stat()
+	if err != nil {
+		return nil, errors.Wrapf(err, "error statting file %q", path)
+	}
+
+	if fhStat.Mode().IsRegular() && fhStat.Size() != 0 {
+		return nil, errors.New("docker-archive doesn't support modifying existing images")
+	}
+
+	succeeded = true
+	return fh, nil
+}

vendor/github.com/containers/image/v5/docker/daemon/daemon_dest.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"io"
 
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/docker/reference"
-	"github.com/containers/image/v5/docker/tarfile"
 	"github.com/containers/image/v5/types"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
@@ -16,6 +16,7 @@ type daemonImageDestination struct {
 	ref                  daemonReference
 	mustMatchRuntimeOS   bool
 	*tarfile.Destination // Implements most of types.ImageDestination
+	archive              *tarfile.Writer
 	// For talking to imageLoadGoroutine
 	goroutineCancel context.CancelFunc
 	statusChannel   <-chan error
@@ -45,6 +46,7 @@ func newImageDestination(ctx context.Context, sys *types.SystemContext, ref daem
 	}
 
 	reader, writer := io.Pipe()
+	archive := tarfile.NewWriter(writer)
 	// Commit() may never be called, so we may never read from this channel; so, make this buffered to allow imageLoadGoroutine to write status and terminate even if we never read it.
 	statusChannel := make(chan error, 1)
 
@@ -54,7 +56,8 @@ func newImageDestination(ctx context.Context, sys *types.SystemContext, ref daem
 	return &daemonImageDestination{
 		ref:                ref,
 		mustMatchRuntimeOS: mustMatchRuntimeOS,
-		Destination:        tarfile.NewDestinationWithContext(sys, writer, namedTaggedRef),
+		Destination:        tarfile.NewDestination(sys, archive, namedTaggedRef),
+		archive:            archive,
 		goroutineCancel:    goroutineCancel,
 		statusChannel:      statusChannel,
 		writer:             writer,
@@ -130,7 +133,7 @@ func (d *daemonImageDestination) Reference() types.ImageReference {
 // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
 func (d *daemonImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
 	logrus.Debugf("docker-daemon: Closing tar stream")
-	if err := d.Destination.Commit(ctx); err != nil {
+	if err := d.archive.Close(); err != nil {
 		return err
 	}
 	if err := d.writer.Close(); err != nil {

vendor/github.com/containers/image/v5/docker/daemon/daemon_src.go

@@ -3,7 +3,7 @@ package daemon
 import (
 	"context"
 
-	"github.com/containers/image/v5/docker/tarfile"
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/types"
 	"github.com/pkg/errors"
 )
@@ -35,10 +35,11 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonRef
 	}
 	defer inputStream.Close()
 
-	src, err := tarfile.NewSourceFromStreamWithSystemContext(sys, inputStream)
+	archive, err := tarfile.NewReaderFromStream(sys, inputStream)
 	if err != nil {
 		return nil, err
 	}
+	src := tarfile.NewSource(archive, true, nil, -1)
 	return &daemonImageSource{
 		ref:    ref,
 		Source: src,

vendor/github.com/containers/image/v5/docker/docker_image_dest.go

@@ -22,7 +22,6 @@ import (
 	"github.com/containers/image/v5/types"
 	"github.com/docker/distribution/registry/api/errcode"
 	v2 "github.com/docker/distribution/registry/api/v2"
-	"github.com/docker/distribution/registry/client"
 	"github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -154,7 +153,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader,
 	defer res.Body.Close()
 	if res.StatusCode != http.StatusAccepted {
 		logrus.Debugf("Error initiating layer upload, response %#v", *res)
-		return types.BlobInfo{}, errors.Wrapf(client.HandleErrorResponse(res), "Error initiating layer upload to %s in %s", uploadPath, d.c.registry)
+		return types.BlobInfo{}, errors.Wrapf(registryHTTPResponseToError(res), "Error initiating layer upload to %s in %s", uploadPath, d.c.registry)
 	}
 	uploadLocation, err := res.Location()
 	if err != nil {
@@ -175,7 +174,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader,
 		}
 		defer res.Body.Close()
 		if !successStatus(res.StatusCode) {
-			return nil, errors.Wrapf(client.HandleErrorResponse(res), "Error uploading layer chunked")
+			return nil, errors.Wrapf(registryHTTPResponseToError(res), "Error uploading layer chunked")
 		}
 		uploadLocation, err := res.Location()
 		if err != nil {
@@ -201,7 +200,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader,
 	defer res.Body.Close()
 	if res.StatusCode != http.StatusCreated {
 		logrus.Debugf("Error uploading layer, response %#v", *res)
-		return types.BlobInfo{}, errors.Wrapf(client.HandleErrorResponse(res), "Error uploading layer to %s", uploadLocation)
+		return types.BlobInfo{}, errors.Wrapf(registryHTTPResponseToError(res), "Error uploading layer to %s", uploadLocation)
 	}
 
 	logrus.Debugf("Upload of layer %s complete", computedDigest)
@@ -226,7 +225,7 @@ func (d *dockerImageDestination) blobExists(ctx context.Context, repo reference.
 		return true, getBlobSize(res), nil
 	case http.StatusUnauthorized:
 		logrus.Debugf("... not authorized")
-		return false, -1, errors.Wrapf(client.HandleErrorResponse(res), "Error checking whether a blob %s exists in %s", digest, repo.Name())
+		return false, -1, errors.Wrapf(registryHTTPResponseToError(res), "Error checking whether a blob %s exists in %s", digest, repo.Name())
 	case http.StatusNotFound:
 		logrus.Debugf("... not present")
 		return false, -1, nil
@@ -277,7 +276,7 @@ func (d *dockerImageDestination) mountBlob(ctx context.Context, srcRepo referenc
 		return fmt.Errorf("Mounting %s from %s to %s started an upload instead", srcDigest, srcRepo.Name(), d.ref.ref.Name())
 	default:
 		logrus.Debugf("Error mounting, response %#v", *res)
-		return errors.Wrapf(client.HandleErrorResponse(res), "Error mounting %s from %s to %s", srcDigest, srcRepo.Name(), d.ref.ref.Name())
+		return errors.Wrapf(registryHTTPResponseToError(res), "Error mounting %s from %s to %s", srcDigest, srcRepo.Name(), d.ref.ref.Name())
 	}
 }
 
@@ -414,7 +413,7 @@ func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, inst
 	}
 	defer res.Body.Close()
 	if !successStatus(res.StatusCode) {
-		err = errors.Wrapf(client.HandleErrorResponse(res), "Error uploading manifest %s to %s", refTail, d.ref.ref.Name())
+		err = errors.Wrapf(registryHTTPResponseToError(res), "Error uploading manifest %s to %s", refTail, d.ref.ref.Name())
 		if isManifestInvalidError(errors.Cause(err)) {
 			err = types.ManifestTypeRejectedError{Err: err}
 		}
@@ -641,7 +640,7 @@ sigExists:
 				logrus.Debugf("Error body %s", string(body))
 			}
 			logrus.Debugf("Error uploading signature, status %d, %#v", res.StatusCode, res)
-			return errors.Wrapf(client.HandleErrorResponse(res), "Error uploading signature to %s in %s", path, d.c.registry)
+			return errors.Wrapf(registryHTTPResponseToError(res), "Error uploading signature to %s in %s", path, d.c.registry)
 		}
 	}
 

vendor/github.com/containers/image/v5/docker/docker_image_src.go

@@ -17,7 +17,6 @@ import (
 	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/pkg/sysregistriesv2"
 	"github.com/containers/image/v5/types"
-	"github.com/docker/distribution/registry/client"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -193,7 +192,7 @@ func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest strin
 	logrus.Debugf("Content-Type from manifest GET is %q", res.Header.Get("Content-Type"))
 	defer res.Body.Close()
 	if res.StatusCode != http.StatusOK {
-		return nil, "", errors.Wrapf(client.HandleErrorResponse(res), "Error reading manifest %s in %s", tagOrDigest, s.physicalRef.ref.Name())
+		return nil, "", errors.Wrapf(registryHTTPResponseToError(res), "Error reading manifest %s in %s", tagOrDigest, s.physicalRef.ref.Name())
 	}
 
 	manblob, err := iolimits.ReadAtMost(res.Body, iolimits.MaxManifestBodySize)
@@ -235,6 +234,9 @@ func (s *dockerImageSource) getExternalBlob(ctx context.Context, urls []string)
 		resp *http.Response
 		err  error
 	)
+	if len(urls) == 0 {
+		return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
+	}
 	for _, url := range urls {
 		resp, err = s.c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, noAuth, nil)
 		if err == nil {

vendor/github.com/containers/image/v5/docker/errors.go

@@ -44,3 +44,17 @@ func httpResponseToError(res *http.Response, context string) error {
 		return perrors.Errorf("%sinvalid status code from registry %d (%s)", context, res.StatusCode, http.StatusText(res.StatusCode))
 	}
 }
+
+// registryHTTPResponseToError creates a Go error from an HTTP error response of a docker/distribution
+// registry
+func registryHTTPResponseToError(res *http.Response) error {
+	errResponse := client.HandleErrorResponse(res)
+	if e, ok := perrors.Cause(errResponse).(*client.UnexpectedHTTPResponseError); ok {
+		response := string(e.Response)
+		if len(response) > 50 {
+			response = response[:50] + "..."
+		}
+		errResponse = fmt.Errorf("StatusCode: %d, %s", e.StatusCode, response)
+	}
+	return errResponse
+}

vendor/github.com/containers/image/v5/docker/internal/tarfile/dest.go

@@ -0,0 +1,217 @@
+package tarfile
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/internal/iolimits"
+	"github.com/containers/image/v5/internal/tmpdir"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/types"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Destination is a partial implementation of types.ImageDestination for writing to an io.Writer.
+type Destination struct {
+	archive  *Writer
+	repoTags []reference.NamedTagged
+	// Other state.
+	config []byte
+	sysCtx *types.SystemContext
+}
+
+// NewDestination returns a tarfile.Destination adding images to the specified Writer.
+func NewDestination(sys *types.SystemContext, archive *Writer, ref reference.NamedTagged) *Destination {
+	repoTags := []reference.NamedTagged{}
+	if ref != nil {
+		repoTags = append(repoTags, ref)
+	}
+	return &Destination{
+		archive:  archive,
+		repoTags: repoTags,
+		sysCtx:   sys,
+	}
+}
+
+// AddRepoTags adds the specified tags to the destination's repoTags.
+func (d *Destination) AddRepoTags(tags []reference.NamedTagged) {
+	d.repoTags = append(d.repoTags, tags...)
+}
+
+// SupportedManifestMIMETypes tells which manifest mime types the destination supports
+// If an empty slice or nil it's returned, then any mime type can be tried to upload
+func (d *Destination) SupportedManifestMIMETypes() []string {
+	return []string{
+		manifest.DockerV2Schema2MediaType, // We rely on the types.Image.UpdatedImage schema conversion capabilities.
+	}
+}
+
+// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
+// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
+func (d *Destination) SupportsSignatures(ctx context.Context) error {
+	return errors.Errorf("Storing signatures for docker tar files is not supported")
+}
+
+// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
+// uploaded to the image destination, true otherwise.
+func (d *Destination) AcceptsForeignLayerURLs() bool {
+	return false
+}
+
+// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise.
+func (d *Destination) MustMatchRuntimeOS() bool {
+	return false
+}
+
+// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(),
+// and would prefer to receive an unmodified manifest instead of one modified for the destination.
+// Does not make a difference if Reference().DockerReference() is nil.
+func (d *Destination) IgnoresEmbeddedDockerReference() bool {
+	return false // N/A, we only accept schema2 images where EmbeddedDockerReferenceConflicts() is always false.
+}
+
+// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
+func (d *Destination) HasThreadSafePutBlob() bool {
+	// The code _is_ actually thread-safe, but apart from computing sizes/digests of layers where
+	// this is unknown in advance, the actual copy is serialized by d.archive, so there probably isn’t
+	// much benefit from concurrency, mostly just extra CPU, memory and I/O contention.
+	return false
+}
+
+// PutBlob writes contents of stream and returns data representing the result (with all data filled in).
+// inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it.
+// inputInfo.Size is the expected length of stream, if known.
+// May update cache.
+// WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
+// to any other readers for download using the supplied digest.
+// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
+func (d *Destination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) {
+	// Ouch, we need to stream the blob into a temporary file just to determine the size.
+	// When the layer is decompressed, we also have to generate the digest on uncompressed datas.
+	if inputInfo.Size == -1 || inputInfo.Digest.String() == "" {
+		logrus.Debugf("docker tarfile: input with unknown size, streaming to disk first ...")
+		streamCopy, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(d.sysCtx), "docker-tarfile-blob")
+		if err != nil {
+			return types.BlobInfo{}, err
+		}
+		defer os.Remove(streamCopy.Name())
+		defer streamCopy.Close()
+
+		digester := digest.Canonical.Digester()
+		tee := io.TeeReader(stream, digester.Hash())
+		// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
+		size, err := io.Copy(streamCopy, tee)
+		if err != nil {
+			return types.BlobInfo{}, err
+		}
+		_, err = streamCopy.Seek(0, io.SeekStart)
+		if err != nil {
+			return types.BlobInfo{}, err
+		}
+		inputInfo.Size = size // inputInfo is a struct, so we are only modifying our copy.
+		if inputInfo.Digest == "" {
+			inputInfo.Digest = digester.Digest()
+		}
+		stream = streamCopy
+		logrus.Debugf("... streaming done")
+	}
+
+	if err := d.archive.lock(); err != nil {
+		return types.BlobInfo{}, err
+	}
+	defer d.archive.unlock()
+
+	// Maybe the blob has been already sent
+	ok, reusedInfo, err := d.archive.tryReusingBlobLocked(inputInfo)
+	if err != nil {
+		return types.BlobInfo{}, err
+	}
+	if ok {
+		return reusedInfo, nil
+	}
+
+	if isConfig {
+		buf, err := iolimits.ReadAtMost(stream, iolimits.MaxConfigBodySize)
+		if err != nil {
+			return types.BlobInfo{}, errors.Wrap(err, "Error reading Config file stream")
+		}
+		d.config = buf
+		if err := d.archive.sendFileLocked(d.archive.configPath(inputInfo.Digest), inputInfo.Size, bytes.NewReader(buf)); err != nil {
+			return types.BlobInfo{}, errors.Wrap(err, "Error writing Config file")
+		}
+	} else {
+		if err := d.archive.sendFileLocked(d.archive.physicalLayerPath(inputInfo.Digest), inputInfo.Size, stream); err != nil {
+			return types.BlobInfo{}, err
+		}
+	}
+	d.archive.recordBlobLocked(types.BlobInfo{Digest: inputInfo.Digest, Size: inputInfo.Size})
+	return types.BlobInfo{Digest: inputInfo.Digest, Size: inputInfo.Size}, nil
+}
+
+// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
+// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
+// info.Digest must not be empty.
+// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input.
+// If the blob has been succesfully reused, returns (true, info, nil); info must contain at least a digest and size.
+// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
+// May use and/or update cache.
+func (d *Destination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
+	if err := d.archive.lock(); err != nil {
+		return false, types.BlobInfo{}, err
+	}
+	defer d.archive.unlock()
+
+	return d.archive.tryReusingBlobLocked(info)
+}
+
+// PutManifest writes manifest to the destination.
+// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so
+// there can be no secondary manifests.
+// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
+// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
+// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
+func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error {
+	if instanceDigest != nil {
+		return errors.New(`Manifest lists are not supported for docker tar files`)
+	}
+	// We do not bother with types.ManifestTypeRejectedError; our .SupportedManifestMIMETypes() above is already providing only one alternative,
+	// so the caller trying a different manifest kind would be pointless.
+	var man manifest.Schema2
+	if err := json.Unmarshal(m, &man); err != nil {
+		return errors.Wrap(err, "Error parsing manifest")
+	}
+	if man.SchemaVersion != 2 || man.MediaType != manifest.DockerV2Schema2MediaType {
+		return errors.Errorf("Unsupported manifest type, need a Docker schema 2 manifest")
+	}
+
+	if err := d.archive.lock(); err != nil {
+		return err
+	}
+	defer d.archive.unlock()
+
+	if err := d.archive.writeLegacyMetadataLocked(man.LayersDescriptors, d.config, d.repoTags); err != nil {
+		return err
+	}
+
+	return d.archive.ensureManifestItemLocked(man.LayersDescriptors, man.ConfigDescriptor.Digest, d.repoTags)
+}
+
+// PutSignatures would add the given signatures to the docker tarfile (currently not supported).
+// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so
+// there can be no secondary manifests.  MUST be called after PutManifest (signatures reference manifest contents).
+func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error {
+	if instanceDigest != nil {
+		return errors.Errorf(`Manifest lists are not supported for docker tar files`)
+	}
+	if len(signatures) != 0 {
+		return errors.Errorf("Storing signatures for docker tar files is not supported")
+	}
+	return nil
+}

vendor/github.com/containers/image/v5/docker/internal/tarfile/reader.go

@@ -0,0 +1,269 @@
+package tarfile
+
+import (
+	"archive/tar"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/internal/iolimits"
+	"github.com/containers/image/v5/internal/tmpdir"
+	"github.com/containers/image/v5/pkg/compression"
+	"github.com/containers/image/v5/types"
+	"github.com/pkg/errors"
+)
+
+// Reader is a ((docker save)-formatted) tar archive that allows random access to any component.
+type Reader struct {
+	// None of the fields below are modified after the archive is created, until .Close();
+	// this allows concurrent readers of the same archive.
+	path          string         // "" if the archive has already been closed.
+	removeOnClose bool           // Remove file on close if true
+	Manifest      []ManifestItem // Guaranteed to exist after the archive is created.
+}
+
+// NewReaderFromFile returns a Reader for the specified path.
+// The caller should call .Close() on the returned archive when done.
+func NewReaderFromFile(sys *types.SystemContext, path string) (*Reader, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error opening file %q", path)
+	}
+	defer file.Close()
+
+	// If the file is already not compressed we can just return the file itself
+	// as a source. Otherwise we pass the stream to NewReaderFromStream.
+	stream, isCompressed, err := compression.AutoDecompress(file)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error detecting compression for file %q", path)
+	}
+	defer stream.Close()
+	if !isCompressed {
+		return newReader(path, false)
+	}
+	return NewReaderFromStream(sys, stream)
+}
+
+// NewReaderFromStream returns a Reader for the specified inputStream,
+// which can be either compressed or uncompressed. The caller can close the
+// inputStream immediately after NewReaderFromFile returns.
+// The caller should call .Close() on the returned archive when done.
+func NewReaderFromStream(sys *types.SystemContext, inputStream io.Reader) (*Reader, error) {
+	// Save inputStream to a temporary file
+	tarCopyFile, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(sys), "docker-tar")
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating temporary file")
+	}
+	defer tarCopyFile.Close()
+
+	succeeded := false
+	defer func() {
+		if !succeeded {
+			os.Remove(tarCopyFile.Name())
+		}
+	}()
+
+	// In order to be compatible with docker-load, we need to support
+	// auto-decompression (it's also a nice quality-of-life thing to avoid
+	// giving users really confusing "invalid tar header" errors).
+	uncompressedStream, _, err := compression.AutoDecompress(inputStream)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error auto-decompressing input")
+	}
+	defer uncompressedStream.Close()
+
+	// Copy the plain archive to the temporary file.
+	//
+	// TODO: This can take quite some time, and should ideally be cancellable
+	//       using a context.Context.
+	if _, err := io.Copy(tarCopyFile, uncompressedStream); err != nil {
+		return nil, errors.Wrapf(err, "error copying contents to temporary file %q", tarCopyFile.Name())
+	}
+	succeeded = true
+
+	return newReader(tarCopyFile.Name(), true)
+}
+
+// newReader creates a Reader for the specified path and removeOnClose flag.
+// The caller should call .Close() on the returned archive when done.
+func newReader(path string, removeOnClose bool) (*Reader, error) {
+	// This is a valid enough archive, except Manifest is not yet filled.
+	r := Reader{
+		path:          path,
+		removeOnClose: removeOnClose,
+	}
+	succeeded := false
+	defer func() {
+		if !succeeded {
+			r.Close()
+		}
+	}()
+
+	// We initialize Manifest immediately when constructing the Reader instead
+	// of later on-demand because every caller will need the data, and because doing it now
+	// removes the need to synchronize the access/creation of the data if the archive is later
+	// used from multiple goroutines to access different images.
+
+	// FIXME? Do we need to deal with the legacy format?
+	bytes, err := r.readTarComponent(manifestFileName, iolimits.MaxTarFileManifestSize)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(bytes, &r.Manifest); err != nil {
+		return nil, errors.Wrap(err, "Error decoding tar manifest.json")
+	}
+
+	succeeded = true
+	return &r, nil
+}
+
+// Close removes resources associated with an initialized Reader, if any.
+func (r *Reader) Close() error {
+	path := r.path
+	r.path = "" // Mark the archive as closed
+	if r.removeOnClose {
+		return os.Remove(path)
+	}
+	return nil
+}
+
+// ChooseManifestItem selects a manifest item from r.Manifest matching (ref, sourceIndex), one or
+// both of which should be (nil, -1).
+// On success, it returns the manifest item and an index of the matching tag, if a tag was used
+// for matching; the index is -1 if a tag was not used.
+func (r *Reader) ChooseManifestItem(ref reference.NamedTagged, sourceIndex int) (*ManifestItem, int, error) {
+	switch {
+	case ref != nil && sourceIndex != -1:
+		return nil, -1, errors.Errorf("Internal error: Cannot have both ref %s and source index @%d",
+			ref.String(), sourceIndex)
+
+	case ref != nil:
+		refString := ref.String()
+		for i := range r.Manifest {
+			for tagIndex, tag := range r.Manifest[i].RepoTags {
+				parsedTag, err := reference.ParseNormalizedNamed(tag)
+				if err != nil {
+					return nil, -1, errors.Wrapf(err, "Invalid tag %#v in manifest.json item @%d", tag, i)
+				}
+				if parsedTag.String() == refString {
+					return &r.Manifest[i], tagIndex, nil
+				}
+			}
+		}
+		return nil, -1, errors.Errorf("Tag %#v not found", refString)
+
+	case sourceIndex != -1:
+		if sourceIndex >= len(r.Manifest) {
+			return nil, -1, errors.Errorf("Invalid source index @%d, only %d manifest items available",
+				sourceIndex, len(r.Manifest))
+		}
+		return &r.Manifest[sourceIndex], -1, nil
+
+	default:
+		if len(r.Manifest) != 1 {
+			return nil, -1, errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(r.Manifest))
+		}
+		return &r.Manifest[0], -1, nil
+	}
+}
+
+// tarReadCloser is a way to close the backing file of a tar.Reader when the user no longer needs the tar component.
+type tarReadCloser struct {
+	*tar.Reader
+	backingFile *os.File
+}
+
+func (t *tarReadCloser) Close() error {
+	return t.backingFile.Close()
+}
+
+// openTarComponent returns a ReadCloser for the specific file within the archive.
+// This is linear scan; we assume that the tar file will have a fairly small amount of files (~layers),
+// and that filesystem caching will make the repeated seeking over the (uncompressed) tarPath cheap enough.
+// It is safe to call this method from multiple goroutines simultaneously.
+// The caller should call .Close() on the returned stream.
+func (r *Reader) openTarComponent(componentPath string) (io.ReadCloser, error) {
+	// This is only a sanity check; if anyone did concurrently close ra, this access is technically
+	// racy against the write in .Close().
+	if r.path == "" {
+		return nil, errors.New("Internal error: trying to read an already closed tarfile.Reader")
+	}
+
+	f, err := os.Open(r.path)
+	if err != nil {
+		return nil, err
+	}
+	succeeded := false
+	defer func() {
+		if !succeeded {
+			f.Close()
+		}
+	}()
+
+	tarReader, header, err := findTarComponent(f, componentPath)
+	if err != nil {
+		return nil, err
+	}
+	if header == nil {
+		return nil, os.ErrNotExist
+	}
+	if header.FileInfo().Mode()&os.ModeType == os.ModeSymlink { // FIXME: untested
+		// We follow only one symlink; so no loops are possible.
+		if _, err := f.Seek(0, io.SeekStart); err != nil {
+			return nil, err
+		}
+		// The new path could easily point "outside" the archive, but we only compare it to existing tar headers without extracting the archive,
+		// so we don't care.
+		tarReader, header, err = findTarComponent(f, path.Join(path.Dir(componentPath), header.Linkname))
+		if err != nil {
+			return nil, err
+		}
+		if header == nil {
+			return nil, os.ErrNotExist
+		}
+	}
+
+	if !header.FileInfo().Mode().IsRegular() {
+		return nil, errors.Errorf("Error reading tar archive component %s: not a regular file", header.Name)
+	}
+	succeeded = true
+	return &tarReadCloser{Reader: tarReader, backingFile: f}, nil
+}
+
+// findTarComponent returns a header and a reader matching componentPath within inputFile,
+// or (nil, nil, nil) if not found.
+func findTarComponent(inputFile io.Reader, componentPath string) (*tar.Reader, *tar.Header, error) {
+	t := tar.NewReader(inputFile)
+	componentPath = path.Clean(componentPath)
+	for {
+		h, err := t.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, nil, err
+		}
+		if path.Clean(h.Name) == componentPath {
+			return t, h, nil
+		}
+	}
+	return nil, nil, nil
+}
+
+// readTarComponent returns full contents of componentPath.
+// It is safe to call this method from multiple goroutines simultaneously.
+func (r *Reader) readTarComponent(path string, limit int) ([]byte, error) {
+	file, err := r.openTarComponent(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error loading tar component %s", path)
+	}
+	defer file.Close()
+	bytes, err := iolimits.ReadAtMost(file, limit)
+	if err != nil {
+		return nil, err
+	}
+	return bytes, nil
+}

vendor/github.com/containers/image/v5/docker/internal/tarfile/src.go

@@ -0,0 +1,331 @@
+package tarfile
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"sync"
+
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/internal/iolimits"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/pkg/compression"
+	"github.com/containers/image/v5/types"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+// Source is a partial implementation of types.ImageSource for reading from tarPath.
+type Source struct {
+	archive      *Reader
+	closeArchive bool // .Close() the archive when the source is closed.
+	// If ref is nil and sourceIndex is -1, indicates the only image in the archive.
+	ref         reference.NamedTagged // May be nil
+	sourceIndex int                   // May be -1
+	// The following data is only available after ensureCachedDataIsPresent() succeeds
+	tarManifest       *ManifestItem // nil if not available yet.
+	configBytes       []byte
+	configDigest      digest.Digest
+	orderedDiffIDList []digest.Digest
+	knownLayers       map[digest.Digest]*layerInfo
+	// Other state
+	generatedManifest []byte    // Private cache for GetManifest(), nil if not set yet.
+	cacheDataLock     sync.Once // Private state for ensureCachedDataIsPresent to make it concurrency-safe
+	cacheDataResult   error     // Private state for ensureCachedDataIsPresent
+}
+
+type layerInfo struct {
+	path string
+	size int64
+}
+
+// NewSource returns a tarfile.Source for an image in the specified archive matching ref
+// and sourceIndex (or the only image if they are (nil, -1)).
+// The archive will be closed if closeArchive
+func NewSource(archive *Reader, closeArchive bool, ref reference.NamedTagged, sourceIndex int) *Source {
+	return &Source{
+		archive:      archive,
+		closeArchive: closeArchive,
+		ref:          ref,
+		sourceIndex:  sourceIndex,
+	}
+}
+
+// ensureCachedDataIsPresent loads data necessary for any of the public accessors.
+// It is safe to call this from multi-threaded code.
+func (s *Source) ensureCachedDataIsPresent() error {
+	s.cacheDataLock.Do(func() {
+		s.cacheDataResult = s.ensureCachedDataIsPresentPrivate()
+	})
+	return s.cacheDataResult
+}
+
+// ensureCachedDataIsPresentPrivate is a private implementation detail of ensureCachedDataIsPresent.
+// Call ensureCachedDataIsPresent instead.
+func (s *Source) ensureCachedDataIsPresentPrivate() error {
+	tarManifest, _, err := s.archive.ChooseManifestItem(s.ref, s.sourceIndex)
+	if err != nil {
+		return err
+	}
+
+	// Read and parse config.
+	configBytes, err := s.archive.readTarComponent(tarManifest.Config, iolimits.MaxConfigBodySize)
+	if err != nil {
+		return err
+	}
+	var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs.
+	if err := json.Unmarshal(configBytes, &parsedConfig); err != nil {
+		return errors.Wrapf(err, "Error decoding tar config %s", tarManifest.Config)
+	}
+	if parsedConfig.RootFS == nil {
+		return errors.Errorf("Invalid image config (rootFS is not set): %s", tarManifest.Config)
+	}
+
+	knownLayers, err := s.prepareLayerData(tarManifest, &parsedConfig)
+	if err != nil {
+		return err
+	}
+
+	// Success; commit.
+	s.tarManifest = tarManifest
+	s.configBytes = configBytes
+	s.configDigest = digest.FromBytes(configBytes)
+	s.orderedDiffIDList = parsedConfig.RootFS.DiffIDs
+	s.knownLayers = knownLayers
+	return nil
+}
+
+// Close removes resources associated with an initialized Source, if any.
+func (s *Source) Close() error {
+	if s.closeArchive {
+		return s.archive.Close()
+	}
+	return nil
+}
+
+// TarManifest returns contents of manifest.json
+func (s *Source) TarManifest() []ManifestItem {
+	return s.archive.Manifest
+}
+
+func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manifest.Schema2Image) (map[digest.Digest]*layerInfo, error) {
+	// Collect layer data available in manifest and config.
+	if len(tarManifest.Layers) != len(parsedConfig.RootFS.DiffIDs) {
+		return nil, errors.Errorf("Inconsistent layer count: %d in manifest, %d in config", len(tarManifest.Layers), len(parsedConfig.RootFS.DiffIDs))
+	}
+	knownLayers := map[digest.Digest]*layerInfo{}
+	unknownLayerSizes := map[string]*layerInfo{} // Points into knownLayers, a "to do list" of items with unknown sizes.
+	for i, diffID := range parsedConfig.RootFS.DiffIDs {
+		if _, ok := knownLayers[diffID]; ok {
+			// Apparently it really can happen that a single image contains the same layer diff more than once.
+			// In that case, the diffID validation ensures that both layers truly are the same, and it should not matter
+			// which of the tarManifest.Layers paths is used; (docker save) actually makes the duplicates symlinks to the original.
+			continue
+		}
+		layerPath := path.Clean(tarManifest.Layers[i])
+		if _, ok := unknownLayerSizes[layerPath]; ok {
+			return nil, errors.Errorf("Layer tarfile %s used for two different DiffID values", layerPath)
+		}
+		li := &layerInfo{ // A new element in each iteration
+			path: layerPath,
+			size: -1,
+		}
+		knownLayers[diffID] = li
+		unknownLayerSizes[layerPath] = li
+	}
+
+	// Scan the tar file to collect layer sizes.
+	file, err := os.Open(s.archive.path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	t := tar.NewReader(file)
+	for {
+		h, err := t.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		layerPath := path.Clean(h.Name)
+		// FIXME: Cache this data across images in Reader.
+		if li, ok := unknownLayerSizes[layerPath]; ok {
+			// Since GetBlob will decompress layers that are compressed we need
+			// to do the decompression here as well, otherwise we will
+			// incorrectly report the size. Pretty critical, since tools like
+			// umoci always compress layer blobs. Obviously we only bother with
+			// the slower method of checking if it's compressed.
+			uncompressedStream, isCompressed, err := compression.AutoDecompress(t)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Error auto-decompressing %s to determine its size", layerPath)
+			}
+			defer uncompressedStream.Close()
+
+			uncompressedSize := h.Size
+			if isCompressed {
+				uncompressedSize, err = io.Copy(ioutil.Discard, uncompressedStream)
+				if err != nil {
+					return nil, errors.Wrapf(err, "Error reading %s to find its size", layerPath)
+				}
+			}
+			li.size = uncompressedSize
+			delete(unknownLayerSizes, layerPath)
+		}
+	}
+	if len(unknownLayerSizes) != 0 {
+		return nil, errors.Errorf("Some layer tarfiles are missing in the tarball") // This could do with a better error reporting, if this ever happened in practice.
+	}
+
+	return knownLayers, nil
+}
+
+// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
+// It may use a remote (= slow) service.
+// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
+// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
+// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil,
+// as the primary manifest can not be a list, so there can be no secondary instances.
+func (s *Source) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
+	if instanceDigest != nil {
+		// How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType.
+		return nil, "", errors.New(`Manifest lists are not supported by "docker-daemon:"`)
+	}
+	if s.generatedManifest == nil {
+		if err := s.ensureCachedDataIsPresent(); err != nil {
+			return nil, "", err
+		}
+		m := manifest.Schema2{
+			SchemaVersion: 2,
+			MediaType:     manifest.DockerV2Schema2MediaType,
+			ConfigDescriptor: manifest.Schema2Descriptor{
+				MediaType: manifest.DockerV2Schema2ConfigMediaType,
+				Size:      int64(len(s.configBytes)),
+				Digest:    s.configDigest,
+			},
+			LayersDescriptors: []manifest.Schema2Descriptor{},
+		}
+		for _, diffID := range s.orderedDiffIDList {
+			li, ok := s.knownLayers[diffID]
+			if !ok {
+				return nil, "", errors.Errorf("Internal inconsistency: Information about layer %s missing", diffID)
+			}
+			m.LayersDescriptors = append(m.LayersDescriptors, manifest.Schema2Descriptor{
+				Digest:    diffID, // diffID is a digest of the uncompressed tarball
+				MediaType: manifest.DockerV2Schema2LayerMediaType,
+				Size:      li.size,
+			})
+		}
+		manifestBytes, err := json.Marshal(&m)
+		if err != nil {
+			return nil, "", err
+		}
+		s.generatedManifest = manifestBytes
+	}
+	return s.generatedManifest, manifest.DockerV2Schema2MediaType, nil
+}
+
+// uncompressedReadCloser is an io.ReadCloser that closes both the uncompressed stream and the underlying input.
+type uncompressedReadCloser struct {
+	io.Reader
+	underlyingCloser   func() error
+	uncompressedCloser func() error
+}
+
+func (r uncompressedReadCloser) Close() error {
+	var res error
+	if err := r.uncompressedCloser(); err != nil {
+		res = err
+	}
+	if err := r.underlyingCloser(); err != nil && res == nil {
+		res = err
+	}
+	return res
+}
+
+// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently.
+func (s *Source) HasThreadSafeGetBlob() bool {
+	return true
+}
+
+// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
+// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
+// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
+func (s *Source) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
+	if err := s.ensureCachedDataIsPresent(); err != nil {
+		return nil, 0, err
+	}
+
+	if info.Digest == s.configDigest { // FIXME? Implement a more general algorithm matching instead of assuming sha256.
+		return ioutil.NopCloser(bytes.NewReader(s.configBytes)), int64(len(s.configBytes)), nil
+	}
+
+	if li, ok := s.knownLayers[info.Digest]; ok { // diffID is a digest of the uncompressed tarball,
+		underlyingStream, err := s.archive.openTarComponent(li.path)
+		if err != nil {
+			return nil, 0, err
+		}
+		closeUnderlyingStream := true
+		defer func() {
+			if closeUnderlyingStream {
+				underlyingStream.Close()
+			}
+		}()
+
+		// In order to handle the fact that digests != diffIDs (and thus that a
+		// caller which is trying to verify the blob will run into problems),
+		// we need to decompress blobs. This is a bit ugly, but it's a
+		// consequence of making everything addressable by their DiffID rather
+		// than by their digest...
+		//
+		// In particular, because the v2s2 manifest being generated uses
+		// DiffIDs, any caller of GetBlob is going to be asking for DiffIDs of
+		// layers not their _actual_ digest. The result is that copy/... will
+		// be verifing a "digest" which is not the actual layer's digest (but
+		// is instead the DiffID).
+
+		uncompressedStream, _, err := compression.AutoDecompress(underlyingStream)
+		if err != nil {
+			return nil, 0, errors.Wrapf(err, "Error auto-decompressing blob %s", info.Digest)
+		}
+
+		newStream := uncompressedReadCloser{
+			Reader:             uncompressedStream,
+			underlyingCloser:   underlyingStream.Close,
+			uncompressedCloser: uncompressedStream.Close,
+		}
+		closeUnderlyingStream = false
+
+		return newStream, li.size, nil
+	}
+
+	return nil, 0, errors.Errorf("Unknown blob %s", info.Digest)
+}
+
+// GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
+// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil,
+// as there can be no secondary manifests.
+func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
+	if instanceDigest != nil {
+		// How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType.
+		return nil, errors.Errorf(`Manifest lists are not supported by "docker-daemon:"`)
+	}
+	return [][]byte{}, nil
+}
+
+// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer
+// blobsums that are listed in the image's manifest.  If values are returned, they should be used when using GetBlob()
+// to read the image's layers.
+// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil,
+// as the primary manifest can not be a list, so there can be no secondary manifests.
+// The Digest field is guaranteed to be provided; Size may be -1.
+// WARNING: The list may contain duplicates, and they are semantically relevant.
+func (s *Source) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) {
+	return nil, nil
+}

vendor/github.com/containers/image/v5/docker/internal/tarfile/types.go

@@ -0,0 +1,28 @@
+package tarfile
+
+import (
+	"github.com/containers/image/v5/manifest"
+	"github.com/opencontainers/go-digest"
+)
+
+// Various data structures.
+
+// Based on github.com/docker/docker/image/tarexport/tarexport.go
+const (
+	manifestFileName           = "manifest.json"
+	legacyLayerFileName        = "layer.tar"
+	legacyConfigFileName       = "json"
+	legacyVersionFileName      = "VERSION"
+	legacyRepositoriesFileName = "repositories"
+)
+
+// ManifestItem is an element of the array stored in the top-level manifest.json file.
+type ManifestItem struct { // NOTE: This is visible as docker/tarfile.ManifestItem, and a part of the stable API.
+	Config       string
+	RepoTags     []string
+	Layers       []string
+	Parent       imageID                                      `json:",omitempty"`
+	LayerSources map[digest.Digest]manifest.Schema2Descriptor `json:",omitempty"`
+}
+
+type imageID string

vendor/github.com/containers/image/v5/docker/internal/tarfile/writer.go

@@ -0,0 +1,381 @@
+package tarfile
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/types"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Writer allows creating a (docker save)-formatted tar archive containing one or more images.
+type Writer struct {
+	mutex sync.Mutex
+	// ALL of the following members can only be accessed with the mutex held.
+	// Use Writer.lock() to obtain the mutex.
+	writer io.Writer
+	tar    *tar.Writer // nil if the Writer has already been closed.
+	// Other state.
+	blobs            map[digest.Digest]types.BlobInfo // list of already-sent blobs
+	repositories     map[string]map[string]string
+	legacyLayers     map[string]struct{} // A set of IDs of legacy layers that have been already sent.
+	manifest         []ManifestItem
+	manifestByConfig map[digest.Digest]int // A map from config digest to an entry index in manifest above.
+}
+
+// NewWriter returns a Writer for the specified io.Writer.
+// The caller must eventually call .Close() on the returned object to create a valid archive.
+func NewWriter(dest io.Writer) *Writer {
+	return &Writer{
+		writer:           dest,
+		tar:              tar.NewWriter(dest),
+		blobs:            make(map[digest.Digest]types.BlobInfo),
+		repositories:     map[string]map[string]string{},
+		legacyLayers:     map[string]struct{}{},
+		manifestByConfig: map[digest.Digest]int{},
+	}
+}
+
+// lock does some sanity checks and locks the Writer.
+// If this function suceeds, the caller must call w.unlock.
+// Do not use Writer.mutex directly.
+func (w *Writer) lock() error {
+	w.mutex.Lock()
+	if w.tar == nil {
+		w.mutex.Unlock()
+		return errors.New("Internal error: trying to use an already closed tarfile.Writer")
+	}
+	return nil
+}
+
+// unlock releases the lock obtained by Writer.lock
+// Do not use Writer.mutex directly.
+func (w *Writer) unlock() {
+	w.mutex.Unlock()
+}
+
+// tryReusingBlobLocked checks whether the transport already contains, a blob, and if so, returns its metadata.
+// info.Digest must not be empty.
+// If the blob has been succesfully reused, returns (true, info, nil); info must contain at least a digest and size.
+// If the transport can not reuse the requested blob, tryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
+// The caller must have locked the Writer.
+func (w *Writer) tryReusingBlobLocked(info types.BlobInfo) (bool, types.BlobInfo, error) {
+	if info.Digest == "" {
+		return false, types.BlobInfo{}, errors.Errorf("Can not check for a blob with unknown digest")
+	}
+	if blob, ok := w.blobs[info.Digest]; ok {
+		return true, types.BlobInfo{Digest: info.Digest, Size: blob.Size}, nil
+	}
+	return false, types.BlobInfo{}, nil
+}
+
+// recordBlob records metadata of a recorded blob, which must contain at least a digest and size.
+// The caller must have locked the Writer.
+func (w *Writer) recordBlobLocked(info types.BlobInfo) {
+	w.blobs[info.Digest] = info
+}
+
+// ensureSingleLegacyLayerLocked writes legacy VERSION and configuration files for a single layer
+// The caller must have locked the Writer.
+func (w *Writer) ensureSingleLegacyLayerLocked(layerID string, layerDigest digest.Digest, configBytes []byte) error {
+	if _, ok := w.legacyLayers[layerID]; !ok {
+		// Create a symlink for the legacy format, where there is one subdirectory per layer ("image").
+		// See also the comment in physicalLayerPath.
+		physicalLayerPath := w.physicalLayerPath(layerDigest)
+		if err := w.sendSymlinkLocked(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
+			return errors.Wrap(err, "Error creating layer symbolic link")
+		}
+
+		b := []byte("1.0")
+		if err := w.sendBytesLocked(filepath.Join(layerID, legacyVersionFileName), b); err != nil {
+			return errors.Wrap(err, "Error writing VERSION file")
+		}
+
+		if err := w.sendBytesLocked(filepath.Join(layerID, legacyConfigFileName), configBytes); err != nil {
+			return errors.Wrap(err, "Error writing config json file")
+		}
+
+		w.legacyLayers[layerID] = struct{}{}
+	}
+	return nil
+}
+
+// writeLegacyMetadataLocked writes legacy layer metadata and records tags for a single image.
+func (w *Writer) writeLegacyMetadataLocked(layerDescriptors []manifest.Schema2Descriptor, configBytes []byte, repoTags []reference.NamedTagged) error {
+	var chainID digest.Digest
+	lastLayerID := ""
+	for i, l := range layerDescriptors {
+		// The legacy format requires a config file per layer
+		layerConfig := make(map[string]interface{})
+
+		// The root layer doesn't have any parent
+		if lastLayerID != "" {
+			layerConfig["parent"] = lastLayerID
+		}
+		// The top layer configuration file is generated by using subpart of the image configuration
+		if i == len(layerDescriptors)-1 {
+			var config map[string]*json.RawMessage
+			err := json.Unmarshal(configBytes, &config)
+			if err != nil {
+				return errors.Wrap(err, "Error unmarshaling config")
+			}
+			for _, attr := range [7]string{"architecture", "config", "container", "container_config", "created", "docker_version", "os"} {
+				layerConfig[attr] = config[attr]
+			}
+		}
+
+		// This chainID value matches the computation in docker/docker/layer.CreateChainID …
+		if chainID == "" {
+			chainID = l.Digest
+		} else {
+			chainID = digest.Canonical.FromString(chainID.String() + " " + l.Digest.String())
+		}
+		// … but note that the image ID does not _exactly_ match docker/docker/image/v1.CreateID, primarily because
+		// we create the image configs differently in details. At least recent versions allocate new IDs on load,
+		// so this is fine as long as the IDs we use are unique / cannot loop.
+		//
+		// For intermediate images, we could just use the chainID as an image ID, but using a digest of ~the created
+		// config makes sure that everything uses the same “namespace”; a bit less efficient but clearer.
+		//
+		// Temporarily add the chainID to the config, only for the purpose of generating the image ID.
+		layerConfig["layer_id"] = chainID
+		b, err := json.Marshal(layerConfig) // Note that layerConfig["id"] is not set yet at this point.
+		if err != nil {
+			return errors.Wrap(err, "Error marshaling layer config")
+		}
+		delete(layerConfig, "layer_id")
+		layerID := digest.Canonical.FromBytes(b).Hex()
+		layerConfig["id"] = layerID
+
+		configBytes, err := json.Marshal(layerConfig)
+		if err != nil {
+			return errors.Wrap(err, "Error marshaling layer config")
+		}
+
+		if err := w.ensureSingleLegacyLayerLocked(layerID, l.Digest, configBytes); err != nil {
+			return err
+		}
+
+		lastLayerID = layerID
+	}
+
+	if lastLayerID != "" {
+		for _, repoTag := range repoTags {
+			if val, ok := w.repositories[repoTag.Name()]; ok {
+				val[repoTag.Tag()] = lastLayerID
+			} else {
+				w.repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): lastLayerID}
+			}
+		}
+	}
+	return nil
+}
+
+// checkManifestItemsMatch checks that a and b describe the same image,
+// and returns an error if that’s not the case (which should never happen).
+func checkManifestItemsMatch(a, b *ManifestItem) error {
+	if a.Config != b.Config {
+		return fmt.Errorf("Internal error: Trying to reuse ManifestItem values with configs %#v vs. %#v", a.Config, b.Config)
+	}
+	if len(a.Layers) != len(b.Layers) {
+		return fmt.Errorf("Internal error: Trying to reuse ManifestItem values with layers %#v vs. %#v", a.Layers, b.Layers)
+	}
+	for i := range a.Layers {
+		if a.Layers[i] != b.Layers[i] {
+			return fmt.Errorf("Internal error: Trying to reuse ManifestItem values with layers[i] %#v vs. %#v", a.Layers[i], b.Layers[i])
+		}
+	}
+	// Ignore RepoTags, that will be built later.
+	// Ignore Parent and LayerSources, which we don’t set to anything meaningful.
+	return nil
+}
+
+// ensureManifestItemLocked ensures that there is a manifest item pointing to (layerDescriptors, configDigest) with repoTags
+// The caller must have locked the Writer.
+func (w *Writer) ensureManifestItemLocked(layerDescriptors []manifest.Schema2Descriptor, configDigest digest.Digest, repoTags []reference.NamedTagged) error {
+	layerPaths := []string{}
+	for _, l := range layerDescriptors {
+		layerPaths = append(layerPaths, w.physicalLayerPath(l.Digest))
+	}
+
+	var item *ManifestItem
+	newItem := ManifestItem{
+		Config:       w.configPath(configDigest),
+		RepoTags:     []string{},
+		Layers:       layerPaths,
+		Parent:       "", // We don’t have this information
+		LayerSources: nil,
+	}
+	if i, ok := w.manifestByConfig[configDigest]; ok {
+		item = &w.manifest[i]
+		if err := checkManifestItemsMatch(item, &newItem); err != nil {
+			return err
+		}
+	} else {
+		i := len(w.manifest)
+		w.manifestByConfig[configDigest] = i
+		w.manifest = append(w.manifest, newItem)
+		item = &w.manifest[i]
+	}
+
+	knownRepoTags := map[string]struct{}{}
+	for _, repoTag := range item.RepoTags {
+		knownRepoTags[repoTag] = struct{}{}
+	}
+	for _, tag := range repoTags {
+		// For github.com/docker/docker consumers, this works just as well as
+		//   refString := ref.String()
+		// because when reading the RepoTags strings, github.com/docker/docker/reference
+		// normalizes both of them to the same value.
+		//
+		// Doing it this way to include the normalized-out `docker.io[/library]` does make
+		// a difference for github.com/projectatomic/docker consumers, with the
+		// “Add --add-registry and --block-registry options to docker daemon” patch.
+		// These consumers treat reference strings which include a hostname and reference
+		// strings without a hostname differently.
+		//
+		// Using the host name here is more explicit about the intent, and it has the same
+		// effect as (docker pull) in projectatomic/docker, which tags the result using
+		// a hostname-qualified reference.
+		// See https://github.com/containers/image/issues/72 for a more detailed
+		// analysis and explanation.
+		refString := fmt.Sprintf("%s:%s", tag.Name(), tag.Tag())
+
+		if _, ok := knownRepoTags[refString]; !ok {
+			item.RepoTags = append(item.RepoTags, refString)
+			knownRepoTags[refString] = struct{}{}
+		}
+	}
+
+	return nil
+}
+
+// Close writes all outstanding data about images to the archive, and finishes writing data
+// to the underlying io.Writer.
+// No more images can be added after this is called.
+func (w *Writer) Close() error {
+	if err := w.lock(); err != nil {
+		return err
+	}
+	defer w.unlock()
+
+	b, err := json.Marshal(&w.manifest)
+	if err != nil {
+		return err
+	}
+	if err := w.sendBytesLocked(manifestFileName, b); err != nil {
+		return err
+	}
+
+	b, err = json.Marshal(w.repositories)
+	if err != nil {
+		return errors.Wrap(err, "Error marshaling repositories")
+	}
+	if err := w.sendBytesLocked(legacyRepositoriesFileName, b); err != nil {
+		return errors.Wrap(err, "Error writing config json file")
+	}
+
+	if err := w.tar.Close(); err != nil {
+		return err
+	}
+	w.tar = nil // Mark the Writer as closed.
+	return nil
+}
+
+// configPath returns a path we choose for storing a config with the specified digest.
+// NOTE: This is an internal implementation detail, not a format property, and can change
+// any time.
+func (w *Writer) configPath(configDigest digest.Digest) string {
+	return configDigest.Hex() + ".json"
+}
+
+// physicalLayerPath returns a path we choose for storing a layer with the specified digest
+// (the actual path, i.e. a regular file, not a symlink that may be used in the legacy format).
+// NOTE: This is an internal implementation detail, not a format property, and can change
+// any time.
+func (w *Writer) physicalLayerPath(layerDigest digest.Digest) string {
+	// Note that this can't be e.g. filepath.Join(l.Digest.Hex(), legacyLayerFileName); due to the way
+	// writeLegacyMetadata constructs layer IDs differently from inputinfo.Digest values (as described
+	// inside it), most of the layers would end up in subdirectories alone without any metadata; (docker load)
+	// tries to load every subdirectory as an image and fails if the config is missing.  So, keep the layers
+	// in the root of the tarball.
+	return layerDigest.Hex() + ".tar"
+}
+
+type tarFI struct {
+	path      string
+	size      int64
+	isSymlink bool
+}
+
+func (t *tarFI) Name() string {
+	return t.path
+}
+func (t *tarFI) Size() int64 {
+	return t.size
+}
+func (t *tarFI) Mode() os.FileMode {
+	if t.isSymlink {
+		return os.ModeSymlink
+	}
+	return 0444
+}
+func (t *tarFI) ModTime() time.Time {
+	return time.Unix(0, 0)
+}
+func (t *tarFI) IsDir() bool {
+	return false
+}
+func (t *tarFI) Sys() interface{} {
+	return nil
+}
+
+// sendSymlinkLocked sends a symlink into the tar stream.
+// The caller must have locked the Writer.
+func (w *Writer) sendSymlinkLocked(path string, target string) error {
+	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: 0, isSymlink: true}, target)
+	if err != nil {
+		return nil
+	}
+	logrus.Debugf("Sending as tar link %s -> %s", path, target)
+	return w.tar.WriteHeader(hdr)
+}
+
+// sendBytesLocked sends a path into the tar stream.
+// The caller must have locked the Writer.
+func (w *Writer) sendBytesLocked(path string, b []byte) error {
+	return w.sendFileLocked(path, int64(len(b)), bytes.NewReader(b))
+}
+
+// sendFileLocked sends a file into the tar stream.
+// The caller must have locked the Writer.
+func (w *Writer) sendFileLocked(path string, expectedSize int64, stream io.Reader) error {
+	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: expectedSize}, "")
+	if err != nil {
+		return nil
+	}
+	logrus.Debugf("Sending as tar file %s", path)
+	if err := w.tar.WriteHeader(hdr); err != nil {
+		return err
+	}
+	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
+	size, err := io.Copy(w.tar, stream)
+	if err != nil {
+		return err
+	}
+	if size != expectedSize {
+		return errors.Errorf("Size mismatch when copying %s, expected %d, got %d", path, expectedSize, size)
+	}
+	return nil
+}

vendor/github.com/containers/image/v5/docker/lookaside.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/types"
+	"github.com/containers/storage/pkg/homedir"
 	"github.com/ghodss/yaml"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -26,6 +27,9 @@ var systemRegistriesDirPath = builtinRegistriesDirPath
 // DO NOT change this, instead see systemRegistriesDirPath above.
 const builtinRegistriesDirPath = "/etc/containers/registries.d"
 
+// userRegistriesDirPath is the path to the per user registries.d.
+var userRegistriesDir = filepath.FromSlash(".config/containers/registries.d")
+
 // registryConfiguration is one of the files in registriesDirPath configuring lookaside locations, or the result of merging them all.
 // NOTE: Keep this in sync with docs/registries.d.md!
 type registryConfiguration struct {
@@ -75,14 +79,17 @@ func configuredSignatureStorageBase(sys *types.SystemContext, ref dockerReferenc
 
 // registriesDirPath returns a path to registries.d
 func registriesDirPath(sys *types.SystemContext) string {
-	if sys != nil {
+	if sys != nil && sys.RegistriesDirPath != "" {
-		if sys.RegistriesDirPath != "" {
+		return sys.RegistriesDirPath
-			return sys.RegistriesDirPath
-		}
-		if sys.RootForImplicitAbsolutePaths != "" {
-			return filepath.Join(sys.RootForImplicitAbsolutePaths, systemRegistriesDirPath)
-		}
 	}
+	userRegistriesDirPath := filepath.Join(homedir.Get(), userRegistriesDir)
+	if _, err := os.Stat(userRegistriesDirPath); err == nil {
+		return userRegistriesDirPath
+	}
+	if sys != nil && sys.RootForImplicitAbsolutePaths != "" {
+		return filepath.Join(sys.RootForImplicitAbsolutePaths, systemRegistriesDirPath)
+	}
+
 	return systemRegistriesDirPath
 }
 

vendor/github.com/containers/image/v5/docker/tarfile/dest.go

@@ -1,36 +1,19 @@
 package tarfile
 
 import (
-	"archive/tar"
-	"bytes"
 	"context"
-	"encoding/json"
-	"fmt"
 	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"time"
 
+	internal "github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/docker/reference"
-	"github.com/containers/image/v5/internal/iolimits"
-	"github.com/containers/image/v5/internal/tmpdir"
-	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/types"
 	"github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 // Destination is a partial implementation of types.ImageDestination for writing to an io.Writer.
 type Destination struct {
-	writer   io.Writer
+	internal *internal.Destination
-	tar      *tar.Writer
+	archive  *internal.Writer
-	repoTags []reference.NamedTagged
-	// Other state.
-	blobs  map[digest.Digest]types.BlobInfo // list of already-sent blobs
-	config []byte
-	sysCtx *types.SystemContext
 }
 
 // NewDestination returns a tarfile.Destination for the specified io.Writer.
@@ -41,59 +24,51 @@ func NewDestination(dest io.Writer, ref reference.NamedTagged) *Destination {
 
 // NewDestinationWithContext returns a tarfile.Destination for the specified io.Writer.
 func NewDestinationWithContext(sys *types.SystemContext, dest io.Writer, ref reference.NamedTagged) *Destination {
-	repoTags := []reference.NamedTagged{}
+	archive := internal.NewWriter(dest)
-	if ref != nil {
-		repoTags = append(repoTags, ref)
-	}
 	return &Destination{
-		writer:   dest,
+		internal: internal.NewDestination(sys, archive, ref),
-		tar:      tar.NewWriter(dest),
+		archive:  archive,
-		repoTags: repoTags,
-		blobs:    make(map[digest.Digest]types.BlobInfo),
-		sysCtx:   sys,
 	}
 }
 
 // AddRepoTags adds the specified tags to the destination's repoTags.
 func (d *Destination) AddRepoTags(tags []reference.NamedTagged) {
-	d.repoTags = append(d.repoTags, tags...)
+	d.internal.AddRepoTags(tags)
 }
 
 // SupportedManifestMIMETypes tells which manifest mime types the destination supports
 // If an empty slice or nil it's returned, then any mime type can be tried to upload
 func (d *Destination) SupportedManifestMIMETypes() []string {
-	return []string{
+	return d.internal.SupportedManifestMIMETypes()
-		manifest.DockerV2Schema2MediaType, // We rely on the types.Image.UpdatedImage schema conversion capabilities.
-	}
 }
 
 // SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
 // Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
 func (d *Destination) SupportsSignatures(ctx context.Context) error {
-	return errors.Errorf("Storing signatures for docker tar files is not supported")
+	return d.internal.SupportsSignatures(ctx)
 }
 
 // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
 // uploaded to the image destination, true otherwise.
 func (d *Destination) AcceptsForeignLayerURLs() bool {
-	return false
+	return d.internal.AcceptsForeignLayerURLs()
 }
 
 // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise.
 func (d *Destination) MustMatchRuntimeOS() bool {
-	return false
+	return d.internal.MustMatchRuntimeOS()
 }
 
 // IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(),
 // and would prefer to receive an unmodified manifest instead of one modified for the destination.
 // Does not make a difference if Reference().DockerReference() is nil.
 func (d *Destination) IgnoresEmbeddedDockerReference() bool {
-	return false // N/A, we only accept schema2 images where EmbeddedDockerReferenceConflicts() is always false.
+	return d.internal.IgnoresEmbeddedDockerReference()
 }
 
 // HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
 func (d *Destination) HasThreadSafePutBlob() bool {
-	return false
+	return d.internal.HasThreadSafePutBlob()
 }
 
 // PutBlob writes contents of stream and returns data representing the result (with all data filled in).
@@ -104,66 +79,7 @@ func (d *Destination) HasThreadSafePutBlob() bool {
 // to any other readers for download using the supplied digest.
 // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
 func (d *Destination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) {
-	// Ouch, we need to stream the blob into a temporary file just to determine the size.
+	return d.internal.PutBlob(ctx, stream, inputInfo, cache, isConfig)
-	// When the layer is decompressed, we also have to generate the digest on uncompressed datas.
-	if inputInfo.Size == -1 || inputInfo.Digest.String() == "" {
-		logrus.Debugf("docker tarfile: input with unknown size, streaming to disk first ...")
-		streamCopy, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(d.sysCtx), "docker-tarfile-blob")
-		if err != nil {
-			return types.BlobInfo{}, err
-		}
-		defer os.Remove(streamCopy.Name())
-		defer streamCopy.Close()
-
-		digester := digest.Canonical.Digester()
-		tee := io.TeeReader(stream, digester.Hash())
-		// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
-		size, err := io.Copy(streamCopy, tee)
-		if err != nil {
-			return types.BlobInfo{}, err
-		}
-		_, err = streamCopy.Seek(0, io.SeekStart)
-		if err != nil {
-			return types.BlobInfo{}, err
-		}
-		inputInfo.Size = size // inputInfo is a struct, so we are only modifying our copy.
-		if inputInfo.Digest == "" {
-			inputInfo.Digest = digester.Digest()
-		}
-		stream = streamCopy
-		logrus.Debugf("... streaming done")
-	}
-
-	// Maybe the blob has been already sent
-	ok, reusedInfo, err := d.TryReusingBlob(ctx, inputInfo, cache, false)
-	if err != nil {
-		return types.BlobInfo{}, err
-	}
-	if ok {
-		return reusedInfo, nil
-	}
-
-	if isConfig {
-		buf, err := iolimits.ReadAtMost(stream, iolimits.MaxConfigBodySize)
-		if err != nil {
-			return types.BlobInfo{}, errors.Wrap(err, "Error reading Config file stream")
-		}
-		d.config = buf
-		if err := d.sendFile(inputInfo.Digest.Hex()+".json", inputInfo.Size, bytes.NewReader(buf)); err != nil {
-			return types.BlobInfo{}, errors.Wrap(err, "Error writing Config file")
-		}
-	} else {
-		// Note that this can't be e.g. filepath.Join(l.Digest.Hex(), legacyLayerFileName); due to the way
-		// writeLegacyLayerMetadata constructs layer IDs differently from inputinfo.Digest values (as described
-		// inside it), most of the layers would end up in subdirectories alone without any metadata; (docker load)
-		// tries to load every subdirectory as an image and fails if the config is missing.  So, keep the layers
-		// in the root of the tarball.
-		if err := d.sendFile(inputInfo.Digest.Hex()+".tar", inputInfo.Size, stream); err != nil {
-			return types.BlobInfo{}, err
-		}
-	}
-	d.blobs[inputInfo.Digest] = types.BlobInfo{Digest: inputInfo.Digest, Size: inputInfo.Size}
-	return types.BlobInfo{Digest: inputInfo.Digest, Size: inputInfo.Size}, nil
 }
 
 // TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
@@ -174,33 +90,7 @@ func (d *Destination) PutBlob(ctx context.Context, stream io.Reader, inputInfo t
 // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
 // May use and/or update cache.
 func (d *Destination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
-	if info.Digest == "" {
+	return d.internal.TryReusingBlob(ctx, info, cache, canSubstitute)
-		return false, types.BlobInfo{}, errors.Errorf("Can not check for a blob with unknown digest")
-	}
-	if blob, ok := d.blobs[info.Digest]; ok {
-		return true, types.BlobInfo{Digest: info.Digest, Size: blob.Size}, nil
-	}
-	return false, types.BlobInfo{}, nil
-}
-
-func (d *Destination) createRepositoriesFile(rootLayerID string) error {
-	repositories := map[string]map[string]string{}
-	for _, repoTag := range d.repoTags {
-		if val, ok := repositories[repoTag.Name()]; ok {
-			val[repoTag.Tag()] = rootLayerID
-		} else {
-			repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID}
-		}
-	}
-
-	b, err := json.Marshal(repositories)
-	if err != nil {
-		return errors.Wrap(err, "Error marshaling repositories")
-	}
-	if err := d.sendBytes(legacyRepositoriesFileName, b); err != nil {
-		return errors.Wrap(err, "Error writing config json file")
-	}
-	return nil
 }
 
 // PutManifest writes manifest to the destination.
@@ -210,215 +100,18 @@ func (d *Destination) createRepositoriesFile(rootLayerID string) error {
 // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
 // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
 func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error {
-	if instanceDigest != nil {
+	return d.internal.PutManifest(ctx, m, instanceDigest)
-		return errors.New(`Manifest lists are not supported for docker tar files`)
-	}
-	// We do not bother with types.ManifestTypeRejectedError; our .SupportedManifestMIMETypes() above is already providing only one alternative,
-	// so the caller trying a different manifest kind would be pointless.
-	var man manifest.Schema2
-	if err := json.Unmarshal(m, &man); err != nil {
-		return errors.Wrap(err, "Error parsing manifest")
-	}
-	if man.SchemaVersion != 2 || man.MediaType != manifest.DockerV2Schema2MediaType {
-		return errors.Errorf("Unsupported manifest type, need a Docker schema 2 manifest")
-	}
-
-	layerPaths, lastLayerID, err := d.writeLegacyLayerMetadata(man.LayersDescriptors)
-	if err != nil {
-		return err
-	}
-
-	if len(man.LayersDescriptors) > 0 {
-		if err := d.createRepositoriesFile(lastLayerID); err != nil {
-			return err
-		}
-	}
-
-	repoTags := []string{}
-	for _, tag := range d.repoTags {
-		// For github.com/docker/docker consumers, this works just as well as
-		//   refString := ref.String()
-		// because when reading the RepoTags strings, github.com/docker/docker/reference
-		// normalizes both of them to the same value.
-		//
-		// Doing it this way to include the normalized-out `docker.io[/library]` does make
-		// a difference for github.com/projectatomic/docker consumers, with the
-		// “Add --add-registry and --block-registry options to docker daemon” patch.
-		// These consumers treat reference strings which include a hostname and reference
-		// strings without a hostname differently.
-		//
-		// Using the host name here is more explicit about the intent, and it has the same
-		// effect as (docker pull) in projectatomic/docker, which tags the result using
-		// a hostname-qualified reference.
-		// See https://github.com/containers/image/issues/72 for a more detailed
-		// analysis and explanation.
-		refString := fmt.Sprintf("%s:%s", tag.Name(), tag.Tag())
-		repoTags = append(repoTags, refString)
-	}
-
-	items := []ManifestItem{{
-		Config:       man.ConfigDescriptor.Digest.Hex() + ".json",
-		RepoTags:     repoTags,
-		Layers:       layerPaths,
-		Parent:       "",
-		LayerSources: nil,
-	}}
-	itemsBytes, err := json.Marshal(&items)
-	if err != nil {
-		return err
-	}
-
-	// FIXME? Do we also need to support the legacy format?
-	return d.sendBytes(manifestFileName, itemsBytes)
-}
-
-// writeLegacyLayerMetadata writes legacy VERSION and configuration files for all layers
-func (d *Destination) writeLegacyLayerMetadata(layerDescriptors []manifest.Schema2Descriptor) (layerPaths []string, lastLayerID string, err error) {
-	var chainID digest.Digest
-	lastLayerID = ""
-	for i, l := range layerDescriptors {
-		// This chainID value matches the computation in docker/docker/layer.CreateChainID …
-		if chainID == "" {
-			chainID = l.Digest
-		} else {
-			chainID = digest.Canonical.FromString(chainID.String() + " " + l.Digest.String())
-		}
-		// … but note that this image ID does not match docker/docker/image/v1.CreateID. At least recent
-		// versions allocate new IDs on load, as long as the IDs we use are unique / cannot loop.
-		//
-		// Overall, the goal of computing a digest dependent on the full history is to avoid reusing an image ID
-		// (and possibly creating a loop in the "parent" links) if a layer with the same DiffID appears two or more
-		// times in layersDescriptors.  The ChainID values are sufficient for this, the v1.CreateID computation
-		// which also mixes in the full image configuration seems unnecessary, at least as long as we are storing
-		// only a single image per tarball, i.e. all DiffID prefixes are unique (can’t differ only with
-		// configuration).
-		layerID := chainID.Hex()
-
-		physicalLayerPath := l.Digest.Hex() + ".tar"
-		// The layer itself has been stored into physicalLayerPath in PutManifest.
-		// So, use that path for layerPaths used in the non-legacy manifest
-		layerPaths = append(layerPaths, physicalLayerPath)
-		// ... and create a symlink for the legacy format;
-		if err := d.sendSymlink(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
-			return nil, "", errors.Wrap(err, "Error creating layer symbolic link")
-		}
-
-		b := []byte("1.0")
-		if err := d.sendBytes(filepath.Join(layerID, legacyVersionFileName), b); err != nil {
-			return nil, "", errors.Wrap(err, "Error writing VERSION file")
-		}
-
-		// The legacy format requires a config file per layer
-		layerConfig := make(map[string]interface{})
-		layerConfig["id"] = layerID
-
-		// The root layer doesn't have any parent
-		if lastLayerID != "" {
-			layerConfig["parent"] = lastLayerID
-		}
-		// The root layer configuration file is generated by using subpart of the image configuration
-		if i == len(layerDescriptors)-1 {
-			var config map[string]*json.RawMessage
-			err := json.Unmarshal(d.config, &config)
-			if err != nil {
-				return nil, "", errors.Wrap(err, "Error unmarshaling config")
-			}
-			for _, attr := range [7]string{"architecture", "config", "container", "container_config", "created", "docker_version", "os"} {
-				layerConfig[attr] = config[attr]
-			}
-		}
-		b, err := json.Marshal(layerConfig)
-		if err != nil {
-			return nil, "", errors.Wrap(err, "Error marshaling layer config")
-		}
-		if err := d.sendBytes(filepath.Join(layerID, legacyConfigFileName), b); err != nil {
-			return nil, "", errors.Wrap(err, "Error writing config json file")
-		}
-
-		lastLayerID = layerID
-	}
-	return layerPaths, lastLayerID, nil
-}
-
-type tarFI struct {
-	path      string
-	size      int64
-	isSymlink bool
-}
-
-func (t *tarFI) Name() string {
-	return t.path
-}
-func (t *tarFI) Size() int64 {
-	return t.size
-}
-func (t *tarFI) Mode() os.FileMode {
-	if t.isSymlink {
-		return os.ModeSymlink
-	}
-	return 0444
-}
-func (t *tarFI) ModTime() time.Time {
-	return time.Unix(0, 0)
-}
-func (t *tarFI) IsDir() bool {
-	return false
-}
-func (t *tarFI) Sys() interface{} {
-	return nil
-}
-
-// sendSymlink sends a symlink into the tar stream.
-func (d *Destination) sendSymlink(path string, target string) error {
-	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: 0, isSymlink: true}, target)
-	if err != nil {
-		return nil
-	}
-	logrus.Debugf("Sending as tar link %s -> %s", path, target)
-	return d.tar.WriteHeader(hdr)
-}
-
-// sendBytes sends a path into the tar stream.
-func (d *Destination) sendBytes(path string, b []byte) error {
-	return d.sendFile(path, int64(len(b)), bytes.NewReader(b))
-}
-
-// sendFile sends a file into the tar stream.
-func (d *Destination) sendFile(path string, expectedSize int64, stream io.Reader) error {
-	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: expectedSize}, "")
-	if err != nil {
-		return nil
-	}
-	logrus.Debugf("Sending as tar file %s", path)
-	if err := d.tar.WriteHeader(hdr); err != nil {
-		return err
-	}
-	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
-	size, err := io.Copy(d.tar, stream)
-	if err != nil {
-		return err
-	}
-	if size != expectedSize {
-		return errors.Errorf("Size mismatch when copying %s, expected %d, got %d", path, expectedSize, size)
-	}
-	return nil
 }
 
 // PutSignatures would add the given signatures to the docker tarfile (currently not supported).
 // The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so
 // there can be no secondary manifests.  MUST be called after PutManifest (signatures reference manifest contents).
 func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error {
-	if instanceDigest != nil {
+	return d.internal.PutSignatures(ctx, signatures, instanceDigest)
-		return errors.Errorf(`Manifest lists are not supported for docker tar files`)
-	}
-	if len(signatures) != 0 {
-		return errors.Errorf("Storing signatures for docker tar files is not supported")
-	}
-	return nil
 }
 
 // Commit finishes writing data to the underlying io.Writer.
 // It is the caller's responsibility to close it, if necessary.
 func (d *Destination) Commit(ctx context.Context) error {
-	return d.tar.Close()
+	return d.archive.Close()
 }

vendor/github.com/containers/image/v5/docker/tarfile/src.go

@@ -1,51 +1,20 @@
 package tarfile
 
 import (
-	"archive/tar"
-	"bytes"
 	"context"
-	"encoding/json"
 	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"sync"
 
-	"github.com/containers/image/v5/internal/iolimits"
+	internal "github.com/containers/image/v5/docker/internal/tarfile"
-	"github.com/containers/image/v5/internal/tmpdir"
-	"github.com/containers/image/v5/manifest"
-	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/types"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 )
 
 // Source is a partial implementation of types.ImageSource for reading from tarPath.
+// Most users should use this via implementations of ImageReference from docker/archive or docker/daemon.
 type Source struct {
-	tarPath              string
+	internal *internal.Source
-	removeTarPathOnClose bool // Remove temp file on close if true
-	// The following data is only available after ensureCachedDataIsPresent() succeeds
-	tarManifest       *ManifestItem // nil if not available yet.
-	configBytes       []byte
-	configDigest      digest.Digest
-	orderedDiffIDList []digest.Digest
-	knownLayers       map[digest.Digest]*layerInfo
-	// Other state
-	generatedManifest []byte    // Private cache for GetManifest(), nil if not set yet.
-	cacheDataLock     sync.Once // Private state for ensureCachedDataIsPresent to make it concurrency-safe
-	cacheDataResult   error     // Private state for ensureCachedDataIsPresent
 }
 
-type layerInfo struct {
-	path string
-	size int64
-}
-
-// TODO: We could add support for multiple images in a single archive, so
-//       that people could use docker-archive:opensuse.tar:opensuse:leap as
-//       the source of an image.
-// 	To do for both the NewSourceFromFile and NewSourceFromStream functions
-
 // NewSourceFromFile returns a tarfile.Source for the specified path.
 // Deprecated: Please use NewSourceFromFileWithContext which will allows you to configure temp directory
 // for big files through SystemContext.BigFilesTemporaryDir
@@ -55,25 +24,12 @@ func NewSourceFromFile(path string) (*Source, error) {
 
 // NewSourceFromFileWithContext returns a tarfile.Source for the specified path.
 func NewSourceFromFileWithContext(sys *types.SystemContext, path string) (*Source, error) {
-	file, err := os.Open(path)
+	archive, err := internal.NewReaderFromFile(sys, path)
 	if err != nil {
-		return nil, errors.Wrapf(err, "error opening file %q", path)
+		return nil, err
 	}
-	defer file.Close()
+	src := internal.NewSource(archive, true, nil, -1)
-
+	return &Source{internal: src}, nil
-	// If the file is already not compressed we can just return the file itself
-	// as a source. Otherwise we pass the stream to NewSourceFromStream.
-	stream, isCompressed, err := compression.AutoDecompress(file)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Error detecting compression for file %q", path)
-	}
-	defer stream.Close()
-	if !isCompressed {
-		return &Source{
-			tarPath: path,
-		}, nil
-	}
-	return NewSourceFromStreamWithSystemContext(sys, stream)
 }
 
 // NewSourceFromStream returns a tarfile.Source for the specified inputStream,
@@ -89,280 +45,22 @@ func NewSourceFromStream(inputStream io.Reader) (*Source, error) {
 // which can be either compressed or uncompressed. The caller can close the
 // inputStream immediately after NewSourceFromFile returns.
 func NewSourceFromStreamWithSystemContext(sys *types.SystemContext, inputStream io.Reader) (*Source, error) {
-	// FIXME: use SystemContext here.
+	archive, err := internal.NewReaderFromStream(sys, inputStream)
-	// Save inputStream to a temporary file
-	tarCopyFile, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(sys), "docker-tar")
-	if err != nil {
-		return nil, errors.Wrap(err, "error creating temporary file")
-	}
-	defer tarCopyFile.Close()
-
-	succeeded := false
-	defer func() {
-		if !succeeded {
-			os.Remove(tarCopyFile.Name())
-		}
-	}()
-
-	// In order to be compatible with docker-load, we need to support
-	// auto-decompression (it's also a nice quality-of-life thing to avoid
-	// giving users really confusing "invalid tar header" errors).
-	uncompressedStream, _, err := compression.AutoDecompress(inputStream)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error auto-decompressing input")
-	}
-	defer uncompressedStream.Close()
-
-	// Copy the plain archive to the temporary file.
-	//
-	// TODO: This can take quite some time, and should ideally be cancellable
-	//       using a context.Context.
-	if _, err := io.Copy(tarCopyFile, uncompressedStream); err != nil {
-		return nil, errors.Wrapf(err, "error copying contents to temporary file %q", tarCopyFile.Name())
-	}
-	succeeded = true
-
-	return &Source{
-		tarPath:              tarCopyFile.Name(),
-		removeTarPathOnClose: true,
-	}, nil
-}
-
-// tarReadCloser is a way to close the backing file of a tar.Reader when the user no longer needs the tar component.
-type tarReadCloser struct {
-	*tar.Reader
-	backingFile *os.File
-}
-
-func (t *tarReadCloser) Close() error {
-	return t.backingFile.Close()
-}
-
-// openTarComponent returns a ReadCloser for the specific file within the archive.
-// This is linear scan; we assume that the tar file will have a fairly small amount of files (~layers),
-// and that filesystem caching will make the repeated seeking over the (uncompressed) tarPath cheap enough.
-// The caller should call .Close() on the returned stream.
-func (s *Source) openTarComponent(componentPath string) (io.ReadCloser, error) {
-	f, err := os.Open(s.tarPath)
 	if err != nil {
 		return nil, err
 	}
-	succeeded := false
+	src := internal.NewSource(archive, true, nil, -1)
-	defer func() {
+	return &Source{internal: src}, nil
-		if !succeeded {
-			f.Close()
-		}
-	}()
-
-	tarReader, header, err := findTarComponent(f, componentPath)
-	if err != nil {
-		return nil, err
-	}
-	if header == nil {
-		return nil, os.ErrNotExist
-	}
-	if header.FileInfo().Mode()&os.ModeType == os.ModeSymlink { // FIXME: untested
-		// We follow only one symlink; so no loops are possible.
-		if _, err := f.Seek(0, io.SeekStart); err != nil {
-			return nil, err
-		}
-		// The new path could easily point "outside" the archive, but we only compare it to existing tar headers without extracting the archive,
-		// so we don't care.
-		tarReader, header, err = findTarComponent(f, path.Join(path.Dir(componentPath), header.Linkname))
-		if err != nil {
-			return nil, err
-		}
-		if header == nil {
-			return nil, os.ErrNotExist
-		}
-	}
-
-	if !header.FileInfo().Mode().IsRegular() {
-		return nil, errors.Errorf("Error reading tar archive component %s: not a regular file", header.Name)
-	}
-	succeeded = true
-	return &tarReadCloser{Reader: tarReader, backingFile: f}, nil
-}
-
-// findTarComponent returns a header and a reader matching path within inputFile,
-// or (nil, nil, nil) if not found.
-func findTarComponent(inputFile io.Reader, path string) (*tar.Reader, *tar.Header, error) {
-	t := tar.NewReader(inputFile)
-	for {
-		h, err := t.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, nil, err
-		}
-		if h.Name == path {
-			return t, h, nil
-		}
-	}
-	return nil, nil, nil
-}
-
-// readTarComponent returns full contents of componentPath.
-func (s *Source) readTarComponent(path string, limit int) ([]byte, error) {
-	file, err := s.openTarComponent(path)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Error loading tar component %s", path)
-	}
-	defer file.Close()
-	bytes, err := iolimits.ReadAtMost(file, limit)
-	if err != nil {
-		return nil, err
-	}
-	return bytes, nil
-}
-
-// ensureCachedDataIsPresent loads data necessary for any of the public accessors.
-// It is safe to call this from multi-threaded code.
-func (s *Source) ensureCachedDataIsPresent() error {
-	s.cacheDataLock.Do(func() {
-		s.cacheDataResult = s.ensureCachedDataIsPresentPrivate()
-	})
-	return s.cacheDataResult
-}
-
-// ensureCachedDataIsPresentPrivate is a private implementation detail of ensureCachedDataIsPresent.
-// Call ensureCachedDataIsPresent instead.
-func (s *Source) ensureCachedDataIsPresentPrivate() error {
-	// Read and parse manifest.json
-	tarManifest, err := s.loadTarManifest()
-	if err != nil {
-		return err
-	}
-
-	// Check to make sure length is 1
-	if len(tarManifest) != 1 {
-		return errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(tarManifest))
-	}
-
-	// Read and parse config.
-	configBytes, err := s.readTarComponent(tarManifest[0].Config, iolimits.MaxConfigBodySize)
-	if err != nil {
-		return err
-	}
-	var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs.
-	if err := json.Unmarshal(configBytes, &parsedConfig); err != nil {
-		return errors.Wrapf(err, "Error decoding tar config %s", tarManifest[0].Config)
-	}
-	if parsedConfig.RootFS == nil {
-		return errors.Errorf("Invalid image config (rootFS is not set): %s", tarManifest[0].Config)
-	}
-
-	knownLayers, err := s.prepareLayerData(&tarManifest[0], &parsedConfig)
-	if err != nil {
-		return err
-	}
-
-	// Success; commit.
-	s.tarManifest = &tarManifest[0]
-	s.configBytes = configBytes
-	s.configDigest = digest.FromBytes(configBytes)
-	s.orderedDiffIDList = parsedConfig.RootFS.DiffIDs
-	s.knownLayers = knownLayers
-	return nil
-}
-
-// loadTarManifest loads and decodes the manifest.json.
-func (s *Source) loadTarManifest() ([]ManifestItem, error) {
-	// FIXME? Do we need to deal with the legacy format?
-	bytes, err := s.readTarComponent(manifestFileName, iolimits.MaxTarFileManifestSize)
-	if err != nil {
-		return nil, err
-	}
-	var items []ManifestItem
-	if err := json.Unmarshal(bytes, &items); err != nil {
-		return nil, errors.Wrap(err, "Error decoding tar manifest.json")
-	}
-	return items, nil
 }
 
 // Close removes resources associated with an initialized Source, if any.
 func (s *Source) Close() error {
-	if s.removeTarPathOnClose {
+	return s.internal.Close()
-		return os.Remove(s.tarPath)
-	}
-	return nil
 }
 
 // LoadTarManifest loads and decodes the manifest.json
 func (s *Source) LoadTarManifest() ([]ManifestItem, error) {
-	return s.loadTarManifest()
+	return s.internal.TarManifest(), nil
-}
-
-func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manifest.Schema2Image) (map[digest.Digest]*layerInfo, error) {
-	// Collect layer data available in manifest and config.
-	if len(tarManifest.Layers) != len(parsedConfig.RootFS.DiffIDs) {
-		return nil, errors.Errorf("Inconsistent layer count: %d in manifest, %d in config", len(tarManifest.Layers), len(parsedConfig.RootFS.DiffIDs))
-	}
-	knownLayers := map[digest.Digest]*layerInfo{}
-	unknownLayerSizes := map[string]*layerInfo{} // Points into knownLayers, a "to do list" of items with unknown sizes.
-	for i, diffID := range parsedConfig.RootFS.DiffIDs {
-		if _, ok := knownLayers[diffID]; ok {
-			// Apparently it really can happen that a single image contains the same layer diff more than once.
-			// In that case, the diffID validation ensures that both layers truly are the same, and it should not matter
-			// which of the tarManifest.Layers paths is used; (docker save) actually makes the duplicates symlinks to the original.
-			continue
-		}
-		layerPath := tarManifest.Layers[i]
-		if _, ok := unknownLayerSizes[layerPath]; ok {
-			return nil, errors.Errorf("Layer tarfile %s used for two different DiffID values", layerPath)
-		}
-		li := &layerInfo{ // A new element in each iteration
-			path: layerPath,
-			size: -1,
-		}
-		knownLayers[diffID] = li
-		unknownLayerSizes[layerPath] = li
-	}
-
-	// Scan the tar file to collect layer sizes.
-	file, err := os.Open(s.tarPath)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-	t := tar.NewReader(file)
-	for {
-		h, err := t.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-		if li, ok := unknownLayerSizes[h.Name]; ok {
-			// Since GetBlob will decompress layers that are compressed we need
-			// to do the decompression here as well, otherwise we will
-			// incorrectly report the size. Pretty critical, since tools like
-			// umoci always compress layer blobs. Obviously we only bother with
-			// the slower method of checking if it's compressed.
-			uncompressedStream, isCompressed, err := compression.AutoDecompress(t)
-			if err != nil {
-				return nil, errors.Wrapf(err, "Error auto-decompressing %s to determine its size", h.Name)
-			}
-			defer uncompressedStream.Close()
-
-			uncompressedSize := h.Size
-			if isCompressed {
-				uncompressedSize, err = io.Copy(ioutil.Discard, uncompressedStream)
-				if err != nil {
-					return nil, errors.Wrapf(err, "Error reading %s to find its size", h.Name)
-				}
-			}
-			li.size = uncompressedSize
-			delete(unknownLayerSizes, h.Name)
-		}
-	}
-	if len(unknownLayerSizes) != 0 {
-		return nil, errors.Errorf("Some layer tarfiles are missing in the tarball") // This could do with a better error reporting, if this ever happened in practice.
-	}
-
-	return knownLayers, nil
 }
 
 // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
@@ -372,130 +70,26 @@ func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manif
 // This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil,
 // as the primary manifest can not be a list, so there can be no secondary instances.
 func (s *Source) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
-	if instanceDigest != nil {
+	return s.internal.GetManifest(ctx, instanceDigest)
-		// How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType.
-		return nil, "", errors.New(`Manifest lists are not supported by "docker-daemon:"`)
-	}
-	if s.generatedManifest == nil {
-		if err := s.ensureCachedDataIsPresent(); err != nil {
-			return nil, "", err
-		}
-		m := manifest.Schema2{
-			SchemaVersion: 2,
-			MediaType:     manifest.DockerV2Schema2MediaType,
-			ConfigDescriptor: manifest.Schema2Descriptor{
-				MediaType: manifest.DockerV2Schema2ConfigMediaType,
-				Size:      int64(len(s.configBytes)),
-				Digest:    s.configDigest,
-			},
-			LayersDescriptors: []manifest.Schema2Descriptor{},
-		}
-		for _, diffID := range s.orderedDiffIDList {
-			li, ok := s.knownLayers[diffID]
-			if !ok {
-				return nil, "", errors.Errorf("Internal inconsistency: Information about layer %s missing", diffID)
-			}
-			m.LayersDescriptors = append(m.LayersDescriptors, manifest.Schema2Descriptor{
-				Digest:    diffID, // diffID is a digest of the uncompressed tarball
-				MediaType: manifest.DockerV2Schema2LayerMediaType,
-				Size:      li.size,
-			})
-		}
-		manifestBytes, err := json.Marshal(&m)
-		if err != nil {
-			return nil, "", err
-		}
-		s.generatedManifest = manifestBytes
-	}
-	return s.generatedManifest, manifest.DockerV2Schema2MediaType, nil
-}
-
-// uncompressedReadCloser is an io.ReadCloser that closes both the uncompressed stream and the underlying input.
-type uncompressedReadCloser struct {
-	io.Reader
-	underlyingCloser   func() error
-	uncompressedCloser func() error
-}
-
-func (r uncompressedReadCloser) Close() error {
-	var res error
-	if err := r.uncompressedCloser(); err != nil {
-		res = err
-	}
-	if err := r.underlyingCloser(); err != nil && res == nil {
-		res = err
-	}
-	return res
 }
 
 // HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently.
 func (s *Source) HasThreadSafeGetBlob() bool {
-	return true
+	return s.internal.HasThreadSafeGetBlob()
 }
 
 // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
 // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
 // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
 func (s *Source) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
-	if err := s.ensureCachedDataIsPresent(); err != nil {
+	return s.internal.GetBlob(ctx, info, cache)
-		return nil, 0, err
-	}
-
-	if info.Digest == s.configDigest { // FIXME? Implement a more general algorithm matching instead of assuming sha256.
-		return ioutil.NopCloser(bytes.NewReader(s.configBytes)), int64(len(s.configBytes)), nil
-	}
-
-	if li, ok := s.knownLayers[info.Digest]; ok { // diffID is a digest of the uncompressed tarball,
-		underlyingStream, err := s.openTarComponent(li.path)
-		if err != nil {
-			return nil, 0, err
-		}
-		closeUnderlyingStream := true
-		defer func() {
-			if closeUnderlyingStream {
-				underlyingStream.Close()
-			}
-		}()
-
-		// In order to handle the fact that digests != diffIDs (and thus that a
-		// caller which is trying to verify the blob will run into problems),
-		// we need to decompress blobs. This is a bit ugly, but it's a
-		// consequence of making everything addressable by their DiffID rather
-		// than by their digest...
-		//
-		// In particular, because the v2s2 manifest being generated uses
-		// DiffIDs, any caller of GetBlob is going to be asking for DiffIDs of
-		// layers not their _actual_ digest. The result is that copy/... will
-		// be verifing a "digest" which is not the actual layer's digest (but
-		// is instead the DiffID).
-
-		uncompressedStream, _, err := compression.AutoDecompress(underlyingStream)
-		if err != nil {
-			return nil, 0, errors.Wrapf(err, "Error auto-decompressing blob %s", info.Digest)
-		}
-
-		newStream := uncompressedReadCloser{
-			Reader:             uncompressedStream,
-			underlyingCloser:   underlyingStream.Close,
-			uncompressedCloser: uncompressedStream.Close,
-		}
-		closeUnderlyingStream = false
-
-		return newStream, li.size, nil
-	}
-
-	return nil, 0, errors.Errorf("Unknown blob %s", info.Digest)
 }
 
 // GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
 // This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil,
 // as there can be no secondary manifests.
 func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
-	if instanceDigest != nil {
+	return s.internal.GetSignatures(ctx, instanceDigest)
-		// How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType.
-		return nil, errors.Errorf(`Manifest lists are not supported by "docker-daemon:"`)
-	}
-	return [][]byte{}, nil
 }
 
 // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer
@@ -505,6 +99,6 @@ func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Diges
 // as the primary manifest can not be a list, so there can be no secondary manifests.
 // The Digest field is guaranteed to be provided; Size may be -1.
 // WARNING: The list may contain duplicates, and they are semantically relevant.
-func (s *Source) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) {
+func (s *Source) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) {
-	return nil, nil
+	return s.internal.LayerInfosForCopy(ctx, instanceDigest)
 }

vendor/github.com/containers/image/v5/docker/tarfile/types.go

@@ -1,28 +1,8 @@
 package tarfile
 
 import (
-	"github.com/containers/image/v5/manifest"
+	internal "github.com/containers/image/v5/docker/internal/tarfile"
-	"github.com/opencontainers/go-digest"
-)
-
-// Various data structures.
-
-// Based on github.com/docker/docker/image/tarexport/tarexport.go
-const (
-	manifestFileName           = "manifest.json"
-	legacyLayerFileName        = "layer.tar"
-	legacyConfigFileName       = "json"
-	legacyVersionFileName      = "VERSION"
-	legacyRepositoriesFileName = "repositories"
 )
 
 // ManifestItem is an element of the array stored in the top-level manifest.json file.
-type ManifestItem struct {
+type ManifestItem = internal.ManifestItem // All public members from the internal package remain accessible.
-	Config       string
-	RepoTags     []string
-	Layers       []string
-	Parent       imageID                                      `json:",omitempty"`
-	LayerSources map[digest.Digest]manifest.Schema2Descriptor `json:",omitempty"`
-}
-
-type imageID string

vendor/github.com/containers/image/v5/oci/layout/oci_src.go

@@ -141,6 +141,10 @@ func (s *ociImageSource) GetSignatures(ctx context.Context, instanceDigest *dige
 }
 
 func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (io.ReadCloser, int64, error) {
+	if len(urls) == 0 {
+		return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
+	}
+
 	errWrap := errors.New("failed fetching external blob from all urls")
 	for _, url := range urls {
 

vendor/github.com/containers/image/v5/pkg/docker/config/config.go

@@ -11,9 +11,9 @@ import (
 	"strings"
 
 	"github.com/containers/image/v5/types"
+	"github.com/containers/storage/pkg/homedir"
 	helperclient "github.com/docker/docker-credential-helpers/client"
 	"github.com/docker/docker-credential-helpers/credentials"
-	"github.com/docker/docker/pkg/homedir"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -345,7 +345,7 @@ func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) (
 			return errors.Wrapf(err, "error marshaling JSON %q", path)
 		}
 
-		if err = ioutil.WriteFile(path, newData, 0755); err != nil {
+		if err = ioutil.WriteFile(path, newData, 0600); err != nil {
 			return errors.Wrapf(err, "error writing to file %q", path)
 		}
 	}

vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go

@@ -338,55 +338,86 @@ func (config *V2RegistriesConf) postProcess() error {
 }
 
 // ConfigPath returns the path to the system-wide registry configuration file.
+// Deprecated: This API implies configuration is read from files, and that there is only one.
+// Please use ConfigurationSourceDescription to obtain a string usable for error messages.
 func ConfigPath(ctx *types.SystemContext) string {
-	if ctx != nil && ctx.SystemRegistriesConfPath != "" {
+	return newConfigWrapper(ctx).configPath
-		return ctx.SystemRegistriesConfPath
-	}
-
-	userRegistriesFilePath := filepath.Join(homedir.Get(), userRegistriesFile)
-	if _, err := os.Stat(userRegistriesFilePath); err == nil {
-		return userRegistriesFilePath
-	}
-
-	if ctx != nil && ctx.RootForImplicitAbsolutePaths != "" {
-		return filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfPath)
-	}
-
-	return systemRegistriesConfPath
 }
 
-// ConfigDirPath returns the path to the system-wide directory for drop-in
+// ConfigDirPath returns the path to the directory for drop-in
 // registry configuration files.
+// Deprecated: This API implies configuration is read from directories, and that there is only one.
+// Please use ConfigurationSourceDescription to obtain a string usable for error messages.
 func ConfigDirPath(ctx *types.SystemContext) string {
-	if ctx != nil && ctx.SystemRegistriesConfDirPath != "" {
+	configWrapper := newConfigWrapper(ctx)
-		return ctx.SystemRegistriesConfDirPath
+	if configWrapper.userConfigDirPath != "" {
+		return configWrapper.userConfigDirPath
 	}
-
+	return configWrapper.configDirPath
-	userRegistriesDirPath := filepath.Join(homedir.Get(), userRegistriesDir)
-	if _, err := os.Stat(userRegistriesDirPath); err == nil {
-		return userRegistriesDirPath
-	}
-
-	if ctx != nil && ctx.RootForImplicitAbsolutePaths != "" {
-		return filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfDirPath)
-	}
-
-	return systemRegistriesConfDirPath
 }
 
 // configWrapper is used to store the paths from ConfigPath and ConfigDirPath
 // and acts as a key to the internal cache.
 type configWrapper struct {
-	configPath    string
+	// path to the registries.conf file
+	configPath string
+	// path to system-wide registries.conf.d directory, or "" if not used
 	configDirPath string
+	// path to user specificed registries.conf.d directory, or "" if not used
+	userConfigDirPath string
 }
 
 // newConfigWrapper returns a configWrapper for the specified SystemContext.
 func newConfigWrapper(ctx *types.SystemContext) configWrapper {
-	return configWrapper{
+	var wrapper configWrapper
-		configPath:    ConfigPath(ctx),
+	userRegistriesFilePath := filepath.Join(homedir.Get(), userRegistriesFile)
-		configDirPath: ConfigDirPath(ctx),
+	userRegistriesDirPath := filepath.Join(homedir.Get(), userRegistriesDir)
+
+	// decide configPath using per-user path or system file
+	if ctx != nil && ctx.SystemRegistriesConfPath != "" {
+		wrapper.configPath = ctx.SystemRegistriesConfPath
+	} else if _, err := os.Stat(userRegistriesFilePath); err == nil {
+		// per-user registries.conf exists, not reading system dir
+		// return config dirs from ctx or per-user one
+		wrapper.configPath = userRegistriesFilePath
+		if ctx != nil && ctx.SystemRegistriesConfDirPath != "" {
+			wrapper.configDirPath = ctx.SystemRegistriesConfDirPath
+		} else {
+			wrapper.userConfigDirPath = userRegistriesDirPath
+		}
+		return wrapper
+	} else if ctx != nil && ctx.RootForImplicitAbsolutePaths != "" {
+		wrapper.configPath = filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfPath)
+	} else {
+		wrapper.configPath = systemRegistriesConfPath
 	}
+
+	// potentially use both system and per-user dirs if not using per-user config file
+	if ctx != nil && ctx.SystemRegistriesConfDirPath != "" {
+		// dir explicitly chosen: use only that one
+		wrapper.configDirPath = ctx.SystemRegistriesConfDirPath
+	} else if ctx != nil && ctx.RootForImplicitAbsolutePaths != "" {
+		wrapper.configDirPath = filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfDirPath)
+		wrapper.userConfigDirPath = userRegistriesDirPath
+	} else {
+		wrapper.configDirPath = systemRegistriesConfDirPath
+		wrapper.userConfigDirPath = userRegistriesDirPath
+	}
+
+	return wrapper
+}
+
+// ConfigurationSourceDescription returns a string containres paths of registries.conf and registries.conf.d
+func ConfigurationSourceDescription(ctx *types.SystemContext) string {
+	wrapper := newConfigWrapper(ctx)
+	configSources := []string{wrapper.configPath}
+	if wrapper.configDirPath != "" {
+		configSources = append(configSources, wrapper.configDirPath)
+	}
+	if wrapper.userConfigDirPath != "" {
+		configSources = append(configSources, wrapper.userConfigDirPath)
+	}
+	return strings.Join(configSources, ", ")
 }
 
 // configMutex is used to synchronize concurrent accesses to configCache.
@@ -422,39 +453,49 @@ func getConfig(ctx *types.SystemContext) (*V2RegistriesConf, error) {
 // dropInConfigs returns a slice of drop-in-configs from the registries.conf.d
 // directory.
 func dropInConfigs(wrapper configWrapper) ([]string, error) {
-	var configs []string
+	var (
-
+		configs  []string
-	err := filepath.Walk(wrapper.configDirPath,
+		dirPaths []string
-		// WalkFunc to read additional configs
-		func(path string, info os.FileInfo, err error) error {
-			switch {
-			case err != nil:
-				// return error (could be a permission problem)
-				return err
-			case info == nil:
-				// this should only happen when err != nil but let's be sure
-				return nil
-			case info.IsDir():
-				if path != wrapper.configDirPath {
-					// make sure to not recurse into sub-directories
-					return filepath.SkipDir
-				}
-				// ignore directories
-				return nil
-			default:
-				// only add *.conf files
-				if strings.HasSuffix(path, ".conf") {
-					configs = append(configs, path)
-				}
-				return nil
-			}
-		},
 	)
+	if wrapper.configDirPath != "" {
+		dirPaths = append(dirPaths, wrapper.configDirPath)
+	}
+	if wrapper.userConfigDirPath != "" {
+		dirPaths = append(dirPaths, wrapper.userConfigDirPath)
+	}
+	for _, dirPath := range dirPaths {
+		err := filepath.Walk(dirPath,
+			// WalkFunc to read additional configs
+			func(path string, info os.FileInfo, err error) error {
+				switch {
+				case err != nil:
+					// return error (could be a permission problem)
+					return err
+				case info == nil:
+					// this should only happen when err != nil but let's be sure
+					return nil
+				case info.IsDir():
+					if path != dirPath {
+						// make sure to not recurse into sub-directories
+						return filepath.SkipDir
+					}
+					// ignore directories
+					return nil
+				default:
+					// only add *.conf files
+					if strings.HasSuffix(path, ".conf") {
+						configs = append(configs, path)
+					}
+					return nil
+				}
+			},
+		)
 
-	if err != nil && !os.IsNotExist(err) {
+		if err != nil && !os.IsNotExist(err) {
-		// Ignore IsNotExist errors: most systems won't have a registries.conf.d
+			// Ignore IsNotExist errors: most systems won't have a registries.conf.d
-		// directory.
+			// directory.
-		return nil, errors.Wrapf(err, "error reading registries.conf.d")
+			return nil, errors.Wrapf(err, "error reading registries.conf.d")
+		}
 	}
 
 	return configs, nil

vendor/github.com/containers/image/v5/version/version.go

@@ -11,7 +11,7 @@ const (
 	VersionPatch = 2
 
 	// VersionDev indicates development branch. Releases will be empty string.
-	VersionDev = ""
+	VersionDev = "-dev"
 )
 
 // Version is the specification version that the package types support.

vendor/github.com/imdario/mergo/README.md

@@ -1,44 +1,54 @@
 # Mergo
 
+
+[![GoDoc][3]][4]
+[![GitHub release][5]][6]
+[![GoCard][7]][8]
+[![Build Status][1]][2]
+[![Coverage Status][9]][10]
+[![Sourcegraph][11]][12]
+[![FOSSA Status][13]][14]
+
+[![GoCenter Kudos][15]][16]
+
+[1]: https://travis-ci.org/imdario/mergo.png
+[2]: https://travis-ci.org/imdario/mergo
+[3]: https://godoc.org/github.com/imdario/mergo?status.svg
+[4]: https://godoc.org/github.com/imdario/mergo
+[5]: https://img.shields.io/github/release/imdario/mergo.svg
+[6]: https://github.com/imdario/mergo/releases
+[7]: https://goreportcard.com/badge/imdario/mergo
+[8]: https://goreportcard.com/report/github.com/imdario/mergo
+[9]: https://coveralls.io/repos/github/imdario/mergo/badge.svg?branch=master
+[10]: https://coveralls.io/github/imdario/mergo?branch=master
+[11]: https://sourcegraph.com/github.com/imdario/mergo/-/badge.svg
+[12]: https://sourcegraph.com/github.com/imdario/mergo?badge
+[13]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield
+[14]: https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield
+[15]: https://search.gocenter.io/api/ui/badge/github.com%2Fimdario%2Fmergo
+[16]: https://search.gocenter.io/github.com/imdario/mergo
+
 A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements.
 
+Mergo merges same-type structs and maps by setting default values in zero-value fields. Mergo won't merge unexported (private) fields. It will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection).
+
 Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the Province of Ancona in the Italian region of Marche.
 
 ## Status
 
 It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild).
 
-[![GoDoc][3]][4]
-[![GoCard][5]][6]
-[![Build Status][1]][2]
-[![Coverage Status][7]][8]
-[![Sourcegraph][9]][10]
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield)
-
-[1]: https://travis-ci.org/imdario/mergo.png
-[2]: https://travis-ci.org/imdario/mergo
-[3]: https://godoc.org/github.com/imdario/mergo?status.svg
-[4]: https://godoc.org/github.com/imdario/mergo
-[5]: https://goreportcard.com/badge/imdario/mergo
-[6]: https://goreportcard.com/report/github.com/imdario/mergo
-[7]: https://coveralls.io/repos/github/imdario/mergo/badge.svg?branch=master
-[8]: https://coveralls.io/github/imdario/mergo?branch=master
-[9]: https://sourcegraph.com/github.com/imdario/mergo/-/badge.svg
-[10]: https://sourcegraph.com/github.com/imdario/mergo?badge
-
-### Latest release
-
-[Release v0.3.7](https://github.com/imdario/mergo/releases/tag/v0.3.7).
-
 ### Important note
 
-Please keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2) Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). An optional/variadic argument has been added, so it won't break existing code.
+Please keep in mind that a problematic PR broke [0.3.9](//github.com/imdario/mergo/releases/tag/0.3.9). I reverted it in [0.3.10](//github.com/imdario/mergo/releases/tag/0.3.10), and I consider it stable but not bug-free. Also, this version adds suppot for go modules.
 
-If you were using Mergo **before** April 6th 2015, please check your project works as intended after updating your local copy with ```go get -u github.com/imdario/mergo```. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause (I hope it won't!) in existing projects after the change (release 0.2.0).
+Keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2), Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). I added an optional/variadic argument so that it won't break the existing code.
+
+If you were using Mergo before April 6th, 2015, please check your project works as intended after updating your local copy with ```go get -u github.com/imdario/mergo```. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause in existing projects after the change (release 0.2.0).
 
 ### Donations
 
-If Mergo is useful to you, consider buying me a coffee, a beer or making a monthly donation so I can keep building great free software. :heart_eyes:
+If Mergo is useful to you, consider buying me a coffee, a beer, or making a monthly donation to allow me to keep building great free software. :heart_eyes:
 
 <a href='https://ko-fi.com/B0B58839' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi1.png?v=0' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
 [![Beerpay](https://beerpay.io/imdario/mergo/badge.svg)](https://beerpay.io/imdario/mergo)
@@ -87,8 +97,9 @@ If Mergo is useful to you, consider buying me a coffee, a beer or making a month
 - [mantasmatelis/whooplist-server](https://github.com/mantasmatelis/whooplist-server)
 - [jnuthong/item_search](https://github.com/jnuthong/item_search)
 - [bukalapak/snowboard](https://github.com/bukalapak/snowboard)
+- [janoszen/containerssh](https://github.com/janoszen/containerssh)
 
-## Installation
+## Install
 
     go get github.com/imdario/mergo
 
@@ -99,7 +110,7 @@ If Mergo is useful to you, consider buying me a coffee, a beer or making a month
 
 ## Usage
 
-You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. It won't merge empty structs value as [they are not considered zero values](https://golang.org/ref/spec#The_zero_value) either. Also maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection).
+You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. It won't merge empty structs value as [they are zero values](https://golang.org/ref/spec#The_zero_value) too. Also, maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection).
 
 ```go
 if err := mergo.Merge(&dst, src); err != nil {
@@ -125,9 +136,7 @@ if err := mergo.Map(&dst, srcMap); err != nil {
 
 Warning: if you map a struct to map, it won't do it recursively. Don't expect Mergo to map struct members of your struct as `map[string]interface{}`. They will be just assigned as values.
 
-More information and examples in [godoc documentation](http://godoc.org/github.com/imdario/mergo).
+Here is a nice example:
-
-### Nice example
 
 ```go
 package main
@@ -175,10 +184,10 @@ import (
         "time"
 )
 
-type timeTransfomer struct {
+type timeTransformer struct {
 }
 
-func (t timeTransfomer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
+func (t timeTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
 	if typ == reflect.TypeOf(time.Time{}) {
 		return func(dst, src reflect.Value) error {
 			if dst.CanSet() {
@@ -202,7 +211,7 @@ type Snapshot struct {
 func main() {
 	src := Snapshot{time.Now()}
 	dest := Snapshot{}
-	mergo.Merge(&dest, src, mergo.WithTransformers(timeTransfomer{}))
+	mergo.Merge(&dest, src, mergo.WithTransformers(timeTransformer{}))
 	fmt.Println(dest)
 	// Will print
 	// { 2018-01-12 01:15:00 +0000 UTC m=+0.000000001 }

vendor/github.com/imdario/mergo/doc.go

@@ -4,41 +4,140 @@
 // license that can be found in the LICENSE file.
 
 /*
-Package mergo merges same-type structs and maps by setting default values in zero-value fields.
+A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements.
 
-Mergo won't merge unexported (private) fields but will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection).
+Mergo merges same-type structs and maps by setting default values in zero-value fields. Mergo won't merge unexported (private) fields. It will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection).
+
+Status
+
+It is ready for production use. It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc.
+
+Important note
+
+Please keep in mind that a problematic PR broke 0.3.9. We reverted it in 0.3.10. We consider 0.3.10 as stable but not bug-free. . Also, this version adds suppot for go modules.
+
+Keep in mind that in 0.3.2, Mergo changed Merge() and Map() signatures to support transformers. We added an optional/variadic argument so that it won't break the existing code.
+
+If you were using Mergo before April 6th, 2015, please check your project works as intended after updating your local copy with go get -u github.com/imdario/mergo. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause in existing projects after the change (release 0.2.0).
+
+Install
+
+Do your usual installation procedure:
+
+    go get github.com/imdario/mergo
+
+    // use in your .go code
+    import (
+        "github.com/imdario/mergo"
+    )
 
 Usage
 
-From my own work-in-progress project:
+You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. It won't merge empty structs value as they are zero values too. Also, maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection).
 
-	type networkConfig struct {
+	if err := mergo.Merge(&dst, src); err != nil {
-		Protocol string
+		// ...
-		Address string
-		ServerType string `json: "server_type"`
-		Port uint16
 	}
 
-	type FssnConfig struct {
+Also, you can merge overwriting values using the transformer WithOverride.
-		Network networkConfig
+
+	if err := mergo.Merge(&dst, src, mergo.WithOverride); err != nil {
+		// ...
 	}
 
-	var fssnDefault = FssnConfig {
+Additionally, you can map a map[string]interface{} to a struct (and otherwise, from struct to map), following the same restrictions as in Merge(). Keys are capitalized to find each corresponding exported field.
-		networkConfig {
+
-			"tcp",
+	if err := mergo.Map(&dst, srcMap); err != nil {
-			"127.0.0.1",
+		// ...
-			"http",
-			31560,
-		},
 	}
 
-	// Inside a function [...]
+Warning: if you map a struct to map, it won't do it recursively. Don't expect Mergo to map struct members of your struct as map[string]interface{}. They will be just assigned as values.
 
-	if err := mergo.Merge(&config, fssnDefault); err != nil {
+Here is a nice example:
-		log.Fatal(err)
+
+	package main
+
+	import (
+		"fmt"
+		"github.com/imdario/mergo"
+	)
+
+	type Foo struct {
+		A string
+		B int64
 	}
 
-	// More code [...]
+	func main() {
+		src := Foo{
+			A: "one",
+			B: 2,
+		}
+		dest := Foo{
+			A: "two",
+		}
+		mergo.Merge(&dest, src)
+		fmt.Println(dest)
+		// Will print
+		// {two 2}
+	}
+
+Transformers
+
+Transformers allow to merge specific types differently than in the default behavior. In other words, now you can customize how some types are merged. For example, time.Time is a struct; it doesn't have zero value but IsZero can return true because it has fields with zero value. How can we merge a non-zero time.Time?
+
+	package main
+
+	import (
+		"fmt"
+		"github.com/imdario/mergo"
+			"reflect"
+			"time"
+	)
+
+	type timeTransformer struct {
+	}
+
+	func (t timeTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
+		if typ == reflect.TypeOf(time.Time{}) {
+			return func(dst, src reflect.Value) error {
+				if dst.CanSet() {
+					isZero := dst.MethodByName("IsZero")
+					result := isZero.Call([]reflect.Value{})
+					if result[0].Bool() {
+						dst.Set(src)
+					}
+				}
+				return nil
+			}
+		}
+		return nil
+	}
+
+	type Snapshot struct {
+		Time time.Time
+		// ...
+	}
+
+	func main() {
+		src := Snapshot{time.Now()}
+		dest := Snapshot{}
+		mergo.Merge(&dest, src, mergo.WithTransformers(timeTransformer{}))
+		fmt.Println(dest)
+		// Will print
+		// { 2018-01-12 01:15:00 +0000 UTC m=+0.000000001 }
+	}
+
+Contact me
+
+If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): https://twitter.com/im_dario
+
+About
+
+Written by Dario Castañé: https://da.rio.hn
+
+License
+
+BSD 3-Clause license, as Go language.
 
 */
 package mergo

vendor/github.com/imdario/mergo/go.mod

@@ -0,0 +1,5 @@
+module github.com/imdario/mergo
+
+go 1.13
+
+require gopkg.in/yaml.v2 v2.3.0

vendor/github.com/imdario/mergo/go.sum

@@ -0,0 +1,4 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/imdario/mergo/map.go

@@ -99,11 +99,11 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf
 				continue
 			}
 			if srcKind == dstKind {
-				if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
+				if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
 					return
 				}
 			} else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface {
-				if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
+				if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
 					return
 				}
 			} else if srcKind == reflect.Map {
@@ -141,6 +141,9 @@ func MapWithOverwrite(dst, src interface{}, opts ...func(*Config)) error {
 }
 
 func _map(dst, src interface{}, opts ...func(*Config)) error {
+	if dst != nil && reflect.ValueOf(dst).Kind() != reflect.Ptr {
+		return ErrNonPointerAgument
+	}
 	var (
 		vDst, vSrc reflect.Value
 		err        error
@@ -157,8 +160,7 @@ func _map(dst, src interface{}, opts ...func(*Config)) error {
 	// To be friction-less, we redirect equal-type arguments
 	// to deepMerge. Only because arguments can be anything.
 	if vSrc.Kind() == vDst.Kind() {
-		_, err := deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
+		return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
-		return err
 	}
 	switch vSrc.Kind() {
 	case reflect.Struct:

vendor/github.com/imdario/mergo/merge.go

@@ -11,26 +11,26 @@ package mergo
 import (
 	"fmt"
 	"reflect"
-	"unsafe"
 )
 
-func hasExportedField(dst reflect.Value) (exported bool) {
+func hasMergeableFields(dst reflect.Value) (exported bool) {
 	for i, n := 0, dst.NumField(); i < n; i++ {
 		field := dst.Type().Field(i)
-		if isExportedComponent(&field) {
+		if field.Anonymous && dst.Field(i).Kind() == reflect.Struct {
-			return true
+			exported = exported || hasMergeableFields(dst.Field(i))
+		} else if isExportedComponent(&field) {
+			exported = exported || len(field.PkgPath) == 0
 		}
 	}
 	return
 }
 
 func isExportedComponent(field *reflect.StructField) bool {
-	name := field.Name
 	pkgPath := field.PkgPath
 	if len(pkgPath) > 0 {
 		return false
 	}
-	c := name[0]
+	c := field.Name[0]
 	if 'a' <= c && c <= 'z' || c == '_' {
 		return false
 	}
@@ -44,6 +44,8 @@ type Config struct {
 	Transformers                 Transformers
 	overwriteWithEmptyValue      bool
 	overwriteSliceWithEmptyValue bool
+	sliceDeepCopy                bool
+	debug                        bool
 }
 
 type Transformers interface {
@@ -53,17 +55,16 @@ type Transformers interface {
 // Traverses recursively both values, assigning src's fields values to dst.
 // The map argument tracks comparisons that have already been seen, which allows
 // short circuiting on recursive types.
-func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (dst reflect.Value, err error) {
+func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) {
-	dst = dstIn
 	overwrite := config.Overwrite
 	typeCheck := config.TypeCheck
 	overwriteWithEmptySrc := config.overwriteWithEmptyValue
 	overwriteSliceWithEmptySrc := config.overwriteSliceWithEmptyValue
+	sliceDeepCopy := config.sliceDeepCopy
 
 	if !src.IsValid() {
 		return
 	}
-
 	if dst.CanAddr() {
 		addr := dst.UnsafeAddr()
 		h := 17 * addr
@@ -71,7 +72,7 @@ func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int,
 		typ := dst.Type()
 		for p := seen; p != nil; p = p.next {
 			if p.ptr == addr && p.typ == typ {
-				return dst, nil
+				return nil
 			}
 		}
 		// Remember, remember...
@@ -85,125 +86,153 @@ func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int,
 		}
 	}
 
-	if dst.IsValid() && src.IsValid() && src.Type() != dst.Type() {
-		err = fmt.Errorf("cannot append two different types (%s, %s)", src.Kind(), dst.Kind())
-		return
-	}
-
 	switch dst.Kind() {
 	case reflect.Struct:
-		if hasExportedField(dst) {
+		if hasMergeableFields(dst) {
-			dstCp := reflect.New(dst.Type()).Elem()
 			for i, n := 0, dst.NumField(); i < n; i++ {
-				dstField := dst.Field(i)
+				if err = deepMerge(dst.Field(i), src.Field(i), visited, depth+1, config); err != nil {
-				structField := dst.Type().Field(i)
-				// copy un-exported struct fields
-				if !isExportedComponent(&structField) {
-					rf := dstCp.Field(i)
-					rf = reflect.NewAt(rf.Type(), unsafe.Pointer(rf.UnsafeAddr())).Elem() //nolint:gosec
-					dstRF := dst.Field(i)
-					if !dst.Field(i).CanAddr() {
-						continue
-					}
-
-					dstRF = reflect.NewAt(dstRF.Type(), unsafe.Pointer(dstRF.UnsafeAddr())).Elem() //nolint:gosec
-					rf.Set(dstRF)
-					continue
-				}
-				dstField, err = deepMerge(dstField, src.Field(i), visited, depth+1, config)
-				if err != nil {
 					return
 				}
-				dstCp.Field(i).Set(dstField)
 			}
-
-			if dst.CanSet() {
-				dst.Set(dstCp)
-			} else {
-				dst = dstCp
-			}
-			return
 		} else {
 			if (isReflectNil(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) {
-				dst = src
+				dst.Set(src)
 			}
 		}
-
 	case reflect.Map:
 		if dst.IsNil() && !src.IsNil() {
-			if dst.CanSet() {
+			dst.Set(reflect.MakeMap(dst.Type()))
-				dst.Set(reflect.MakeMap(dst.Type()))
-			} else {
-				dst = src
-				return
-			}
 		}
+
+		if src.Kind() != reflect.Map {
+			if overwrite {
+				dst.Set(src)
+			}
+			return
+		}
+
 		for _, key := range src.MapKeys() {
 			srcElement := src.MapIndex(key)
-			dstElement := dst.MapIndex(key)
 			if !srcElement.IsValid() {
 				continue
 			}
-			if dst.MapIndex(key).IsValid() {
+			dstElement := dst.MapIndex(key)
-				k := dstElement.Interface()
+			switch srcElement.Kind() {
-				dstElement = reflect.ValueOf(k)
+			case reflect.Chan, reflect.Func, reflect.Map, reflect.Interface, reflect.Slice:
-			}
+				if srcElement.IsNil() {
-			if isReflectNil(srcElement) {
+					if overwrite {
-				if overwrite || isReflectNil(dstElement) {
+						dst.SetMapIndex(key, srcElement)
-					dst.SetMapIndex(key, srcElement)
+					}
+					continue
+				}
+				fallthrough
+			default:
+				if !srcElement.CanInterface() {
+					continue
+				}
+				switch reflect.TypeOf(srcElement.Interface()).Kind() {
+				case reflect.Struct:
+					fallthrough
+				case reflect.Ptr:
+					fallthrough
+				case reflect.Map:
+					srcMapElm := srcElement
+					dstMapElm := dstElement
+					if srcMapElm.CanInterface() {
+						srcMapElm = reflect.ValueOf(srcMapElm.Interface())
+						if dstMapElm.IsValid() {
+							dstMapElm = reflect.ValueOf(dstMapElm.Interface())
+						}
+					}
+					if err = deepMerge(dstMapElm, srcMapElm, visited, depth+1, config); err != nil {
+						return
+					}
+				case reflect.Slice:
+					srcSlice := reflect.ValueOf(srcElement.Interface())
+
+					var dstSlice reflect.Value
+					if !dstElement.IsValid() || dstElement.IsNil() {
+						dstSlice = reflect.MakeSlice(srcSlice.Type(), 0, srcSlice.Len())
+					} else {
+						dstSlice = reflect.ValueOf(dstElement.Interface())
+					}
+
+					if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice && !sliceDeepCopy {
+						if typeCheck && srcSlice.Type() != dstSlice.Type() {
+							return fmt.Errorf("cannot override two slices with different type (%s, %s)", srcSlice.Type(), dstSlice.Type())
+						}
+						dstSlice = srcSlice
+					} else if config.AppendSlice {
+						if srcSlice.Type() != dstSlice.Type() {
+							return fmt.Errorf("cannot append two slices with different type (%s, %s)", srcSlice.Type(), dstSlice.Type())
+						}
+						dstSlice = reflect.AppendSlice(dstSlice, srcSlice)
+					} else if sliceDeepCopy {
+						i := 0
+						for ; i < srcSlice.Len() && i < dstSlice.Len(); i++ {
+							srcElement := srcSlice.Index(i)
+							dstElement := dstSlice.Index(i)
+
+							if srcElement.CanInterface() {
+								srcElement = reflect.ValueOf(srcElement.Interface())
+							}
+							if dstElement.CanInterface() {
+								dstElement = reflect.ValueOf(dstElement.Interface())
+							}
+
+							if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
+								return
+							}
+						}
+
+					}
+					dst.SetMapIndex(key, dstSlice)
 				}
-				continue
 			}
-			if !srcElement.CanInterface() {
+			if dstElement.IsValid() && !isEmptyValue(dstElement) && (reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map || reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Slice) {
 				continue
 			}
 
-			if srcElement.CanInterface() {
+			if srcElement.IsValid() && ((srcElement.Kind() != reflect.Ptr && overwrite) || !dstElement.IsValid() || isEmptyValue(dstElement)) {
-				srcElement = reflect.ValueOf(srcElement.Interface())
+				if dst.IsNil() {
-				if dstElement.IsValid() {
+					dst.Set(reflect.MakeMap(dst.Type()))
-					dstElement = reflect.ValueOf(dstElement.Interface())
 				}
+				dst.SetMapIndex(key, srcElement)
 			}
-			dstElement, err = deepMerge(dstElement, srcElement, visited, depth+1, config)
-			if err != nil {
-				return
-			}
-			dst.SetMapIndex(key, dstElement)
-
 		}
 	case reflect.Slice:
-		newSlice := dst
+		if !dst.CanSet() {
-		if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
-			if typeCheck && src.Type() != dst.Type() {
-				return dst, fmt.Errorf("cannot override two slices with different type (%s, %s)", src.Type(), dst.Type())
-			}
-			newSlice = src
-		} else if config.AppendSlice {
-			if typeCheck && src.Type() != dst.Type() {
-				err = fmt.Errorf("cannot append two slice with different type (%s, %s)", src.Type(), dst.Type())
-				return
-			}
-			newSlice = reflect.AppendSlice(dst, src)
-		}
-		if dst.CanSet() {
-			dst.Set(newSlice)
-		} else {
-			dst = newSlice
-		}
-	case reflect.Ptr, reflect.Interface:
-		if isReflectNil(src) {
 			break
 		}
-
+		if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice && !sliceDeepCopy {
-		if dst.Kind() != reflect.Ptr && src.Type().AssignableTo(dst.Type()) {
+			dst.Set(src)
-			if dst.IsNil() || overwrite {
+		} else if config.AppendSlice {
-				if overwrite || isEmptyValue(dst) {
+			if src.Type() != dst.Type() {
-					if dst.CanSet() {
+				return fmt.Errorf("cannot append two slice with different type (%s, %s)", src.Type(), dst.Type())
-						dst.Set(src)
+			}
-					} else {
+			dst.Set(reflect.AppendSlice(dst, src))
-						dst = src
+		} else if sliceDeepCopy {
-					}
+			for i := 0; i < src.Len() && i < dst.Len(); i++ {
+				srcElement := src.Index(i)
+				dstElement := dst.Index(i)
+				if srcElement.CanInterface() {
+					srcElement = reflect.ValueOf(srcElement.Interface())
 				}
+				if dstElement.CanInterface() {
+					dstElement = reflect.ValueOf(dstElement.Interface())
+				}
+
+				if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
+					return
+				}
+			}
+		}
+	case reflect.Ptr:
+		fallthrough
+	case reflect.Interface:
+		if isReflectNil(src) {
+			if overwriteWithEmptySrc && dst.CanSet() && src.Type().AssignableTo(dst.Type()) {
+				dst.Set(src)
 			}
 			break
 		}
@@ -214,33 +243,35 @@ func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int,
 					dst.Set(src)
 				}
 			} else if src.Kind() == reflect.Ptr {
-				if dst, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
+				if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
 					return
 				}
-				dst = dst.Addr()
 			} else if dst.Elem().Type() == src.Type() {
-				if dst, err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil {
+				if err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil {
 					return
 				}
 			} else {
-				return dst, ErrDifferentArgumentsTypes
+				return ErrDifferentArgumentsTypes
 			}
 			break
 		}
+
 		if dst.IsNil() || overwrite {
-			if (overwrite || isEmptyValue(dst)) && (overwriteWithEmptySrc || !isEmptyValue(src)) {
+			if dst.CanSet() && (overwrite || isEmptyValue(dst)) {
-				if dst.CanSet() {
+				dst.Set(src)
-					dst.Set(src)
-				} else {
-					dst = src
-				}
 			}
-		} else if _, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
+			break
-			return
+		}
+
+		if dst.Elem().Kind() == src.Elem().Kind() {
+			if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
+				return
+			}
+			break
 		}
 	default:
-		overwriteFull := (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst))
+		mustSet := (isEmptyValue(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc)
-		if overwriteFull {
+		if mustSet {
 			if dst.CanSet() {
 				dst.Set(src)
 			} else {
@@ -281,6 +312,7 @@ func WithOverride(config *Config) {
 
 // WithOverwriteWithEmptyValue will make merge override non empty dst attributes with empty src attributes values.
 func WithOverwriteWithEmptyValue(config *Config) {
+	config.Overwrite = true
 	config.overwriteWithEmptyValue = true
 }
 
@@ -299,7 +331,16 @@ func WithTypeCheck(config *Config) {
 	config.TypeCheck = true
 }
 
+// WithSliceDeepCopy will merge slice element one by one with Overwrite flag.
+func WithSliceDeepCopy(config *Config) {
+	config.sliceDeepCopy = true
+	config.Overwrite = true
+}
+
 func merge(dst, src interface{}, opts ...func(*Config)) error {
+	if dst != nil && reflect.ValueOf(dst).Kind() != reflect.Ptr {
+		return ErrNonPointerAgument
+	}
 	var (
 		vDst, vSrc reflect.Value
 		err        error
@@ -314,14 +355,10 @@ func merge(dst, src interface{}, opts ...func(*Config)) error {
 	if vDst, vSrc, err = resolveValues(dst, src); err != nil {
 		return err
 	}
-	if !vDst.CanSet() {
-		return fmt.Errorf("cannot set dst, needs reference")
-	}
 	if vDst.Type() != vSrc.Type() {
 		return ErrDifferentArgumentsTypes
 	}
-	_, err = deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
+	return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
-	return err
 }
 
 // IsReflectNil is the reflect value provided nil

vendor/github.com/imdario/mergo/mergo.go

@@ -20,6 +20,7 @@ var (
 	ErrNotSupported                = errors.New("only structs and maps are supported")
 	ErrExpectedMapAsDestination    = errors.New("dst was expected to be a map")
 	ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct")
+	ErrNonPointerAgument           = errors.New("dst must be a pointer")
 )
 
 // During deepMerge, must keep track of checks that are
@@ -75,23 +76,3 @@ func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) {
 	}
 	return
 }
-
-// Traverses recursively both values, assigning src's fields values to dst.
-// The map argument tracks comparisons that have already been seen, which allows
-// short circuiting on recursive types.
-func deeper(dst, src reflect.Value, visited map[uintptr]*visit, depth int) (err error) {
-	if dst.CanAddr() {
-		addr := dst.UnsafeAddr()
-		h := 17 * addr
-		seen := visited[h]
-		typ := dst.Type()
-		for p := seen; p != nil; p = p.next {
-			if p.ptr == addr && p.typ == typ {
-				return nil
-			}
-		}
-		// Remember, remember...
-		visited[h] = &visit{addr, typ, seen}
-	}
-	return // TODO refactor
-}

vendor/github.com/mattn/go-isatty/.travis.yml

@@ -1,14 +0,0 @@
-language: go
-sudo: false
-go:
-  - 1.13.x
-  - tip
-
-before_install:
-  - go get -t -v ./...
-
-script:
-  - ./go.test.sh
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)

vendor/github.com/mattn/go-isatty/LICENSE

@@ -1,9 +0,0 @@
-Copyright (c) Yasuhiro MATSUMOTO <mattn.jp@gmail.com>
-
-MIT License (Expat)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/mattn/go-isatty/README.md

@@ -1,50 +0,0 @@
-# go-isatty
-
-[![Godoc Reference](https://godoc.org/github.com/mattn/go-isatty?status.svg)](http://godoc.org/github.com/mattn/go-isatty)
-[![Codecov](https://codecov.io/gh/mattn/go-isatty/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-isatty)
-[![Coverage Status](https://coveralls.io/repos/github/mattn/go-isatty/badge.svg?branch=master)](https://coveralls.io/github/mattn/go-isatty?branch=master)
-[![Go Report Card](https://goreportcard.com/badge/mattn/go-isatty)](https://goreportcard.com/report/mattn/go-isatty)
-
-isatty for golang
-
-## Usage
-
-```go
-package main
-
-import (
-	"fmt"
-	"github.com/mattn/go-isatty"
-	"os"
-)
-
-func main() {
-	if isatty.IsTerminal(os.Stdout.Fd()) {
-		fmt.Println("Is Terminal")
-	} else if isatty.IsCygwinTerminal(os.Stdout.Fd()) {
-		fmt.Println("Is Cygwin/MSYS2 Terminal")
-	} else {
-		fmt.Println("Is Not Terminal")
-	}
-}
-```
-
-## Installation
-
-```
-$ go get github.com/mattn/go-isatty
-```
-
-## License
-
-MIT
-
-## Author
-
-Yasuhiro Matsumoto (a.k.a mattn)
-
-## Thanks
-
-* k-takata: base idea for IsCygwinTerminal
-
-    https://github.com/k-takata/go-iscygpty

vendor/github.com/mattn/go-isatty/doc.go

@@ -1,2 +0,0 @@
-// Package isatty implements interface to isatty
-package isatty

vendor/github.com/mattn/go-isatty/go.mod

@@ -1,5 +0,0 @@
-module github.com/mattn/go-isatty
-
-go 1.12
-
-require golang.org/x/sys v0.0.0-20200116001909-b77594299b42

vendor/github.com/mattn/go-isatty/go.sum

@@ -1,2 +0,0 @@
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/mattn/go-isatty/go.test.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-echo "" > coverage.txt
-
-for d in $(go list ./... | grep -v vendor); do
-    go test -race -coverprofile=profile.out -covermode=atomic "$d"
-    if [ -f profile.out ]; then
-        cat profile.out >> coverage.txt
-        rm profile.out
-    fi
-done

vendor/github.com/mattn/go-isatty/isatty_bsd.go

@@ -1,18 +0,0 @@
-// +build darwin freebsd openbsd netbsd dragonfly
-// +build !appengine
-
-package isatty
-
-import "golang.org/x/sys/unix"
-
-// IsTerminal return true if the file descriptor is terminal.
-func IsTerminal(fd uintptr) bool {
-	_, err := unix.IoctlGetTermios(int(fd), unix.TIOCGETA)
-	return err == nil
-}
-
-// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
-// terminal. This is also always false on this environment.
-func IsCygwinTerminal(fd uintptr) bool {
-	return false
-}

vendor/github.com/mattn/go-isatty/isatty_others.go

@@ -1,15 +0,0 @@
-// +build appengine js nacl
-
-package isatty
-
-// IsTerminal returns true if the file descriptor is terminal which
-// is always false on js and appengine classic which is a sandboxed PaaS.
-func IsTerminal(fd uintptr) bool {
-	return false
-}
-
-// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2
-// terminal. This is also always false on this environment.
-func IsCygwinTerminal(fd uintptr) bool {
-	return false
-}

vendor/github.com/mattn/go-isatty/isatty_plan9.go

@@ -1,22 +0,0 @@
-// +build plan9
-
-package isatty
-
-import (
-	"syscall"
-)
-
-// IsTerminal returns true if the given file descriptor is a terminal.
-func IsTerminal(fd uintptr) bool {
-	path, err := syscall.Fd2path(int(fd))
-	if err != nil {
-		return false
-	}
-	return path == "/dev/cons" || path == "/mnt/term/dev/cons"
-}
-
-// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
-// terminal. This is also always false on this environment.
-func IsCygwinTerminal(fd uintptr) bool {
-	return false
-}

vendor/github.com/mattn/go-isatty/isatty_solaris.go

@@ -1,22 +0,0 @@
-// +build solaris
-// +build !appengine
-
-package isatty
-
-import (
-	"golang.org/x/sys/unix"
-)
-
-// IsTerminal returns true if the given file descriptor is a terminal.
-// see: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libbc/libc/gen/common/isatty.c
-func IsTerminal(fd uintptr) bool {
-	var termio unix.Termio
-	err := unix.IoctlSetTermio(int(fd), unix.TCGETA, &termio)
-	return err == nil
-}
-
-// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
-// terminal. This is also always false on this environment.
-func IsCygwinTerminal(fd uintptr) bool {
-	return false
-}

vendor/github.com/mattn/go-isatty/isatty_tcgets.go

@@ -1,18 +0,0 @@
-// +build linux aix
-// +build !appengine
-
-package isatty
-
-import "golang.org/x/sys/unix"
-
-// IsTerminal return true if the file descriptor is terminal.
-func IsTerminal(fd uintptr) bool {
-	_, err := unix.IoctlGetTermios(int(fd), unix.TCGETS)
-	return err == nil
-}
-
-// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
-// terminal. This is also always false on this environment.
-func IsCygwinTerminal(fd uintptr) bool {
-	return false
-}

vendor/github.com/mattn/go-isatty/isatty_windows.go

@@ -1,125 +0,0 @@
-// +build windows
-// +build !appengine
-
-package isatty
-
-import (
-	"errors"
-	"strings"
-	"syscall"
-	"unicode/utf16"
-	"unsafe"
-)
-
-const (
-	objectNameInfo uintptr = 1
-	fileNameInfo           = 2
-	fileTypePipe           = 3
-)
-
-var (
-	kernel32                         = syscall.NewLazyDLL("kernel32.dll")
-	ntdll                            = syscall.NewLazyDLL("ntdll.dll")
-	procGetConsoleMode               = kernel32.NewProc("GetConsoleMode")
-	procGetFileInformationByHandleEx = kernel32.NewProc("GetFileInformationByHandleEx")
-	procGetFileType                  = kernel32.NewProc("GetFileType")
-	procNtQueryObject                = ntdll.NewProc("NtQueryObject")
-)
-
-func init() {
-	// Check if GetFileInformationByHandleEx is available.
-	if procGetFileInformationByHandleEx.Find() != nil {
-		procGetFileInformationByHandleEx = nil
-	}
-}
-
-// IsTerminal return true if the file descriptor is terminal.
-func IsTerminal(fd uintptr) bool {
-	var st uint32
-	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, fd, uintptr(unsafe.Pointer(&st)), 0)
-	return r != 0 && e == 0
-}
-
-// Check pipe name is used for cygwin/msys2 pty.
-// Cygwin/MSYS2 PTY has a name like:
-//   \{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master
-func isCygwinPipeName(name string) bool {
-	token := strings.Split(name, "-")
-	if len(token) < 5 {
-		return false
-	}
-
-	if token[0] != `\msys` &&
-		token[0] != `\cygwin` &&
-		token[0] != `\Device\NamedPipe\msys` &&
-		token[0] != `\Device\NamedPipe\cygwin` {
-		return false
-	}
-
-	if token[1] == "" {
-		return false
-	}
-
-	if !strings.HasPrefix(token[2], "pty") {
-		return false
-	}
-
-	if token[3] != `from` && token[3] != `to` {
-		return false
-	}
-
-	if token[4] != "master" {
-		return false
-	}
-
-	return true
-}
-
-// getFileNameByHandle use the undocomented ntdll NtQueryObject to get file full name from file handler
-// since GetFileInformationByHandleEx is not avilable under windows Vista and still some old fashion
-// guys are using Windows XP, this is a workaround for those guys, it will also work on system from
-// Windows vista to 10
-// see https://stackoverflow.com/a/18792477 for details
-func getFileNameByHandle(fd uintptr) (string, error) {
-	if procNtQueryObject == nil {
-		return "", errors.New("ntdll.dll: NtQueryObject not supported")
-	}
-
-	var buf [4 + syscall.MAX_PATH]uint16
-	var result int
-	r, _, e := syscall.Syscall6(procNtQueryObject.Addr(), 5,
-		fd, objectNameInfo, uintptr(unsafe.Pointer(&buf)), uintptr(2*len(buf)), uintptr(unsafe.Pointer(&result)), 0)
-	if r != 0 {
-		return "", e
-	}
-	return string(utf16.Decode(buf[4 : 4+buf[0]/2])), nil
-}
-
-// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2
-// terminal.
-func IsCygwinTerminal(fd uintptr) bool {
-	if procGetFileInformationByHandleEx == nil {
-		name, err := getFileNameByHandle(fd)
-		if err != nil {
-			return false
-		}
-		return isCygwinPipeName(name)
-	}
-
-	// Cygwin/msys's pty is a pipe.
-	ft, _, e := syscall.Syscall(procGetFileType.Addr(), 1, fd, 0, 0)
-	if ft != fileTypePipe || e != 0 {
-		return false
-	}
-
-	var buf [2 + syscall.MAX_PATH]uint16
-	r, _, e := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(),
-		4, fd, fileNameInfo, uintptr(unsafe.Pointer(&buf)),
-		uintptr(len(buf)*2), 0, 0)
-	if r == 0 || e != 0 {
-		return false
-	}
-
-	l := *(*uint32)(unsafe.Pointer(&buf))
-	return isCygwinPipeName(string(utf16.Decode(buf[2 : 2+l/2])))
-}

vendor/github.com/mattn/go-isatty/renovate.json

@@ -1,8 +0,0 @@
-{
-  "extends": [
-    "config:base"
-  ],
-  "postUpdateOptions": [
-    "gomodTidy"
-  ]
-}

vendor/github.com/ulikunitz/xz/TODO.md

@@ -90,6 +90,11 @@
 
 ## Log
 
+## 2020-08-19
+
+Release v0.5.8 fixes issue
+[issue #35](https://github.com/ulikunitz/xz/issues/35).
+
 ### 2020-02-24
 
 Release v0.5.7 supports the check-ID None and fixes

vendor/github.com/ulikunitz/xz/bits.go

@@ -54,6 +54,8 @@ var errOverflowU64 = errors.New("xz: uvarint overflows 64-bit unsigned integer")
 
 // readUvarint reads a uvarint from the given byte reader.
 func readUvarint(r io.ByteReader) (x uint64, n int, err error) {
+	const maxUvarintLen = 10
+
 	var s uint
 	i := 0
 	for {
@@ -62,8 +64,11 @@ func readUvarint(r io.ByteReader) (x uint64, n int, err error) {
 			return x, i, err
 		}
 		i++
+		if i > maxUvarintLen {
+			return x, i, errOverflowU64
+		}
 		if b < 0x80 {
-			if i > 10 || i == 10 && b > 1 {
+			if i == maxUvarintLen && b > 1 {
 				return x, i, errOverflowU64
 			}
 			return x | uint64(b)<<s, i, nil

vendor/github.com/vbauerster/mpb/v5/bar.go

@@ -86,7 +86,7 @@ func newBar(container *Progress, bs *bState) *Bar {
 		noPop:        bs.noPop,
 		operateState: make(chan func(*bState)),
 		frameCh:      make(chan io.Reader, 1),
-		syncTableCh:  make(chan [][]chan int),
+		syncTableCh:  make(chan [][]chan int, 1),
 		completed:    make(chan bool, 1),
 		done:         make(chan struct{}),
 		cancel:       cancel,
@@ -132,14 +132,18 @@ func (b *Bar) Current() int64 {
 // Given default bar style is "[=>-]<+", refill rune is '+'.
 // To set bar style use mpb.BarStyle(string) BarOption.
 func (b *Bar) SetRefill(amount int64) {
-	b.operateState <- func(s *bState) {
+	select {
+	case b.operateState <- func(s *bState) {
 		s.refill = amount
+	}:
+	case <-b.done:
 	}
 }
 
 // TraverseDecorators traverses all available decorators and calls cb func on each.
 func (b *Bar) TraverseDecorators(cb func(decor.Decorator)) {
-	b.operateState <- func(s *bState) {
+	select {
+	case b.operateState <- func(s *bState) {
 		for _, decorators := range [...][]decor.Decorator{
 			s.pDecorators,
 			s.aDecorators,
@@ -148,6 +152,8 @@ func (b *Bar) TraverseDecorators(cb func(decor.Decorator)) {
 				cb(extractBaseDecorator(d))
 			}
 		}
+	}:
+	case <-b.done:
 	}
 }
 
@@ -174,6 +180,7 @@ func (b *Bar) SetTotal(total int64, complete bool) {
 }
 
 // SetCurrent sets progress' current to an arbitrary value.
+// Setting a negative value will cause a panic.
 func (b *Bar) SetCurrent(current int64) {
 	select {
 	case b.operateState <- func(s *bState) {
@@ -305,11 +312,13 @@ func (b *Bar) render(tw int) {
 		defer func() {
 			// recovering if user defined decorator panics for example
 			if p := recover(); p != nil {
-				s.extender = makePanicExtender(p)
+				if b.recoveredPanic == nil {
+					s.extender = makePanicExtender(p)
+					b.toShutdown = !b.toShutdown
+					b.recoveredPanic = p
+				}
 				frame, lines := s.extender(nil, s.reqWidth, stat)
 				b.extendedLines = lines
-				b.toShutdown = !b.toShutdown
-				b.recoveredPanic = p
 				b.frameCh <- frame
 				b.dlogger.Println(p)
 			}
@@ -348,12 +357,15 @@ func (b *Bar) subscribeDecorators() {
 			shutdownListeners = append(shutdownListeners, d)
 		}
 	})
-	b.operateState <- func(s *bState) {
+	select {
+	case b.operateState <- func(s *bState) {
 		s.averageDecorators = averageDecorators
 		s.ewmaDecorators = ewmaDecorators
 		s.shutdownListeners = shutdownListeners
+	}:
+		b.hasEwmaDecorators = len(ewmaDecorators) != 0
+	case <-b.done:
 	}
-	b.hasEwmaDecorators = len(ewmaDecorators) != 0
 }
 
 func (b *Bar) refreshTillShutdown() {

vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go

@@ -0,0 +1,7 @@
+// +build darwin dragonfly freebsd netbsd openbsd
+
+package cwriter
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TIOCGETA

vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go

@@ -0,0 +1,7 @@
+// +build aix linux
+
+package cwriter
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TCGETS

vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go

@@ -0,0 +1,7 @@
+// +build solaris
+
+package cwriter
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TCGETA

vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go

@@ -3,17 +3,19 @@ package cwriter
 import (
 	"bytes"
 	"errors"
-	"fmt"
 	"io"
 	"os"
-
+	"strconv"
-	"github.com/mattn/go-isatty"
 )
 
 // NotATTY not a TeleTYpewriter error.
 var NotATTY = errors.New("not a terminal")
 
-var cuuAndEd = fmt.Sprintf("%c[%%dA%[1]c[J", 27)
+// http://ascii-table.com/ansi-escape-sequences.php
+const (
+	escOpen  = "\x1b["
+	cuuAndEd = "A\x1b[J"
+)
 
 // Writer is a buffered the writer that updates the terminal. The
 // contents of writer will be flushed when Flush is called.
@@ -21,7 +23,7 @@ type Writer struct {
 	out        io.Writer
 	buf        bytes.Buffer
 	lineCount  int
-	fd         uintptr
+	fd         int
 	isTerminal bool
 }
 
@@ -29,8 +31,8 @@ type Writer struct {
 func New(out io.Writer) *Writer {
 	w := &Writer{out: out}
 	if f, ok := out.(*os.File); ok {
-		w.fd = f.Fd()
+		w.fd = int(f.Fd())
-		w.isTerminal = isatty.IsTerminal(w.fd)
+		w.isTerminal = IsTerminal(w.fd)
 	}
 	return w
 }
@@ -39,7 +41,10 @@ func New(out io.Writer) *Writer {
 func (w *Writer) Flush(lineCount int) (err error) {
 	// some terminals interpret clear 0 lines as clear 1
 	if w.lineCount > 0 {
-		w.clearLines()
+		err = w.clearLines()
+		if err != nil {
+			return
+		}
 	}
 	w.lineCount = lineCount
 	_, err = w.buf.WriteTo(w.out)
@@ -70,3 +75,10 @@ func (w *Writer) GetWidth() (int, error) {
 	tw, _, err := GetSize(w.fd)
 	return tw, err
 }
+
+func (w *Writer) ansiCuuAndEd() (err error) {
+	buf := make([]byte, 8)
+	buf = strconv.AppendInt(buf[:copy(buf, escOpen)], int64(w.lineCount), 10)
+	_, err = w.out.Write(append(buf, cuuAndEd...))
+	return
+}

vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go

@@ -3,20 +3,24 @@
 package cwriter
 
 import (
-	"fmt"
-
 	"golang.org/x/sys/unix"
 )
 
-func (w *Writer) clearLines() {
+func (w *Writer) clearLines() error {
-	fmt.Fprintf(w.out, cuuAndEd, w.lineCount)
+	return w.ansiCuuAndEd()
 }
 
 // GetSize returns the dimensions of the given terminal.
-func GetSize(fd uintptr) (width, height int, err error) {
+func GetSize(fd int) (width, height int, err error) {
-	ws, err := unix.IoctlGetWinsize(int(fd), unix.TIOCGWINSZ)
+	ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ)
 	if err != nil {
 		return -1, -1, err
 	}
 	return int(ws.Col), int(ws.Row), nil
 }
+
+// IsTerminal returns whether the given file descriptor is a terminal.
+func IsTerminal(fd int) bool {
+	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	return err == nil
+}

vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go

@@ -3,67 +3,71 @@
 package cwriter
 
 import (
-	"fmt"
-	"syscall"
 	"unsafe"
+
+	"golang.org/x/sys/windows"
 )
 
-var kernel32 = syscall.NewLazyDLL("kernel32.dll")
+var kernel32 = windows.NewLazySystemDLL("kernel32.dll")
 
 var (
-	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
 	procSetConsoleCursorPosition   = kernel32.NewProc("SetConsoleCursorPosition")
 	procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW")
 )
 
-type coord struct {
+func (w *Writer) clearLines() error {
-	x int16
-	y int16
-}
-
-type smallRect struct {
-	left   int16
-	top    int16
-	right  int16
-	bottom int16
-}
-
-type consoleScreenBufferInfo struct {
-	size              coord
-	cursorPosition    coord
-	attributes        uint16
-	window            smallRect
-	maximumWindowSize coord
-}
-
-func (w *Writer) clearLines() {
 	if !w.isTerminal {
-		fmt.Fprintf(w.out, cuuAndEd, w.lineCount)
+		// hope it's cygwin or similar
+		return w.ansiCuuAndEd()
 	}
 
-	info := new(consoleScreenBufferInfo)
+	var info windows.ConsoleScreenBufferInfo
-	procGetConsoleScreenBufferInfo.Call(w.fd, uintptr(unsafe.Pointer(info)))
+	if err := windows.GetConsoleScreenBufferInfo(windows.Handle(w.fd), &info); err != nil {
-
+		return err
-	info.cursorPosition.y -= int16(w.lineCount)
-	if info.cursorPosition.y < 0 {
-		info.cursorPosition.y = 0
 	}
-	procSetConsoleCursorPosition.Call(w.fd, uintptr(uint32(uint16(info.cursorPosition.y))<<16|uint32(uint16(info.cursorPosition.x))))
+
+	info.CursorPosition.Y -= int16(w.lineCount)
+	if info.CursorPosition.Y < 0 {
+		info.CursorPosition.Y = 0
+	}
+	_, _, _ = procSetConsoleCursorPosition.Call(
+		uintptr(w.fd),
+		uintptr(uint32(uint16(info.CursorPosition.Y))<<16|uint32(uint16(info.CursorPosition.X))),
+	)
 
 	// clear the lines
-	cursor := &coord{
+	cursor := &windows.Coord{
-		x: info.window.left,
+		X: info.Window.Left,
-		y: info.cursorPosition.y,
+		Y: info.CursorPosition.Y,
 	}
-	count := uint32(info.size.x) * uint32(w.lineCount)
+	count := uint32(info.Size.X) * uint32(w.lineCount)
-	procFillConsoleOutputCharacter.Call(w.fd, uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(cursor)), uintptr(unsafe.Pointer(new(uint32))))
+	_, _, _ = procFillConsoleOutputCharacter.Call(
+		uintptr(w.fd),
+		uintptr(' '),
+		uintptr(count),
+		*(*uintptr)(unsafe.Pointer(cursor)),
+		uintptr(unsafe.Pointer(new(uint32))),
+	)
+	return nil
 }
 
 // GetSize returns the visible dimensions of the given terminal.
 //
 // These dimensions don't include any scrollback buffer height.
-func GetSize(fd uintptr) (width, height int, err error) {
+func GetSize(fd int) (width, height int, err error) {
-	info := new(consoleScreenBufferInfo)
+	var info windows.ConsoleScreenBufferInfo
-	procGetConsoleScreenBufferInfo.Call(fd, uintptr(unsafe.Pointer(info)))
+	if err := windows.GetConsoleScreenBufferInfo(windows.Handle(fd), &info); err != nil {
-	return int(info.window.right - info.window.left), int(info.window.bottom - info.window.top), nil
+		return 0, 0, err
+	}
+	// terminal.GetSize from crypto/ssh adds "+ 1" to both width and height:
+	// https://go.googlesource.com/crypto/+/refs/heads/release-branch.go1.14/ssh/terminal/util_windows.go#75
+	// but looks like this is a root cause of issue #66, so removing both "+ 1" have fixed it.
+	return int(info.Window.Right - info.Window.Left), int(info.Window.Bottom - info.Window.Top), nil
+}
+
+// IsTerminal returns whether the given file descriptor is a terminal.
+func IsTerminal(fd int) bool {
+	var st uint32
+	err := windows.GetConsoleMode(windows.Handle(fd), &st)
+	return err == nil
 }

vendor/github.com/vbauerster/mpb/v5/decor/counters.go

@@ -2,6 +2,7 @@ package decor
 
 import (
 	"fmt"
+	"strings"
 )
 
 const (
@@ -31,7 +32,7 @@ func CountersKiloByte(pairFmt string, wcc ...WC) Decorator {
 //
 //	`unit` one of [0|UnitKiB|UnitKB] zero for no unit
 //
-//	`pairFmt` printf compatible verbs for current and total, like "%f" or "%d"
+//	`pairFmt` printf compatible verbs for current and total pair
 //
 //	`wcc` optional WC config
 //
@@ -43,25 +44,200 @@ func CountersKiloByte(pairFmt string, wcc ...WC) Decorator {
 //	pairFmt="% d / % d"     output: "1 MB / 12 MB"
 //
 func Counters(unit int, pairFmt string, wcc ...WC) Decorator {
-	return Any(chooseSizeProducer(unit, pairFmt), wcc...)
+	producer := func(unit int, pairFmt string) DecorFunc {
+		if pairFmt == "" {
+			pairFmt = "%d / %d"
+		} else if strings.Count(pairFmt, "%") != 2 {
+			panic("expected pairFmt with exactly 2 verbs")
+		}
+		switch unit {
+		case UnitKiB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(pairFmt, SizeB1024(s.Current), SizeB1024(s.Total))
+			}
+		case UnitKB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(pairFmt, SizeB1000(s.Current), SizeB1000(s.Total))
+			}
+		default:
+			return func(s Statistics) string {
+				return fmt.Sprintf(pairFmt, s.Current, s.Total)
+			}
+		}
+	}
+	return Any(producer(unit, pairFmt), wcc...)
 }
 
-func chooseSizeProducer(unit int, format string) DecorFunc {
+// TotalNoUnit is a wrapper around Total with no unit param.
-	if format == "" {
+func TotalNoUnit(format string, wcc ...WC) Decorator {
-		format = "%d / %d"
+	return Total(0, format, wcc...)
-	}
+}
-	switch unit {
+
-	case UnitKiB:
+// TotalKibiByte is a wrapper around Total with predefined unit
-		return func(s Statistics) string {
+// UnitKiB (bytes/1024).
-			return fmt.Sprintf(format, SizeB1024(s.Current), SizeB1024(s.Total))
+func TotalKibiByte(format string, wcc ...WC) Decorator {
-		}
+	return Total(UnitKiB, format, wcc...)
-	case UnitKB:
+}
-		return func(s Statistics) string {
+
-			return fmt.Sprintf(format, SizeB1000(s.Current), SizeB1000(s.Total))
+// TotalKiloByte is a wrapper around Total with predefined unit
-		}
+// UnitKB (bytes/1000).
-	default:
+func TotalKiloByte(format string, wcc ...WC) Decorator {
-		return func(s Statistics) string {
+	return Total(UnitKB, format, wcc...)
-			return fmt.Sprintf(format, s.Current, s.Total)
+}
-		}
+
-	}
+// Total decorator with dynamic unit measure adjustment.
+//
+//	`unit` one of [0|UnitKiB|UnitKB] zero for no unit
+//
+//	`format` printf compatible verb for Total
+//
+//	`wcc` optional WC config
+//
+// format example if unit=UnitKiB:
+//
+//	format="%.1f"  output: "12.0MiB"
+//	format="% .1f" output: "12.0 MiB"
+//	format="%d"    output: "12MiB"
+//	format="% d"   output: "12 MiB"
+//
+func Total(unit int, format string, wcc ...WC) Decorator {
+	producer := func(unit int, format string) DecorFunc {
+		if format == "" {
+			format = "%d"
+		} else if strings.Count(format, "%") != 1 {
+			panic("expected format with exactly 1 verb")
+		}
+
+		switch unit {
+		case UnitKiB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, SizeB1024(s.Total))
+			}
+		case UnitKB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, SizeB1000(s.Total))
+			}
+		default:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, s.Total)
+			}
+		}
+	}
+	return Any(producer(unit, format), wcc...)
+}
+
+// CurrentNoUnit is a wrapper around Current with no unit param.
+func CurrentNoUnit(format string, wcc ...WC) Decorator {
+	return Current(0, format, wcc...)
+}
+
+// CurrentKibiByte is a wrapper around Current with predefined unit
+// UnitKiB (bytes/1024).
+func CurrentKibiByte(format string, wcc ...WC) Decorator {
+	return Current(UnitKiB, format, wcc...)
+}
+
+// CurrentKiloByte is a wrapper around Current with predefined unit
+// UnitKB (bytes/1000).
+func CurrentKiloByte(format string, wcc ...WC) Decorator {
+	return Current(UnitKB, format, wcc...)
+}
+
+// Current decorator with dynamic unit measure adjustment.
+//
+//	`unit` one of [0|UnitKiB|UnitKB] zero for no unit
+//
+//	`format` printf compatible verb for Current
+//
+//	`wcc` optional WC config
+//
+// format example if unit=UnitKiB:
+//
+//	format="%.1f"  output: "12.0MiB"
+//	format="% .1f" output: "12.0 MiB"
+//	format="%d"    output: "12MiB"
+//	format="% d"   output: "12 MiB"
+//
+func Current(unit int, format string, wcc ...WC) Decorator {
+	producer := func(unit int, format string) DecorFunc {
+		if format == "" {
+			format = "%d"
+		} else if strings.Count(format, "%") != 1 {
+			panic("expected format with exactly 1 verb")
+		}
+
+		switch unit {
+		case UnitKiB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, SizeB1024(s.Current))
+			}
+		case UnitKB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, SizeB1000(s.Current))
+			}
+		default:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, s.Current)
+			}
+		}
+	}
+	return Any(producer(unit, format), wcc...)
+}
+
+// InvertedCurrentNoUnit is a wrapper around InvertedCurrent with no unit param.
+func InvertedCurrentNoUnit(format string, wcc ...WC) Decorator {
+	return InvertedCurrent(0, format, wcc...)
+}
+
+// InvertedCurrentKibiByte is a wrapper around InvertedCurrent with predefined unit
+// UnitKiB (bytes/1024).
+func InvertedCurrentKibiByte(format string, wcc ...WC) Decorator {
+	return InvertedCurrent(UnitKiB, format, wcc...)
+}
+
+// InvertedCurrentKiloByte is a wrapper around InvertedCurrent with predefined unit
+// UnitKB (bytes/1000).
+func InvertedCurrentKiloByte(format string, wcc ...WC) Decorator {
+	return InvertedCurrent(UnitKB, format, wcc...)
+}
+
+// InvertedCurrent decorator with dynamic unit measure adjustment.
+//
+//	`unit` one of [0|UnitKiB|UnitKB] zero for no unit
+//
+//	`format` printf compatible verb for InvertedCurrent
+//
+//	`wcc` optional WC config
+//
+// format example if unit=UnitKiB:
+//
+//	format="%.1f"  output: "12.0MiB"
+//	format="% .1f" output: "12.0 MiB"
+//	format="%d"    output: "12MiB"
+//	format="% d"   output: "12 MiB"
+//
+func InvertedCurrent(unit int, format string, wcc ...WC) Decorator {
+	producer := func(unit int, format string) DecorFunc {
+		if format == "" {
+			format = "%d"
+		} else if strings.Count(format, "%") != 1 {
+			panic("expected format with exactly 1 verb")
+		}
+
+		switch unit {
+		case UnitKiB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, SizeB1024(s.Total-s.Current))
+			}
+		case UnitKB:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, SizeB1000(s.Total-s.Current))
+			}
+		default:
+			return func(s Statistics) string {
+				return fmt.Sprintf(format, s.Total-s.Current)
+			}
+		}
+	}
+	return Any(producer(unit, format), wcc...)
 }

vendor/github.com/vbauerster/mpb/v5/go.mod

@@ -3,9 +3,8 @@ module github.com/vbauerster/mpb/v5
 require (
 	github.com/VividCortex/ewma v1.1.1
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
-	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-runewidth v0.0.9
-	golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
+	golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed
 )
 
 go 1.14

vendor/github.com/vbauerster/mpb/v5/go.sum

@@ -2,10 +2,7 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed h1:WBkVNH1zd9jg/dK4HCM4lNANnmd12EHC9z+LmcCG4ns=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
+golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/golang.org/x/sys/cpu/cpu.go

@@ -6,6 +6,11 @@
 // various CPU architectures.
 package cpu
 
+import (
+	"os"
+	"strings"
+)
+
 // Initialized reports whether the CPU features were initialized.
 //
 // For some GOOS/GOARCH combinations initialization of the CPU features depends
@@ -169,3 +174,94 @@ var S390X struct {
 	HasVXE    bool // vector-enhancements facility 1
 	_         CacheLinePad
 }
+
+func init() {
+	archInit()
+	initOptions()
+	processOptions()
+}
+
+// options contains the cpu debug options that can be used in GODEBUG.
+// Options are arch dependent and are added by the arch specific initOptions functions.
+// Features that are mandatory for the specific GOARCH should have the Required field set
+// (e.g. SSE2 on amd64).
+var options []option
+
+// Option names should be lower case. e.g. avx instead of AVX.
+type option struct {
+	Name      string
+	Feature   *bool
+	Specified bool // whether feature value was specified in GODEBUG
+	Enable    bool // whether feature should be enabled
+	Required  bool // whether feature is mandatory and can not be disabled
+}
+
+func processOptions() {
+	env := os.Getenv("GODEBUG")
+field:
+	for env != "" {
+		field := ""
+		i := strings.IndexByte(env, ',')
+		if i < 0 {
+			field, env = env, ""
+		} else {
+			field, env = env[:i], env[i+1:]
+		}
+		if len(field) < 4 || field[:4] != "cpu." {
+			continue
+		}
+		i = strings.IndexByte(field, '=')
+		if i < 0 {
+			print("GODEBUG sys/cpu: no value specified for \"", field, "\"\n")
+			continue
+		}
+		key, value := field[4:i], field[i+1:] // e.g. "SSE2", "on"
+
+		var enable bool
+		switch value {
+		case "on":
+			enable = true
+		case "off":
+			enable = false
+		default:
+			print("GODEBUG sys/cpu: value \"", value, "\" not supported for cpu option \"", key, "\"\n")
+			continue field
+		}
+
+		if key == "all" {
+			for i := range options {
+				options[i].Specified = true
+				options[i].Enable = enable || options[i].Required
+			}
+			continue field
+		}
+
+		for i := range options {
+			if options[i].Name == key {
+				options[i].Specified = true
+				options[i].Enable = enable
+				continue field
+			}
+		}
+
+		print("GODEBUG sys/cpu: unknown cpu feature \"", key, "\"\n")
+	}
+
+	for _, o := range options {
+		if !o.Specified {
+			continue
+		}
+
+		if o.Enable && !*o.Feature {
+			print("GODEBUG sys/cpu: can not enable \"", o.Name, "\", missing CPU support\n")
+			continue
+		}
+
+		if !o.Enable && o.Required {
+			print("GODEBUG sys/cpu: can not disable \"", o.Name, "\", required CPU feature\n")
+			continue
+		}
+
+		*o.Feature = o.Enable
+	}
+}

vendor/golang.org/x/sys/cpu/cpu_aix.go

@@ -6,8 +6,6 @@
 
 package cpu
 
-const cacheLineSize = 128
-
 const (
 	// getsystemcfg constants
 	_SC_IMPL     = 2
@@ -15,7 +13,7 @@ const (
 	_IMPL_POWER9 = 0x20000
 )
 
-func init() {
+func archInit() {
 	impl := getsystemcfg(_SC_IMPL)
 	if impl&_IMPL_POWER8 != 0 {
 		PPC64.IsPOWER8 = true

vendor/golang.org/x/sys/cpu/cpu_arm.go

@@ -38,3 +38,36 @@ const (
 	hwcap2_SHA2  = 1 << 3
 	hwcap2_CRC32 = 1 << 4
 )
+
+func initOptions() {
+	options = []option{
+		{Name: "pmull", Feature: &ARM.HasPMULL},
+		{Name: "sha1", Feature: &ARM.HasSHA1},
+		{Name: "sha2", Feature: &ARM.HasSHA2},
+		{Name: "swp", Feature: &ARM.HasSWP},
+		{Name: "thumb", Feature: &ARM.HasTHUMB},
+		{Name: "thumbee", Feature: &ARM.HasTHUMBEE},
+		{Name: "tls", Feature: &ARM.HasTLS},
+		{Name: "vfp", Feature: &ARM.HasVFP},
+		{Name: "vfpd32", Feature: &ARM.HasVFPD32},
+		{Name: "vfpv3", Feature: &ARM.HasVFPv3},
+		{Name: "vfpv3d16", Feature: &ARM.HasVFPv3D16},
+		{Name: "vfpv4", Feature: &ARM.HasVFPv4},
+		{Name: "half", Feature: &ARM.HasHALF},
+		{Name: "26bit", Feature: &ARM.Has26BIT},
+		{Name: "fastmul", Feature: &ARM.HasFASTMUL},
+		{Name: "fpa", Feature: &ARM.HasFPA},
+		{Name: "edsp", Feature: &ARM.HasEDSP},
+		{Name: "java", Feature: &ARM.HasJAVA},
+		{Name: "iwmmxt", Feature: &ARM.HasIWMMXT},
+		{Name: "crunch", Feature: &ARM.HasCRUNCH},
+		{Name: "neon", Feature: &ARM.HasNEON},
+		{Name: "idivt", Feature: &ARM.HasIDIVT},
+		{Name: "idiva", Feature: &ARM.HasIDIVA},
+		{Name: "lpae", Feature: &ARM.HasLPAE},
+		{Name: "evtstrm", Feature: &ARM.HasEVTSTRM},
+		{Name: "aes", Feature: &ARM.HasAES},
+		{Name: "crc32", Feature: &ARM.HasCRC32},
+	}
+
+}

vendor/golang.org/x/sys/cpu/cpu_arm64.go

@@ -8,7 +8,36 @@ import "runtime"
 
 const cacheLineSize = 64
 
-func init() {
+func initOptions() {
+	options = []option{
+		{Name: "fp", Feature: &ARM64.HasFP},
+		{Name: "asimd", Feature: &ARM64.HasASIMD},
+		{Name: "evstrm", Feature: &ARM64.HasEVTSTRM},
+		{Name: "aes", Feature: &ARM64.HasAES},
+		{Name: "fphp", Feature: &ARM64.HasFPHP},
+		{Name: "jscvt", Feature: &ARM64.HasJSCVT},
+		{Name: "lrcpc", Feature: &ARM64.HasLRCPC},
+		{Name: "pmull", Feature: &ARM64.HasPMULL},
+		{Name: "sha1", Feature: &ARM64.HasSHA1},
+		{Name: "sha2", Feature: &ARM64.HasSHA2},
+		{Name: "sha3", Feature: &ARM64.HasSHA3},
+		{Name: "sha512", Feature: &ARM64.HasSHA512},
+		{Name: "sm3", Feature: &ARM64.HasSM3},
+		{Name: "sm4", Feature: &ARM64.HasSM4},
+		{Name: "sve", Feature: &ARM64.HasSVE},
+		{Name: "crc32", Feature: &ARM64.HasCRC32},
+		{Name: "atomics", Feature: &ARM64.HasATOMICS},
+		{Name: "asimdhp", Feature: &ARM64.HasASIMDHP},
+		{Name: "cpuid", Feature: &ARM64.HasCPUID},
+		{Name: "asimrdm", Feature: &ARM64.HasASIMDRDM},
+		{Name: "fcma", Feature: &ARM64.HasFCMA},
+		{Name: "dcpop", Feature: &ARM64.HasDCPOP},
+		{Name: "asimddp", Feature: &ARM64.HasASIMDDP},
+		{Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM},
+	}
+}
+
+func archInit() {
 	switch runtime.GOOS {
 	case "android", "darwin", "netbsd":
 		// Android and iOS don't seem to allow reading these registers.

vendor/golang.org/x/sys/cpu/cpu_linux.go

@@ -6,7 +6,7 @@
 
 package cpu
 
-func init() {
+func archInit() {
 	if err := readHWCAP(); err != nil {
 		return
 	}

vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go

@@ -7,8 +7,6 @@
 
 package cpu
 
-const cacheLineSize = 128
-
 // HWCAP/HWCAP2 bits. These are exposed by the kernel.
 const (
 	// ISA Level

vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go

@@ -4,8 +4,6 @@
 
 package cpu
 
-const cacheLineSize = 256
-
 const (
 	// bit mask values from /usr/include/bits/hwcap.h
 	hwcap_ZARCH  = 2

vendor/golang.org/x/sys/cpu/cpu_mips64x.go

@@ -7,3 +7,9 @@
 package cpu
 
 const cacheLineSize = 32
+
+func initOptions() {
+	options = []option{
+		{Name: "msa", Feature: &MIPS64X.HasMSA},
+	}
+}

vendor/golang.org/x/sys/cpu/cpu_mipsx.go

@@ -7,3 +7,5 @@
 package cpu
 
 const cacheLineSize = 32
+
+func initOptions() {}

vendor/golang.org/x/sys/cpu/cpu_other_arm.go

@@ -0,0 +1,9 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !linux,arm
+
+package cpu
+
+func archInit() {}

vendor/golang.org/x/sys/cpu/cpu_ppc64x.go

@@ -0,0 +1,16 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ppc64 ppc64le
+
+package cpu
+
+const cacheLineSize = 128
+
+func initOptions() {
+	options = []option{
+		{Name: "darn", Feature: &PPC64.HasDARN},
+		{Name: "scv", Feature: &PPC64.HasSCV},
+	}
+}

vendor/golang.org/x/sys/cpu/cpu_riscv64.go

@@ -7,3 +7,5 @@
 package cpu
 
 const cacheLineSize = 32
+
+func initOptions() {}

vendor/golang.org/x/sys/cpu/cpu_s390x.go

@@ -0,0 +1,30 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+const cacheLineSize = 256
+
+func initOptions() {
+	options = []option{
+		{Name: "zarch", Feature: &S390X.HasZARCH},
+		{Name: "stfle", Feature: &S390X.HasSTFLE},
+		{Name: "ldisp", Feature: &S390X.HasLDISP},
+		{Name: "eimm", Feature: &S390X.HasEIMM},
+		{Name: "dfp", Feature: &S390X.HasDFP},
+		{Name: "etf3eh", Feature: &S390X.HasETF3EH},
+		{Name: "msa", Feature: &S390X.HasMSA},
+		{Name: "aes", Feature: &S390X.HasAES},
+		{Name: "aescbc", Feature: &S390X.HasAESCBC},
+		{Name: "aesctr", Feature: &S390X.HasAESCTR},
+		{Name: "aesgcm", Feature: &S390X.HasAESGCM},
+		{Name: "ghash", Feature: &S390X.HasGHASH},
+		{Name: "sha1", Feature: &S390X.HasSHA1},
+		{Name: "sha256", Feature: &S390X.HasSHA256},
+		{Name: "sha3", Feature: &S390X.HasSHA3},
+		{Name: "sha512", Feature: &S390X.HasSHA512},
+		{Name: "vx", Feature: &S390X.HasVX},
+		{Name: "vxe", Feature: &S390X.HasVXE},
+	}
+}

vendor/golang.org/x/sys/cpu/cpu_wasm.go

@@ -11,3 +11,7 @@ package cpu
 // rules are good enough.
 
 const cacheLineSize = 0
+
+func initOptions() {}
+
+func archInit() {}

vendor/golang.org/x/sys/cpu/cpu_x86.go

@@ -6,9 +6,37 @@
 
 package cpu
 
+import "runtime"
+
 const cacheLineSize = 64
 
-func init() {
+func initOptions() {
+	options = []option{
+		{Name: "adx", Feature: &X86.HasADX},
+		{Name: "aes", Feature: &X86.HasAES},
+		{Name: "avx", Feature: &X86.HasAVX},
+		{Name: "avx2", Feature: &X86.HasAVX2},
+		{Name: "bmi1", Feature: &X86.HasBMI1},
+		{Name: "bmi2", Feature: &X86.HasBMI2},
+		{Name: "erms", Feature: &X86.HasERMS},
+		{Name: "fma", Feature: &X86.HasFMA},
+		{Name: "osxsave", Feature: &X86.HasOSXSAVE},
+		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
+		{Name: "popcnt", Feature: &X86.HasPOPCNT},
+		{Name: "rdrand", Feature: &X86.HasRDRAND},
+		{Name: "rdseed", Feature: &X86.HasRDSEED},
+		{Name: "sse3", Feature: &X86.HasSSE3},
+		{Name: "sse41", Feature: &X86.HasSSE41},
+		{Name: "sse42", Feature: &X86.HasSSE42},
+		{Name: "ssse3", Feature: &X86.HasSSSE3},
+
+		// These capabilities should always be enabled on amd64:
+		{Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"},
+	}
+}
+
+func archInit() {
+
 	Initialized = true
 
 	maxID, _, _, _ := cpuid(0, 0)
@@ -52,6 +80,7 @@ func init() {
 	X86.HasERMS = isSet(9, ebx7)
 	X86.HasRDSEED = isSet(18, ebx7)
 	X86.HasADX = isSet(19, ebx7)
+
 }
 
 func isSet(bitpos uint, value uint32) bool {

vendor/golang.org/x/sys/unix/syscall_linux.go

@@ -1965,10 +1965,15 @@ func isGroupMember(gid int) bool {
 }
 
 //sys	faccessat(dirfd int, path string, mode uint32) (err error)
+//sys	Faccessat2(dirfd int, path string, mode uint32, flags int) (err error)
 
 func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) {
-	if flags & ^(AT_SYMLINK_NOFOLLOW|AT_EACCESS) != 0 {
+	if flags == 0 {
-		return EINVAL
+		return faccessat(dirfd, path, mode)
+	}
+
+	if err := Faccessat2(dirfd, path, mode, flags); err != ENOSYS && err != EPERM {
+		return err
 	}
 
 	// The Linux kernel faccessat system call does not take any flags.
@@ -1977,8 +1982,8 @@ func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) {
 	// Because people naturally expect syscall.Faccessat to act
 	// like C faccessat, we do the same.
 
-	if flags == 0 {
+	if flags & ^(AT_SYMLINK_NOFOLLOW|AT_EACCESS) != 0 {
-		return faccessat(dirfd, path, mode)
+		return EINVAL
 	}
 
 	var st Stat_t

vendor/golang.org/x/sys/unix/zerrors_linux.go

@@ -265,6 +265,7 @@ const (
 	CAP_AUDIT_READ                              = 0x25
 	CAP_AUDIT_WRITE                             = 0x1d
 	CAP_BLOCK_SUSPEND                           = 0x24
+	CAP_BPF                                     = 0x27
 	CAP_CHOWN                                   = 0x0
 	CAP_DAC_OVERRIDE                            = 0x1
 	CAP_DAC_READ_SEARCH                         = 0x2
@@ -273,7 +274,7 @@ const (
 	CAP_IPC_LOCK                                = 0xe
 	CAP_IPC_OWNER                               = 0xf
 	CAP_KILL                                    = 0x5
-	CAP_LAST_CAP                                = 0x25
+	CAP_LAST_CAP                                = 0x27
 	CAP_LEASE                                   = 0x1c
 	CAP_LINUX_IMMUTABLE                         = 0x9
 	CAP_MAC_ADMIN                               = 0x21
@@ -283,6 +284,7 @@ const (
 	CAP_NET_BIND_SERVICE                        = 0xa
 	CAP_NET_BROADCAST                           = 0xb
 	CAP_NET_RAW                                 = 0xd
+	CAP_PERFMON                                 = 0x26
 	CAP_SETFCAP                                 = 0x1f
 	CAP_SETGID                                  = 0x6
 	CAP_SETPCAP                                 = 0x8
@@ -372,6 +374,7 @@ const (
 	DEVLINK_GENL_NAME                           = "devlink"
 	DEVLINK_GENL_VERSION                        = 0x1
 	DEVLINK_SB_THRESHOLD_TO_ALPHA_MAX           = 0x14
+	DEVMEM_MAGIC                                = 0x454d444d
 	DEVPTS_SUPER_MAGIC                          = 0x1cd1
 	DMA_BUF_MAGIC                               = 0x444d4142
 	DT_BLK                                      = 0x6
@@ -475,6 +478,7 @@ const (
 	ETH_P_MOBITEX                               = 0x15
 	ETH_P_MPLS_MC                               = 0x8848
 	ETH_P_MPLS_UC                               = 0x8847
+	ETH_P_MRP                                   = 0x88e3
 	ETH_P_MVRP                                  = 0x88f5
 	ETH_P_NCSI                                  = 0x88f8
 	ETH_P_NSH                                   = 0x894f
@@ -602,8 +606,9 @@ const (
 	FSCRYPT_POLICY_FLAGS_PAD_4                  = 0x0
 	FSCRYPT_POLICY_FLAGS_PAD_8                  = 0x1
 	FSCRYPT_POLICY_FLAGS_PAD_MASK               = 0x3
-	FSCRYPT_POLICY_FLAGS_VALID                  = 0xf
+	FSCRYPT_POLICY_FLAGS_VALID                  = 0x1f
 	FSCRYPT_POLICY_FLAG_DIRECT_KEY              = 0x4
+	FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32          = 0x10
 	FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64          = 0x8
 	FSCRYPT_POLICY_V1                           = 0x0
 	FSCRYPT_POLICY_V2                           = 0x2
@@ -632,7 +637,7 @@ const (
 	FS_POLICY_FLAGS_PAD_4                       = 0x0
 	FS_POLICY_FLAGS_PAD_8                       = 0x1
 	FS_POLICY_FLAGS_PAD_MASK                    = 0x3
-	FS_POLICY_FLAGS_VALID                       = 0xf
+	FS_POLICY_FLAGS_VALID                       = 0x1f
 	FS_VERITY_FL                                = 0x100000
 	FS_VERITY_HASH_ALG_SHA256                   = 0x1
 	FS_VERITY_HASH_ALG_SHA512                   = 0x2
@@ -834,6 +839,7 @@ const (
 	IPPROTO_EGP                                 = 0x8
 	IPPROTO_ENCAP                               = 0x62
 	IPPROTO_ESP                                 = 0x32
+	IPPROTO_ETHERNET                            = 0x8f
 	IPPROTO_FRAGMENT                            = 0x2c
 	IPPROTO_GRE                                 = 0x2f
 	IPPROTO_HOPOPTS                             = 0x0
@@ -847,6 +853,7 @@ const (
 	IPPROTO_L2TP                                = 0x73
 	IPPROTO_MH                                  = 0x87
 	IPPROTO_MPLS                                = 0x89
+	IPPROTO_MPTCP                               = 0x106
 	IPPROTO_MTP                                 = 0x5c
 	IPPROTO_NONE                                = 0x3b
 	IPPROTO_PIM                                 = 0x67
@@ -1016,6 +1023,7 @@ const (
 	KEYCTL_CAPS0_PERSISTENT_KEYRINGS            = 0x2
 	KEYCTL_CAPS0_PUBLIC_KEY                     = 0x8
 	KEYCTL_CAPS0_RESTRICT_KEYRING               = 0x40
+	KEYCTL_CAPS1_NOTIFICATIONS                  = 0x4
 	KEYCTL_CAPS1_NS_KEYRING_NAME                = 0x1
 	KEYCTL_CAPS1_NS_KEY_TAG                     = 0x2
 	KEYCTL_CHOWN                                = 0x4
@@ -1053,6 +1061,7 @@ const (
 	KEYCTL_SUPPORTS_VERIFY                      = 0x8
 	KEYCTL_UNLINK                               = 0x9
 	KEYCTL_UPDATE                               = 0x2
+	KEYCTL_WATCH_KEY                            = 0x20
 	KEY_REQKEY_DEFL_DEFAULT                     = 0x0
 	KEY_REQKEY_DEFL_GROUP_KEYRING               = 0x6
 	KEY_REQKEY_DEFL_NO_CHANGE                   = -0x1
@@ -1096,6 +1105,8 @@ const (
 	LOOP_SET_FD                                 = 0x4c00
 	LOOP_SET_STATUS                             = 0x4c02
 	LOOP_SET_STATUS64                           = 0x4c04
+	LOOP_SET_STATUS_CLEARABLE_FLAGS             = 0x4
+	LOOP_SET_STATUS_SETTABLE_FLAGS              = 0xc
 	LO_KEY_SIZE                                 = 0x20
 	LO_NAME_SIZE                                = 0x40
 	MADV_COLD                                   = 0x14
@@ -1992,8 +2003,10 @@ const (
 	STATX_ATTR_APPEND                           = 0x20
 	STATX_ATTR_AUTOMOUNT                        = 0x1000
 	STATX_ATTR_COMPRESSED                       = 0x4
+	STATX_ATTR_DAX                              = 0x2000
 	STATX_ATTR_ENCRYPTED                        = 0x800
 	STATX_ATTR_IMMUTABLE                        = 0x10
+	STATX_ATTR_MOUNT_ROOT                       = 0x2000
 	STATX_ATTR_NODUMP                           = 0x40
 	STATX_ATTR_VERITY                           = 0x100000
 	STATX_BASIC_STATS                           = 0x7ff
@@ -2002,6 +2015,7 @@ const (
 	STATX_CTIME                                 = 0x80
 	STATX_GID                                   = 0x10
 	STATX_INO                                   = 0x100
+	STATX_MNT_ID                                = 0x1000
 	STATX_MODE                                  = 0x2
 	STATX_MTIME                                 = 0x40
 	STATX_NLINK                                 = 0x4

vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go

@@ -192,6 +192,7 @@ const (
 	PPPIOCSRASYNCMAP                 = 0x40047454
 	PPPIOCSXASYNCMAP                 = 0x4020744f
 	PPPIOCXFERUNIT                   = 0x744e
+	PROT_BTI                         = 0x10
 	PR_SET_PTRACER_ANY               = 0xffffffffffffffff
 	PTRACE_SYSEMU                    = 0x1f
 	PTRACE_SYSEMU_SINGLESTEP         = 0x20

vendor/golang.org/x/sys/unix/zsyscall_linux.go

@@ -1821,6 +1821,21 @@ func faccessat(dirfd int, path string, mode uint32) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Faccessat2(dirfd int, path string, mode uint32, flags int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	_, _, e1 := Syscall6(SYS_FACCESSAT2, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func nameToHandleAt(dirFD int, pathname string, fh *fileHandle, mountID *_C_int, flags int) (err error) {
 	var _p0 *byte
 	_p0, err = BytePtrFromString(pathname)