Clean up cached machine images

When initing machines, we download a machine image, and uncompress and
copy the image for the actual vm image. When a user constantly pulls new
machines, there may be a buildup of old, unused machine images. This
commit cleans ups the unused cached images.

Changes:
- If the machine is pulled from a URL or from the FCOS releases, we pull
  them into XDG_DATA_HOME/containers/podman/machine/vmType/cache
- Cache cleanups only happen if there is a cache miss, and we need to
  pull a new image
- For Fedora and FCOS, we actually use the cache, so we go through the
  cache dir and remove any images older than 2 weeks (FCOS's release cycle), on a cache miss.
- For generic files pulled from a URL, we don't actually cache, so we
  delete the pulled file immediately after creating a machine image
- For generic files from a local path, the original file will never be
  cleaned up

Note that because we cache in a different dir, this will not clean up
old images pulled before this commit.

[NO NEW TESTS NEEDED]

Signed-off-by: Ashley Cui <acui@redhat.com>
This commit is contained in:
Ashley Cui
2022-07-08 20:10:25 -04:00
committed by Matthew Heon
parent e473c5e4b7
commit 17dbce2fb0
4 changed files with 94 additions and 16 deletions

View File

@ -73,6 +73,7 @@ type Download struct {
Arch string
Artifact string
CompressionType string
CacheDir string
Format string
ImageName string
LocalPath string
@ -139,6 +140,7 @@ type VM interface {
type DistributionDownload interface {
HasUsableCache() (bool, error)
Get() *Download
CleanCache() error
}
type InspectInfo struct {
ConfigPath VMFile
@ -172,6 +174,19 @@ func (rc RemoteConnectionType) MakeSSHURL(host, path, port, userName string) url
return uri
}
// GetCacheDir returns the dir where VM images are downladed into when pulled
func GetCacheDir(vmType string) (string, error) {
dataDir, err := GetDataDir(vmType)
if err != nil {
return "", err
}
cacheDir := filepath.Join(dataDir, "cache")
if _, err := os.Stat(cacheDir); !errors.Is(err, os.ErrNotExist) {
return cacheDir, nil
}
return cacheDir, os.MkdirAll(cacheDir, 0755)
}
// GetDataDir returns the filepath where vm images should
// live for podman-machine.
func GetDataDir(vmType string) (string, error) {
@ -180,7 +195,7 @@ func GetDataDir(vmType string) (string, error) {
return "", err
}
dataDir := filepath.Join(dataDirPrefix, vmType)
if _, err := os.Stat(dataDir); !os.IsNotExist(err) {
if _, err := os.Stat(dataDir); !errors.Is(err, os.ErrNotExist) {
return dataDir, nil
}
mkdirErr := os.MkdirAll(dataDir, 0755)
@ -205,7 +220,7 @@ func GetConfDir(vmType string) (string, error) {
return "", err
}
confDir := filepath.Join(confDirPrefix, vmType)
if _, err := os.Stat(confDir); !os.IsNotExist(err) {
if _, err := os.Stat(confDir); !errors.Is(err, os.ErrNotExist) {
return confDir, nil
}
mkdirErr := os.MkdirAll(confDir, 0755)

View File

@ -13,6 +13,7 @@ import (
"path/filepath"
"runtime"
"strings"
"time"
"github.com/coreos/stream-metadata-go/fedoracoreos"
"github.com/coreos/stream-metadata-go/release"
@ -53,7 +54,7 @@ func NewFcosDownloader(vmType, vmName, imageStream string) (DistributionDownload
return nil, err
}
dataDir, err := GetDataDir(vmType)
cacheDir, err := GetCacheDir(vmType)
if err != nil {
return nil, err
}
@ -62,15 +63,20 @@ func NewFcosDownloader(vmType, vmName, imageStream string) (DistributionDownload
Download: Download{
Arch: getFcosArch(),
Artifact: artifact,
CacheDir: cacheDir,
Format: Format,
ImageName: imageName,
LocalPath: filepath.Join(dataDir, imageName),
LocalPath: filepath.Join(cacheDir, imageName),
Sha256sum: info.Sha256Sum,
URL: url,
VMName: vmName,
},
}
fcd.Download.LocalUncompressedFile = fcd.getLocalUncompressedName()
dataDir, err := GetDataDir(vmType)
if err != nil {
return nil, err
}
fcd.Download.LocalUncompressedFile = fcd.getLocalUncompressedFile(dataDir)
return fcd, nil
}
@ -108,6 +114,13 @@ func (f FcosDownload) HasUsableCache() (bool, error) {
return sum.Encoded() == f.Sha256sum, nil
}
func (f FcosDownload) CleanCache() error {
// Set cached image to expire after 2 weeks
// FCOS refreshes around every 2 weeks, assume old images aren't needed
expire := 14 * 24 * time.Hour
return removeImageAfterExpire(f.CacheDir, expire)
}
func getFcosArch() string {
var arch string
// TODO fill in more architectures

View File

@ -11,6 +11,7 @@ import (
"net/http"
"net/url"
"path/filepath"
"time"
)
const (
@ -27,7 +28,7 @@ func NewFedoraDownloader(vmType, vmName, releaseStream string) (DistributionDown
return nil, err
}
dataDir, err := GetDataDir(vmType)
cacheDir, err := GetCacheDir(vmType)
if err != nil {
return nil, err
}
@ -38,15 +39,20 @@ func NewFedoraDownloader(vmType, vmName, releaseStream string) (DistributionDown
Download: Download{
Arch: getFcosArch(),
Artifact: artifact,
CacheDir: cacheDir,
Format: Format,
ImageName: imageName,
LocalPath: filepath.Join(dataDir, imageName),
LocalPath: filepath.Join(cacheDir, imageName),
URL: downloadURL,
VMName: vmName,
Size: size,
},
}
f.Download.LocalUncompressedFile = f.getLocalUncompressedName()
dataDir, err := GetDataDir(vmType)
if err != nil {
return nil, err
}
f.Download.LocalUncompressedFile = f.getLocalUncompressedFile(dataDir)
return f, nil
}
@ -65,6 +71,12 @@ func (f FedoraDownload) HasUsableCache() (bool, error) {
return info.Size() == f.Size, nil
}
func (f FedoraDownload) CleanCache() error {
// Set cached image to expire after 2 weeks
expire := 14 * 24 * time.Hour
return removeImageAfterExpire(f.CacheDir, expire)
}
func getFedoraDownload(releaseURL string) (*url.URL, int64, error) {
downloadURL, err := url.Parse(releaseURL)
if err != nil {

View File

@ -5,6 +5,7 @@ package machine
import (
"bufio"
"errors"
"fmt"
"io"
"io/ioutil"
@ -39,6 +40,10 @@ func NewGenericDownloader(vmType, vmName, pullPath string) (DistributionDownload
if err != nil {
return nil, err
}
cacheDir, err := GetCacheDir(vmType)
if err != nil {
return nil, err
}
dl := Download{}
// Is pullpath a file or url?
getURL, err := url2.Parse(pullPath)
@ -48,25 +53,23 @@ func NewGenericDownloader(vmType, vmName, pullPath string) (DistributionDownload
if len(getURL.Scheme) > 0 {
urlSplit := strings.Split(getURL.Path, "/")
imageName = urlSplit[len(urlSplit)-1]
dl.LocalUncompressedFile = filepath.Join(dataDir, imageName)
dl.URL = getURL
dl.LocalPath = filepath.Join(dataDir, imageName)
dl.LocalPath = filepath.Join(cacheDir, imageName)
} else {
// Dealing with FilePath
imageName = filepath.Base(pullPath)
dl.LocalUncompressedFile = filepath.Join(dataDir, imageName)
dl.LocalPath = pullPath
}
dl.VMName = vmName
dl.ImageName = imageName
dl.LocalUncompressedFile = filepath.Join(dataDir, imageName)
// The download needs to be pulled into the datadir
gd := GenericDownload{Download: dl}
gd.LocalUncompressedFile = gd.getLocalUncompressedName()
return gd, nil
}
func (d Download) getLocalUncompressedName() string {
func (d Download) getLocalUncompressedFile(dataDir string) string {
var (
extension string
)
@ -78,8 +81,8 @@ func (d Download) getLocalUncompressedName() string {
case strings.HasSuffix(d.LocalPath, ".xz"):
extension = ".xz"
}
uncompressedFilename := filepath.Join(filepath.Dir(d.LocalPath), d.VMName+"_"+d.ImageName)
return strings.TrimSuffix(uncompressedFilename, extension)
uncompressedFilename := d.VMName + "_" + d.ImageName
return filepath.Join(dataDir, strings.TrimSuffix(uncompressedFilename, extension))
}
func (g GenericDownload) Get() *Download {
@ -91,6 +94,18 @@ func (g GenericDownload) HasUsableCache() (bool, error) {
return g.URL == nil, nil
}
// CleanCache cleans out downloaded uncompressed image files
func (g GenericDownload) CleanCache() error {
// Remove any image that has been downloaded via URL
// We never read from cache for generic downloads
if g.URL != nil {
if err := os.Remove(g.LocalPath); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
return nil
}
func DownloadImage(d DistributionDownload) error {
// check if the latest image is already present
ok, err := d.HasUsableCache()
@ -101,8 +116,14 @@ func DownloadImage(d DistributionDownload) error {
if err := DownloadVMImage(d.Get().URL, d.Get().LocalPath); err != nil {
return err
}
// Clean out old cached images, since we didn't find needed image in cache
defer func() {
if err = d.CleanCache(); err != nil {
logrus.Warnf("error cleaning machine image cache: %s", err)
}
}()
}
return Decompress(d.Get().LocalPath, d.Get().getLocalUncompressedName())
return Decompress(d.Get().LocalPath, d.Get().LocalUncompressedFile)
}
// DownloadVMImage downloads a VM image from url to given path
@ -253,3 +274,20 @@ func decompressEverythingElse(src string, output io.WriteCloser) error {
_, err = io.Copy(output, uncompressStream)
return err
}
func removeImageAfterExpire(dir string, expire time.Duration) error {
now := time.Now()
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
// Delete any cache files that are older than expiry date
if !info.IsDir() && (now.Sub(info.ModTime()) > expire) {
err := os.Remove(path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
logrus.Warnf("unable to clean up cached image: %s", path)
} else {
logrus.Debugf("cleaning up cached image: %s", path)
}
}
return nil
})
return err
}