Respect artifact extension when populating cache

This caused cache to not hit correctly
This commit is contained in:
Ettore Di Giacinto 2021-01-22 11:48:54 +01:00
parent 9a2ff0a3e2
commit a6e7a3059c
3 changed files with 31 additions and 7 deletions

View File

@ -325,7 +325,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
} }
defer original.Close() defer original.Close()
zstdFile := a.Path + ".zst" zstdFile := a.getCompressedName()
bufferedReader := bufio.NewReader(original) bufferedReader := bufio.NewReader(original)
// Open a file for writing. // Open a file for writing.
@ -348,6 +348,8 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
} }
os.RemoveAll(a.Path) // Remove original os.RemoveAll(a.Path) // Remove original
Debug("Removed artifact", a.Path)
a.Path = zstdFile a.Path = zstdFile
return nil return nil
case GZip: case GZip:
@ -361,7 +363,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
} }
defer original.Close() defer original.Close()
gzipfile := a.Path + ".gz" gzipfile := a.getCompressedName()
bufferedReader := bufio.NewReader(original) bufferedReader := bufio.NewReader(original)
// Open a file for writing. // Open a file for writing.
@ -380,6 +382,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
} }
w.Close() w.Close()
os.RemoveAll(a.Path) // Remove original os.RemoveAll(a.Path) // Remove original
Debug("Removed artifact", a.Path)
// a.CompressedPath = gzipfile // a.CompressedPath = gzipfile
a.Path = gzipfile a.Path = gzipfile
return nil return nil
@ -387,12 +390,31 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
// Defaults to tar only (covers when "none" is supplied) // Defaults to tar only (covers when "none" is supplied)
default: default:
return helpers.Tar(src, a.Path) return helpers.Tar(src, a.getCompressedName())
} }
return errors.New("Compression type must be supplied") return errors.New("Compression type must be supplied")
} }
func (a *PackageArtifact) getCompressedName() string {
switch a.CompressionType {
case Zstandard:
return a.Path + ".zst"
case GZip:
return a.Path + ".gz"
}
return a.Path
}
// GetUncompressedName returns the artifact path without the extension suffix
func (a *PackageArtifact) GetUncompressedName() string {
switch a.CompressionType {
case Zstandard, GZip:
return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
}
return a.Path
}
func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) { func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
// If the destination path already exists I rename target file name with postfix. // If the destination path already exists I rename target file name with postfix.
var destPath string var destPath string

View File

@ -117,6 +117,7 @@ type Artifact interface {
SetChecksums(c Checksums) SetChecksums(c Checksums)
GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error) GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error)
GetUncompressedName() string
} }
type ArtifactNode struct { type ArtifactNode struct {

View File

@ -74,7 +74,7 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
var resultingArtifact compiler.Artifact var resultingArtifact compiler.Artifact
artifactName := path.Base(artifact.GetPath()) artifactName := path.Base(artifact.GetPath())
cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName)
Debug("Cache file", cacheFile)
if err := helpers.EnsureDir(cacheFile); err != nil { if err := helpers.EnsureDir(cacheFile); err != nil {
return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile) return nil, errors.Wrapf(err, "could not create cache folder %s for %s", config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), cacheFile)
} }
@ -89,7 +89,7 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
// Check if file is already in cache // Check if file is already in cache
if helpers.Exists(cacheFile) { if helpers.Exists(cacheFile) {
Debug("Use artifact", artifactName, "from cache.") Debug("Cache hit for artifact", artifactName)
resultingArtifact = artifact resultingArtifact = artifact
resultingArtifact.SetPath(cacheFile) resultingArtifact.SetPath(cacheFile)
resultingArtifact.SetChecksums(compiler.Checksums{}) resultingArtifact.SetChecksums(compiler.Checksums{})
@ -117,7 +117,8 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar
newart := artifact newart := artifact
// We discard checksum, that are checked while during pull and unpack // We discard checksum, that are checked while during pull and unpack
newart.SetChecksums(compiler.Checksums{}) newart.SetChecksums(compiler.Checksums{})
newart.SetPath(cacheFile) newart.SetPath(cacheFile) // First set to cache file
newart.SetPath(newart.GetUncompressedName()) // Calculate the real path from cacheFile
err = newart.Compress(temp, 1) err = newart.Compress(temp, 1)
if err != nil { if err != nil {
Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error())) Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error()))