diff --git a/pkg/compiler/artifact.go b/pkg/compiler/artifact.go
index 2e801d00..3aca36e6 100644
--- a/pkg/compiler/artifact.go
+++ b/pkg/compiler/artifact.go
@@ -330,35 +330,23 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 
 func (a *PackageArtifact) GetProtectFiles() []string {
 	ans := []string{}
+	annotationDir := ""
 
 	if !LuetCfg.ConfigProtectSkip &&
 		LuetCfg.GetConfigProtectConfFiles() != nil &&
 		len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
 
-		for _, file := range a.Files {
-			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
-				for _, dir := range conf.Directories {
-					// Note file is without / at begin.
-					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
-						// docker archive modifier works with path without / at begin.
-						ans = append(ans, file)
-						goto nextFile
-					}
-				}
+		if a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
 			}
-
-			if a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
-				dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
-				if ok {
-					if strings.HasPrefix("/"+file, filepath.Clean(dir)) {
-						ans = append(ans, file)
-						goto nextFile
-					}
-				}
-			}
-
-		nextFile:
 		}
+
+		cp := NewConfigProtect(annotationDir)
+		cp.Map(a.Files)
+
+		ans = cp.GetProtectFiles()
 	}
 
 	return ans
diff --git a/pkg/config/config_protect.go b/pkg/config/config_protect.go
index 41c37e2b..c287bf78 100644
--- a/pkg/config/config_protect.go
+++ b/pkg/config/config_protect.go
@@ -18,6 +18,8 @@ package config
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 )
 
 type ConfigProtectConfFile struct {
@@ -39,3 +41,68 @@ func (c *ConfigProtectConfFile) String() string {
 	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
 		c.Directories)
 }
+
+type ConfigProtect struct {
+	AnnotationDir string
+	MapProtected  map[string]bool
+}
+
+func NewConfigProtect(annotationDir string) *ConfigProtect {
+	return &ConfigProtect{
+		AnnotationDir: annotationDir,
+		MapProtected:  make(map[string]bool, 0),
+	}
+}
+
+func (c *ConfigProtect) AddAnnotationDir(d string) {
+	c.AnnotationDir = d
+}
+
+func (c *ConfigProtect) GetAnnotationDir() string {
+	return c.AnnotationDir
+}
+
+func (c *ConfigProtect) Map(files []string) {
+	if LuetCfg.ConfigProtectSkip {
+		return
+	}
+
+	for _, file := range files {
+
+		if file[0:1] != "/" {
+			file = "/" + file
+		}
+
+		if len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin (on unpack)
+					if strings.HasPrefix(file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						c.MapProtected[file] = true
+						goto nextFile
+					}
+				}
+			}
+		}
+
+		if c.AnnotationDir != "" && strings.HasPrefix(file, filepath.Clean(c.AnnotationDir)) {
+			c.MapProtected[file] = true
+		}
+	nextFile:
+	}
+}
+
+func (c *ConfigProtect) Protected(file string) bool {
+	_, ans := c.MapProtected[file]
+	return ans
+}
+
+func (c *ConfigProtect) GetProtectFiles() []string {
+	ans := []string{}
+
+	for key, _ := range c.MapProtected {
+		ans = append(ans, key)
+	}
+	return ans
+}
diff --git a/pkg/installer/installer.go b/pkg/installer/installer.go
index d87e4ae0..f7b97f27 100644
--- a/pkg/installer/installer.go
+++ b/pkg/installer/installer.go
@@ -610,16 +610,37 @@ func (l *LuetInstaller) installerWorker(i int, wg *sync.WaitGroup, c <-chan Arti
 }
 
 func (l *LuetInstaller) uninstall(p pkg.Package, s *System) error {
+	var cp *config.ConfigProtect
+	annotationDir := ""
+
 	files, err := s.Database.GetPackageFiles(p)
 	if err != nil {
 		return errors.Wrap(err, "Failed getting installed files")
 	}
 
+	if !config.LuetCfg.ConfigProtectSkip {
+
+		if p.HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := p.GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+
+		cp = config.NewConfigProtect(annotationDir)
+		cp.Map(files)
+	}
+
 	// Remove from target
 	for _, f := range files {
 		target := filepath.Join(s.Target, f)
-		Debug("Removing", target)
 
+		if !config.LuetCfg.ConfigProtectSkip && cp.Protected(f) {
+			Debug("Protected file found", f)
+			continue
+		}
+
+		Debug("Removing", target)
 		if l.Options.PreserveSystemEssentialData &&
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath()) ||
 			strings.HasPrefix(f, config.LuetCfg.GetSystem().GetSystemRepoDatabaseDirPath()) {