diff --git a/cli/factory.go b/cli/factory.go
index 4da8ef54c..999f2f5cd 100644
--- a/cli/factory.go
+++ b/cli/factory.go
@@ -158,20 +158,21 @@ var initFactoryCommand = cli.Command{
 			return errors.New("invalid runtime config")
 		}
 
+		factoryConfig := vf.Config{
+			Template: runtimeConfig.FactoryConfig.Template,
+			Cache:    runtimeConfig.FactoryConfig.VMCacheNumber,
+			VMCache:  runtimeConfig.FactoryConfig.VMCacheNumber > 0,
+			VMConfig: vc.VMConfig{
+				HypervisorType:   runtimeConfig.HypervisorType,
+				HypervisorConfig: runtimeConfig.HypervisorConfig,
+				AgentType:        runtimeConfig.AgentType,
+				AgentConfig:      runtimeConfig.AgentConfig,
+				ProxyType:        runtimeConfig.ProxyType,
+				ProxyConfig:      runtimeConfig.ProxyConfig,
+			},
+		}
+
 		if runtimeConfig.FactoryConfig.VMCacheNumber > 0 {
-			factoryConfig := vf.Config{
-				Template: runtimeConfig.FactoryConfig.Template,
-				Cache:    runtimeConfig.FactoryConfig.VMCacheNumber,
-				VMCache:  true,
-				VMConfig: vc.VMConfig{
-					HypervisorType:   runtimeConfig.HypervisorType,
-					HypervisorConfig: runtimeConfig.HypervisorConfig,
-					AgentType:        runtimeConfig.AgentType,
-					AgentConfig:      runtimeConfig.AgentConfig,
-					ProxyType:        runtimeConfig.ProxyType,
-					ProxyConfig:      runtimeConfig.ProxyConfig,
-				},
-			}
 			f, err := vf.NewFactory(ctx, factoryConfig, false)
 			if err != nil {
 				return err
@@ -204,16 +205,6 @@ var initFactoryCommand = cli.Command{
 		}
 
 		if runtimeConfig.FactoryConfig.Template {
-			factoryConfig := vf.Config{
-				Template: true,
-				VMConfig: vc.VMConfig{
-					HypervisorType:   runtimeConfig.HypervisorType,
-					HypervisorConfig: runtimeConfig.HypervisorConfig,
-					AgentType:        runtimeConfig.AgentType,
-					AgentConfig:      runtimeConfig.AgentConfig,
-					ProxyType:        runtimeConfig.ProxyType,
-				},
-			}
 			kataLog.WithField("factory", factoryConfig).Info("create vm factory")
 			_, err := vf.NewFactory(ctx, factoryConfig, false)
 			if err != nil {
@@ -222,8 +213,9 @@ var initFactoryCommand = cli.Command{
 			}
 			fmt.Fprintln(defaultOutputFile, "vm factory initialized")
 		} else {
-			kataLog.Error("vm factory is not enabled")
-			fmt.Fprintln(defaultOutputFile, "vm factory is not enabled")
+			const errstring = "vm factory or VMCache is not enabled"
+			kataLog.Error(errstring)
+			fmt.Fprintln(defaultOutputFile, errstring)
 		}
 
 		return nil
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 4adcd2a69..06dff7d89 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -945,10 +945,6 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
 
 // checkFactoryConfig ensures the VM factory configuration is valid.
 func checkFactoryConfig(config oci.RuntimeConfig) error {
-	if config.FactoryConfig.Template && config.FactoryConfig.VMCacheNumber > 0 {
-		return errors.New("VM factory cannot work together with VM cache")
-	}
-
 	if config.FactoryConfig.Template {
 		if config.HypervisorConfig.InitrdPath == "" {
 			return errors.New("Factory option enable_template requires an initrd image")
diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go
index c9f9848f5..13880330e 100644
--- a/pkg/katautils/create.go
+++ b/pkg/katautils/create.go
@@ -122,7 +122,6 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon
 	if !runtimeConfig.FactoryConfig.Template && runtimeConfig.FactoryConfig.VMCacheNumber == 0 {
 		return
 	}
-
 	factoryConfig := vf.Config{
 		Template:        runtimeConfig.FactoryConfig.Template,
 		VMCache:         runtimeConfig.FactoryConfig.VMCacheNumber > 0,
@@ -142,13 +141,13 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon
 	kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory")
 
 	f, err := vf.NewFactory(ctx, factoryConfig, true)
-	if err != nil {
+	if err != nil && !factoryConfig.VMCache {
 		kataUtilsLogger.WithError(err).Warn("load vm factory failed, about to create new one")
 		f, err = vf.NewFactory(ctx, factoryConfig, false)
-		if err != nil {
-			kataUtilsLogger.WithError(err).Warn("create vm factory failed")
-			return
-		}
+	}
+	if err != nil {
+		kataUtilsLogger.WithError(err).Warn("create vm factory failed")
+		return
 	}
 
 	vci.SetFactory(ctx, f)
diff --git a/virtcontainers/factory/factory.go b/virtcontainers/factory/factory.go
index ea32d26aa..8c897d9dd 100644
--- a/virtcontainers/factory/factory.go
+++ b/virtcontainers/factory/factory.go
@@ -61,29 +61,32 @@ func NewFactory(ctx context.Context, config Config, fetchOnly bool) (vc.Factory,
 	}
 
 	var b base.FactoryBase
-	if config.Template {
-		if fetchOnly {
-			b, err = template.Fetch(config.VMConfig)
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			b, err = template.New(ctx, config.VMConfig)
-			if err != nil {
-				return nil, err
-			}
-		}
-	} else if config.VMCache && config.Cache == 0 {
+	if config.VMCache && config.Cache == 0 {
+		// For VMCache client
 		b, err = grpccache.New(ctx, config.VMCacheEndpoint)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		b = direct.New(ctx, config.VMConfig)
-	}
+		if config.Template {
+			if fetchOnly {
+				b, err = template.Fetch(config.VMConfig)
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				b, err = template.New(ctx, config.VMConfig)
+				if err != nil {
+					return nil, err
+				}
+			}
+		} else {
+			b = direct.New(ctx, config.VMConfig)
+		}
 
-	if config.Cache > 0 {
-		b = cache.New(ctx, config.Cache, b)
+		if config.Cache > 0 {
+			b = cache.New(ctx, config.Cache, b)
+		}
 	}
 
 	return &factory{b}, nil