diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index e39cfe3eee..d6fb118108 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -973,6 +973,16 @@ func (s *Sandbox) createNetwork() error {
 	span, _ := s.trace("createNetwork")
 	defer span.Finish()
 
+	// In case there is a factory, the network should be handled
+	// through some calls at the API level, in order to add or
+	// remove interfaces and routes.
+	// This prevents from any assumptions that could be made from
+	// virtcontainers, in particular that the VM has not been started
+	// before it starts to scan the current network.
+	if s.factory != nil {
+		return nil
+	}
+
 	// Add the network
 	if err := s.network.add(s); err != nil {
 		return err
@@ -986,6 +996,13 @@ func (s *Sandbox) removeNetwork() error {
 	span, _ := s.trace("removeNetwork")
 	defer span.Finish()
 
+	// In case there is a factory, the network has been handled through
+	// some API calls to hotplug some interfaces and routes. This means
+	// the removal of the network should follow the same logic.
+	if s.factory != nil {
+		return nil
+	}
+
 	return s.network.remove(s)
 }