diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go
index a14f386833e..1b14b51a0c2 100644
--- a/cmd/kube-proxy/app/server_others.go
+++ b/cmd/kube-proxy/app/server_others.go
@@ -47,11 +47,11 @@ import (
 	proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
 	"k8s.io/kubernetes/pkg/proxy/iptables"
 	"k8s.io/kubernetes/pkg/proxy/ipvs"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 	utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
 	proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics"
 	proxyutil "k8s.io/kubernetes/pkg/proxy/util"
 	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 	"k8s.io/utils/exec"
 	netutils "k8s.io/utils/net"
diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go
index f77280d6bcf..5147c112c4f 100644
--- a/pkg/proxy/ipvs/ipset.go
+++ b/pkg/proxy/ipvs/ipset.go
@@ -19,7 +19,7 @@ package ipvs
 import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 
 	"fmt"
 	"strings"
diff --git a/pkg/util/ipset/ipset.go b/pkg/proxy/ipvs/ipset/ipset.go
similarity index 94%
rename from pkg/util/ipset/ipset.go
rename to pkg/proxy/ipvs/ipset/ipset.go
index c82fe0c310f..f6532dc090e 100644
--- a/pkg/util/ipset/ipset.go
+++ b/pkg/proxy/ipvs/ipset/ipset.go
@@ -28,6 +28,8 @@ import (
 	netutils "k8s.io/utils/net"
 )
 
+var validationError = fmt.Errorf("failed to validate entry for ipset")
+
 // Interface is an injectable interface for running ipset commands.  Implementations must be goroutine-safe.
 type Interface interface {
 	// FlushSet deletes all entries from a named set.
@@ -165,7 +167,7 @@ type Entry struct {
 // Validate checks if a given ipset entry is valid or not.  The set parameter is the ipset that entry belongs to.
 func (e *Entry) Validate(set *IPSet) bool {
 	if e.Port < 0 {
-		klog.Errorf("Entry %v port number %d should be >=0 for ipset %v", e, e.Port, set)
+		klog.ErrorS(validationError, "port number should be >=0", "entry", e, "port", e.Port, "ipset", set)
 		return false
 	}
 	switch e.SetType {
@@ -187,7 +189,7 @@ func (e *Entry) Validate(set *IPSet) bool {
 
 		// IP2 can not be empty for `hash:ip,port,ip` type ip set
 		if netutils.ParseIPSloppy(e.IP2) == nil {
-			klog.Errorf("Error parsing entry %v second ip address %v for ipset %v", e, e.IP2, set)
+			klog.ErrorS(validationError, "error parsing second ip address", "entry", e, "ip", e.IP2, "ipset", set)
 			return false
 		}
 	case HashIPPortNet:
@@ -198,22 +200,22 @@ func (e *Entry) Validate(set *IPSet) bool {
 
 		// Net can not be empty for `hash:ip,port,net` type ip set
 		if _, ipNet, err := netutils.ParseCIDRSloppy(e.Net); ipNet == nil {
-			klog.Errorf("Error parsing entry %v ip net %v for ipset %v, error: %v", e, e.Net, set, err)
+			klog.ErrorS(err, "error parsing ip net", "entry", e, "net", e.Net, "set", set)
 			return false
 		}
 	case BitmapPort:
 		// check if port number satisfies its ipset's requirement of port range
 		if set == nil {
-			klog.Errorf("Unable to reference ip set where the entry %v exists", e)
+			klog.ErrorS(validationError, "unable to reference ip set where the entry exists", "entry", e)
 			return false
 		}
 		begin, end, err := parsePortRange(set.PortRange)
 		if err != nil {
-			klog.Errorf("Failed to parse set %v port range %s for ipset %v, error: %v", set, set.PortRange, set, err)
+			klog.ErrorS(err, "failed to parse set port range", "ipset", set, "portRange", set.PortRange)
 			return false
 		}
 		if e.Port < begin || e.Port > end {
-			klog.Errorf("Entry %v port number %d is not in the port range %s of its ipset %v", e, e.Port, set.PortRange, set)
+			klog.ErrorS(validationError, "port number is not in the port range of its ipset", "entry", e, "port", e.Port, "portRange", set.PortRange, "ipset", set)
 			return false
 		}
 	}
@@ -261,7 +263,7 @@ func (e *Entry) checkIPandProtocol(set *IPSet) bool {
 // checkIP checks if IP of Entry is valid.
 func (e *Entry) checkIP(set *IPSet) bool {
 	if netutils.ParseIPSloppy(e.IP) == nil {
-		klog.Errorf("Error parsing entry %v ip address %v for ipset %v", e, e.IP, set)
+		klog.ErrorS(validationError, "error parsing ip address", "entry", e, "ip", e.IP, "ipset", set)
 		return false
 	}
 
@@ -489,7 +491,7 @@ func validateProtocol(protocol string) bool {
 	if protocol == ProtocolTCP || protocol == ProtocolUDP || protocol == ProtocolSCTP {
 		return true
 	}
-	klog.Errorf("Invalid entry's protocol: %s, supported protocols are [%s, %s, %s]", protocol, ProtocolTCP, ProtocolUDP, ProtocolSCTP)
+	klog.ErrorS(validationError, "invalid protocol", "protocol", protocol, "supportedProtocols", []string{ProtocolTCP, ProtocolUDP, ProtocolSCTP})
 	return false
 }
 
diff --git a/pkg/util/ipset/ipset_test.go b/pkg/proxy/ipvs/ipset/ipset_test.go
similarity index 100%
rename from pkg/util/ipset/ipset_test.go
rename to pkg/proxy/ipvs/ipset/ipset_test.go
diff --git a/pkg/util/ipset/testing/fake.go b/pkg/proxy/ipvs/ipset/testing/fake.go
similarity index 98%
rename from pkg/util/ipset/testing/fake.go
rename to pkg/proxy/ipvs/ipset/testing/fake.go
index 79f8f38554d..a94c709de79 100644
--- a/pkg/util/ipset/testing/fake.go
+++ b/pkg/proxy/ipvs/ipset/testing/fake.go
@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/kubernetes/pkg/util/ipset"
+	"k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 // FakeIPSet is a no-op implementation of ipset Interface
diff --git a/pkg/util/ipset/testing/fake_test.go b/pkg/proxy/ipvs/ipset/testing/fake_test.go
similarity index 99%
rename from pkg/util/ipset/testing/fake_test.go
rename to pkg/proxy/ipvs/ipset/testing/fake_test.go
index 433e5970671..0d07dfa9e0f 100644
--- a/pkg/util/ipset/testing/fake_test.go
+++ b/pkg/proxy/ipvs/ipset/testing/fake_test.go
@@ -20,7 +20,7 @@ import (
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/kubernetes/pkg/util/ipset"
+	"k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 const testVersion = "v6.19"
diff --git a/pkg/util/ipset/types.go b/pkg/proxy/ipvs/ipset/types.go
similarity index 100%
rename from pkg/util/ipset/types.go
rename to pkg/proxy/ipvs/ipset/types.go
diff --git a/pkg/proxy/ipvs/ipset_test.go b/pkg/proxy/ipvs/ipset_test.go
index 6ae74252dda..dafde0b4808 100644
--- a/pkg/proxy/ipvs/ipset_test.go
+++ b/pkg/proxy/ipvs/ipset_test.go
@@ -22,8 +22,8 @@ package ipvs
 import (
 	"testing"
 
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
-	fakeipset "k8s.io/kubernetes/pkg/util/ipset/testing"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
+	fakeipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset/testing"
 )
 
 func TestCheckIPSetVersion(t *testing.T) {
diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index 81e940be6e5..cf3236bf1a5 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -45,13 +45,13 @@ import (
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/proxy/conntrack"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 	utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
 	"k8s.io/kubernetes/pkg/proxy/metaproxier"
 	"k8s.io/kubernetes/pkg/proxy/metrics"
 	proxyutil "k8s.io/kubernetes/pkg/proxy/util"
 	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
 	"k8s.io/kubernetes/pkg/util/async"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 )
 
diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go
index b672ed649e8..134276f9aec 100644
--- a/pkg/proxy/ipvs/proxier_test.go
+++ b/pkg/proxy/ipvs/proxier_test.go
@@ -39,6 +39,8 @@ import (
 	"k8s.io/component-base/metrics/testutil"
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
+	ipsettest "k8s.io/kubernetes/pkg/proxy/ipvs/ipset/testing"
 	netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing"
 	utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
 	ipvstest "k8s.io/kubernetes/pkg/proxy/ipvs/util/testing"
@@ -47,8 +49,6 @@ import (
 	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
 	proxyutiltest "k8s.io/kubernetes/pkg/proxy/util/testing"
 	"k8s.io/kubernetes/pkg/util/async"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
-	ipsettest "k8s.io/kubernetes/pkg/util/ipset/testing"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 	iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
 	"k8s.io/utils/exec"
diff --git a/pkg/proxy/ipvs/safe_ipset.go b/pkg/proxy/ipvs/safe_ipset.go
index 2be9900d96d..1dbad0eb5cd 100644
--- a/pkg/proxy/ipvs/safe_ipset.go
+++ b/pkg/proxy/ipvs/safe_ipset.go
@@ -19,7 +19,7 @@ package ipvs
 import (
 	"sync"
 
-	"k8s.io/kubernetes/pkg/util/ipset"
+	"k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 type safeIpset struct {
diff --git a/pkg/proxy/ipvs/testing/util.go b/pkg/proxy/ipvs/testing/util.go
index c312923ac2b..cd3bb7ebf0f 100644
--- a/pkg/proxy/ipvs/testing/util.go
+++ b/pkg/proxy/ipvs/testing/util.go
@@ -17,7 +17,7 @@ limitations under the License.
 package testing
 
 import (
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 // ExpectedVirtualServer is the expected ipvs rules with VirtualServer and RealServer
diff --git a/pkg/util/ipset/OWNERS b/pkg/util/ipset/OWNERS
deleted file mode 100644
index c70337b9a47..00000000000
--- a/pkg/util/ipset/OWNERS
+++ /dev/null
@@ -1,11 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - sig-network-reviewers
-approvers:
-  - sig-network-approvers
-labels:
-  - sig/network
-emeritus_approvers:
-  - brendandburns
-  - m1093782566