diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go
index 96b52495155..b716b641d5f 100644
--- a/cmd/kube-proxy/app/server_others.go
+++ b/cmd/kube-proxy/app/server_others.go
@@ -47,11 +47,11 @@ import (
 	proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config"
 	"k8s.io/kubernetes/pkg/proxy/iptables"
 	"k8s.io/kubernetes/pkg/proxy/ipvs"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 	utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
 	proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics"
 	proxyutil "k8s.io/kubernetes/pkg/proxy/util"
 	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 	"k8s.io/utils/exec"
 	netutils "k8s.io/utils/net"
diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go
index f77280d6bcf..5147c112c4f 100644
--- a/pkg/proxy/ipvs/ipset.go
+++ b/pkg/proxy/ipvs/ipset.go
@@ -19,7 +19,7 @@ package ipvs
 import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 
 	"fmt"
 	"strings"
diff --git a/pkg/util/ipset/ipset.go b/pkg/proxy/ipvs/ipset/ipset.go
similarity index 100%
rename from pkg/util/ipset/ipset.go
rename to pkg/proxy/ipvs/ipset/ipset.go
diff --git a/pkg/util/ipset/ipset_test.go b/pkg/proxy/ipvs/ipset/ipset_test.go
similarity index 100%
rename from pkg/util/ipset/ipset_test.go
rename to pkg/proxy/ipvs/ipset/ipset_test.go
diff --git a/pkg/util/ipset/testing/fake.go b/pkg/proxy/ipvs/ipset/testing/fake.go
similarity index 98%
rename from pkg/util/ipset/testing/fake.go
rename to pkg/proxy/ipvs/ipset/testing/fake.go
index 79f8f38554d..a94c709de79 100644
--- a/pkg/util/ipset/testing/fake.go
+++ b/pkg/proxy/ipvs/ipset/testing/fake.go
@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/kubernetes/pkg/util/ipset"
+	"k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 // FakeIPSet is a no-op implementation of ipset Interface
diff --git a/pkg/util/ipset/testing/fake_test.go b/pkg/proxy/ipvs/ipset/testing/fake_test.go
similarity index 99%
rename from pkg/util/ipset/testing/fake_test.go
rename to pkg/proxy/ipvs/ipset/testing/fake_test.go
index 433e5970671..0d07dfa9e0f 100644
--- a/pkg/util/ipset/testing/fake_test.go
+++ b/pkg/proxy/ipvs/ipset/testing/fake_test.go
@@ -20,7 +20,7 @@ import (
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/kubernetes/pkg/util/ipset"
+	"k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 const testVersion = "v6.19"
diff --git a/pkg/util/ipset/types.go b/pkg/proxy/ipvs/ipset/types.go
similarity index 100%
rename from pkg/util/ipset/types.go
rename to pkg/proxy/ipvs/ipset/types.go
diff --git a/pkg/proxy/ipvs/ipset_test.go b/pkg/proxy/ipvs/ipset_test.go
index 6ae74252dda..dafde0b4808 100644
--- a/pkg/proxy/ipvs/ipset_test.go
+++ b/pkg/proxy/ipvs/ipset_test.go
@@ -22,8 +22,8 @@ package ipvs
 import (
 	"testing"
 
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
-	fakeipset "k8s.io/kubernetes/pkg/util/ipset/testing"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
+	fakeipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset/testing"
 )
 
 func TestCheckIPSetVersion(t *testing.T) {
diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index d319a12e9c0..95c7e6dea91 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -45,13 +45,13 @@ import (
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/proxy/conntrack"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 	utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
 	"k8s.io/kubernetes/pkg/proxy/metaproxier"
 	"k8s.io/kubernetes/pkg/proxy/metrics"
 	utilproxy "k8s.io/kubernetes/pkg/proxy/util"
 	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
 	"k8s.io/kubernetes/pkg/util/async"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 )
 
diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go
index 8b1628e2768..b92c34917d7 100644
--- a/pkg/proxy/ipvs/proxier_test.go
+++ b/pkg/proxy/ipvs/proxier_test.go
@@ -39,6 +39,8 @@ import (
 	"k8s.io/component-base/metrics/testutil"
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
+	ipsettest "k8s.io/kubernetes/pkg/proxy/ipvs/ipset/testing"
 	netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing"
 	utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
 	ipvstest "k8s.io/kubernetes/pkg/proxy/ipvs/util/testing"
@@ -47,8 +49,6 @@ import (
 	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
 	proxyutiltest "k8s.io/kubernetes/pkg/proxy/util/testing"
 	"k8s.io/kubernetes/pkg/util/async"
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
-	ipsettest "k8s.io/kubernetes/pkg/util/ipset/testing"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 	iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
 	"k8s.io/utils/exec"
diff --git a/pkg/proxy/ipvs/safe_ipset.go b/pkg/proxy/ipvs/safe_ipset.go
index 2be9900d96d..1dbad0eb5cd 100644
--- a/pkg/proxy/ipvs/safe_ipset.go
+++ b/pkg/proxy/ipvs/safe_ipset.go
@@ -19,7 +19,7 @@ package ipvs
 import (
 	"sync"
 
-	"k8s.io/kubernetes/pkg/util/ipset"
+	"k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 type safeIpset struct {
diff --git a/pkg/proxy/ipvs/testing/util.go b/pkg/proxy/ipvs/testing/util.go
index c312923ac2b..cd3bb7ebf0f 100644
--- a/pkg/proxy/ipvs/testing/util.go
+++ b/pkg/proxy/ipvs/testing/util.go
@@ -17,7 +17,7 @@ limitations under the License.
 package testing
 
 import (
-	utilipset "k8s.io/kubernetes/pkg/util/ipset"
+	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
 )
 
 // ExpectedVirtualServer is the expected ipvs rules with VirtualServer and RealServer
diff --git a/pkg/util/ipset/OWNERS b/pkg/util/ipset/OWNERS
deleted file mode 100644
index c70337b9a47..00000000000
--- a/pkg/util/ipset/OWNERS
+++ /dev/null
@@ -1,11 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - sig-network-reviewers
-approvers:
-  - sig-network-approvers
-labels:
-  - sig/network
-emeritus_approvers:
-  - brendandburns
-  - m1093782566