mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Merge pull request #2319 from brendandburns/external
Externalized services v2
This commit is contained in:
commit
e987f11838
@ -127,16 +127,10 @@ being aware of which `pods` they are accessing.
|
|||||||
|
|
||||||

|

|
||||||
|
|
||||||
|
## External Services
|
||||||
|
For some parts of your application (e.g. your frontend) you want to expose a service on an external (publically visible) IP address. To achieve this, you can set the ```createExternalLoadBalancer``` flag on the service. This sets up a cloud provider specific load balancer (assuming that it is supported by your cloud provider) and also sets up IPTables rules on each host that map packets from the specified External IP address to the service proxy in the same manner as internal service IP addresses.
|
||||||
|
|
||||||
## Shortcomings
|
## Shortcomings
|
||||||
|
|
||||||
Part of the `service` specification is a `createExternalLoadBalancer` flag,
|
|
||||||
which tells the master to make an external load balancer that points to the
|
|
||||||
service. In order to do this today, the service proxy must answer on a known
|
|
||||||
(i.e. not random) port. In this case, the service port is promoted to the
|
|
||||||
proxy port. This means that it is still possible for users to collide with
|
|
||||||
each other's services or with other pods. We expect most `services` will not
|
|
||||||
set this flag, mitigating the exposure.
|
|
||||||
|
|
||||||
We expect that using iptables for portals will work at small scale, but will
|
We expect that using iptables for portals will work at small scale, but will
|
||||||
not scale to large clusters with thousands of services. See [the original
|
not scale to large clusters with thousands of services. See [the original
|
||||||
design proposal for
|
design proposal for
|
||||||
|
@ -576,6 +576,8 @@ type ServiceSpec struct {
|
|||||||
|
|
||||||
// CreateExternalLoadBalancer indicates whether a load balancer should be created for this service.
|
// CreateExternalLoadBalancer indicates whether a load balancer should be created for this service.
|
||||||
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
||||||
|
// PublicIPs are used by external load balancers.
|
||||||
|
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
|
||||||
|
|
||||||
// ContainerPort is the name of the port on the container to direct traffic to.
|
// ContainerPort is the name of the port on the container to direct traffic to.
|
||||||
// Optional, if unspecified use the first port on the container.
|
// Optional, if unspecified use the first port on the container.
|
||||||
|
@ -370,6 +370,7 @@ func init() {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer
|
out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer
|
||||||
|
out.PublicIPs = in.Spec.PublicIPs
|
||||||
out.ContainerPort = in.Spec.ContainerPort
|
out.ContainerPort = in.Spec.ContainerPort
|
||||||
out.PortalIP = in.Spec.PortalIP
|
out.PortalIP = in.Spec.PortalIP
|
||||||
out.ProxyPort = in.Spec.ProxyPort
|
out.ProxyPort = in.Spec.ProxyPort
|
||||||
@ -392,6 +393,7 @@ func init() {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer
|
out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer
|
||||||
|
out.Spec.PublicIPs = in.PublicIPs
|
||||||
out.Spec.ContainerPort = in.ContainerPort
|
out.Spec.ContainerPort = in.ContainerPort
|
||||||
out.Spec.PortalIP = in.PortalIP
|
out.Spec.PortalIP = in.PortalIP
|
||||||
out.Spec.ProxyPort = in.ProxyPort
|
out.Spec.ProxyPort = in.ProxyPort
|
||||||
|
@ -467,6 +467,8 @@ type Service struct {
|
|||||||
// This service will route traffic to pods having labels matching this selector.
|
// This service will route traffic to pods having labels matching this selector.
|
||||||
Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"`
|
Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"`
|
||||||
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
||||||
|
// PublicIPs are used by external load balancers.
|
||||||
|
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
|
||||||
|
|
||||||
// ContainerPort is the name of the port on the container to direct traffic to.
|
// ContainerPort is the name of the port on the container to direct traffic to.
|
||||||
// Optional, if unspecified use the first port on the container.
|
// Optional, if unspecified use the first port on the container.
|
||||||
|
@ -299,6 +299,7 @@ func init() {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer
|
out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer
|
||||||
|
out.PublicIPs = in.Spec.PublicIPs
|
||||||
out.ContainerPort = in.Spec.ContainerPort
|
out.ContainerPort = in.Spec.ContainerPort
|
||||||
out.PortalIP = in.Spec.PortalIP
|
out.PortalIP = in.Spec.PortalIP
|
||||||
out.ProxyPort = in.Spec.ProxyPort
|
out.ProxyPort = in.Spec.ProxyPort
|
||||||
@ -322,6 +323,7 @@ func init() {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer
|
out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer
|
||||||
|
out.Spec.PublicIPs = in.PublicIPs
|
||||||
out.Spec.ContainerPort = in.ContainerPort
|
out.Spec.ContainerPort = in.ContainerPort
|
||||||
out.Spec.PortalIP = in.PortalIP
|
out.Spec.PortalIP = in.PortalIP
|
||||||
out.Spec.ProxyPort = in.ProxyPort
|
out.Spec.ProxyPort = in.ProxyPort
|
||||||
|
@ -432,6 +432,8 @@ type Service struct {
|
|||||||
// This service will route traffic to pods having labels matching this selector.
|
// This service will route traffic to pods having labels matching this selector.
|
||||||
Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"`
|
Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"`
|
||||||
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
||||||
|
// PublicIPs are used by external load balancers.
|
||||||
|
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
|
||||||
|
|
||||||
// ContainerPort is the name of the port on the container to direct traffic to.
|
// ContainerPort is the name of the port on the container to direct traffic to.
|
||||||
// Optional, if unspecified use the first port on the container.
|
// Optional, if unspecified use the first port on the container.
|
||||||
|
@ -614,6 +614,8 @@ type ServiceSpec struct {
|
|||||||
|
|
||||||
// CreateExternalLoadBalancer indicates whether a load balancer should be created for this service.
|
// CreateExternalLoadBalancer indicates whether a load balancer should be created for this service.
|
||||||
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"`
|
||||||
|
// PublicIPs are used by external load balancers.
|
||||||
|
PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"`
|
||||||
|
|
||||||
// ContainerPort is the name of the port on the container to direct traffic to.
|
// ContainerPort is the name of the port on the container to direct traffic to.
|
||||||
// Optional, if unspecified use the first port on the container.
|
// Optional, if unspecified use the first port on the container.
|
||||||
|
@ -47,8 +47,8 @@ type TCPLoadBalancer interface {
|
|||||||
// TCPLoadBalancerExists returns whether the specified load balancer exists.
|
// TCPLoadBalancerExists returns whether the specified load balancer exists.
|
||||||
// TODO: Break this up into different interfaces (LB, etc) when we have more than one type of service
|
// TODO: Break this up into different interfaces (LB, etc) when we have more than one type of service
|
||||||
TCPLoadBalancerExists(name, region string) (bool, error)
|
TCPLoadBalancerExists(name, region string) (bool, error)
|
||||||
// CreateTCPLoadBalancer creates a new tcp load balancer.
|
// CreateTCPLoadBalancer creates a new tcp load balancer. Returns the IP address of the balancer
|
||||||
CreateTCPLoadBalancer(name, region string, port int, hosts []string) error
|
CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error)
|
||||||
// UpdateTCPLoadBalancer updates hosts under the specified load balancer.
|
// UpdateTCPLoadBalancer updates hosts under the specified load balancer.
|
||||||
UpdateTCPLoadBalancer(name, region string, hosts []string) error
|
UpdateTCPLoadBalancer(name, region string, hosts []string) error
|
||||||
// DeleteTCPLoadBalancer deletes a specified load balancer.
|
// DeleteTCPLoadBalancer deletes a specified load balancer.
|
||||||
|
@ -34,6 +34,7 @@ type FakeCloud struct {
|
|||||||
NodeResources *api.NodeResources
|
NodeResources *api.NodeResources
|
||||||
ClusterList []string
|
ClusterList []string
|
||||||
MasterName string
|
MasterName string
|
||||||
|
ExternalIP net.IP
|
||||||
|
|
||||||
cloudprovider.Zone
|
cloudprovider.Zone
|
||||||
}
|
}
|
||||||
@ -83,9 +84,9 @@ func (f *FakeCloud) TCPLoadBalancerExists(name, region string) (bool, error) {
|
|||||||
|
|
||||||
// CreateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.CreateTCPLoadBalancer.
|
// CreateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.CreateTCPLoadBalancer.
|
||||||
// It adds an entry "create" into the internal method call record.
|
// It adds an entry "create" into the internal method call record.
|
||||||
func (f *FakeCloud) CreateTCPLoadBalancer(name, region string, port int, hosts []string) error {
|
func (f *FakeCloud) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) {
|
||||||
f.addCall("create")
|
f.addCall("create")
|
||||||
return f.Err
|
return f.ExternalIP, f.Err
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.UpdateTCPLoadBalancer.
|
// UpdateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.UpdateTCPLoadBalancer.
|
||||||
|
@ -192,10 +192,10 @@ func (gce *GCECloud) TCPLoadBalancerExists(name, region string) (bool, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CreateTCPLoadBalancer is an implementation of TCPLoadBalancer.CreateTCPLoadBalancer.
|
// CreateTCPLoadBalancer is an implementation of TCPLoadBalancer.CreateTCPLoadBalancer.
|
||||||
func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, port int, hosts []string) error {
|
func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) {
|
||||||
pool, err := gce.makeTargetPool(name, region, hosts)
|
pool, err := gce.makeTargetPool(name, region, hosts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
req := &compute.ForwardingRule{
|
req := &compute.ForwardingRule{
|
||||||
Name: name,
|
Name: name,
|
||||||
@ -203,8 +203,22 @@ func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, port int, hosts
|
|||||||
PortRange: strconv.Itoa(port),
|
PortRange: strconv.Itoa(port),
|
||||||
Target: pool,
|
Target: pool,
|
||||||
}
|
}
|
||||||
_, err = gce.service.ForwardingRules.Insert(gce.projectID, region, req).Do()
|
if len(externalIP) > 0 {
|
||||||
return err
|
req.IPAddress = externalIP.String()
|
||||||
|
}
|
||||||
|
op, err := gce.service.ForwardingRules.Insert(gce.projectID, region, req).Do()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
err = gce.waitForRegionOp(op, region)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
fwd, err := gce.service.ForwardingRules.Get(gce.projectID, region, name).Do()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return net.ParseIP(fwd.IPAddress), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateTCPLoadBalancer is an implementation of TCPLoadBalancer.UpdateTCPLoadBalancer.
|
// UpdateTCPLoadBalancer is an implementation of TCPLoadBalancer.UpdateTCPLoadBalancer.
|
||||||
|
@ -40,6 +40,8 @@ type serviceInfo struct {
|
|||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
mu sync.Mutex // protects active
|
mu sync.Mutex // protects active
|
||||||
active bool
|
active bool
|
||||||
|
// TODO: make this an net.IP address
|
||||||
|
publicIP []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (si *serviceInfo) isActive() bool {
|
func (si *serviceInfo) isActive() bool {
|
||||||
@ -443,7 +445,7 @@ func (proxier *Proxier) OnUpdate(services []api.Service) {
|
|||||||
if exists && info.isActive() && info.portalPort == service.Spec.Port && info.portalIP.Equal(serviceIP) {
|
if exists && info.isActive() && info.portalPort == service.Spec.Port && info.portalIP.Equal(serviceIP) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if exists && (info.portalPort != service.Spec.Port || !info.portalIP.Equal(serviceIP)) {
|
if exists && (info.portalPort != service.Spec.Port || !info.portalIP.Equal(serviceIP) || service.Spec.CreateExternalLoadBalancer != (len(info.publicIP) > 0)) {
|
||||||
glog.V(4).Infof("Something changed for service %q: stopping it", service.Name)
|
glog.V(4).Infof("Something changed for service %q: stopping it", service.Name)
|
||||||
err := proxier.closePortal(service.Name, info)
|
err := proxier.closePortal(service.Name, info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -462,6 +464,9 @@ func (proxier *Proxier) OnUpdate(services []api.Service) {
|
|||||||
}
|
}
|
||||||
info.portalIP = serviceIP
|
info.portalIP = serviceIP
|
||||||
info.portalPort = service.Spec.Port
|
info.portalPort = service.Spec.Port
|
||||||
|
if service.Spec.CreateExternalLoadBalancer {
|
||||||
|
info.publicIP = service.Spec.PublicIPs
|
||||||
|
}
|
||||||
err = proxier.openPortal(service.Name, info)
|
err = proxier.openPortal(service.Name, info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Errorf("Failed to open portal for %q: %s", service.Name, err)
|
glog.Errorf("Failed to open portal for %q: %s", service.Name, err)
|
||||||
@ -494,6 +499,24 @@ func (proxier *Proxier) openPortal(service string, info *serviceInfo) error {
|
|||||||
if !existed {
|
if !existed {
|
||||||
glog.Infof("Opened iptables portal for service %q on %s:%d", service, info.portalIP, info.portalPort)
|
glog.Infof("Opened iptables portal for service %q on %s:%d", service, info.portalIP, info.portalPort)
|
||||||
}
|
}
|
||||||
|
if len(info.publicIP) > 0 {
|
||||||
|
return proxier.openExternalPortal(service, info)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (proxier *Proxier) openExternalPortal(service string, info *serviceInfo) error {
|
||||||
|
for _, publicIP := range info.publicIP {
|
||||||
|
args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service)
|
||||||
|
existed, err := proxier.iptables.EnsureRule(iptables.TableNAT, iptablesProxyChain, args...)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("Failed to install iptables %s rule for service %q", iptablesProxyChain, service)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !existed {
|
||||||
|
glog.Infof("Opened iptables external portal for service %q on %s:%d", service, publicIP, info.proxyPort)
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -503,10 +526,25 @@ func (proxier *Proxier) closePortal(service string, info *serviceInfo) error {
|
|||||||
glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesProxyChain, service)
|
glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesProxyChain, service)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(info.publicIP) > 0 {
|
||||||
|
return proxier.closeExternalPortal(service, info)
|
||||||
|
}
|
||||||
glog.Infof("Closed iptables portal for service %q", service)
|
glog.Infof("Closed iptables portal for service %q", service)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (proxier *Proxier) closeExternalPortal(service string, info *serviceInfo) error {
|
||||||
|
for _, publicIP := range info.publicIP {
|
||||||
|
args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service)
|
||||||
|
if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesProxyChain, args...); err != nil {
|
||||||
|
glog.Errorf("Failed to delete external iptables %s rule for service %q", iptablesProxyChain, service)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
glog.Infof("Closed external iptables portal for service %q", service)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
var iptablesProxyChain iptables.Chain = "KUBE-PROXY"
|
var iptablesProxyChain iptables.Chain = "KUBE-PROXY"
|
||||||
|
|
||||||
// Ensure that the iptables infrastructure we use is set up. This can safely be called periodically.
|
// Ensure that the iptables infrastructure we use is set up. This can safely be called periodically.
|
||||||
|
@ -112,6 +112,7 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (<-chan apiserver.RE
|
|||||||
return apiserver.MakeAsync(func() (runtime.Object, error) {
|
return apiserver.MakeAsync(func() (runtime.Object, error) {
|
||||||
// TODO: Consider moving this to a rectification loop, so that we make/remove external load balancers
|
// TODO: Consider moving this to a rectification loop, so that we make/remove external load balancers
|
||||||
// correctly no matter what http operations happen.
|
// correctly no matter what http operations happen.
|
||||||
|
// TODO: Get rid of ProxyPort.
|
||||||
service.Spec.ProxyPort = 0
|
service.Spec.ProxyPort = 0
|
||||||
if service.Spec.CreateExternalLoadBalancer {
|
if service.Spec.CreateExternalLoadBalancer {
|
||||||
if rs.cloud == nil {
|
if rs.cloud == nil {
|
||||||
@ -133,13 +134,21 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (<-chan apiserver.RE
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, service.Spec.Port, hostsFromMinionList(hosts))
|
var ip net.IP
|
||||||
|
if len(service.Spec.PublicIPs) > 0 {
|
||||||
|
for _, publicIP := range service.Spec.PublicIPs {
|
||||||
|
ip, err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, net.ParseIP(publicIP), service.Spec.Port, hostsFromMinionList(hosts))
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ip, err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, nil, service.Spec.Port, hostsFromMinionList(hosts))
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// External load-balancers require a known port for the service proxy.
|
service.Spec.PublicIPs = []string{ip.String()}
|
||||||
// TODO: If we end up brokering HostPorts between Pods and Services, this can be any port.
|
|
||||||
service.Spec.ProxyPort = service.Spec.Port
|
|
||||||
}
|
}
|
||||||
err := rs.registry.CreateService(ctx, service)
|
err := rs.registry.CreateService(ctx, service)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -638,7 +638,7 @@ func TestServiceRegistryIPExternalLoadBalancer(t *testing.T) {
|
|||||||
if created_service.Spec.PortalIP != "1.2.3.1" {
|
if created_service.Spec.PortalIP != "1.2.3.1" {
|
||||||
t.Errorf("Unexpected PortalIP: %s", created_service.Spec.PortalIP)
|
t.Errorf("Unexpected PortalIP: %s", created_service.Spec.PortalIP)
|
||||||
}
|
}
|
||||||
if created_service.Spec.ProxyPort != 6502 {
|
if created_service.Spec.ProxyPort != 0 {
|
||||||
t.Errorf("Unexpected ProxyPort: %d", created_service.Spec.ProxyPort)
|
t.Errorf("Unexpected ProxyPort: %d", created_service.Spec.ProxyPort)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -54,8 +54,9 @@ const (
|
|||||||
type Chain string
|
type Chain string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ChainPrerouting Chain = "PREROUTING"
|
ChainPostrouting Chain = "POSTROUTING"
|
||||||
ChainOutput Chain = "OUTPUT"
|
ChainPrerouting Chain = "PREROUTING"
|
||||||
|
ChainOutput Chain = "OUTPUT"
|
||||||
)
|
)
|
||||||
|
|
||||||
// runner implements Interface in terms of exec("iptables").
|
// runner implements Interface in terms of exec("iptables").
|
||||||
|
Loading…
Reference in New Issue
Block a user