mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Make master service IP static (no longer randomly assigned)
This commit is contained in:
saadali
parent
b40d079551
commit
e83fd7b8e7
@ -162,7 +162,7 @@ func startComponents(manifestURL string) (apiServerURL string) {
|
||||
AdmissionControl: admit.NewAlwaysAdmit(),
|
||||
ReadWritePort: portNumber,
|
||||
ReadOnlyPort: portNumber,
|
||||
PublicAddress: host,
|
||||
PublicAddress: net.ParseIP(host),
|
||||
})
|
||||
handler.delegate = m.Handler
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
kubeletapp "github.com/GoogleCloudPlatform/kubernetes/cmd/kubelet/app"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
|
||||
@ -46,7 +47,7 @@ var (
|
||||
masterServiceNamespace = flag.String("master_service_namespace", api.NamespaceDefault, "The namespace from which the kubernetes master services should be injected into pods")
|
||||
)
|
||||
|
||||
func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr string, port int) {
|
||||
func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr net.IP, port int) {
|
||||
machineList := []string{"localhost"}
|
||||
|
||||
standalone.RunApiServer(cl, etcdClient, addr, port, *masterServiceNamespace)
|
||||
@ -57,7 +58,7 @@ func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr string
|
||||
standalone.SimpleRunKubelet(cl, nil, dockerClient, machineList[0], "/tmp/kubernetes", "", "127.0.0.1", 10250, *masterServiceNamespace, kubeletapp.ProbeVolumePlugins())
|
||||
}
|
||||
|
||||
func newApiClient(addr string, port int) *client.Client {
|
||||
func newApiClient(addr net.IP, port int) *client.Client {
|
||||
apiServerURL := fmt.Sprintf("http://%s:%d", addr, port)
|
||||
cl := client.NewOrDie(&client.Config{Host: apiServerURL, Version: testapi.Version()})
|
||||
return cl
|
||||
@ -73,7 +74,8 @@ func main() {
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to connect to etcd: %v", err)
|
||||
}
|
||||
startComponents(etcdClient, newApiClient(*addr, *port), *addr, *port)
|
||||
address := net.ParseIP(*addr)
|
||||
startComponents(etcdClient, newApiClient(address, *port), address, *port)
|
||||
glog.Infof("Kubernetes API Server is up and running on http://%s:%d", *addr, *port)
|
||||
|
||||
select {}
|
||||
|
||||
@ -97,8 +97,8 @@ type Config struct {
|
||||
// Defaults to 443 if not set.
|
||||
ReadWritePort int
|
||||
|
||||
// If empty, the first result from net.InterfaceAddrs will be used.
|
||||
PublicAddress string
|
||||
// If nil, the first result from net.InterfaceAddrs will be used.
|
||||
PublicAddress net.IP
|
||||
}
|
||||
|
||||
// Master contains state for a Kubernetes cluster master/api server.
|
||||
@ -133,9 +133,14 @@ type Master struct {
|
||||
v1beta3 bool
|
||||
nodeIPCache IPGetter
|
||||
|
||||
readOnlyServer string
|
||||
readWriteServer string
|
||||
masterServices *util.Runner
|
||||
publicIP net.IP
|
||||
publicReadOnlyPort int
|
||||
publicReadWritePort int
|
||||
serviceReadOnlyIP net.IP
|
||||
serviceReadOnlyPort int
|
||||
serviceReadWriteIP net.IP
|
||||
serviceReadWritePort int
|
||||
masterServices *util.Runner
|
||||
|
||||
// "Outputs"
|
||||
Handler http.Handler
|
||||
@ -176,7 +181,7 @@ func setDefaults(c *Config) {
|
||||
if c.ReadWritePort == 0 {
|
||||
c.ReadWritePort = 443
|
||||
}
|
||||
for c.PublicAddress == "" {
|
||||
for c.PublicAddress == nil {
|
||||
// Find and use the first non-loopback address.
|
||||
// TODO: potentially it'd be useful to skip the docker interface if it
|
||||
// somehow is first in the list.
|
||||
@ -196,7 +201,7 @@ func setDefaults(c *Config) {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
c.PublicAddress = ip.String()
|
||||
c.PublicAddress = ip
|
||||
glog.Infof("Will report %v as public IP address.", ip)
|
||||
break
|
||||
}
|
||||
@ -244,6 +249,17 @@ func New(c *Config) *Master {
|
||||
glog.Fatalf("master.New() called with config.KubeletClient == nil")
|
||||
}
|
||||
|
||||
// Select the first two valid IPs from portalNet to use as the master service portalIPs
|
||||
serviceReadOnlyIP, err := service.GetIndexedIP(c.PortalNet, 1)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to generate service read-only IP for master service: %v", err)
|
||||
}
|
||||
serviceReadWriteIP, err := service.GetIndexedIP(c.PortalNet, 2)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to generate service read-write IP for master service: %v", err)
|
||||
}
|
||||
glog.Infof("Setting master service IPs based on PortalNet subnet to %q (read-only) and %q (read-write).", serviceReadOnlyIP, serviceReadWriteIP)
|
||||
|
||||
m := &Master{
|
||||
podRegistry: etcd.NewRegistry(c.EtcdHelper, boundPodFactory),
|
||||
controllerRegistry: etcd.NewRegistry(c.EtcdHelper, nil),
|
||||
@ -268,9 +284,16 @@ func New(c *Config) *Master {
|
||||
v1beta3: c.EnableV1Beta3,
|
||||
nodeIPCache: NewIPCache(c.Cloud, util.RealClock{}, 30*time.Second),
|
||||
|
||||
masterCount: c.MasterCount,
|
||||
readOnlyServer: net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadOnlyPort))),
|
||||
readWriteServer: net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadWritePort))),
|
||||
masterCount: c.MasterCount,
|
||||
publicIP: c.PublicAddress,
|
||||
publicReadOnlyPort: c.ReadOnlyPort,
|
||||
publicReadWritePort: c.ReadWritePort,
|
||||
serviceReadOnlyIP: serviceReadOnlyIP,
|
||||
// TODO: serviceReadOnlyPort should be passed in as an argument, it may not always be 80
|
||||
serviceReadOnlyPort: 80,
|
||||
serviceReadWriteIP: serviceReadWriteIP,
|
||||
// TODO: serviceReadWritePort should be passed in as an argument, it may not always be 443
|
||||
serviceReadWritePort: 443,
|
||||
}
|
||||
|
||||
if c.RestfulContainer != nil {
|
||||
@ -443,7 +466,7 @@ func (m *Master) init(c *Config) {
|
||||
func (m *Master) InstallSwaggerAPI() {
|
||||
// Enable swagger UI and discovery API
|
||||
swaggerConfig := swagger.Config{
|
||||
WebServicesUrl: m.readWriteServer,
|
||||
WebServicesUrl: net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadWritePort))),
|
||||
WebServices: m.handlerContainer.RegisteredWebServices(),
|
||||
// TODO: Parameterize the path?
|
||||
ApiPath: "/swaggerapi/",
|
||||
|
||||
@ -18,6 +18,8 @@ package master
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
|
||||
@ -32,12 +34,11 @@ func (m *Master) serviceWriterLoop(stop chan struct{}) {
|
||||
// TODO: when it becomes possible to change this stuff,
|
||||
// stop polling and start watching.
|
||||
// TODO: add endpoints of all replicas, not just the elected master.
|
||||
if m.readWriteServer != "" {
|
||||
// TODO: the public port should be part of the argument here, port will not always be 443
|
||||
if err := m.createMasterServiceIfNeeded("kubernetes", 443); err != nil {
|
||||
if m.serviceReadWriteIP != nil {
|
||||
if err := m.createMasterServiceIfNeeded("kubernetes", m.serviceReadWriteIP, m.serviceReadWritePort); err != nil {
|
||||
glog.Errorf("Can't create rw service: %v", err)
|
||||
}
|
||||
if err := m.ensureEndpointsContain("kubernetes", m.readWriteServer); err != nil {
|
||||
if err := m.ensureEndpointsContain("kubernetes", net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadWritePort)))); err != nil {
|
||||
glog.Errorf("Can't create rw endpoints: %v", err)
|
||||
}
|
||||
}
|
||||
@ -55,12 +56,11 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) {
|
||||
// Update service & endpoint records.
|
||||
// TODO: when it becomes possible to change this stuff,
|
||||
// stop polling and start watching.
|
||||
if m.readOnlyServer != "" {
|
||||
// TODO: the public port should be part of the argument here, port will not always be 80
|
||||
if err := m.createMasterServiceIfNeeded("kubernetes-ro", 80); err != nil {
|
||||
if m.serviceReadOnlyIP != nil {
|
||||
if err := m.createMasterServiceIfNeeded("kubernetes-ro", m.serviceReadOnlyIP, m.serviceReadOnlyPort); err != nil {
|
||||
glog.Errorf("Can't create ro service: %v", err)
|
||||
}
|
||||
if err := m.ensureEndpointsContain("kubernetes-ro", m.readOnlyServer); err != nil {
|
||||
if err := m.ensureEndpointsContain("kubernetes-ro", net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadOnlyPort)))); err != nil {
|
||||
glog.Errorf("Can't create ro endpoints: %v", err)
|
||||
}
|
||||
}
|
||||
@ -75,7 +75,7 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) {
|
||||
|
||||
// createMasterServiceIfNeeded will create the specified service if it
|
||||
// doesn't already exist.
|
||||
func (m *Master) createMasterServiceIfNeeded(serviceName string, port int) error {
|
||||
func (m *Master) createMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePort int) error {
|
||||
ctx := api.NewDefaultContext()
|
||||
if _, err := m.serviceRegistry.GetService(ctx, serviceName); err == nil {
|
||||
// The service already exists.
|
||||
@ -88,9 +88,10 @@ func (m *Master) createMasterServiceIfNeeded(serviceName string, port int) error
|
||||
Labels: map[string]string{"provider": "kubernetes", "component": "apiserver"},
|
||||
},
|
||||
Spec: api.ServiceSpec{
|
||||
Port: port,
|
||||
Port: servicePort,
|
||||
// maintained by this code, not by the pod selector
|
||||
Selector: nil,
|
||||
PortalIP: serviceIP.String(),
|
||||
},
|
||||
}
|
||||
// Kids, don't do this at home: this is a hack. There's no good way to call the business
|
||||
|
||||
@ -47,7 +47,7 @@ import (
|
||||
type APIServer struct {
|
||||
Port int
|
||||
Address util.IP
|
||||
PublicAddressOverride string
|
||||
PublicAddressOverride util.IP
|
||||
ReadOnlyPort int
|
||||
APIRate float32
|
||||
APIBurst int
|
||||
@ -80,6 +80,7 @@ func NewAPIServer() *APIServer {
|
||||
s := APIServer{
|
||||
Port: 8080,
|
||||
Address: util.IP(net.ParseIP("127.0.0.1")),
|
||||
PublicAddressOverride: util.IP(net.ParseIP("")),
|
||||
ReadOnlyPort: 7080,
|
||||
APIRate: 10.0,
|
||||
APIBurst: 200,
|
||||
@ -127,11 +128,10 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
|
||||
"further assumed that port 443 on the cluster's public address is proxied to this "+
|
||||
"port. This is performed by nginx in the default setup.")
|
||||
fs.Var(&s.Address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces)")
|
||||
fs.StringVar(&s.PublicAddressOverride, "public_address_override", s.PublicAddressOverride, ""+
|
||||
"Public serving address. Read only port will be opened on this address, "+
|
||||
"and it is assumed that port 443 at this address will be proxied/redirected "+
|
||||
"to '-address':'-port'. If blank, the address in the first listed interface "+
|
||||
"will be used.")
|
||||
fs.Var(&s.PublicAddressOverride, "public_address_override", "Public serving address."+
|
||||
"Read only port will be opened on this address, and it is assumed that port "+
|
||||
"443 at this address will be proxied/redirected to '-address':'-port'. If "+
|
||||
"blank, the address in the first listed interface will be used.")
|
||||
fs.IntVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, ""+
|
||||
"The port from which to serve read-only resources. If 0, don't serve on a "+
|
||||
"read-only address. It is assumed that firewall rules are set up such that "+
|
||||
@ -251,7 +251,7 @@ func (s *APIServer) Run(_ []string) error {
|
||||
CorsAllowedOriginList: s.CorsAllowedOriginList,
|
||||
ReadOnlyPort: s.ReadOnlyPort,
|
||||
ReadWritePort: s.Port,
|
||||
PublicAddress: s.PublicAddressOverride,
|
||||
PublicAddress: net.IP(s.PublicAddressOverride),
|
||||
Authenticator: authenticator,
|
||||
Authorizer: authorizer,
|
||||
AdmissionControl: admissionController,
|
||||
@ -263,11 +263,11 @@ func (s *APIServer) Run(_ []string) error {
|
||||
// We serve on 3 ports. See docs/reaching_the_api.md
|
||||
roLocation := ""
|
||||
if s.ReadOnlyPort != 0 {
|
||||
roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort))
|
||||
roLocation = net.JoinHostPort(config.PublicAddress.String(), strconv.Itoa(config.ReadOnlyPort))
|
||||
}
|
||||
secureLocation := ""
|
||||
if s.SecurePort != 0 {
|
||||
secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(s.SecurePort))
|
||||
secureLocation = net.JoinHostPort(config.PublicAddress.String(), strconv.Itoa(s.SecurePort))
|
||||
}
|
||||
rwLocation := net.JoinHostPort(s.Address.String(), strconv.Itoa(int(s.Port)))
|
||||
|
||||
@ -317,7 +317,7 @@ func (s *APIServer) Run(_ []string) error {
|
||||
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
|
||||
s.TLSCertFile = "/var/run/kubernetes/apiserver.crt"
|
||||
s.TLSPrivateKeyFile = "/var/run/kubernetes/apiserver.key"
|
||||
if err := util.GenerateSelfSignedCert(config.PublicAddress, s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
|
||||
if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
|
||||
glog.Errorf("Unable to generate self signed cert: %v", err)
|
||||
} else {
|
||||
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
|
||||
|
||||
@ -174,6 +174,18 @@ func (ipa *ipAllocator) AllocateNext() (net.IP, error) {
|
||||
return nil, fmt.Errorf("can't find a free IP in %s", ipa.subnet)
|
||||
}
|
||||
|
||||
// Returns the index-th IP from the specified subnet range.
|
||||
// For example, subnet "10.0.0.0/24" with index "2" will return the IP "10.0.0.2".
|
||||
// TODO(saad-ali): Move this (and any other functions that are independent of ipAllocator) to some
|
||||
// place more generic.
|
||||
func GetIndexedIP(subnet *net.IPNet, index int) (net.IP, error) {
|
||||
ip := ipAdd(subnet.IP, index /* offset */)
|
||||
if !subnet.Contains(ip) {
|
||||
return nil, fmt.Errorf("can't generate IP with index %d from subnet. subnet too small. subnet: %q", index, subnet)
|
||||
}
|
||||
return ip, nil
|
||||
}
|
||||
|
||||
func (ipa *ipAllocator) createRandomIp() net.IP {
|
||||
ip := ipa.subnet.IP
|
||||
mask := ipa.subnet.Mask
|
||||
|
||||
@ -214,6 +214,75 @@ func TestIPAdd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateFirstTwoIPsFromSubnet(t *testing.T) {
|
||||
// Arrange
|
||||
testCases := []struct {
|
||||
subnet string
|
||||
expected1stIP string
|
||||
expected2ndIP string
|
||||
}{
|
||||
{"10.0.0.0/24", "10.0.0.1", "10.0.0.2"},
|
||||
{"10.10.10.10/24", "10.10.10.1", "10.10.10.2"},
|
||||
{"10.10.10.10/16", "10.10.0.1", "10.10.0.2"},
|
||||
{"10.10.10.10/8", "10.0.0.1", "10.0.0.2"},
|
||||
{"10.10.10.10/0", "0.0.0.1", "0.0.0.2"},
|
||||
{"192.168.100.1/16", "192.168.0.1", "192.168.0.2"},
|
||||
{"153.15.250.5/23", "153.15.250.1", "153.15.250.2"},
|
||||
{"2001:db8::/48", "2001:db8::1", "2001:db8::2"},
|
||||
{"2001:db8:123:255::/48", "2001:db8:123::1", "2001:db8:123::2"},
|
||||
{"12.12.0.0/30", "12.12.0.1", "12.12.0.2"},
|
||||
}
|
||||
|
||||
// Act & Assert
|
||||
for _, testCase := range testCases {
|
||||
_, subnet, _ := net.ParseCIDR(testCase.subnet)
|
||||
firstIP, err := GetIndexedIP(subnet, 1)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
secondIP, err := GetIndexedIP(subnet, 2)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
if firstIP.String() != testCase.expected1stIP {
|
||||
t.Errorf("Unexpected first IP: Expected <%q> Actual <%q>", testCase.expected1stIP, firstIP.String())
|
||||
}
|
||||
if secondIP.String() != testCase.expected2ndIP {
|
||||
t.Errorf("Unexpected second IP: Expected <%q> Actual <%q>", testCase.expected2ndIP, secondIP.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetIndexedIPSubnetTooSmall(t *testing.T) {
|
||||
// Arrange
|
||||
testCases := []struct {
|
||||
subnet string
|
||||
}{
|
||||
{"12.12.0.0/32"},
|
||||
{"12.12.0.0/31"},
|
||||
}
|
||||
|
||||
// Act & Assert
|
||||
for _, testCase := range testCases {
|
||||
_, subnet, _ := net.ParseCIDR(testCase.subnet)
|
||||
secondIP, err := GetIndexedIP(subnet, 2)
|
||||
if err == nil {
|
||||
t.Errorf("Expected error but no error occured for subnet: ", testCase.subnet)
|
||||
}
|
||||
thirdIP, err := GetIndexedIP(subnet, 3)
|
||||
if err == nil {
|
||||
t.Errorf("Expected error but no error occured for subnet: ", testCase.subnet)
|
||||
}
|
||||
if secondIP != nil {
|
||||
t.Errorf("Unexpected second IP: Expected nil Actual <%q>", thirdIP.String())
|
||||
}
|
||||
if thirdIP != nil {
|
||||
t.Errorf("Unexpected third IP: Expected nil Actual <%q>", secondIP.String())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyIP(t *testing.T) {
|
||||
ip1 := net.ParseIP("1.2.3.4")
|
||||
ip2 := copyIP(ip1)
|
||||
|
||||
@ -85,7 +85,7 @@ func GetAPIServerClient(authPath string, apiServerList util.StringList) (*client
|
||||
}
|
||||
|
||||
// RunApiServer starts an API server in a go routine.
|
||||
func RunApiServer(cl *client.Client, etcdClient tools.EtcdClient, addr string, port int, masterServiceNamespace string) {
|
||||
func RunApiServer(cl *client.Client, etcdClient tools.EtcdClient, addr net.IP, port int, masterServiceNamespace string) {
|
||||
handler := delegateHandler{}
|
||||
|
||||
helper, err := master.NewEtcdHelper(etcdClient, "")
|
||||
|
||||
@ -106,7 +106,7 @@ var aService string = `
|
||||
"apiVersion": "v1beta1",
|
||||
"id": "a",
|
||||
"port": 8000,
|
||||
"portalIP": "10.0.0.1",
|
||||
"portalIP": "10.0.0.100",
|
||||
"labels": { "name": "a" },
|
||||
"selector": { "name": "a" }
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user