// Copyright (c) 2023 Multus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This is a entrypoint for thin (stand-alone) images.
package main

import (
	"bytes"
	"crypto/sha256"
	b64 "encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"github.com/containernetworking/cni/libcni"
	"github.com/spf13/pflag"

	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/signals"
)

// Options stores command line options
type Options struct {
	CNIBinDir                string
	CNIConfDir               string
	CNIVersion               string
	MultusConfFile           string
	MultusBinFile            string // may be hidden or remove?
	MultusCNIConfDir         string
	SkipMultusBinaryCopy     bool
	MultusKubeConfigFileHost string
	MultusMasterCNIFileName  string
	NamespaceIsolation       bool
	GlobalNamespaces         string
	MultusAutoconfigDir      string
	MultusLogToStderr        bool
	MultusLogLevel           string
	MultusLogFile            string
	OverrideNetworkName      bool
	CleanupConfigOnExit      bool
	RenameConfFile           bool
	ReadinessIndicatorFile   string
	AdditionalBinDir         string
	ForceCNIVersion          bool
	SkipTLSVerify            bool
	SkipMultusConfWatch      bool
}

const (
	serviceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
	serviceAccountCAFile    = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
)

func (o *Options) addFlags() {
	pflag.ErrHelp = nil // suppress error message for help
	fs := pflag.CommandLine
	fs.StringVar(&o.CNIBinDir, "cni-bin-dir", "/host/opt/cni/bin", "CNI binary directory")
	fs.StringVar(&o.CNIConfDir, "cni-conf-dir", "/host/etc/cni/net.d", "CNI config directory")
	fs.StringVar(&o.CNIVersion, "cni-version", "", "CNI version for multus CNI config (e.g. '0.3.1')")
	fs.StringVar(&o.MultusConfFile, "multus-conf-file", "auto", "multus CNI config file")
	fs.StringVar(&o.MultusBinFile, "multus-bin-file", "/usr/src/multus-cni/bin/multus", "multus binary file path")
	fs.StringVar(&o.MultusCNIConfDir, "multus-cni-conf-dir", "/host/etc/cni/multus/net.d", "multus specific CNI config directory")
	fs.BoolVar(&o.SkipMultusBinaryCopy, "skip-multus-binary-copy", false, "skip multus binary file copy")

	fs.StringVar(&o.MultusKubeConfigFileHost, "multus-kubeconfig-file-host", "/etc/cni/net.d/multus.d/multus.kubeconfig", "kubeconfig for multus (used only with --multus-conf-file=auto)")
	fs.StringVar(&o.MultusMasterCNIFileName, "multus-master-cni-file-name", "", "master CNI file in multus-autoconfig-dir")
	fs.BoolVar(&o.NamespaceIsolation, "namespace-isolation", false, "namespace isolation")
	fs.StringVar(&o.GlobalNamespaces, "global-namespaces", "", "global namespaces, comma separated (used only with --namespace-isolation=true)")
	fs.StringVar(&o.MultusAutoconfigDir, "multus-autoconfig-dir", "/host/etc/cni/net.d", "multus autoconfig dir (used only with --multus-conf-file=auto)")
	fs.BoolVar(&o.MultusLogToStderr, "multus-log-to-stderr", true, "log to stderr")
	fs.StringVar(&o.MultusLogLevel, "multus-log-level", "", "multus log level")
	fs.StringVar(&o.MultusLogFile, "multus-log-file", "", "multus log file")
	fs.BoolVar(&o.OverrideNetworkName, "override-network-name", false, "override network name from master cni file (used only with --multus-conf-file=auto)")
	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit")
	fs.BoolVar(&o.SkipMultusConfWatch, "skip-config-watch", false, "dont watch for config (master cni and kubeconfig) changes (used only with --multus-conf-file=auto)")
	fs.BoolVar(&o.RenameConfFile, "rename-conf-file", false, "rename master config file to invalidate (used only with --multus-conf-file=auto)")
	fs.StringVar(&o.ReadinessIndicatorFile, "readiness-indicator-file", "", "readiness indicator file (used only with --multus-conf-file=auto)")
	fs.StringVar(&o.AdditionalBinDir, "additional-bin-dir", "", "adds binDir option to configuration (used only with --multus-conf-file=auto)")
	fs.BoolVar(&o.SkipTLSVerify, "skip-tls-verify", false, "skip TLS verify")
	fs.BoolVar(&o.ForceCNIVersion, "force-cni-version", false, "force cni version to '--cni-version' (only for e2e-kind testing)")
	fs.MarkHidden("force-cni-version")
	fs.MarkHidden("skip-tls-verify")
}

func (o *Options) verifyFileExists() error {
	// CNIConfDir
	if _, err := os.Stat(o.CNIConfDir); err != nil {
		return fmt.Errorf("cni-conf-dir is not found: %v", err)
	}

	// CNIBinDir
	if _, err := os.Stat(o.CNIBinDir); err != nil {
		return fmt.Errorf("cni-bin-dir is not found: %v", err)
	}

	// MultusBinFile
	if _, err := os.Stat(o.MultusBinFile); err != nil {
		return fmt.Errorf("multus-bin-file is not found: %v", err)
	}

	if o.MultusConfFile != "auto" {
		// MultusConfFile
		if _, err := os.Stat(o.MultusConfFile); err != nil {
			return fmt.Errorf("multus-conf-file is not found: %v", err)
		}
	}
	return nil
}

const kubeConfigTemplate = `# Kubeconfig file for Multus CNI plugin.
apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    server: {{ .KubeConfigHost }}
    {{ .KubeServerTLS }}
users:
- name: multus
  user:
    token: "{{ .KubeServiceAccountToken }}"
contexts:
- name: multus-context
  context:
    cluster: local
    user: multus
current-context: multus-context
`

func getFileAndHash(filepath string) ([]byte, []byte, error) {
	if _, err := os.Stat(filepath); err != nil {
		return nil, nil, fmt.Errorf("file %s not found: %v", filepath, err)
	}
	content, err := os.ReadFile(filepath)
	if err != nil {
		return nil, nil, fmt.Errorf("cannot read %s file: %v", filepath, err)
	}

	hash := sha256.New()
	hash.Write(content)
	return content, hash.Sum(nil), nil
}

func (o *Options) createKubeConfig(prevCAHash, prevSATokenHash []byte) ([]byte, []byte, error) {
	caFileByte, caHash, err := getFileAndHash(serviceAccountCAFile)
	if err != nil {
		return nil, nil, err
	}
	saTokenByte, saTokenHash, err := getFileAndHash(serviceAccountTokenFile)
	if err != nil {
		return nil, nil, err
	}

	caUnchanged := prevCAHash != nil && bytes.Equal(prevCAHash, caHash)
	saUnchanged := prevSATokenHash != nil && bytes.Equal(prevSATokenHash, saTokenHash)

	if o.SkipTLSVerify {
		if saUnchanged {
			return caHash, saTokenHash, nil
		}
	} else {
		if caUnchanged && saUnchanged {
			return caHash, saTokenHash, nil
		}
	}

	if prevSATokenHash != nil {
		// don't log "recreating" on first function execution
		fmt.Printf("CA (%v) or SA token (%v) changed - recreating kubeconfig\n", !caUnchanged, !saUnchanged)
	}

	// create multus.d directory
	if err := os.MkdirAll(fmt.Sprintf("%s/multus.d", o.CNIConfDir), 0755); err != nil {
		return nil, nil, fmt.Errorf("cannot create multus.d directory: %v", err)
	}

	// create multus cni conf directory
	if err := os.MkdirAll(o.MultusCNIConfDir, 0755); err != nil {
		return nil, nil, fmt.Errorf("cannot create multus-cni-conf-dir(%s) directory: %v", o.MultusCNIConfDir, err)
	}

	// get Kubernetes service protocol/host/port
	kubeProtocol := os.Getenv("KUBERNETES_SERVICE_PROTOCOL")
	if kubeProtocol == "" {
		kubeProtocol = "https"
	}
	kubeHost := os.Getenv("KUBERNETES_SERVICE_HOST")
	kubePort := os.Getenv("KUBERNETES_SERVICE_PORT")

	// check tlsConfig
	tlsConfig := ""
	if o.SkipTLSVerify {
		tlsConfig = "insecure-skip-tls-verify: true"
	} else {
		// create tlsConfig by service account CA file
		caFileB64 := bytes.ReplaceAll([]byte(b64.StdEncoding.EncodeToString(caFileByte)), []byte("\n"), []byte(""))
		tlsConfig = fmt.Sprintf("certificate-authority-data: %s", string(caFileB64))
	}

	// create kubeconfig by template and replace it by atomic
	tempKubeConfigFile := fmt.Sprintf("%s/multus.d/multus.kubeconfig.new", o.CNIConfDir)
	multusKubeConfig := fmt.Sprintf("%s/multus.d/multus.kubeconfig", o.CNIConfDir)
	fp, err := os.OpenFile(tempKubeConfigFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return nil, nil, fmt.Errorf("cannot create kubeconfig temp file: %v", err)
	}

	templateKubeconfig, err := template.New("kubeconfig").Parse(kubeConfigTemplate)
	if err != nil {
		return nil, nil, fmt.Errorf("template parse error: %v", err)
	}
	templateData := map[string]string{
		"KubeConfigHost":          fmt.Sprintf("%s://[%s]:%s", kubeProtocol, kubeHost, kubePort),
		"KubeServerTLS":           tlsConfig,
		"KubeServiceAccountToken": string(saTokenByte),
	}

	// generate kubeconfig from template
	if err = templateKubeconfig.Execute(fp, templateData); err != nil {
		return nil, nil, fmt.Errorf("cannot create kubeconfig: %v", err)
	}

	if err := fp.Sync(); err != nil {
		os.Remove(fp.Name())
		return nil, nil, fmt.Errorf("cannot flush kubeconfig temp file: %v", err)
	}
	if err := fp.Close(); err != nil {
		os.Remove(fp.Name())
		return nil, nil, fmt.Errorf("cannot close kubeconfig temp file: %v", err)
	}

	// replace file with tempfile
	if err := os.Rename(tempKubeConfigFile, multusKubeConfig); err != nil {
		return nil, nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusKubeConfig, tempKubeConfigFile, err)
	}

	fmt.Printf("kubeconfig is created in %s\n", multusKubeConfig)
	return caHash, saTokenHash, nil
}

const multusConflistTemplate = `{
    "cniVersion": "{{ .CNIVersion }}",
    "name": "{{ .MasterPluginNetworkName }}",
    "plugins": [ {
        "type": "multus",{{
            .NestedCapabilities
        }}{{
            .NamespaceIsolationConfig
        }}{{
            .GlobalNamespacesConfig
        }}{{
            .LogToStderrConfig
        }}{{
            .LogLevelConfig
        }}{{
            .LogFileConfig
        }}{{
            .AdditionalBinDirConfig
        }}{{
            .MultusCNIConfDirConfig
        }}{{
            .ReadinessIndicatorFileConfig
        }}
        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
        "delegates": [
            {{ .MasterPluginJSON }}
        ]
    }]
}
`

const multusConfTemplate = `{
        "cniVersion": "{{ .CNIVersion }}",
        "name": "{{ .MasterPluginNetworkName }}",
        "type": "multus",{{
            .NestedCapabilities
        }}{{
            .NamespaceIsolationConfig
        }}{{
            .GlobalNamespacesConfig
        }}{{
            .LogToStderrConfig
        }}{{
            .LogLevelConfig
        }}{{
            .LogFileConfig
        }}{{
            .AdditionalBinDirConfig
        }}{{
            .MultusCNIConfDirConfig
        }}{{
            .ReadinessIndicatorFileConfig
        }}
        "kubeconfig": "{{ .MultusKubeConfigFileHost }}",
        "delegates": [
                {{ .MasterPluginJSON }}
        ]
}
`

func (o *Options) getMasterConfigPath() (string, error) {
	// Master config file is specified
	if o.MultusMasterCNIFileName != "" {
		return filepath.Join(o.MultusAutoconfigDir, o.MultusMasterCNIFileName), nil
	}

	// Pick the alphabetically first config file from MultusAutoconfigDir
	files, err := libcni.ConfFiles(o.MultusAutoconfigDir, []string{".conf", ".conflist"})
	if err != nil {
		return "", fmt.Errorf("cannot find master CNI config in %q: %v", o.MultusAutoconfigDir, err)
	}

	for _, filename := range files {
		if !strings.HasPrefix(filepath.Base(filename), "00-multus.conf") {
			return filename, nil
		}
	}

	// No config file found
	return "", fmt.Errorf("cannot find valid master CNI config in %q", o.MultusAutoconfigDir)
}

func (o *Options) createMultusConfig(prevMasterConfigFileHash []byte) (string, []byte, error) {
	masterConfigPath, err := o.getMasterConfigPath()
	if err != nil {
		return "", nil, err
	}

	masterConfigBytes, masterConfigFileHash, err := getFileAndHash(masterConfigPath)
	if err != nil {
		return "", nil, err
	}

	if prevMasterConfigFileHash != nil && bytes.Equal(prevMasterConfigFileHash, masterConfigFileHash) {
		return masterConfigPath, masterConfigFileHash, nil
	}

	if prevMasterConfigFileHash != nil {
		// don't log "recreating" on first function execution
		fmt.Printf("master config changed - recreating multus config\n")
	}

	masterConfig := map[string]interface{}{}
	if err = json.Unmarshal(masterConfigBytes, &masterConfig); err != nil {
		return "", nil, fmt.Errorf("cannot read master CNI config json: %v", err)
	}

	// check CNIVersion
	masterCNIVersionElem, ok := masterConfig["cniVersion"]
	if !ok {
		return "", nil, fmt.Errorf("cannot get cniVersion in master CNI config file %q: %v", masterConfigPath, err)
	}

	if o.ForceCNIVersion {
		masterConfig["cniVersion"] = o.CNIVersion
		fmt.Printf("force CNI version to %q\n", o.CNIVersion)
	} else {
		masterCNIVersion := masterCNIVersionElem.(string)
		if o.CNIVersion != "" && masterCNIVersion != o.CNIVersion {
			return "", nil, fmt.Errorf("Multus cni version is %q while master plugin cni version is %q", o.CNIVersion, masterCNIVersion)
		}
		o.CNIVersion = masterCNIVersion
	}
	cniVersionConfig := o.CNIVersion

	// check OverrideNetworkName (if true, get master plugin name, otherwise 'multus-cni-network'
	masterPluginNetworkName := "multus-cni-network"
	if o.OverrideNetworkName {
		masterPluginNetworkElem, ok := masterConfig["name"]
		if !ok {
			return "", nil, fmt.Errorf("cannot get name in master CNI config file %q: %v", masterConfigPath, err)
		}

		masterPluginNetworkName = masterPluginNetworkElem.(string)
		fmt.Printf("master plugin name is overrided to %q\n", masterPluginNetworkName)
	}

	// check capabilities (from master conf, top and 'plugins')
	masterCapabilities := map[string]bool{}
	_, isMasterConfList := masterConfig["plugins"]

	if isMasterConfList {
		masterPluginsElem, ok := masterConfig["plugins"]
		if !ok {
			return "", nil, fmt.Errorf("cannot get 'plugins' field in master CNI config file %q: %v", masterConfigPath, err)
		}
		masterPlugins := masterPluginsElem.([]interface{})
		for _, v := range masterPlugins {
			pluginFields := v.(map[string]interface{})
			capabilitiesElem, ok := pluginFields["capabilities"]
			if ok {
				capabilities := capabilitiesElem.(map[string]interface{})
				for k, v := range capabilities {
					masterCapabilities[k] = v.(bool)
				}
			}
		}
		fmt.Printf("master capabilities is get from conflist\n")
	} else {
		masterCapabilitiesElem, ok := masterConfig["capabilities"]
		if ok {
			for k, v := range masterCapabilitiesElem.(map[string]interface{}) {
				masterCapabilities[k] = v.(bool)
			}
		}
		fmt.Printf("master capabilities is get from conffile\n")
	}
	nestedCapabilitiesConf := ""
	if len(masterCapabilities) != 0 {
		capabilitiesByte, err := json.Marshal(masterCapabilities)
		if err != nil {
			return "", nil, fmt.Errorf("cannot get capabilities map: %v", err)
		}
		nestedCapabilitiesConf = fmt.Sprintf("\n        \"capabilities\": %s,", string(capabilitiesByte))
	}

	// check NamespaceIsolation
	namespaceIsolationConfig := ""
	if o.NamespaceIsolation {
		namespaceIsolationConfig = "\n        \"namespaceIsolation\": true,"
	}

	// check GlobalNamespaces
	globalNamespaceConfig := ""
	if o.GlobalNamespaces != "" {
		globalNamespaceConfig = fmt.Sprintf("\n        \"globalNamespaces\": %q,", o.GlobalNamespaces)
	}

	// check MultusLogToStderr
	logToStderrConfig := ""
	if !o.MultusLogToStderr {
		logToStderrConfig = "\n        \"logToStderr\": false,"
	}

	// check MultusLogLevel (debug/error/panic/verbose) and reject others
	logLevelConfig := ""
	logLevelStr := strings.ToLower(o.MultusLogLevel)
	switch logLevelStr {
	case "debug", "error", "panic", "verbose":
		logLevelConfig = fmt.Sprintf("\n        \"logLevel\": %q,", logLevelStr)
	case "":
		// no logLevel config, skipped
	default:
		return "", nil, fmt.Errorf("Log levels should be one of: debug/verbose/error/panic, did not understand: %q", o.MultusLogLevel)
	}

	// check MultusLogFile
	logFileConfig := ""
	if o.MultusLogFile != "" {
		logFileConfig = fmt.Sprintf("\n        \"logFile\": %q,", o.MultusLogFile)
	}

	// check AdditionalBinDir
	additionalBinDirConfig := ""
	if o.AdditionalBinDir != "" {
		additionalBinDirConfig = fmt.Sprintf("\n        \"binDir\": %q,", o.AdditionalBinDir)
	}

	// check MultusCNIConfDir
	multusCNIConfDirConfig := ""
	if o.MultusCNIConfDir != "" {
		multusCNIConfDirConfig = fmt.Sprintf("\n        \"cniConf\": %q,", o.MultusCNIConfDir)
	}

	// check ReadinessIndicatorFile
	readinessIndicatorFileConfig := ""
	if o.ReadinessIndicatorFile != "" {
		readinessIndicatorFileConfig = fmt.Sprintf("\n        \"readinessindicatorfile\": %q,", o.ReadinessIndicatorFile)
	}

	// fill .MasterPluginJSON
	masterPluginByte, err := json.Marshal(masterConfig)
	if err != nil {
		return "", nil, fmt.Errorf("cannot encode master CNI config: %v", err)
	}

	// generate multus config
	tempFileName := fmt.Sprintf("%s/00-multus.conf.new", o.CNIConfDir)
	fp, err := os.OpenFile(tempFileName, os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return "", nil, fmt.Errorf("cannot create multus cni temp file: %v", err)
	}

	// use conflist template if cniVersionConfig == "1.0.0"
	multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", o.CNIConfDir)
	templateMultusConfig, err := template.New("multusCNIConfig").Parse(multusConfTemplate)
	if err != nil {
		return "", nil, fmt.Errorf("template parse error: %v", err)
	}

	if o.CNIVersion == "1.0.0" { //Check 1.0.0 or above!
		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", o.CNIConfDir)
		templateMultusConfig, err = template.New("multusCNIConfig").Parse(multusConflistTemplate)
		if err != nil {
			return "", nil, fmt.Errorf("template parse error: %v", err)
		}
	}

	templateData := map[string]string{
		"CNIVersion":                   cniVersionConfig,
		"MasterPluginNetworkName":      masterPluginNetworkName,
		"NestedCapabilities":           nestedCapabilitiesConf,
		"NamespaceIsolationConfig":     namespaceIsolationConfig,
		"GlobalNamespacesConfig":       globalNamespaceConfig,
		"LogToStderrConfig":            logToStderrConfig,
		"LogLevelConfig":               logLevelConfig,
		"LogFileConfig":                logFileConfig,
		"AdditionalBinDirConfig":       additionalBinDirConfig,
		"MultusCNIConfDirConfig":       multusCNIConfDirConfig,
		"ReadinessIndicatorFileConfig": readinessIndicatorFileConfig,
		"MultusKubeConfigFileHost":     o.MultusKubeConfigFileHost, // be fixed?
		"MasterPluginJSON":             string(masterPluginByte),
	}
	if err = templateMultusConfig.Execute(fp, templateData); err != nil {
		return "", nil, fmt.Errorf("cannot create multus cni config: %v", err)
	}

	if err := fp.Sync(); err != nil {
		os.Remove(tempFileName)
		return "", nil, fmt.Errorf("cannot flush multus cni config: %v", err)
	}
	if err := fp.Close(); err != nil {
		os.Remove(tempFileName)
		return "", nil, fmt.Errorf("cannot close multus cni config: %v", err)
	}

	if err := os.Rename(tempFileName, multusConfFilePath); err != nil {
		return "", nil, fmt.Errorf("cannot replace %q with temp file %q: %v", multusConfFilePath, tempFileName, err)
	}

	if o.RenameConfFile {
		//masterConfigPath
		renamedMasterConfigPath := fmt.Sprintf("%s.old", masterConfigPath)
		if err := os.Rename(masterConfigPath, renamedMasterConfigPath); err != nil {
			return "", nil, fmt.Errorf("cannot move original master file to %q", renamedMasterConfigPath)
		}
		fmt.Printf("Original master file moved to %q\n", renamedMasterConfigPath)
	}

	return masterConfigPath, masterConfigFileHash, nil
}

func main() {
	opt := Options{}
	opt.addFlags()
	helpFlag := pflag.BoolP("help", "h", false, "show help message and quit")

	pflag.Parse()
	if *helpFlag {
		pflag.PrintDefaults()
		os.Exit(1)
	}

	err := opt.verifyFileExists()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}

	// copy multus binary
	if !opt.SkipMultusBinaryCopy {
		// Copy
		if err = cmdutils.CopyFileAtomic(opt.MultusBinFile, opt.CNIBinDir, "_multus", "multus"); err != nil {
			fmt.Fprintf(os.Stderr, "failed at multus copy: %v\n", err)
			return
		}
	}

	var masterConfigHash, caHash, saTokenHash []byte
	var masterConfigFilePath string
	// copy user specified multus conf to CNI conf directory
	if opt.MultusConfFile != "auto" {
		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
			return
		}
		confFileName := filepath.Base(opt.MultusConfFile)
		tempConfFileName := fmt.Sprintf("%s.temp", confFileName)
		if err = cmdutils.CopyFileAtomic(opt.MultusConfFile, opt.CNIConfDir, tempConfFileName, confFileName); err != nil {
			fmt.Fprintf(os.Stderr, "failed at copy multus conf file: %v\n", err)
			return
		}
		fmt.Printf("multus config file %s is copied.\n", opt.MultusConfFile)
	} else { // auto generate multus config
		caHash, saTokenHash, err = opt.createKubeConfig(nil, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to create multus kubeconfig: %v\n", err)
			return
		}
		fmt.Printf("kubeconfig file is created.\n")
		masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
			return
		}
		fmt.Printf("multus config file is created.\n")
	}

	ctx := signals.SetupSignalHandler()

	if opt.CleanupConfigOnExit {
		defer cleanupMultusConf(&opt)
	}

	watchChanges := opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" && !opt.SkipMultusConfWatch
	if watchChanges {
		fmt.Printf("Entering watch loop...\n")
		masterConfigExists := true

	outer:
		for range time.Tick(1 * time.Second) {
			select {
			case <-ctx.Done():
				// signal received break from loop
				break outer
			default:
				// Check kubeconfig and update if different (i.e. service account updated)
				caHash, saTokenHash, err = opt.createKubeConfig(caHash, saTokenHash)
				if err != nil {
					fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
					return
				}

				// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
				_, err = os.Stat(masterConfigFilePath)

				// if masterConfigFilePath is no longer exists
				if os.IsNotExist(err) {
					if masterConfigExists {
						fmt.Printf("Master plugin @ %q has been deleted. waiting for its restoration...\n", masterConfigFilePath)
					}
					masterConfigExists = false
					continue
				}

				if !masterConfigExists {
					fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
					masterConfigExists = true
				}

				masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(masterConfigHash)
				if err != nil {
					fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
					return
				}
			}
		}
	} else {
		// wait until signal received
		<-ctx.Done()
	}
}

func cleanupMultusConf(opt *Options) {
	// try remove multus conf
	if opt.MultusConfFile == "auto" {
		multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", opt.CNIConfDir)
		_ = os.Remove(multusConfFilePath)

		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", opt.CNIConfDir)
		_ = os.Remove(multusConfFilePath)
	} else {
		confFileName := filepath.Base(opt.MultusConfFile)
		_ = os.Remove(filepath.Join(opt.CNIConfDir, confFileName))
	}

}