Add a file store utility package in kubelet

More and more components checkpoints (i.e., persist their states) in
kubelet. Refurbish and move the implementation in dockershim to a
utility package to improve code reusability.

pkg/kubelet/util/store/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// A store interface.
+package store // import "k8s.io/kubernetes/pkg/kubelet/util/store"

pkg/kubelet/util/store/filestore.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	utilfs "k8s.io/kubernetes/pkg/util/filesystem"
+)
+
+const (
+	// Name prefix for the temporary files.
+	tmpPrefix = "."
+)
+
+// FileStore is an implementation of the Store interface which stores data in files.
+type FileStore struct {
+	// Absolute path to the base directory for storing data files.
+	directoryPath string
+
+	// filesystem to use.
+	filesystem utilfs.Filesystem
+}
+
+func NewFileStore(path string, fs utilfs.Filesystem) (Store, error) {
+	if err := ensureDirectory(fs, path); err != nil {
+		return nil, err
+	}
+	return &FileStore{directoryPath: path, filesystem: fs}, nil
+}
+
+func (f *FileStore) Write(key string, data []byte) error {
+	if err := ValidateKey(key); err != nil {
+		return err
+	}
+	if err := ensureDirectory(f.filesystem, f.directoryPath); err != nil {
+		return err
+	}
+
+	return writeFile(f.filesystem, f.getPathByKey(key), data)
+}
+
+func (f *FileStore) Read(key string) ([]byte, error) {
+	if err := ValidateKey(key); err != nil {
+		return nil, err
+	}
+	bytes, err := f.filesystem.ReadFile(f.getPathByKey(key))
+	if os.IsNotExist(err) {
+		return bytes, KeyNotFoundError
+	}
+	return bytes, err
+}
+
+func (f *FileStore) Delete(key string) error {
+	if err := ValidateKey(key); err != nil {
+		return err
+	}
+	return removePath(f.filesystem, f.getPathByKey(key))
+}
+
+func (f *FileStore) List() ([]string, error) {
+	keys := make([]string, 0)
+	files, err := f.filesystem.ReadDir(f.directoryPath)
+	if err != nil {
+		return keys, err
+	}
+	for _, f := range files {
+		if !strings.HasPrefix(f.Name(), tmpPrefix) {
+			keys = append(keys, f.Name())
+		}
+	}
+	return keys, nil
+}
+
+func (f *FileStore) getPathByKey(key string) string {
+	return filepath.Join(f.directoryPath, key)
+}
+
+// ensureDirectory creates the directory if it does not exist.
+func ensureDirectory(fs utilfs.Filesystem, path string) error {
+	if _, err := fs.Stat(path); err != nil {
+		// MkdirAll returns nil if directory already exists.
+		return fs.MkdirAll(path, 0755)
+	}
+	return nil
+}
+
+// writeFile writes data to path in a single transaction.
+func writeFile(fs utilfs.Filesystem, path string, data []byte) (retErr error) {
+	// Create a temporary file in the base directory of `path` with a prefix.
+	tmpFile, err := fs.TempFile(filepath.Dir(path), tmpPrefix)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		// Close the file.
+		if err := tmpFile.Close(); err != nil {
+			if retErr == nil {
+				retErr = err
+			} else {
+				retErr = fmt.Errorf("failed to close temp file after error %v", retErr)
+			}
+		}
+	}()
+
+	tmpPath := tmpFile.Name()
+	defer func() {
+		// Clean up the temp file on error.
+		if retErr != nil && tmpPath != "" {
+			if err := removePath(fs, tmpPath); err != nil {
+				retErr = fmt.Errorf("failed to remove the temporary file after error %v", retErr)
+			}
+		}
+	}()
+
+	// Write data.
+	if _, err := tmpFile.Write(data); err != nil {
+		return err
+	}
+
+	// Sync file.
+	if err := tmpFile.Sync(); err != nil {
+		return err
+	}
+
+	return fs.Rename(tmpPath, path)
+}
+
+func removePath(fs utilfs.Filesystem, path string) error {
+	if err := fs.Remove(path); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}

pkg/kubelet/util/store/filestore_test.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"io/ioutil"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/kubernetes/pkg/util/filesystem"
+)
+
+func TestFileStore(t *testing.T) {
+	path, err := ioutil.TempDir("", "FileStore")
+	assert.NoError(t, err)
+	store, err := NewFileStore(path, filesystem.NewFakeFs())
+	assert.NoError(t, err)
+
+	testData := []struct {
+		key       string
+		data      string
+		expectErr bool
+	}{
+		{
+			"id1",
+			"data1",
+			false,
+		},
+		{
+			"id2",
+			"data2",
+			false,
+		},
+		{
+			"/id1",
+			"data1",
+			true,
+		},
+		{
+			".id1",
+			"data1",
+			true,
+		},
+		{
+			"   ",
+			"data2",
+			true,
+		},
+		{
+			"___",
+			"data2",
+			true,
+		},
+	}
+
+	// Test add data.
+	for _, c := range testData {
+		_, err = store.Read(c.key)
+		assert.Error(t, err)
+
+		err = store.Write(c.key, []byte(c.data))
+		if c.expectErr {
+			assert.Error(t, err)
+			continue
+		} else {
+			assert.NoError(t, err)
+		}
+
+		// Test read data by key.
+		data, err := store.Read(c.key)
+		assert.NoError(t, err)
+		assert.Equal(t, string(data), c.data)
+	}
+
+	// Test list keys.
+	keys, err := store.List()
+	assert.NoError(t, err)
+	sort.Strings(keys)
+	assert.Equal(t, keys, []string{"id1", "id2"})
+
+	// Test Delete data
+	for _, c := range testData {
+		if c.expectErr {
+			continue
+		}
+
+		err = store.Delete(c.key)
+		assert.NoError(t, err)
+		_, err = store.Read(c.key)
+		assert.EqualValues(t, KeyNotFoundError, err)
+	}
+
+	// Test delete non-existent key.
+	err = store.Delete("id1")
+	assert.NoError(t, err)
+
+	// Test list keys.
+	keys, err = store.List()
+	assert.NoError(t, err)
+	assert.Equal(t, len(keys), 0)
+}

pkg/kubelet/util/store/store.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"fmt"
+	"regexp"
+)
+
+const keyMaxLength = 250
+
+var keyRegex = regexp.MustCompile("^[a-zA-Z0-9]+$")
+
+var (
+	KeyNotFoundError = fmt.Errorf("key is not found.")
+)
+
+// Store provides the interface for storing keyed data.
+// Store must be thread-safe
+type Store interface {
+	// key must contain one or more characters in [A-Za-z0-9]
+	// Write writes data with key.
+	Write(key string, data []byte) error
+	// Read retrieves data with key
+	// Read must return KeyNotFoundError if key is not found.
+	Read(key string) ([]byte, error)
+	// Delete deletes data by key
+	// Delete must not return error if key does not exist
+	Delete(key string) error
+	// List lists all existing keys.
+	List() ([]string, error)
+}
+
+func ValidateKey(key string) error {
+	if len(key) <= keyMaxLength && keyRegex.MatchString(key) {
+		return nil
+	}
+	return fmt.Errorf("invalid key: %q", key)
+}

pkg/kubelet/util/store/store_test.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsValidKey(t *testing.T) {
+	testcases := []struct {
+		key   string
+		valid bool
+	}{
+		{
+			"    ",
+			false,
+		},
+		{
+			"/foo/bar",
+			false,
+		},
+		{
+			".foo",
+			false,
+		},
+		{
+			"a78768279290d33d0b82eaea43cb8346f500057cb5bd250e88c97a5585385d66",
+			true,
+		},
+	}
+
+	for _, tc := range testcases {
+		if tc.valid {
+			assert.NoError(t, ValidateKey(tc.key))
+		} else {
+			assert.Error(t, ValidateKey(tc.key))
+		}
+	}
+}