diff --git a/pkg/kubelet/dockershim/docker_image.go b/pkg/kubelet/dockershim/docker_image.go
index 70bca7fa5e2..64ec4711d3e 100644
--- a/pkg/kubelet/dockershim/docker_image.go
+++ b/pkg/kubelet/dockershim/docker_image.go
@@ -94,15 +94,20 @@ func (ds *dockerService) RemoveImage(image *runtimeapi.ImageSpec) error {
 	imageInspect, err := ds.client.InspectImageByID(image.Image)
 	if err == nil && imageInspect != nil && len(imageInspect.RepoTags) > 1 {
 		for _, tag := range imageInspect.RepoTags {
-			if _, err := ds.client.RemoveImage(tag, dockertypes.ImageRemoveOptions{PruneChildren: true}); err != nil {
+			if _, err := ds.client.RemoveImage(tag, dockertypes.ImageRemoveOptions{PruneChildren: true}); err != nil && !libdocker.IsImageNotFoundError(err) {
 				return err
 			}
 		}
 		return nil
+	} else if err != nil && libdocker.IsImageNotFoundError(err) {
+		return nil
 	}
 
 	_, err = ds.client.RemoveImage(image.Image, dockertypes.ImageRemoveOptions{PruneChildren: true})
-	return err
+	if err != nil && !libdocker.IsImageNotFoundError(err) {
+		return err
+	}
+	return nil
 }
 
 // getImageRef returns the image digest if exists, or else returns the image ID.
diff --git a/pkg/kubelet/dockershim/libdocker/helpers.go b/pkg/kubelet/dockershim/libdocker/helpers.go
index 28c69a22ace..1dc778ef7b4 100644
--- a/pkg/kubelet/dockershim/libdocker/helpers.go
+++ b/pkg/kubelet/dockershim/libdocker/helpers.go
@@ -129,3 +129,9 @@ func matchImageIDOnly(inspected dockertypes.ImageInspect, image string) bool {
 	glog.V(4).Infof("The reference %s does not directly refer to the given image's ID (%q)", image, inspected.ID)
 	return false
 }
+
+// isImageNotFoundError returns whether the err is caused by image not found in docker
+// TODO: Use native error tester once ImageNotFoundError is supported in docker-engine client(eg. ImageRemove())
+func isImageNotFoundError(err error) bool {
+	return strings.Contains(err.Error(), "No such image:")
+}
diff --git a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go
index 98434a1453d..f3bbc3f0536 100644
--- a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go
+++ b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go
@@ -382,6 +382,9 @@ func (d *kubeDockerClient) RemoveImage(image string, opts dockertypes.ImageRemov
 	if ctxErr := contextError(ctx); ctxErr != nil {
 		return nil, ctxErr
 	}
+	if isImageNotFoundError(err) {
+		return nil, ImageNotFoundError{ID: image}
+	}
 	return resp, err
 }