diff --git a/.travis.yml b/.travis.yml
index 4f5008e6..096053d1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,8 @@ go:
 env:
   global:
     - "GO15VENDOREXPERIMENT=1"
+  jobs:
+    - "DOCKER_BUILDKIT=0"
     - "DOCKER_BUILDKIT=1"
 before_install:
   - sudo rm -rf /var/lib/apt/lists/*
diff --git a/pkg/compiler/artifact_test.go b/pkg/compiler/artifact_test.go
index cbd6d82e..3dae63e4 100644
--- a/pkg/compiler/artifact_test.go
+++ b/pkg/compiler/artifact_test.go
@@ -113,7 +113,7 @@ RUN echo bar > /test2`))
 			Expect(err).ToNot(HaveOccurred())
 
 			artifacts := []ArtifactNode{}
-			if os.Getenv("DOCKER_BUILDKIT") != "" {
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
 				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
 			}
 			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
diff --git a/pkg/compiler/backend/simpledocker_test.go b/pkg/compiler/backend/simpledocker_test.go
index dd6c9af0..0638af00 100644
--- a/pkg/compiler/backend/simpledocker_test.go
+++ b/pkg/compiler/backend/simpledocker_test.go
@@ -102,7 +102,7 @@ RUN echo bar > /test2`))
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
 			artifacts := []ArtifactNode{}
-			if os.Getenv("DOCKER_BUILDKIT") != "" {
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
 				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
 			}
 			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})