diff --git a/pkg/conversion/code_generation_test.go b/pkg/conversion/code_generation_test.go
index 3706dcdeca1..f0d6b7ac757 100644
--- a/pkg/conversion/code_generation_test.go
+++ b/pkg/conversion/code_generation_test.go
@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -114,7 +115,8 @@ func bufferExistingConversions(t *testing.T, fileName string) (bytes.Buffer, byt
 	return functions, names
 }
 
-func compareBuffers(t *testing.T, existing, generated bytes.Buffer) {
+func compareBuffers(t *testing.T, generatedFile string, existing, generated bytes.Buffer) bool {
+	ok := true
 	for {
 		existingLine, existingErr := existing.ReadString('\n')
 		generatedLine, generatedErr := generated.ReadString('\n')
@@ -122,18 +124,22 @@ func compareBuffers(t *testing.T, existing, generated bytes.Buffer) {
 			break
 		}
 		if existingErr != generatedErr {
+			ok = false
 			t.Errorf("reading errors: existing %v generated %v", existingErr, generatedErr)
-			return
+			return ok
 		}
 		if existingErr != nil {
+			ok = false
 			t.Errorf("error while reading: %v", existingErr)
 		}
 		if existingLine != generatedLine {
+			ok = false
 			diff := fmt.Sprintf("first difference: expected %s, got %s", generatedLine, existingLine)
-			t.Errorf("please regenerate conversion functions; %s", diff)
-			return
+			t.Errorf("please update conversion functions; generated: %s; diff: %s", generatedFile, diff)
+			return ok
 		}
 	}
+	return ok
 }
 
 func TestNoManualChangesToGenerateConversions(t *testing.T) {
@@ -145,7 +151,17 @@ func TestNoManualChangesToGenerateConversions(t *testing.T) {
 		existingFunctions, existingNames := bufferExistingConversions(t, fileName)
 		generatedFunctions, generatedNames := generateConversions(t, version)
 
-		compareBuffers(t, existingFunctions, generatedFunctions)
-		compareBuffers(t, existingNames, generatedNames)
+		functionsTxt := fmt.Sprintf("%s.functions.txt", version)
+		ioutil.WriteFile(functionsTxt, generatedFunctions.Bytes(), os.FileMode(0644))
+
+		namesTxt := fmt.Sprintf("%s.names.txt", version)
+		ioutil.WriteFile(namesTxt, generatedNames.Bytes(), os.FileMode(0644))
+
+		if ok := compareBuffers(t, functionsTxt, existingFunctions, generatedFunctions); ok {
+			os.Remove(functionsTxt)
+		}
+		if ok := compareBuffers(t, namesTxt, existingNames, generatedNames); ok {
+			os.Remove(namesTxt)
+		}
 	}
 }