From 007d469034f7c9b171c84934202f146305e266c1 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Mon, 26 Feb 2024 13:09:01 -0500 Subject: [PATCH] bert: fix layer norm epsilon value (#1946) Signed-off-by: Jared Van Bortel --- gpt4all-backend/bert.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gpt4all-backend/bert.cpp b/gpt4all-backend/bert.cpp index bad5a422..955faae1 100644 --- a/gpt4all-backend/bert.cpp +++ b/gpt4all-backend/bert.cpp @@ -343,7 +343,7 @@ void bert_eval( // embd norm { - inpL = ggml_norm(ctx0, inpL, 1e-5f); + inpL = ggml_norm(ctx0, inpL, 1e-12f); inpL = ggml_add(ctx0, ggml_mul(ctx0, @@ -403,7 +403,7 @@ void bert_eval( // attention norm { - cur = ggml_norm(ctx0, cur, 1e-5f); + cur = ggml_norm(ctx0, cur, 1e-12f); cur = ggml_add(ctx0, ggml_mul(ctx0, @@ -429,7 +429,7 @@ void bert_eval( // output norm { - cur = ggml_norm(ctx0, cur, 1e-5f); + cur = ggml_norm(ctx0, cur, 1e-12f); cur = ggml_add(ctx0, ggml_mul(ctx0,