mirror of
https://github.com/nomic-ai/gpt4all.git
synced 2025-07-05 11:36:16 +00:00
Initial Library Loader for .NET Bindings / Update bindings to support newest changes (#763)
* Initial Library Loader * Load library as part of Model factory * Dynamically search and find the dlls * Update tests to use locally built runtimes * Fix dylib loading, add macos runtime support for sample/tests * Bypass automatic loading by default. * Only set CMAKE_OSX_ARCHITECTURES if not already set, allow cross-compile * Switch Loading again * Update build scripts for mac/linux * Update bindings to support newest breaking changes * Fix build * Use llmodel for Windows * Actually, it does need to be libllmodel * Name * Remove TFMs, bypass loading by default * Fix script * Delete mac script --------- Co-authored-by: Tim Miller <innerlogic4321@ghmail.com>
This commit is contained in:
Tim Miller
GitHub
parent
88616fde7f
commit
797891c995
@ -9,7 +9,9 @@ if(APPLE)
|
||||
set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
|
||||
else()
|
||||
# Build for the host architecture on macOS
|
||||
set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
|
||||
if(NOT CMAKE_OSX_ARCHITECTURES)
|
||||
set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@ -1,18 +1,31 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Properties\" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<!-- Windows -->
|
||||
<None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
|
||||
<!-- Linux -->
|
||||
<None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
|
||||
<!-- MacOS -->
|
||||
<None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Windows -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
<!-- Linux -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
<!-- MacOS -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@ -21,7 +21,24 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
|
||||
<ProjectReference Include="..\Gpt4All\Gpt4All.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Windows -->
|
||||
<None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
|
||||
<!-- Linux -->
|
||||
<None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
|
||||
<!-- MacOS -->
|
||||
<None Include="..\runtimes\osx\native\*.dylib" Pack="true" PackagePath="runtimes\osx\native\%(Filename)%(Extension)" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Windows -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
<!-- Linux -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
<!-- MacOS -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="..\runtimes\osx\native\*.dylib" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@ -14,18 +14,18 @@ public class ModelFactoryTests
|
||||
[Fact]
|
||||
public void CanLoadLlamaModel()
|
||||
{
|
||||
using var model = _modelFactory.LoadLlamaModel(Constants.LLAMA_MODEL_PATH);
|
||||
using var model = _modelFactory.LoadModel(Constants.LLAMA_MODEL_PATH);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanLoadGptjModel()
|
||||
{
|
||||
using var model = _modelFactory.LoadGptjModel(Constants.GPTJ_MODEL_PATH);
|
||||
using var model = _modelFactory.LoadModel(Constants.GPTJ_MODEL_PATH);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CanLoadMptModel()
|
||||
{
|
||||
using var model = _modelFactory.LoadMptModel(Constants.MPT_MODEL_PATH);
|
||||
using var model = _modelFactory.LoadModel(Constants.MPT_MODEL_PATH);
|
||||
}
|
||||
}
|
||||
|
||||
@ -195,22 +195,6 @@ public class LLModel : ILLModel
|
||||
{
|
||||
NativeMethods.llmodel_model_destroy(_handle);
|
||||
}
|
||||
|
||||
protected void DestroyLLama()
|
||||
{
|
||||
NativeMethods.llmodel_llama_destroy(_handle);
|
||||
}
|
||||
|
||||
protected void DestroyGptj()
|
||||
{
|
||||
NativeMethods.llmodel_gptj_destroy(_handle);
|
||||
}
|
||||
|
||||
protected void DestroyMtp()
|
||||
{
|
||||
NativeMethods.llmodel_mpt_destroy(_handle);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (_disposed) return;
|
||||
@ -222,15 +206,6 @@ public class LLModel : ILLModel
|
||||
|
||||
switch (_modelType)
|
||||
{
|
||||
case ModelType.LLAMA:
|
||||
DestroyLLama();
|
||||
break;
|
||||
case ModelType.GPTJ:
|
||||
DestroyGptj();
|
||||
break;
|
||||
case ModelType.MPT:
|
||||
DestroyMtp();
|
||||
break;
|
||||
default:
|
||||
Destroy();
|
||||
break;
|
||||
|
||||
@ -56,31 +56,12 @@ internal static unsafe partial class NativeMethods
|
||||
[return: MarshalAs(UnmanagedType.I1)]
|
||||
public delegate bool LlmodelRecalculateCallback(bool isRecalculating);
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
[return: NativeTypeName("llmodel_model")]
|
||||
public static extern IntPtr llmodel_gptj_create();
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
public static extern void llmodel_gptj_destroy([NativeTypeName("llmodel_model")] IntPtr gptj);
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
[return: NativeTypeName("llmodel_model")]
|
||||
public static extern IntPtr llmodel_mpt_create();
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
public static extern void llmodel_mpt_destroy([NativeTypeName("llmodel_model")] IntPtr mpt);
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
[return: NativeTypeName("llmodel_model")]
|
||||
public static extern IntPtr llmodel_llama_create();
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
public static extern void llmodel_llama_destroy([NativeTypeName("llmodel_model")] IntPtr llama);
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, BestFitMapping = false, ThrowOnUnmappableChar = true)]
|
||||
[return: NativeTypeName("llmodel_model")]
|
||||
public static extern IntPtr llmodel_model_create(
|
||||
[NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
|
||||
public static extern IntPtr llmodel_model_create2(
|
||||
[NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
|
||||
[NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string build_variant,
|
||||
out IntPtr error);
|
||||
|
||||
[DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
public static extern void llmodel_model_destroy([NativeTypeName("llmodel_model")] IntPtr model);
|
||||
|
||||
@ -1,27 +1,11 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFrameworks>net6.0</TargetFrameworks>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Windows -->
|
||||
<None Include="..\runtimes\win-x64\native\*.dll" Pack="true" PackagePath="runtimes\win-x64\native\%(Filename)%(Extension)" />
|
||||
<!-- Linux -->
|
||||
<None Include="..\runtimes\linux-x64\native\*.so" Pack="true" PackagePath="runtimes\linux-x64\native\%(Filename)%(Extension)" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Windows -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="..\runtimes\win-x64\native\*.dll" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
<!-- Linux -->
|
||||
<None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="..\runtimes\linux-x64\native\*.so" Visible="False" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
namespace Gpt4All.LibraryLoader;
|
||||
|
||||
public interface ILibraryLoader
|
||||
{
|
||||
LoadResult OpenLibrary(string? fileName);
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Gpt4All.LibraryLoader;
|
||||
|
||||
internal class LinuxLibraryLoader : ILibraryLoader
|
||||
{
|
||||
#pragma warning disable CA2101
|
||||
[DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
|
||||
#pragma warning restore CA2101
|
||||
public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
|
||||
|
||||
#pragma warning disable CA2101
|
||||
[DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
|
||||
#pragma warning restore CA2101
|
||||
public static extern IntPtr NativeOpenLibraryLibdl2(string? filename, int flags);
|
||||
|
||||
[DllImport("libdl.so", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
|
||||
public static extern IntPtr GetLoadError();
|
||||
|
||||
[DllImport("libdl.so.2", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
|
||||
public static extern IntPtr GetLoadError2();
|
||||
|
||||
public LoadResult OpenLibrary(string? fileName)
|
||||
{
|
||||
IntPtr loadedLib;
|
||||
try
|
||||
{
|
||||
// open with rtls lazy flag
|
||||
loadedLib = NativeOpenLibraryLibdl2(fileName, 0x00001);
|
||||
}
|
||||
catch (DllNotFoundException)
|
||||
{
|
||||
loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
|
||||
}
|
||||
|
||||
if (loadedLib == IntPtr.Zero)
|
||||
{
|
||||
string errorMessage;
|
||||
try
|
||||
{
|
||||
errorMessage = Marshal.PtrToStringAnsi(GetLoadError2()) ?? "Unknown error";
|
||||
}
|
||||
catch (DllNotFoundException)
|
||||
{
|
||||
errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
|
||||
}
|
||||
|
||||
return LoadResult.Failure(errorMessage);
|
||||
}
|
||||
|
||||
return LoadResult.Success;
|
||||
}
|
||||
}
|
||||
20
gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LoadResult.cs
Normal file
20
gpt4all-bindings/csharp/Gpt4All/LibraryLoader/LoadResult.cs
Normal file
@ -0,0 +1,20 @@
|
||||
namespace Gpt4All.LibraryLoader;
|
||||
|
||||
public class LoadResult
|
||||
{
|
||||
private LoadResult(bool isSuccess, string? errorMessage)
|
||||
{
|
||||
IsSuccess = isSuccess;
|
||||
ErrorMessage = errorMessage;
|
||||
}
|
||||
|
||||
public static LoadResult Success { get; } = new(true, null);
|
||||
|
||||
public static LoadResult Failure(string errorMessage)
|
||||
{
|
||||
return new(false, errorMessage);
|
||||
}
|
||||
|
||||
public bool IsSuccess { get; }
|
||||
public string? ErrorMessage { get; }
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Gpt4All.LibraryLoader;
|
||||
|
||||
internal class MacOsLibraryLoader : ILibraryLoader
|
||||
{
|
||||
#pragma warning disable CA2101
|
||||
[DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlopen")]
|
||||
#pragma warning restore CA2101
|
||||
public static extern IntPtr NativeOpenLibraryLibdl(string? filename, int flags);
|
||||
|
||||
[DllImport("libdl.dylib", ExactSpelling = true, CharSet = CharSet.Auto, EntryPoint = "dlerror")]
|
||||
public static extern IntPtr GetLoadError();
|
||||
|
||||
public LoadResult OpenLibrary(string? fileName)
|
||||
{
|
||||
var loadedLib = NativeOpenLibraryLibdl(fileName, 0x00001);
|
||||
|
||||
if (loadedLib == IntPtr.Zero)
|
||||
{
|
||||
var errorMessage = Marshal.PtrToStringAnsi(GetLoadError()) ?? "Unknown error";
|
||||
|
||||
return LoadResult.Failure(errorMessage);
|
||||
}
|
||||
|
||||
return LoadResult.Success;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
#if !IOS && !MACCATALYST && !TVOS && !ANDROID
|
||||
using System.Runtime.InteropServices;
|
||||
#endif
|
||||
|
||||
namespace Gpt4All.LibraryLoader;
|
||||
|
||||
public static class NativeLibraryLoader
|
||||
{
|
||||
private static ILibraryLoader? defaultLibraryLoader;
|
||||
|
||||
/// <summary>
|
||||
/// Sets the library loader used to load the native libraries. Overwrite this only if you want some custom loading.
|
||||
/// </summary>
|
||||
/// <param name="libraryLoader">The library loader to be used.</param>
|
||||
public static void SetLibraryLoader(ILibraryLoader libraryLoader)
|
||||
{
|
||||
defaultLibraryLoader = libraryLoader;
|
||||
}
|
||||
|
||||
internal static LoadResult LoadNativeLibrary(string? path = default, bool bypassLoading = true)
|
||||
{
|
||||
// If the user has handled loading the library themselves, we don't need to do anything.
|
||||
if (bypassLoading)
|
||||
{
|
||||
return LoadResult.Success;
|
||||
}
|
||||
|
||||
var architecture = RuntimeInformation.OSArchitecture switch
|
||||
{
|
||||
Architecture.X64 => "x64",
|
||||
Architecture.X86 => "x86",
|
||||
Architecture.Arm => "arm",
|
||||
Architecture.Arm64 => "arm64",
|
||||
_ => throw new PlatformNotSupportedException(
|
||||
$"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
|
||||
};
|
||||
|
||||
var (platform, extension) = Environment.OSVersion.Platform switch
|
||||
{
|
||||
_ when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => ("win", "dll"),
|
||||
_ when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => ("linux", "so"),
|
||||
_ when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => ("osx", "dylib"),
|
||||
_ => throw new PlatformNotSupportedException(
|
||||
$"Unsupported OS platform, architecture: {RuntimeInformation.OSArchitecture}")
|
||||
};
|
||||
|
||||
// If the user hasn't set the path, we'll try to find it ourselves.
|
||||
if (string.IsNullOrEmpty(path))
|
||||
{
|
||||
var libraryName = "libllmodel";
|
||||
var assemblySearchPath = new[]
|
||||
{
|
||||
AppDomain.CurrentDomain.RelativeSearchPath,
|
||||
Path.GetDirectoryName(typeof(NativeLibraryLoader).Assembly.Location),
|
||||
Path.GetDirectoryName(Environment.GetCommandLineArgs()[0])
|
||||
}.FirstOrDefault(it => !string.IsNullOrEmpty(it));
|
||||
// Search for the library dll within the assembly search path. If it doesn't exist, for whatever reason, use the default path.
|
||||
path = Directory.EnumerateFiles(assemblySearchPath ?? string.Empty, $"{libraryName}.{extension}", SearchOption.AllDirectories).FirstOrDefault() ?? Path.Combine("runtimes", $"{platform}-{architecture}", $"{libraryName}.{extension}");
|
||||
}
|
||||
|
||||
if (defaultLibraryLoader != null)
|
||||
{
|
||||
return defaultLibraryLoader.OpenLibrary(path);
|
||||
}
|
||||
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
throw new FileNotFoundException($"Native Library not found in path {path}. " +
|
||||
$"Verify you have have included the native Gpt4All library in your application.");
|
||||
}
|
||||
|
||||
ILibraryLoader libraryLoader = platform switch
|
||||
{
|
||||
"win" => new WindowsLibraryLoader(),
|
||||
"osx" => new MacOsLibraryLoader(),
|
||||
"linux" => new LinuxLibraryLoader(),
|
||||
_ => throw new PlatformNotSupportedException($"Currently {platform} platform is not supported")
|
||||
};
|
||||
return libraryLoader.OpenLibrary(path);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
using System.ComponentModel;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Gpt4All.LibraryLoader;
|
||||
|
||||
internal class WindowsLibraryLoader : ILibraryLoader
|
||||
{
|
||||
public LoadResult OpenLibrary(string? fileName)
|
||||
{
|
||||
var loadedLib = LoadLibrary(fileName);
|
||||
|
||||
if (loadedLib == IntPtr.Zero)
|
||||
{
|
||||
var errorCode = Marshal.GetLastWin32Error();
|
||||
var errorMessage = new Win32Exception(errorCode).Message;
|
||||
return LoadResult.Failure(errorMessage);
|
||||
}
|
||||
|
||||
return LoadResult.Success;
|
||||
}
|
||||
|
||||
[DllImport("kernel32", SetLastError = true, CharSet = CharSet.Auto)]
|
||||
private static extern IntPtr LoadLibrary([MarshalAs(UnmanagedType.LPWStr)] string? lpFileName);
|
||||
}
|
||||
@ -1,7 +1,8 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Gpt4All.Bindings;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Gpt4All.LibraryLoader;
|
||||
|
||||
namespace Gpt4All;
|
||||
|
||||
@ -9,41 +10,43 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
|
||||
{
|
||||
private readonly ILoggerFactory _loggerFactory;
|
||||
private readonly ILogger _logger;
|
||||
private static bool bypassLoading;
|
||||
private static string? libraryPath;
|
||||
|
||||
public Gpt4AllModelFactory(ILoggerFactory? loggerFactory = null)
|
||||
private static readonly Lazy<LoadResult> libraryLoaded = new(() =>
|
||||
{
|
||||
return NativeLibraryLoader.LoadNativeLibrary(Gpt4AllModelFactory.libraryPath, Gpt4AllModelFactory.bypassLoading);
|
||||
}, true);
|
||||
|
||||
public Gpt4AllModelFactory(string? libraryPath = default, bool bypassLoading = true, ILoggerFactory? loggerFactory = null)
|
||||
{
|
||||
_loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
|
||||
_logger = _loggerFactory.CreateLogger<Gpt4AllModelFactory>();
|
||||
Gpt4AllModelFactory.libraryPath = libraryPath;
|
||||
Gpt4AllModelFactory.bypassLoading = bypassLoading;
|
||||
|
||||
if (!libraryLoaded.Value.IsSuccess)
|
||||
{
|
||||
throw new Exception($"Failed to load native gpt4all library. Error: {libraryLoaded.Value.ErrorMessage}");
|
||||
}
|
||||
}
|
||||
|
||||
private IGpt4AllModel CreateModel(string modelPath, ModelType? modelType = null)
|
||||
private IGpt4AllModel CreateModel(string modelPath)
|
||||
{
|
||||
var modelType_ = modelType ?? ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
|
||||
|
||||
var modelType_ = ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
|
||||
_logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
|
||||
|
||||
var handle = modelType_ switch
|
||||
{
|
||||
ModelType.LLAMA => NativeMethods.llmodel_llama_create(),
|
||||
ModelType.GPTJ => NativeMethods.llmodel_gptj_create(),
|
||||
ModelType.MPT => NativeMethods.llmodel_mpt_create(),
|
||||
_ => NativeMethods.llmodel_model_create(modelPath),
|
||||
};
|
||||
|
||||
IntPtr error;
|
||||
var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
|
||||
_logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
|
||||
_logger.LogInformation("Model loading started");
|
||||
|
||||
var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
|
||||
|
||||
_logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
|
||||
|
||||
if (loadedSuccessfully == false)
|
||||
if (!loadedSuccessfully)
|
||||
{
|
||||
throw new Exception($"Failed to load model: '{modelPath}'");
|
||||
}
|
||||
|
||||
var logger = _loggerFactory.CreateLogger<LLModel>();
|
||||
|
||||
var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
|
||||
|
||||
Debug.Assert(underlyingModel.IsLoaded());
|
||||
@ -51,11 +54,5 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
|
||||
return new Gpt4All(underlyingModel, logger: logger);
|
||||
}
|
||||
|
||||
public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath, modelType: null);
|
||||
|
||||
public IGpt4AllModel LoadMptModel(string modelPath) => CreateModel(modelPath, ModelType.MPT);
|
||||
|
||||
public IGpt4AllModel LoadGptjModel(string modelPath) => CreateModel(modelPath, ModelType.GPTJ);
|
||||
|
||||
public IGpt4AllModel LoadLlamaModel(string modelPath) => CreateModel(modelPath, ModelType.LLAMA);
|
||||
public IGpt4AllModel LoadModel(string modelPath) => CreateModel(modelPath);
|
||||
}
|
||||
|
||||
@ -2,11 +2,5 @@
|
||||
|
||||
public interface IGpt4AllModelFactory
|
||||
{
|
||||
IGpt4AllModel LoadGptjModel(string modelPath);
|
||||
|
||||
IGpt4AllModel LoadLlamaModel(string modelPath);
|
||||
|
||||
IGpt4AllModel LoadModel(string modelPath);
|
||||
|
||||
IGpt4AllModel LoadMptModel(string modelPath);
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@ public interface ITextPrediction
|
||||
/// </summary>
|
||||
/// <param name="text">The text to complete</param>
|
||||
/// <param name="opts">The prediction settings</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
|
||||
/// <param name="cancellation">The <see cref="CancellationToken"/> for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
|
||||
/// <returns>The prediction result generated by the model</returns>
|
||||
Task<ITextPredictionResult> GetPredictionAsync(
|
||||
string text,
|
||||
|
||||
@ -5,4 +5,6 @@ mkdir runtimes/linux-x64/build
|
||||
cmake -S ../../gpt4all-backend -B runtimes/linux-x64/build
|
||||
cmake --build runtimes/linux-x64/build --parallel --config Release
|
||||
cp runtimes/linux-x64/build/libllmodel.so runtimes/linux-x64/native/libllmodel.so
|
||||
cp runtimes/linux-x64/build/llama.cpp/libllama.so runtimes/linux-x64/native/libllama.so
|
||||
cp runtimes/linux-x64/build/libgptj*.so runtimes/linux-x64/native/
|
||||
cp runtimes/linux-x64/build/libllama*.so runtimes/linux-x64/native/
|
||||
cp runtimes/linux-x64/build/libmpt*.so runtimes/linux-x64/native/
|
||||
|
||||
@ -13,4 +13,5 @@ cmake --build $BUILD_DIR --parallel --config Release
|
||||
|
||||
# copy native dlls
|
||||
cp "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll" $LIBS_DIR
|
||||
cp "$BUILD_DIR\*.dll" $LIBS_DIR
|
||||
cp "$BUILD_DIR\bin\*.dll" $LIBS_DIR
|
||||
mv $LIBS_DIR\llmodel.dll $LIBS_DIR\libllmodel.dll
|
||||
@ -3,3 +3,4 @@ mkdir .\runtimes\win-x64\msvc\build | Out-Null
|
||||
cmake -G "Visual Studio 17 2022" -A X64 -S ..\..\gpt4all-backend -B .\runtimes\win-x64\msvc\build
|
||||
cmake --build .\runtimes\win-x64\msvc\build --parallel --config Release
|
||||
cp .\runtimes\win-x64\msvc\build\bin\Release\*.dll .\runtimes\win-x64
|
||||
mv .\runtimes\win-x64\llmodel.dll .\runtimes\win-x64\libllmodel.dll
|
||||
Loading…
Reference in New Issue
Block a user