This commit is contained in:
Jared Van Bortel 2025-03-19 18:08:49 -04:00
parent 9772027e5e
commit b359c9245c
11 changed files with 250 additions and 162 deletions

View File

@ -79,25 +79,6 @@ ColumnLayout {
})[provider.id.toString()] })[provider.id.toString()]
} }
} }
RemoteModelCard {
width: parent.childWidth
height: parent.childHeight
providerUsesApiKey: false
providerName: qsTr("Ollama (Custom)")
providerImage: "qrc:/gpt4all/icons/antenna_3.svg"
providerDesc: qsTr("Configure a custom Ollama provider.")
}
// TODO(jared): add custom openai back to the list
/*
RemoteModelCard {
width: parent.childWidth
height: parent.childHeight
providerIsCustom: true
providerName: qsTr("Custom")
providerImage: "qrc:/gpt4all/icons/antenna_3.svg"
providerDesc: qsTr("The custom provider option allows users to connect their own OpenAI-compatible AI models or third-party inference services. This is useful for organizations with proprietary models or those leveraging niche AI providers not listed here.")
}
*/
} }
} }
} }

View File

@ -4,17 +4,12 @@ import QtQuick.Layouts
Rectangle { Rectangle {
id: remoteModelCard id: remoteModelCard
property var provider: null property var provider // required
property alias providerName: providerNameLabel.text property alias providerName: providerNameLabel.text
property alias providerImage: myimage.source property alias providerImage: myimage.source
property alias providerDesc: providerDescLabel.text property alias providerDesc: providerDescLabel.text
property bool providerUsesApiKey: true property bool providerUsesApiKey: true
// for internal use
property bool apiKeyRequired: provider === null ? providerUsesApiKey : "apiKey" in provider
property bool apiKeyGood: !apiKeyRequired // (overwritten later if required)
property bool baseUrlGood: provider !== null // (overwritten later if custom)
color: theme.conversationBackground color: theme.conversationBackground
radius: 10 radius: 10
border.width: 1 border.width: 1
@ -78,7 +73,38 @@ Rectangle {
spacing: 30 spacing: 30
ColumnLayout { ColumnLayout {
visible: apiKeyRequired visible: !provider.isBuiltin
MySettingsLabel {
text: qsTr("Name")
font.bold: true
font.pixelSize: theme.fontSizeLarge
color: theme.settingsTitleTextColor
}
MyTextField {
id: nameField
property bool initialized: false
property bool ok: true
Layout.fillWidth: true
font.pixelSize: theme.fontSizeLarge
wrapMode: Text.WrapAnywhere
Component.onCompleted: {
text = provider.name;
initialized = true;
}
onTextChanged: {
if (!initialized) return;
ok = provider.setNameQml(text.trim());
}
placeholderText: qsTr("Provider Name")
Accessible.role: Accessible.EditableText
Accessible.name: placeholderText
}
}
ColumnLayout {
visible: "apiKey" in provider
MySettingsLabel { MySettingsLabel {
text: qsTr("API Key") text: qsTr("API Key")
@ -89,27 +115,32 @@ Rectangle {
MyTextField { MyTextField {
id: apiKeyField id: apiKeyField
property bool initialized: false
property bool ok: false
Layout.fillWidth: true Layout.fillWidth: true
font.pixelSize: theme.fontSizeLarge font.pixelSize: theme.fontSizeLarge
wrapMode: Text.WrapAnywhere wrapMode: Text.WrapAnywhere
echoMode: TextField.Password echoMode: TextField.Password
function showError() { Component.onCompleted: {
messageToast.show(qsTr("ERROR: $API_KEY is empty.")); if (parent.visible) {
apiKeyField.placeholderTextColor = theme.textErrorColor; text = provider.apiKey;
ok = text.trim() != "";
} else
ok = true;
initialized = true;
} }
Component.onCompleted: { if (parent.visible && provider !== null) { text = provider.apiKey; } }
onTextChanged: { onTextChanged: {
apiKeyField.placeholderTextColor = theme.mutedTextColor; if (!initialized) return;
if (provider !== null) { apiKeyGood = provider.setApiKeyQml(text) && text !== ""; } ok = provider.setApiKeyQml(text.trim()) && text.trim() !== "";
} }
placeholderText: qsTr("enter $API_KEY") placeholderText: qsTr("Provider API Key")
Accessible.role: Accessible.EditableText Accessible.role: Accessible.EditableText
Accessible.name: placeholderText Accessible.name: placeholderText
} }
} }
ColumnLayout { ColumnLayout {
visible: provider === null visible: !provider.isBuiltin
MySettingsLabel { MySettingsLabel {
text: qsTr("Base Url") text: qsTr("Base Url")
font.bold: true font.bold: true
@ -118,18 +149,20 @@ Rectangle {
} }
MyTextField { MyTextField {
id: baseUrlField id: baseUrlField
property bool initialized: false
property bool ok: true
Layout.fillWidth: true Layout.fillWidth: true
font.pixelSize: theme.fontSizeLarge font.pixelSize: theme.fontSizeLarge
wrapMode: Text.WrapAnywhere wrapMode: Text.WrapAnywhere
function showError() { Component.onCompleted: {
messageToast.show(qsTr("ERROR: $BASE_URL is empty.")); text = provider.baseUrl;
baseUrlField.placeholderTextColor = theme.textErrorColor; initialized = true;
} }
onTextChanged: { onTextChanged: {
baseUrlField.placeholderTextColor = theme.mutedTextColor; if (!initialized) return;
baseUrlGood = text.trim() !== ""; ok = provider.setBaseUrlQml(text.trim()) && text.trim() !== "";
} }
placeholderText: qsTr("enter $BASE_URL") placeholderText: qsTr("Provider Base URL")
Accessible.role: Accessible.EditableText Accessible.role: Accessible.EditableText
Accessible.name: placeholderText Accessible.name: placeholderText
} }
@ -152,18 +185,9 @@ Rectangle {
Layout.fillWidth: true Layout.fillWidth: true
id: myModelList id: myModelList
currentIndex: -1 currentIndex: -1
property bool ready: baseUrlGood && apiKeyGood property bool ready: nameField.ok && baseUrlField.ok && apiKeyField.ok
onReadyChanged: { onReadyChanged: {
if (!ready) { return; } if (!ready) return;
let providerRef = null; // owns the new provider
let provider = remoteModelCard.provider;
if (provider === null) {
// TODO: custom OpenAI
providerRef = QmlFunctions.newCustomOllamaProvider("foo", baseUrlField.text.trim());
if (providerRef !== null)
provider = providerRef.get();
}
if (provider !== null) {
provider.listModelsQml().then(modelList => { provider.listModelsQml().then(modelList => {
if (modelList !== null) { if (modelList !== null) {
model = modelList; model = modelList;
@ -174,7 +198,6 @@ Rectangle {
} }
} }
} }
}
MySettingsButton { MySettingsButton {
id: installButton id: installButton
@ -183,7 +206,6 @@ Rectangle {
font.pixelSize: theme.fontSizeLarge font.pixelSize: theme.fontSizeLarge
property string apiKeyText: apiKeyField.text.trim() property string apiKeyText: apiKeyField.text.trim()
property string baseUrlText: provider === null ? baseUrlField.text.trim() : provider.baseUrl
property string modelNameText: myModelList.currentText.trim() property string modelNameText: myModelList.currentText.trim()
enabled: baseUrlGood && apiKeyGood && modelNameText !== "" enabled: baseUrlGood && apiKeyGood && modelNameText !== ""

View File

@ -47,6 +47,7 @@ protected:
class OllamaProvider : public QObject, public virtual ModelProvider { class OllamaProvider : public QObject, public virtual ModelProvider {
Q_OBJECT Q_OBJECT
Q_PROPERTY(QUuid id READ id CONSTANT) Q_PROPERTY(QUuid id READ id CONSTANT)
Q_PROPERTY(bool isBuiltin READ isBuiltin CONSTANT)
protected: protected:
explicit OllamaProvider(); explicit OllamaProvider();

View File

@ -98,14 +98,6 @@ OpenaiProvider::OpenaiProvider(QString apiKey)
OpenaiProvider::~OpenaiProvider() noexcept = default; OpenaiProvider::~OpenaiProvider() noexcept = default;
Q_INVOKABLE bool OpenaiProvider::setApiKeyQml(QString value)
{
auto res = setApiKey(std::move(value));
if (!res)
qWarning().noquote() << "setApiKey failed:" << res.error().errorString();
return bool(res);
}
auto OpenaiProvider::supportedGenerationParams() const -> QSet<GenerationParam> auto OpenaiProvider::supportedGenerationParams() const -> QSet<GenerationParam>
{ {
using enum GenerationParam; using enum GenerationParam;

View File

@ -52,6 +52,7 @@ class OpenaiProvider : public QObject, public virtual ModelProvider {
Q_OBJECT Q_OBJECT
Q_PROPERTY(QUuid id READ id CONSTANT ) Q_PROPERTY(QUuid id READ id CONSTANT )
Q_PROPERTY(QString apiKey READ apiKey NOTIFY apiKeyChanged) Q_PROPERTY(QString apiKey READ apiKey NOTIFY apiKeyChanged)
Q_PROPERTY(bool isBuiltin READ isBuiltin CONSTANT )
protected: protected:
explicit OpenaiProvider(); explicit OpenaiProvider();
@ -66,7 +67,8 @@ public:
[[nodiscard]] const QString &apiKey() const { return m_apiKey; } [[nodiscard]] const QString &apiKey() const { return m_apiKey; }
[[nodiscard]] virtual DataStoreResult<> setApiKey(QString value) = 0; [[nodiscard]] virtual DataStoreResult<> setApiKey(QString value) = 0;
Q_INVOKABLE bool setApiKeyQml(QString value); Q_INVOKABLE bool setApiKeyQml(QString value)
{ return wrapQmlFunc(this, &OpenaiProvider::setApiKey, u"setApiKey", std::move(value)); }
auto supportedGenerationParams() const -> QSet<GenerationParam> override; auto supportedGenerationParams() const -> QSet<GenerationParam> override;
auto makeGenerationParams(const QMap<GenerationParam, QVariant> &values) const -> OpenaiGenerationParams * override; auto makeGenerationParams(const QMap<GenerationParam, QVariant> &values) const -> OpenaiGenerationParams * override;
@ -107,7 +109,7 @@ public:
std::unordered_set<QString> modelWhitelist); std::unordered_set<QString> modelWhitelist);
[[nodiscard]] DataStoreResult<> setApiKey(QString value) override [[nodiscard]] DataStoreResult<> setApiKey(QString value) override
{ return setMemberProp<QString>(&OpenaiProviderBuiltin::m_apiKey, "apiKey", std::move(value), /*createName*/ m_name); } { return setMemberProp<QString>(&OpenaiProviderBuiltin::m_apiKey, "apiKey", std::move(value), /*create*/ true); }
// override for model whitelist // override for model whitelist
auto listModels() -> QCoro::Task<backend::DataOrRespErr<QStringList>> override; auto listModels() -> QCoro::Task<backend::DataOrRespErr<QStringList>> override;

View File

@ -82,6 +82,13 @@ ModelProviderMutable::~ModelProviderMutable() noexcept
res.error().raise(); // should not happen - will terminate program res.error().raise(); // should not happen - will terminate program
} }
DataStoreResult<> ModelProviderCustom::setName(QString value)
{
if (value.isEmpty())
return std::unexpected(u"name cannot be empty"_s);
return setMemberProp<QString>(&ModelProviderCustom::m_name, "name", std::move(value));
}
auto ModelProviderCustom::persist() -> DataStoreResult<> auto ModelProviderCustom::persist() -> DataStoreResult<>
{ {
if (auto res = m_store->create(asData()); !res) if (auto res = m_store->create(asData()); !res)
@ -112,10 +119,10 @@ void ProviderRegistry::load()
auto registerListener = [this](ModelProvider *provider) { auto registerListener = [this](ModelProvider *provider) {
// listen for any change in the provider so we can tell the model about it // listen for any change in the provider so we can tell the model about it
if (auto *mut = dynamic_cast<ModelProviderMutable *>(provider)) if (auto *mut = dynamic_cast<ModelProviderMutable *>(provider))
connect(mut->asQObject(), "apiKeyChanged", this, "onProviderChanged"); connect(mut->asQObject(), SIGNAL(apiKeyChanged (const QString &)), this, SLOT(onProviderChanged()));
if (auto *cust = dynamic_cast<ModelProviderCustom *>(provider)) { if (auto *cust = dynamic_cast<ModelProviderCustom *>(provider)) {
connect(cust->asQObject(), "nameChanged", this, "onProviderChanged"); connect(cust->asQObject(), SIGNAL(nameChanged (const QString &)), this, SLOT(onProviderChanged()));
connect(cust->asQObject(), "baseUrlChanged", this, "onProviderChanged"); connect(cust->asQObject(), SIGNAL(baseUrlChanged(const QUrl &)), this, SLOT(onProviderChanged()));
} }
}; };
for (auto &p : s_builtinProviders) { // (not all builtin providers are stored) for (auto &p : s_builtinProviders) { // (not all builtin providers are stored)
@ -273,8 +280,8 @@ void ProviderList::onAboutToBeCleared()
bool ProviderListSort::lessThan(const QModelIndex &left, const QModelIndex &right) const bool ProviderListSort::lessThan(const QModelIndex &left, const QModelIndex &right) const
{ {
auto *leftData = sourceModel()->data(left ).value<ModelProvider *>(); auto *leftData = dynamic_cast<ModelProvider *>(sourceModel()->data(left ).value<QObject *>());
auto *rightData = sourceModel()->data(right).value<ModelProvider *>(); auto *rightData = dynamic_cast<ModelProvider *>(sourceModel()->data(right).value<QObject *>());
if (leftData && rightData) { if (leftData && rightData) {
if (leftData->isBuiltin() != rightData->isBuiltin()) if (leftData->isBuiltin() != rightData->isBuiltin())
return leftData->isBuiltin() > rightData->isBuiltin(); // builtins first return leftData->isBuiltin() > rightData->isBuiltin(); // builtins first

View File

@ -61,11 +61,15 @@ concept is_expected = is_expected_impl<std::remove_cvref_t<T>>::value;
/// Drop the type and error information from a QCoro::Task<DataOrRespErr<T>> so it can be used by QML. /// Drop the type and error information from a QCoro::Task<DataOrRespErr<T>> so it can be used by QML.
template <typename C, typename F, typename... Args> template <typename C, typename F, typename... Args>
requires (!detail::is_expected<typename std::invoke_result_t<F, C *, Args...>::value_type>) requires (!detail::is_expected<typename std::invoke_result_t<F, C *, Args...>::value_type>)
QCoro::QmlTask wrapQmlTask(std::shared_ptr<C> c, F f, QString prefix, Args &&...args); QCoro::QmlTask wrapQmlTask(C *obj, F f, QString prefix, Args &&...args);
template <typename C, typename F, typename... Args> template <typename C, typename F, typename... Args>
requires detail::is_expected<typename std::invoke_result_t<F, C *, Args...>::value_type> requires detail::is_expected<typename std::invoke_result_t<F, C *, Args...>::value_type>
QCoro::QmlTask wrapQmlTask(std::shared_ptr<C> c, F f, QString prefix, Args &&...args); QCoro::QmlTask wrapQmlTask(C *obj, F f, QString prefix, Args &&...args);
/// Drop the error information from a DataOrRespErr<T> so it can be used by QML.
template <typename C, typename F, typename... Args>
bool wrapQmlFunc(C *obj, F &&f, QStringView prefix, Args &&...args);
enum class GenerationParam { enum class GenerationParam {
NPredict, NPredict,
@ -182,7 +186,7 @@ protected:
template <typename T, typename S, typename C> template <typename T, typename S, typename C>
[[nodiscard]] DataStoreResult<> setMemberProp(this S &self, T C::* member, std::string_view name, T value, [[nodiscard]] DataStoreResult<> setMemberProp(this S &self, T C::* member, std::string_view name, T value,
std::optional<QString> createName = {}); bool create = false);
[[nodiscard]] virtual bool persisted() const { return true; } [[nodiscard]] virtual bool persisted() const { return true; }
@ -198,11 +202,16 @@ public:
bool isBuiltin() const final { return false; } bool isBuiltin() const final { return false; }
// setters // setters
[[nodiscard]] DataStoreResult<> setName (QString value) [[nodiscard]] DataStoreResult<> setName (QString value);
{ return setMemberProp<QString>(&ModelProviderCustom::m_name, "name", std::move(value)); }
[[nodiscard]] DataStoreResult<> setBaseUrl(QUrl value) [[nodiscard]] DataStoreResult<> setBaseUrl(QUrl value)
{ return setMemberProp<QUrl >(&ModelProviderCustom::m_baseUrl, "baseUrl", std::move(value)); } { return setMemberProp<QUrl >(&ModelProviderCustom::m_baseUrl, "baseUrl", std::move(value)); }
// QML setters
Q_INVOKABLE bool setNameQml (QString value)
{ return wrapQmlFunc(this, &ModelProviderCustom::setName, u"setName", std::move(value)); }
Q_INVOKABLE bool setBaseUrlQml(QString value)
{ return wrapQmlFunc(this, &ModelProviderCustom::setBaseUrl, u"setBaseUrl", std::move(value)); }
[[nodiscard]] auto persist() -> DataStoreResult<>; [[nodiscard]] auto persist() -> DataStoreResult<>;
protected: protected:
@ -264,7 +273,7 @@ private:
ProviderStore m_customStore; ProviderStore m_customStore;
ProviderStore m_builtinStore; ProviderStore m_builtinStore;
std::unordered_map<QUuid, std::shared_ptr<ModelProvider>> m_providers; std::unordered_map<QUuid, std::shared_ptr<ModelProvider>> m_providers;
std::vector<const QUuid *> m_providerList; // TODO: implement std::vector<const QUuid *> m_providerList;
}; };
class ProviderList : public QAbstractListModel { class ProviderList : public QAbstractListModel {
@ -296,7 +305,7 @@ class ProviderListSort : public QSortFilterProxyModel {
QML_ELEMENT QML_ELEMENT
private: private:
explicit ProviderListSort() { setSourceModel(&m_model); } explicit ProviderListSort() { setSourceModel(&m_model); sort(0); }
public: public:
static ProviderListSort *create(QQmlEngine *, QJSEngine *) { return new ProviderListSort(); } static ProviderListSort *create(QQmlEngine *, QJSEngine *) { return new ProviderListSort(); }

View File

@ -38,6 +38,14 @@ QCoro::QmlTask wrapQmlTask(C *obj, F f, QString prefix, Args &&...args)
}(std::move(ptr), std::move(f), std::move(prefix), std::forward<Args>(args)...); }(std::move(ptr), std::move(f), std::move(prefix), std::forward<Args>(args)...);
} }
template <typename C, typename F, typename... Args>
bool wrapQmlFunc(C *obj, F &&f, QStringView prefix, Args &&...args)
{
auto res = std::invoke(std::forward<F>(f), obj, std::forward<Args>(args)...);
if (!res) { qWarning().noquote() << prefix << "failed:" << res.error().errorString(); }
return bool(res);
}
template <typename T, typename S, typename C> template <typename T, typename S, typename C>
void GenerationParams::tryParseValue(this S &self, QMap<GenerationParam, QVariant> &values, GenerationParam key, void GenerationParams::tryParseValue(this S &self, QMap<GenerationParam, QVariant> &values, GenerationParam key,
T C::* dest) T C::* dest)
@ -48,15 +56,15 @@ void GenerationParams::tryParseValue(this S &self, QMap<GenerationParam, QVarian
template <typename T, typename S, typename C> template <typename T, typename S, typename C>
auto ModelProviderMutable::setMemberProp(this S &self, T C::* member, std::string_view name, T value, auto ModelProviderMutable::setMemberProp(this S &self, T C::* member, std::string_view name, T value,
std::optional<QString> createName) -> DataStoreResult<> bool create) -> DataStoreResult<>
{ {
auto &mpc = static_cast<ModelProviderMutable &>(self); auto &mpm = static_cast<ModelProviderMutable &>(self);
auto &cur = self.*member; auto &cur = self.*member;
if (cur != value) { if (cur != value) {
cur = std::move(value); cur = std::move(value);
if (mpc.persisted()) { if (mpm.persisted()) {
auto data = mpc.asData(); auto data = mpm.asData();
if (auto res = mpc.m_store->setData(std::move(data), createName); !res) if (auto res = mpm.m_store->setData(std::move(data), mpm.name(), create); !res)
return res; return res;
} }
QMetaObject::invokeMethod(self.asQObject(), fmt::format("{}Changed", name).c_str(), cur); QMetaObject::invokeMethod(self.asQObject(), fmt::format("{}Changed", name).c_str(), cur);

View File

@ -82,7 +82,7 @@ auto DataStoreBase::reload() -> DataStoreResult<>
} }
for (auto &entry : it) { for (auto &entry : it) {
if (!entry.is_regular_file()) if (!entry.is_regular_file() || entry.path().extension() != ".json")
continue; // skip directories and such continue; // skip directories and such
file.setFileName(entry.path()); file.setFileName(entry.path());
if (!file.open(QFile::ReadOnly)) { if (!file.open(QFile::ReadOnly)) {
@ -93,7 +93,7 @@ auto DataStoreBase::reload() -> DataStoreResult<>
if (!jv) { if (!jv) {
(qWarning().nospace() << "skipping " << file.fileName() << " because of read error: ").noquote() (qWarning().nospace() << "skipping " << file.fileName() << " because of read error: ").noquote()
<< jv.error().errorString(); << jv.error().errorString();
} else if (auto [unique, uuid] = cacheInsert(*jv); !unique) } else if (auto [unique, uuid] = cacheInsert(entry.path().stem(), *jv); !unique)
qWarning() << "skipping duplicate data store entry:" << uuid; qWarning() << "skipping duplicate data store entry:" << uuid;
file.close(); file.close();
} }
@ -109,12 +109,23 @@ auto DataStoreBase::setPath(fs::path path) -> DataStoreResult<>
return {}; return {};
} }
auto DataStoreBase::getFilePath(const QString &name) -> fs::path QByteArray DataStoreBase::normalizeName(const QString &name)
{ return m_path / fmt::format("{}.json", QLatin1StringView(normalizeName(name))); }
auto DataStoreBase::openNew(const QString &name) -> DataStoreResult<std::unique_ptr<QFile>>
{ {
auto path = getFilePath(name); auto lower = name.toLower();
auto norm = QUrl::toPercentEncoding(lower, /*exclude*/ " !#$%&'()+,;=@[]^`{}"_ba, /*include*/ "~"_ba);
// leading dot indicates a hidden file on Unix
if (norm.startsWith('.'))
norm = "%2E"_ba.append(QByteArrayView(norm).slice(1));
return norm;
}
auto DataStoreBase::getFilePath(const QByteArray &normName) -> fs::path
{ return m_path / fmt::format("{}.json", QLatin1StringView(normName)); }
auto DataStoreBase::openNew(const QByteArray &normName) -> DataStoreResult<std::unique_ptr<QFile>>
{
auto path = getFilePath(normName);
auto file = std::make_unique<QFile>(path); auto file = std::make_unique<QFile>(path);
if (file->exists()) if (file->exists())
return std::unexpected(sys::system_error(std::make_error_code(std::errc::file_exists), path.string())); return std::unexpected(sys::system_error(std::make_error_code(std::errc::file_exists), path.string()));
@ -123,9 +134,9 @@ auto DataStoreBase::openNew(const QString &name) -> DataStoreResult<std::unique_
return file; return file;
} }
auto DataStoreBase::openExisting(const QString &name, bool allowCreate) -> DataStoreResult<std::unique_ptr<QSaveFile>> auto DataStoreBase::openExisting(const QByteArray &normName, bool allowCreate) -> DataStoreResult<std::unique_ptr<QSaveFile>>
{ {
auto path = getFilePath(name); auto path = getFilePath(normName);
if (!allowCreate && !QFile::exists(path)) if (!allowCreate && !QFile::exists(path))
return std::unexpected(sys::system_error( return std::unexpected(sys::system_error(
std::make_error_code(std::errc::no_such_file_or_directory), path.string() std::make_error_code(std::errc::no_such_file_or_directory), path.string()
@ -213,16 +224,5 @@ auto DataStoreBase::write(const json::value &value, QFileDevice &file) -> DataSt
return {}; return {};
} }
QByteArray DataStoreBase::normalizeName(const QString &name)
{
auto lower = name.toLower();
auto norm = QUrl::toPercentEncoding(lower, /*exclude*/ " !#$%&'()+,;=@[]^`{}"_ba, /*include*/ "~"_ba);
// "." and ".." are special filenames
return norm == "."_ba ? "%2E"_ba :
norm == ".."_ba ? "%2E%2E"_ba :
norm;
}
} // namespace gpt4all::ui } // namespace gpt4all::ui

View File

@ -70,21 +70,20 @@ public:
protected: protected:
auto reload() -> DataStoreResult<>; auto reload() -> DataStoreResult<>;
virtual auto clear() -> DataStoreResult<> = 0; virtual auto clear() -> DataStoreResult<> = 0;
struct CacheInsertResult { bool unique; QUuid uuid; }; struct CacheInsertResult { bool unique; QUuid id; };
virtual CacheInsertResult cacheInsert(const boost::json::value &jv) = 0; virtual CacheInsertResult cacheInsert(const std::filesystem::path &stem, const boost::json::value &jv) = 0;
// helpers // helpers
auto getFilePath(const QString &name) -> std::filesystem::path; static QByteArray normalizeName(const QString &name);
auto openNew(const QString &name) -> DataStoreResult<std::unique_ptr<QFile>>; auto getFilePath(const QByteArray &normName) -> std::filesystem::path;
auto openExisting(const QString &name, bool allowCreate = false) -> DataStoreResult<std::unique_ptr<QSaveFile>>; auto openNew(const QByteArray &normName) -> DataStoreResult<std::unique_ptr<QFile>>;
auto openExisting(const QByteArray &normName, bool allowCreate = false) -> DataStoreResult<std::unique_ptr<QSaveFile>>;
static auto read(QFileDevice &file, boost::json::stream_parser &parser) -> DataStoreResult<boost::json::value>; static auto read(QFileDevice &file, boost::json::stream_parser &parser) -> DataStoreResult<boost::json::value>;
auto write(const boost::json::value &value, QFileDevice &file) -> DataStoreResult<>; auto write(const boost::json::value &value, QFileDevice &file) -> DataStoreResult<>;
private: private:
static constexpr uint JSON_BUFSIZ = 16384; // default QFILE_WRITEBUFFER_SIZE static constexpr uint JSON_BUFSIZ = 16384; // default QFILE_WRITEBUFFER_SIZE
static QByteArray normalizeName(const QString &name);
protected: protected:
std::filesystem::path m_path; std::filesystem::path m_path;
@ -98,7 +97,7 @@ public:
explicit DataStore(std::filesystem::path path); explicit DataStore(std::filesystem::path path);
auto list() { return m_entries | std::views::transform([](auto &e) { return e.second; }); } auto list() { return m_entries | std::views::transform([](auto &e) { return e.second; }); }
auto setData(T data, std::optional<QString> createName = {}) -> DataStoreResult<>; auto setData(T data, const QString &name, bool create = false) -> DataStoreResult<>;
auto remove(const QUuid &id) -> DataStoreResult<>; auto remove(const QUuid &id) -> DataStoreResult<>;
auto acquire(QUuid id) -> DataStoreResult<std::optional<const T *>>; auto acquire(QUuid id) -> DataStoreResult<std::optional<const T *>>;
@ -112,12 +111,13 @@ public:
protected: protected:
auto createImpl(T data, const QString &name) -> DataStoreResult<>; auto createImpl(T data, const QString &name) -> DataStoreResult<>;
auto clear() -> DataStoreResult<> final; auto clear() -> DataStoreResult<> final;
CacheInsertResult cacheInsert(const boost::json::value &jv) override; CacheInsertResult cacheInsert(const std::filesystem::path &stem, const boost::json::value &jv) override;
private: private:
std::unordered_map<QUuid, T> m_entries; std::unordered_map<QUuid, T > m_entries;
std::unordered_map<QUuid, QByteArray> m_normNames;
std::unordered_map<QByteArray, QUuid > m_normNameToId;
std::unordered_set<QUuid> m_acquired; std::unordered_set<QUuid> m_acquired;
std::unordered_map<QUuid, QString> m_names;
}; };

View File

@ -3,13 +3,13 @@
#include <boost/json.hpp> // IWYU pragma: keep #include <boost/json.hpp> // IWYU pragma: keep
#include <gpt4all-backend/json-helpers.h> // IWYU pragma: keep #include <gpt4all-backend/json-helpers.h> // IWYU pragma: keep
#include <QDebug>
#include <QSaveFile> #include <QSaveFile>
#include <QtAssert> #include <QtAssert>
#include <QtLogging>
#include <ranges> #include <algorithm>
#include <system_error> #include <tuple>
namespace views = std::views;
namespace gpt4all::ui { namespace gpt4all::ui {
@ -26,35 +26,74 @@ DataStore<T>::DataStore(std::filesystem::path path)
template <typename T> template <typename T>
auto DataStore<T>::createImpl(T data, const QString &name) -> DataStoreResult<> auto DataStore<T>::createImpl(T data, const QString &name) -> DataStoreResult<>
{ {
// acquire id
typename decltype(m_entries)::iterator entry;
bool unique;
std::tie(entry, unique) = m_entries.emplace(data.id, std::move(data));
if (!unique)
return std::unexpected(QStringLiteral("id not unique: %1").arg(data.id.toString()));
auto *pdata = &entry->second;
auto normName = normalizeName(name);
auto [nameIt, unique2] = m_normNames.emplace(pdata->id, normName);
Q_UNUSED(unique2)
Q_ASSERT(unique2);
// acquire name
typename decltype(m_normNameToId)::iterator n2idIt;
std::tie(n2idIt, unique) = m_normNameToId.emplace(std::move(normName), pdata->id);
if (!unique) {
m_entries.erase(entry);
m_normNames.erase(nameIt);
return std::unexpected(QStringLiteral("name not unique: %1").arg(QLatin1StringView(normName)));
}
// acquire path // acquire path
auto file = openNew(name); auto file = openNew(normName);
if (!file) if (!file) {
m_entries.erase(entry);
m_normNames.erase(nameIt);
m_normNameToId.erase(n2idIt);
return std::unexpected(file.error()); return std::unexpected(file.error());
}
// serialize // serialize
if (auto res = write(boost::json::value_from(data), **file); !res) if (auto res = write(boost::json::value_from(*pdata), **file); !res) {
m_entries.erase(entry);
m_normNames.erase(nameIt);
m_normNameToId.erase(n2idIt);
if (!(*file)->remove())
(qWarning().nospace() << "failed to remove " << (*file)->fileName()).noquote() << ": " << (*file)->errorString();
return std::unexpected(res.error()); return std::unexpected(res.error());
}
// insert
auto [it, unique] = m_entries.emplace(data.id, std::move(data));
Q_ASSERT(unique);
return {}; return {};
} }
template <typename T> template <typename T>
auto DataStore<T>::setData(T data, std::optional<QString> createName) -> DataStoreResult<> auto DataStore<T>::setData(T data, const QString &name, bool create) -> DataStoreResult<>
{ {
const QString *openName; // acquire name
auto name_it = m_names.find(data.id); auto normName = normalizeName(name);
if (name_it != m_names.end()) { auto n2idIt = m_normNameToId.find(normName);
openName = &name_it->second; if (n2idIt != m_normNameToId.end() && n2idIt->second != data.id)
} else if (createName) { return std::unexpected(QStringLiteral("name is not unique: %1").arg(QLatin1StringView(normName)));
openName = &*createName;
// determine filename to open
bool isNew = false;
QByteArray *openName;
auto nameIt = m_normNames.find(data.id);
if (nameIt != m_normNames.end()) {
openName = &nameIt->second;
} else if (create) {
isNew = true;
openName = &normName;
} else } else
return std::unexpected(QStringLiteral("id not found: %1").arg(data.id.toString())); return std::unexpected(QStringLiteral("id not found: %1").arg(data.id.toString()));
bool isRename = !isNew && n2idIt == m_normNameToId.end();
// acquire path // acquire path
auto file = openExisting(*openName, !!createName); auto file = openExisting(*openName, create);
if (!file) if (!file)
return std::unexpected(file.error()); return std::unexpected(file.error());
@ -64,21 +103,33 @@ auto DataStore<T>::setData(T data, std::optional<QString> createName) -> DataSto
if (!(*file)->commit()) if (!(*file)->commit())
return std::unexpected(file->get()); return std::unexpected(file->get());
// update // update cache
m_entries[data.id] = std::move(data); auto id = data.id;
if (isNew) {
// rename if necessary [[maybe_unused]] bool unique;
if (name_it == m_names.end()) { std::tie(std::ignore, unique) = m_entries .emplace(data.id, std::move(data));
m_names.emplace(data.id, std::move(*createName)); Q_ASSERT(unique);
} else if (*createName != name_it->second) { std::tie(std::ignore, unique) = m_normNames.emplace(data.id, normName );
std::error_code ec; Q_ASSERT(unique);
auto newPath = getFilePath(*createName); } else {
std::filesystem::rename(getFilePath(name_it->second), newPath, ec); m_entries.at(data.id) = std::move(data);
if (ec) nameIt->second = normName;
return std::unexpected(ec);
m_names.at(data.id) = std::move(*createName);
} }
// update name
if (isRename) {
[[maybe_unused]] auto nRemoved = m_normNameToId.erase(*openName); // remove old name
Q_ASSERT(nRemoved);
std::error_code ec;
std::filesystem::rename(getFilePath(*openName), getFilePath(normName), ec);
if (ec)
return std::unexpected(ec); // FIXME(jared): attempt rollback? the state is now inconsistent.
}
if (isNew || isRename) {
auto [_, unique] = m_normNameToId.emplace(*openName, id);
Q_ASSERT(unique);
}
return {}; return {};
} }
@ -86,20 +137,24 @@ template <typename T>
auto DataStore<T>::remove(const QUuid &id) -> DataStoreResult<> auto DataStore<T>::remove(const QUuid &id) -> DataStoreResult<>
{ {
// acquire UUID // acquire UUID
auto it = m_entries.find(id); auto nameIt = m_normNames.find(id);
if (it == m_entries.end()) if (nameIt == m_normNames.end())
return std::unexpected(QStringLiteral("id not found: %1").arg(id.toString())); return std::unexpected(QStringLiteral("id not found: %1").arg(id.toString()));
auto &[_, data] = *it;
// remove the path // remove the path
auto path = getFilePath(data.name); auto path = getFilePath(nameIt->second);
QFile file(path); QFile file(path);
if (!file.remove()) if (!file.remove())
throw std::unexpected(&file); throw std::unexpected(&file);
// update cache // update cache
m_entries.erase(it); auto normName = std::move(nameIt->second);
m_normNames.erase(nameIt);
[[maybe_unused]] size_t nRemoved;
nRemoved = m_entries.erase(id);
Q_ASSERT(nRemoved);
nRemoved = m_normNameToId.erase(normName);
Q_ASSERT(nRemoved);
return {}; return {};
} }
@ -126,16 +181,27 @@ auto DataStore<T>::clear() -> DataStoreResult<>
if (!m_acquired.empty()) if (!m_acquired.empty())
return std::unexpected(QStringLiteral("cannot clear data store with living references")); return std::unexpected(QStringLiteral("cannot clear data store with living references"));
m_entries.clear(); m_entries.clear();
m_normNameToId.clear();
return {}; return {};
} }
template <typename T> template <typename T>
auto DataStore<T>::cacheInsert(const boost::json::value &jv) -> CacheInsertResult auto DataStore<T>::cacheInsert(const std::filesystem::path &stem, const boost::json::value &jv) -> CacheInsertResult
{ {
auto data = boost::json::value_to<T>(jv); auto data = boost::json::value_to<T>(jv);
auto id = data.id; auto id = data.id;
auto [_, ok] = m_entries.emplace(id, std::move(data)); auto [entryIt, unique] = m_entries.emplace(id, std::move(data));
return { ok, std::move(id) }; if (!unique)
return { .unique = false, .id = std::move(id) };
auto normName = toQString(stem).toUtf8(); // FIXME(jared): better not to trust the filename
std::tie(std::ignore, unique) = m_normNameToId.emplace(normName, id);
if (!unique) {
m_entries.erase(entryIt);
return { .unique = false, .id = std::move(id) };
}
std::tie(std::ignore, unique) = m_normNames.emplace(id, std::move(normName));
Q_ASSERT(unique);
return { .unique = true, .id = std::move(id) };
} }