Force tool usage and refactor.

Signed-off-by: Adam Treat <treat.adam@gmail.com>

gpt4all-chat/bravesearch.cpp

@@ -22,10 +22,16 @@ BraveSearch::BraveSearch()
 {
     connect(MySettings::globalInstance(), &MySettings::webSearchUsageModeChanged,
         this, &Tool::usageModeChanged);
+    connect(MySettings::globalInstance(), &MySettings::webSearchConfirmationModeChanged,
+        this, &Tool::confirmationModeChanged);
 }
 
 QString BraveSearch::run(const QJsonObject &parameters, qint64 timeout)
 {
+    // Reset the error state
+    m_error = ToolEnums::Error::NoError;
+    m_errorString = QString();
+
     const QString apiKey = MySettings::globalInstance()->braveSearchAPIKey();
     const QString query = parameters["query"].toString();
     const int count = MySettings::globalInstance()->webSearchRetrievalSize();
@@ -93,6 +99,11 @@ ToolEnums::UsageMode BraveSearch::usageMode() const
     return MySettings::globalInstance()->webSearchUsageMode();
 }
 
+ToolEnums::ConfirmationMode BraveSearch::confirmationMode() const
+{
+    return MySettings::globalInstance()->webSearchConfirmationMode();
+}
+
 void BraveAPIWorker::request(const QString &apiKey, const QString &query, int count)
 {
     // Documentation on the brave web search:
@@ -181,10 +192,12 @@ QString BraveAPIWorker::cleanBraveResponse(const QByteArray& jsonResponse)
                 QJsonObject excerpt;
                 excerpt.insert("text", resultObj["description"]);
             }
+            if (!excerpts.isEmpty()) {
                 result.insert("excerpts", excerpts);
                 cleanArray.append(QJsonValue(result));
             }
         }
+    }
 
     QJsonObject cleanResponse;
     cleanResponse.insert("query", query);

gpt4all-chat/bravesearch.h

@@ -56,6 +56,7 @@ public:
     QJsonObject exampleParams() const override;
     bool isBuiltin() const override { return true; }
     ToolEnums::UsageMode usageMode() const override;
+    ToolEnums::ConfirmationMode confirmationMode() const override;
     bool excerpts() const override { return true; }
 
 private:

gpt4all-chat/chatllm.cpp

@@ -37,6 +37,7 @@
 #include <vector>
 
 using namespace Qt::Literals::StringLiterals;
+using namespace ToolEnums;
 
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
@@ -761,52 +762,218 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
     int32_t repeat_penalty_tokens)
 {
-    // FIXME: Honor the ask before running feature
+    QString toolCallingTemplate = MySettings::globalInstance()->modelToolTemplate(m_modelInfo);
-    // FIXME: The only localdocs specific thing here should be the injection of the parameters
+    Q_ASSERT(toolCallingTemplate.isEmpty() || toolCallingTemplate.contains("%1"));
-    // FIXME: Get the list of tools ... if force usage is set, then we *try* and force usage here.
+    if (toolCallingTemplate.isEmpty() || !toolCallingTemplate.contains("%1"))
-    QList<SourceExcerpt> localDocsExcerpts;
+        toolCallingTemplate = u"### Context:\n%1\n\n"_s;
-    if (!collectionList.isEmpty()) {
-        LocalDocsSearch localdocs;
-        QJsonObject parameters;
-        parameters.insert("text", prompt);
-        parameters.insert("count", MySettings::globalInstance()->localDocsRetrievalSize());
-        parameters.insert("collections", QJsonArray::fromStringList(collectionList));
 
-        // FIXME: This has to handle errors of the tool call
+    const bool isToolCallingModel = MySettings::globalInstance()->modelIsToolCalling(m_modelInfo);
-        const QString localDocsResponse = localdocs.run(parameters, 2000 /*msecs to timeout*/);
+
+    // Iterate over the list of tools and if force usage is set, then we *try* and force usage here
+    QList<QString> toolResponses;
+    qint64 totalTime = 0;
+    bool producedSourceExcerpts = false;
+    const int toolCount = ToolModel::globalInstance()->count();
+    for (int i = 0; i < toolCount; ++i) {
+        Tool *t = ToolModel::globalInstance()->get(i);
+        if (t->usageMode() != UsageMode::ForceUsage)
+            continue;
+
+        // Local docs search is unique. It is the _only_ tool where we try and force usage even if
+        // the model does not support tool calling.
+        if (!isToolCallingModel && t->function() != "localdocs_search")
+            continue;
+
+        // If this is the localdocs tool call, then we perform the search with the entire prompt as
+        // the query
+        if (t->function() == "localdocs_search") {
+            if (collectionList.isEmpty())
+                continue;
+
+            QJsonObject parameters;
+            parameters.insert("collections", QJsonArray::fromStringList(collectionList));
+            parameters.insert("query", prompt);
+            parameters.insert("count", MySettings::globalInstance()->localDocsRetrievalSize());
+
+            // FIXME: Honor the confirmation mode feature
+            const QString response = t->run(parameters, 2000 /*msecs to timeout*/);
+            if (t->error() != Error::NoError) {
+                qWarning() << "ERROR: LocalDocs call produced error:" << t->errorString();
+                continue;
+            }
 
             QString parseError;
-        localDocsExcerpts = SourceExcerpt::fromJson(localDocsResponse, parseError);
+            QList<SourceExcerpt> localDocsExcerpts = SourceExcerpt::fromJson(response, parseError);
             if (!parseError.isEmpty()) {
                 qWarning() << "ERROR: Could not parse source excerpts for localdocs response:" << parseError;
-        } else if (!localDocsExcerpts.isEmpty()) {
+            } else {
+                producedSourceExcerpts = true;
                 emit sourceExcerptsChanged(localDocsExcerpts);
             }
+            toolResponses << QString(toolCallingTemplate).arg(response);
+            continue;
         }
 
-    // Augment the prompt template with the results if any
+        // For all other cases we should have a tool calling model
-    QString docsContext;
+        Q_ASSERT(isToolCallingModel);
-    if (!localDocsExcerpts.isEmpty()) {
+
-        // FIXME(adam): we should be using the new tool template if available otherwise this I guess
+        // Create the tool calling response as if the model has chosen this particular tool
-        QString json = SourceExcerpt::toJson(localDocsExcerpts);
+        const QString toolCallingResponse = QString("<tool_call>{\"name\": \"%1\", \"parameters\": {\"").arg(t->function());
-        docsContext = u"### Context:\n%1\n\n"_s.arg(json);
+
+        // Mimic that the model has already responded like this to trigger our tool calling detection
+        // code and then rely upon it to complete the parameters correctly
+        m_response = toolCallingResponse.toStdString();
+
+        // Insert this response as the tool prompt
+        const QString toolPrompt = QString(promptTemplate).arg(prompt, toolCallingResponse);
+
+        const QString toolCall = completeToolCall(toolPrompt, n_predict, top_k, top_p,
+            min_p, temp, n_batch, repeat_penalty, repeat_penalty_tokens, totalTime);
+
+        // If the tool call is empty, then we failed in our attempt to force usage
+        if (toolCall.isEmpty()) {
+            qWarning() << "WARNING: Attempt to force usage of toolcall" << t->function() << "failed:"
+                << "model could not complete parameters for" << toolPrompt;
+            continue;
         }
 
-    qint64 totalTime = 0;
+        QString errorString;
-    bool producedSourceExcerpts;
+        const QString response = executeToolCall(toolCall, producedSourceExcerpts, errorString);
-    bool success = promptRecursive({ docsContext }, prompt, promptTemplate, n_predict, top_k, top_p,
+        if (response.isEmpty()) {
+            qWarning() << "WARNING: Attempt to force usage of toolcall" << t->function() << "failed:" << errorString;
+            continue;
+        }
+
+        toolResponses << QString(toolCallingTemplate).arg(response);
+    }
+
+    bool success = promptRecursive({ toolResponses }, prompt, promptTemplate, n_predict, top_k, top_p,
         min_p, temp, n_batch, repeat_penalty, repeat_penalty_tokens, totalTime, producedSourceExcerpts);
-
+    Q_ASSERT(success);
     SuggestionMode mode = MySettings::globalInstance()->suggestionMode();
-    if (mode == SuggestionMode::On || (mode == SuggestionMode::SourceExcerptsOnly && (!localDocsExcerpts.isEmpty() || producedSourceExcerpts)))
+    if (mode == SuggestionMode::On || (mode == SuggestionMode::SourceExcerptsOnly && producedSourceExcerpts))
         generateQuestions(totalTime);
     else
         emit responseStopped(totalTime);
-
     return success;
 }
 
-bool ChatLLM::promptRecursive(const QList<QString> &toolContexts, const QString &prompt,
+QString ChatLLM::completeToolCall(const QString &prompt, int32_t n_predict, int32_t top_k, float top_p,
+    float min_p, float temp, int32_t n_batch, float repeat_penalty, int32_t repeat_penalty_tokens,
+    qint64 &totalTime)
+{
+    if (!isModelLoaded())
+        return QString();
+
+    int n_threads = MySettings::globalInstance()->threadCount();
+
+    m_stopGenerating = false;
+    auto promptFunc = std::bind(&ChatLLM::handlePrompt, this, std::placeholders::_1);
+    auto responseFunc = std::bind(&ChatLLM::handleResponse, this, std::placeholders::_1,
+        std::placeholders::_2);
+    emit promptProcessing();
+    m_ctx.n_predict = n_predict;
+    m_ctx.top_k = top_k;
+    m_ctx.top_p = top_p;
+    m_ctx.min_p = min_p;
+    m_ctx.temp = temp;
+    m_ctx.n_batch = n_batch;
+    m_ctx.repeat_penalty = repeat_penalty;
+    m_ctx.repeat_last_n = repeat_penalty_tokens;
+    m_llModelInfo.model->setThreadCount(n_threads);
+#if defined(DEBUG)
+    printf("%s", qPrintable(prompt));
+    fflush(stdout);
+#endif
+
+    QElapsedTimer elapsedTimer;
+    elapsedTimer.start();
+    m_timer->start();
+
+    m_checkToolCall = true;
+
+    // We pass in the prompt as the completed template as we're mimicking that the respone has already
+    // started
+    LLModel::PromptContext ctx = m_ctx;
+    m_llModelInfo.model->prompt(prompt.toStdString(), "%1", promptFunc, responseFunc,
+                                /*allowContextShift*/ false, ctx);
+
+    // After the response has been handled reset this state
+    m_checkToolCall = false;
+    m_maybeToolCall = false;
+
+    m_timer->stop();
+    totalTime = elapsedTimer.elapsed();
+
+    const QString toolCall = QString::fromStdString(trim_whitespace(m_response));
+    m_promptResponseTokens = 0;
+    m_promptTokens = 0;
+    m_response = std::string();
+
+    if (!m_foundToolCall)
+        return QString();
+
+    m_foundToolCall = false;
+    return toolCall;
+}
+
+QString ChatLLM::executeToolCall(const QString &toolCall, bool &producedSourceExcerpts, QString &errorString)
+{
+    const QString toolTemplate = MySettings::globalInstance()->modelToolTemplate(m_modelInfo);
+    if (toolTemplate.isEmpty()) {
+        errorString = QString("ERROR: No valid tool template for this model %1").arg(toolCall);
+        return QString();
+    }
+
+    QJsonParseError err;
+    const QJsonDocument toolCallDoc = QJsonDocument::fromJson(toolCall.toUtf8(), &err);
+
+    if (toolCallDoc.isNull() || err.error != QJsonParseError::NoError || !toolCallDoc.isObject()) {
+        errorString = QString("ERROR: The tool call had null or invalid json %1").arg(toolCall);
+        return QString();
+    }
+
+    QJsonObject rootObject = toolCallDoc.object();
+    if (!rootObject.contains("name") || !rootObject.contains("parameters")) {
+        errorString = QString("ERROR: The tool call did not have required name and argument objects %1").arg(toolCall);
+        return QString();
+    }
+
+    const QString tool = toolCallDoc["name"].toString();
+    const QJsonObject args = toolCallDoc["parameters"].toObject();
+
+    Tool *toolInstance = ToolModel::globalInstance()->get(tool);
+    if (!toolInstance) {
+        errorString = QString("ERROR: Could not find the tool for %1").arg(toolCall);
+        return QString();
+    }
+
+    // FIXME: Honor the confirmation mode feature
+    // Inform the chat that we're executing a tool call
+    emit toolCalled(toolInstance->name().toLower());
+
+    const QString response = toolInstance->run(args, 2000 /*msecs to timeout*/);
+    if (toolInstance->error() != Error::NoError) {
+        errorString = QString("ERROR: Tool call produced error: %1").arg(toolInstance->errorString());
+        return QString();
+    }
+
+    // If the tool supports excerpts then try to parse them here, but it isn't strictly an error
+    // but rather a warning
+    if (toolInstance->excerpts()) {
+        QString parseError;
+        QList<SourceExcerpt> sourceExcerpts = SourceExcerpt::fromJson(response, parseError);
+        if (!parseError.isEmpty()) {
+            qWarning() << "WARNING: Could not parse source excerpts for response:" << parseError;
+        } else if (!sourceExcerpts.isEmpty()) {
+            producedSourceExcerpts = true;
+            emit sourceExcerptsChanged(sourceExcerpts);
+        }
+    }
+    return response;
+}
+
+bool ChatLLM::promptRecursive(const QList<QString> &toolResponses, const QString &prompt,
     const QString &promptTemplate, int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp,
     int32_t n_batch, float repeat_penalty, int32_t repeat_penalty_tokens, qint64 &totalTime, bool &producedSourceExcerpts, bool isRecursiveCall)
 {
@@ -838,8 +1005,8 @@ bool ChatLLM::promptRecursive(const QList<QString> &toolContexts, const QString
     elapsedTimer.start();
     m_timer->start();
 
-    // The list of possible additional contexts that come from previous usage of tool calls
+    // The list of possible additional responses that come from previous usage of tool calls
-    for (const QString &context : toolContexts) {
+    for (const QString &context : toolResponses) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode context without a response
         m_llModelInfo.model->prompt(context.toStdString(), "%1", promptFunc, responseFunc,
                                     /*allowContextShift*/ true, m_ctx);
@@ -869,65 +1036,27 @@ bool ChatLLM::promptRecursive(const QList<QString> &toolContexts, const QString
     if (m_foundToolCall) {
         m_foundToolCall = false;
 
+        QString errorString;
         const QString toolCall = QString::fromStdString(trimmed);
-        const QString toolTemplate = MySettings::globalInstance()->modelToolTemplate(m_modelInfo);
+        const QString toolResponse = executeToolCall(toolCall, producedSourceExcerpts, errorString);
-        if (toolTemplate.isEmpty()) {
+        if (toolResponse.isEmpty()) {
-            qWarning() << "ERROR: No valid tool template for this model" << toolCall;
+            // FIXME: Need to surface errors to the UI
-            return handleFailedToolCall(trimmed, totalTime);
+            // Restore the strings that we excluded previously when detecting the tool call
-        }
+            qWarning() << errorString;
-
+            m_response = "<tool_call>" + toolCall.toStdString() + "</tool_call>";
-        QJsonParseError err;
+            emit responseChanged(QString::fromStdString(m_response));
-        const QJsonDocument toolCallDoc = QJsonDocument::fromJson(toolCall.toUtf8(), &err);
+            emit responseStopped(totalTime);
-
+            m_pristineLoadedState = false;
-        if (toolCallDoc.isNull() || err.error != QJsonParseError::NoError || !toolCallDoc.isObject()) {
+            return false;
-            qWarning() << "ERROR: The tool call had null or invalid json " << toolCall;
-            return handleFailedToolCall(trimmed, totalTime);
-        }
-
-        QJsonObject rootObject = toolCallDoc.object();
-        if (!rootObject.contains("name") || !rootObject.contains("parameters")) {
-            qWarning() << "ERROR: The tool call did not have required name and argument objects " << toolCall;
-            return handleFailedToolCall(trimmed, totalTime);
-        }
-
-        const QString tool = toolCallDoc["name"].toString();
-        const QJsonObject args = toolCallDoc["parameters"].toObject();
-
-        Tool *toolInstance = ToolModel::globalInstance()->get(tool);
-        if (!toolInstance) {
-            qWarning() << "ERROR: Could not find the tool for " << toolCall;
-            return handleFailedToolCall(trimmed, totalTime);
-        }
-
-        // FIXME: Honor the ask before running feature
-        // Inform the chat that we're executing a tool call
-        emit toolCalled(toolInstance->name().toLower());
-
-        const QString response = toolInstance->run(args, 2000 /*msecs to timeout*/);
-        if (toolInstance->error() != ToolEnums::Error::NoError) {
-            qWarning() << "ERROR: Tool call produced error:" << toolInstance->errorString();
-            return handleFailedToolCall(trimmed, totalTime);
-        }
-
-        // If the tool supports excerpts then try to parse them here
-        if (toolInstance->excerpts()) {
-            QString parseError;
-            QList<SourceExcerpt> sourceExcerpts = SourceExcerpt::fromJson(response, parseError);
-            if (!parseError.isEmpty()) {
-                qWarning() << "ERROR: Could not parse source excerpts for response:" << parseError;
-            } else if (!sourceExcerpts.isEmpty()) {
-                producedSourceExcerpts = true;
-                emit sourceExcerptsChanged(sourceExcerpts);
-            }
         }
 
+        // Reset the state now that we've had a successful tool call response
         m_promptResponseTokens = 0;
         m_promptTokens = 0;
         m_response = std::string();
 
-        // This is a recursive call but isRecursiveCall is checked above to arrest infinite recursive
+        // This is a recursive call but flag is checked above to arrest infinite recursive tool calls
-        // tool calls
+        return promptRecursive({ toolResponse }, prompt, promptTemplate,
-        return promptRecursive(QList<QString>()/*tool context*/, response, toolTemplate,
             n_predict, top_k, top_p, min_p, temp, n_batch, repeat_penalty, repeat_penalty_tokens, totalTime,
             producedSourceExcerpts, true /*isRecursiveCall*/);
     } else {
@@ -940,17 +1069,6 @@ bool ChatLLM::promptRecursive(const QList<QString> &toolContexts, const QString
     }
 }
 
-bool ChatLLM::handleFailedToolCall(const std::string &response, qint64 elapsed)
-{
-    // FIXME: Need to surface errors to the UI
-    // Restore the strings that we excluded previously when detecting the tool call
-    m_response = "<tool_call>" + response + "</tool_call>";
-    emit responseChanged(QString::fromStdString(m_response));
-    emit responseStopped(elapsed);
-    m_pristineLoadedState = false;
-    return true;
-}
-
 void ChatLLM::setShouldBeLoaded(bool b)
 {
 #if defined(DEBUG_MODEL_LOADING)

gpt4all-chat/chatllm.h

@@ -200,7 +200,6 @@ protected:
     bool promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
         int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
         int32_t repeat_penalty_tokens);
-    bool handleFailedToolCall(const std::string &toolCall, qint64 elapsed);
     bool handlePrompt(int32_t token);
     bool handleResponse(int32_t token, const std::string &response);
     bool handleNamePrompt(int32_t token);
@@ -220,6 +219,10 @@ protected:
     quint32 m_promptResponseTokens;
 
 private:
+    QString completeToolCall(const QString &promptTemplate, int32_t n_predict, int32_t top_k, float top_p,
+        float min_p, float temp, int32_t n_batch, float repeat_penalty, int32_t repeat_penalty_tokens,
+        qint64 &totalTime);
+    QString executeToolCall(const QString &toolCall, bool &producedSourceExcerpts, QString &errorString);
     bool promptRecursive(const QList<QString> &toolContexts, const QString &prompt, const QString &promptTemplate,
         int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
         int32_t repeat_penalty_tokens, qint64 &totalTime, bool &producedSourceExcerpts, bool isRecursiveCall = false);

gpt4all-chat/localdocssearch.cpp

@@ -1,6 +1,7 @@
 #include "localdocssearch.h"
 #include "database.h"
 #include "localdocs.h"
+#include "mysettings.h"
 
 #include <QCoreApplication>
 #include <QDebug>
@@ -14,12 +15,16 @@ using namespace Qt::Literals::StringLiterals;
 
 QString LocalDocsSearch::run(const QJsonObject &parameters, qint64 timeout)
 {
+    // Reset the error state
+    m_error = ToolEnums::Error::NoError;
+    m_errorString = QString();
+
     QList<QString> collections;
     QJsonArray collectionsArray = parameters["collections"].toArray();
     for (int i = 0; i < collectionsArray.size(); ++i)
         collections.append(collectionsArray[i].toString());
-    const QString text = parameters["text"].toString();
+    const QString text = parameters["query"].toString();
-    const int count = parameters["count"].toInt();
+    const int count = MySettings::globalInstance()->localDocsRetrievalSize();
     QThread workerThread;
     LocalDocsWorker worker;
     worker.moveToThread(&workerThread);
@@ -71,6 +76,16 @@ QJsonObject LocalDocsSearch::paramSchema() const
     return localJsonDoc.object();
 }
 
+QJsonObject LocalDocsSearch::exampleParams() const
+{
+    static const QString example = R"({
+        "query": "the 44th president of the United States"
+      })";
+    static const QJsonDocument exampleDoc = QJsonDocument::fromJson(example.toUtf8());
+    Q_ASSERT(!exampleDoc.isNull() && exampleDoc.isObject());
+    return exampleDoc.object();
+}
+
 LocalDocsWorker::LocalDocsWorker()
     : QObject(nullptr)
 {

gpt4all-chat/localdocssearch.h

@@ -38,6 +38,7 @@ public:
     QString description() const override { return tr("Search the local docs"); }
     QString function() const override { return "localdocs_search"; }
     ToolEnums::PrivacyScope privacyScope() const override { return ToolEnums::PrivacyScope::Local; }
+    QJsonObject exampleParams() const override;
     QJsonObject paramSchema() const override;
     bool isBuiltin() const override { return true; }
     ToolEnums::UsageMode usageMode() const override { return ToolEnums::UsageMode::ForceUsage; }

gpt4all-chat/modellist.cpp

@@ -367,6 +367,17 @@ void ModelInfo::setSuggestedFollowUpPrompt(const QString &p)
     m_suggestedFollowUpPrompt = p;
 }
 
+bool ModelInfo::isToolCalling() const
+{
+    return MySettings::globalInstance()->modelIsToolCalling(*this);
+}
+
+void ModelInfo::setIsToolCalling(bool b)
+{
+    if (shouldSaveMetadata()) MySettings::globalInstance()->setModelIsToolCalling(*this, b, true /*force*/);
+    m_isToolCalling = b;
+}
+
 bool ModelInfo::shouldSaveMetadata() const
 {
     return installed && (isClone() || isDiscovered() || description() == "" /*indicates sideloaded*/);
@@ -400,6 +411,7 @@ QVariantMap ModelInfo::getFields() const
         { "systemPromptTemplate",m_systemPromptTemplate },
         { "chatNamePrompt",      m_chatNamePrompt },
         { "suggestedFollowUpPrompt", m_suggestedFollowUpPrompt },
+        { "isToolCalling",       m_isToolCalling },
     };
 }
 
@@ -518,6 +530,7 @@ ModelList::ModelList()
     connect(MySettings::globalInstance(), &MySettings::promptTemplateChanged, this, &ModelList::updateDataForSettings);
     connect(MySettings::globalInstance(), &MySettings::toolTemplateChanged, this, &ModelList::updateDataForSettings);
     connect(MySettings::globalInstance(), &MySettings::systemPromptChanged, this, &ModelList::updateDataForSettings);
+    connect(MySettings::globalInstance(), &MySettings::isToolCallingChanged, this, &ModelList::updateDataForSettings);
     connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this, &ModelList::handleSslErrors);
 
     updateModelsFromJson();
@@ -803,7 +816,8 @@ QVariant ModelList::dataInternal(const ModelInfo *info, int role) const
             return info->downloads();
         case RecencyRole:
             return info->recency();
-
+        case IsToolCallingRole:
+            return info->isToolCalling();
     }
 
     return QVariant();
@@ -999,6 +1013,8 @@ void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>
                     }
                     break;
                 }
+            case IsToolCallingRole:
+                info->setIsToolCalling(value.toBool()); break;
             }
         }
 
@@ -1573,6 +1589,8 @@ void ModelList::parseModelsJsonFile(const QByteArray &jsonData, bool save)
             data.append({ ModelList::ToolTemplateRole, obj["toolTemplate"].toString() });
         if (obj.contains("systemPrompt"))
             data.append({ ModelList::SystemPromptRole, obj["systemPrompt"].toString() });
+        if (obj.contains("isToolCalling"))
+            data.append({ ModelList::IsToolCallingRole, obj["isToolCalling"].toBool() });
         updateData(id, data);
     }
 
@@ -1888,6 +1906,10 @@ void ModelList::updateModelsFromSettings()
             const QString suggestedFollowUpPrompt = settings.value(g + "/suggestedFollowUpPrompt").toString();
             data.append({ ModelList::SuggestedFollowUpPromptRole, suggestedFollowUpPrompt });
         }
+        if (settings.contains(g + "/isToolCalling")) {
+            const bool isToolCalling = settings.value(g + "/isToolCalling").toBool();
+            data.append({ ModelList::IsToolCallingRole, isToolCalling });
+        }
         updateData(id, data);
     }
 }

gpt4all-chat/modellist.h

@@ -75,6 +75,7 @@ struct ModelInfo {
     Q_PROPERTY(int likes READ likes WRITE setLikes)
     Q_PROPERTY(int downloads READ downloads WRITE setDownloads)
     Q_PROPERTY(QDateTime recency READ recency WRITE setRecency)
+    Q_PROPERTY(bool isToolCalling READ isToolCalling WRITE setIsToolCalling)
 
 public:
     enum HashAlgorithm {
@@ -118,6 +119,9 @@ public:
     QDateTime recency() const;
     void setRecency(const QDateTime &r);
 
+    bool isToolCalling() const;
+    void setIsToolCalling(bool b);
+
     QString dirpath;
     QString filesize;
     QByteArray hash;
@@ -223,6 +227,7 @@ private:
     QString m_systemPromptTemplate    = "### System:\nYou are an AI assistant who gives a quality response to whatever humans ask of you.\n\n";
     QString m_chatNamePrompt          = "Describe the above conversation in seven words or less.";
     QString m_suggestedFollowUpPrompt = "Suggest three very short factual follow-up questions that have not been answered yet or cannot be found inspired by the previous conversation and excerpts.";
+    bool    m_isToolCalling           = false;
     friend class MySettings;
 };
 Q_DECLARE_METATYPE(ModelInfo)
@@ -351,7 +356,8 @@ public:
         MinPRole,
         LikesRole,
         DownloadsRole,
-        RecencyRole
+        RecencyRole,
+        IsToolCallingRole
     };
 
     QHash<int, QByteArray> roleNames() const override

gpt4all-chat/mysettings.cpp

@@ -64,9 +64,9 @@ static const QVariantMap basicDefaults {
     { "localdocs/nomicAPIKey",      "" },
     { "localdocs/embedDevice",      "Auto" },
     { "network/attribution",        "" },
-    { "websearch/usageMode",        QVariant::fromValue(UsageMode::Disabled) },
     { "websearch/retrievalSize",    2 },
-    { "websearch/askBeforeRunning", false },
+    { "websearch/usageMode",        QVariant::fromValue(UsageMode::Disabled) },
+    { "websearch/confirmationMode", QVariant::fromValue(ConfirmationMode::NoConfirmation) },
     { "bravesearch/APIKey",         "" },
 };
 
@@ -203,6 +203,7 @@ void MySettings::restoreModelDefaults(const ModelInfo &info)
     setModelRepeatPenaltyTokens(info, info.m_repeatPenaltyTokens);
     setModelPromptTemplate(info, info.m_promptTemplate);
     setModelToolTemplate(info, info.m_toolTemplate);
+    setModelIsToolCalling(info, info.m_isToolCalling);
     setModelSystemPromptTemplate(info, info.m_systemPromptTemplate);
     setModelChatNamePrompt(info, info.m_chatNamePrompt);
     setModelSuggestedFollowUpPrompt(info, info.m_suggestedFollowUpPrompt);
@@ -239,7 +240,7 @@ void MySettings::restoreWebSearchDefaults()
 {
     setWebSearchUsageMode(basicDefaults.value("websearch/usageMode").value<UsageMode>());
     setWebSearchRetrievalSize(basicDefaults.value("websearch/retrievalSize").toInt());
-    setWebSearchAskBeforeRunning(basicDefaults.value("websearch/askBeforeRunning").toBool());
+    setWebSearchConfirmationMode(basicDefaults.value("websearch/confirmationMode").value<ConfirmationMode>());
     setBraveSearchAPIKey(basicDefaults.value("bravesearch/APIKey").toString());
 }
 
@@ -314,6 +315,7 @@ double    MySettings::modelRepeatPenalty          (const ModelInfo &info) const
 int       MySettings::modelRepeatPenaltyTokens    (const ModelInfo &info) const { return getModelSetting("repeatPenaltyTokens",     info).toInt(); }
 QString   MySettings::modelPromptTemplate         (const ModelInfo &info) const { return getModelSetting("promptTemplate",          info).toString(); }
 QString   MySettings::modelToolTemplate           (const ModelInfo &info) const { return getModelSetting("toolTemplate",            info).toString(); }
+bool      MySettings::modelIsToolCalling          (const ModelInfo &info) const { return getModelSetting("isToolCalling",           info).toBool(); }
 QString   MySettings::modelSystemPromptTemplate   (const ModelInfo &info) const { return getModelSetting("systemPrompt",            info).toString(); }
 QString   MySettings::modelChatNamePrompt         (const ModelInfo &info) const { return getModelSetting("chatNamePrompt",          info).toString(); }
 QString   MySettings::modelSuggestedFollowUpPrompt(const ModelInfo &info) const { return getModelSetting("suggestedFollowUpPrompt", info).toString(); }
@@ -428,6 +430,11 @@ void MySettings::setModelToolTemplate(const ModelInfo &info, const QString &valu
     setModelSetting("toolTemplate", info, value, force, true);
 }
 
+void MySettings::setModelIsToolCalling(const ModelInfo &info, bool value, bool force)
+{
+    setModelSetting("isToolCalling", info, value, force, true);
+}
+
 void MySettings::setModelSystemPromptTemplate(const ModelInfo &info, const QString &value, bool force)
 {
     setModelSetting("systemPrompt", info, value, force, true);
@@ -481,8 +488,8 @@ QString     MySettings::localDocsEmbedDevice() const    { return getBasicSetting
 QString     MySettings::networkAttribution() const      { return getBasicSetting("network/attribution"     ).toString(); }
 QString     MySettings::braveSearchAPIKey() const       { return getBasicSetting("bravesearch/APIKey"   ).toString(); }
 int         MySettings::webSearchRetrievalSize() const      { return getBasicSetting("websearch/retrievalSize").toInt(); }
-bool        MySettings::webSearchAskBeforeRunning() const   { return getBasicSetting("websearch/askBeforeRunning").toBool(); }
 UsageMode        MySettings::webSearchUsageMode() const          { return getBasicSetting("websearch/usageMode").value<UsageMode>(); }
+ConfirmationMode MySettings::webSearchConfirmationMode() const   { return getBasicSetting("websearch/confirmationMode").value<ConfirmationMode>(); }
 
 ChatTheme      MySettings::chatTheme() const      { return ChatTheme     (getEnumSetting("chatTheme", chatThemeNames)); }
 FontSize       MySettings::fontSize() const       { return FontSize      (getEnumSetting("fontSize",  fontSizeNames)); }
@@ -502,9 +509,9 @@ void MySettings::setLocalDocsNomicAPIKey(const QString &value)        { setBasic
 void MySettings::setLocalDocsEmbedDevice(const QString &value)        { setBasicSetting("localdocs/embedDevice",    value, "localDocsEmbedDevice"); }
 void MySettings::setNetworkAttribution(const QString &value)          { setBasicSetting("network/attribution",      value, "networkAttribution"); }
 void MySettings::setBraveSearchAPIKey(const QString &value)           { setBasicSetting("bravesearch/APIKey",       value, "braveSearchAPIKey"); }
-void MySettings::setWebSearchUsageMode(ToolEnums::UsageMode value)    { setBasicSetting("websearch/usageMode",      int(value), "webSearchUsageMode"); }
 void MySettings::setWebSearchRetrievalSize(int value)                 { setBasicSetting("websearch/retrievalSize",  value, "webSearchRetrievalSize"); }
-void MySettings::setWebSearchAskBeforeRunning(bool value)             { setBasicSetting("websearch/askBeforeRunning", value, "webSearchAskBeforeRunning"); }
+void MySettings::setWebSearchUsageMode(ToolEnums::UsageMode value)                  { setBasicSetting("websearch/usageMode",        int(value), "webSearchUsageMode"); }
+void MySettings::setWebSearchConfirmationMode(ToolEnums::ConfirmationMode value)    { setBasicSetting("websearch/confirmationMode", int(value), "webSearchConfirmationMode"); }
 
 void MySettings::setChatTheme(ChatTheme value)           { setBasicSetting("chatTheme",      chatThemeNames     .value(int(value))); }
 void MySettings::setFontSize(FontSize value)             { setBasicSetting("fontSize",       fontSizeNames      .value(int(value))); }
@@ -717,10 +724,14 @@ QString MySettings::systemPromptInternal(const QString &proposedTemplate, QStrin
     params.insert({"currentDate", QDate::currentDate().toString().toStdString()});
 
     jinja2::ValuesList toolList;
-    int c = ToolModel::globalInstance()->count();
+    const int toolCount = ToolModel::globalInstance()->count();
-    for (int i = 0; i < c; ++i) {
+    for (int i = 0; i < toolCount; ++i) {
         Tool *t = ToolModel::globalInstance()->get(i);
-        if (t->usageMode() == UsageMode::Enabled)
+        // FIXME: For now we don't tell the model about the localdocs search in the system prompt because
+        // it will try to call the localdocs search even if no collection is selected. Ideally, we need
+        // away to update model to whether a tool is enabled/disabled either via reprocessing the system
+        // prompt or sending a system message as it happens
+        if (t->usageMode() != UsageMode::Disabled && t->function() != "localdocs_search")
             toolList.push_back(t->jinjaValue());
     }
     params.insert({"toolList", toolList});

gpt4all-chat/mysettings.h

@@ -73,9 +73,9 @@ class MySettings : public QObject
     Q_PROPERTY(int networkPort READ networkPort WRITE setNetworkPort NOTIFY networkPortChanged)
     Q_PROPERTY(SuggestionMode suggestionMode READ suggestionMode WRITE setSuggestionMode NOTIFY suggestionModeChanged)
     Q_PROPERTY(QStringList uiLanguages MEMBER m_uiLanguages CONSTANT)
-    Q_PROPERTY(ToolEnums::UsageMode webSearchUsageMode READ webSearchUsageMode WRITE setWebSearchUsageMode NOTIFY webSearchUsageModeChanged)
     Q_PROPERTY(int webSearchRetrievalSize READ webSearchRetrievalSize WRITE setWebSearchRetrievalSize NOTIFY webSearchRetrievalSizeChanged)
-    Q_PROPERTY(bool webSearchAskBeforeRunning READ webSearchAskBeforeRunning WRITE setWebSearchAskBeforeRunning NOTIFY webSearchAskBeforeRunningChanged)
+    Q_PROPERTY(ToolEnums::UsageMode webSearchUsageMode READ webSearchUsageMode WRITE setWebSearchUsageMode NOTIFY webSearchUsageModeChanged)
+    Q_PROPERTY(ToolEnums::ConfirmationMode webSearchConfirmationMode READ webSearchConfirmationMode WRITE setWebSearchConfirmationMode NOTIFY webSearchConfirmationModeChanged)
     Q_PROPERTY(QString braveSearchAPIKey READ braveSearchAPIKey WRITE setBraveSearchAPIKey NOTIFY braveSearchAPIKeyChanged)
 
 public:
@@ -133,6 +133,8 @@ public:
     Q_INVOKABLE void setModelPromptTemplate(const ModelInfo &info, const QString &value, bool force = false);
     QString modelToolTemplate(const ModelInfo &info) const;
     Q_INVOKABLE void setModelToolTemplate(const ModelInfo &info, const QString &value, bool force = false);
+    bool modelIsToolCalling(const ModelInfo &info) const;
+    Q_INVOKABLE void setModelIsToolCalling(const ModelInfo &info, bool value, bool force = false);
     QString modelSystemPromptTemplate(const ModelInfo &info) const;
     Q_INVOKABLE void setModelSystemPromptTemplate(const ModelInfo &info, const QString &value, bool force = false);
     int modelContextLength(const ModelInfo &info) const;
@@ -194,12 +196,12 @@ public:
     void setLocalDocsEmbedDevice(const QString &value);
 
     // Web search settings
-    ToolEnums::UsageMode webSearchUsageMode() const;
-    void setWebSearchUsageMode(ToolEnums::UsageMode value);
     int webSearchRetrievalSize() const;
     void setWebSearchRetrievalSize(int value);
-    bool webSearchAskBeforeRunning() const;
+    ToolEnums::UsageMode webSearchUsageMode() const;
-    void setWebSearchAskBeforeRunning(bool value);
+    void setWebSearchUsageMode(ToolEnums::UsageMode value);
+    ToolEnums::ConfirmationMode webSearchConfirmationMode() const;
+    void setWebSearchConfirmationMode(ToolEnums::ConfirmationMode value);
     QString braveSearchAPIKey() const;
     void setBraveSearchAPIKey(const QString &value);
 
@@ -238,6 +240,7 @@ Q_SIGNALS:
     void systemPromptChanged(const ModelInfo &info);
     void chatNamePromptChanged(const ModelInfo &info);
     void suggestedFollowUpPromptChanged(const ModelInfo &info);
+    void isToolCallingChanged(const ModelInfo &info);
     void threadCountChanged();
     void saveChatsContextChanged();
     void serverChatChanged();
@@ -262,9 +265,10 @@ Q_SIGNALS:
     void deviceChanged();
     void suggestionModeChanged();
     void languageAndLocaleChanged();
+    void webSearchRetrievalSizeChanged();
+    // FIXME: These are never emitted along with a lot of the signals above probably with all kinds of bugs!!
     void webSearchUsageModeChanged();
-    void webSearchRetrievalSizeChanged() const;
+    void webSearchConfirmationModeChanged();
-    void webSearchAskBeforeRunningChanged() const;
     void braveSearchAPIKeyChanged();
 
 private:

gpt4all-chat/qml/ModelSettings.qml

@@ -153,9 +153,30 @@ MySettingsTab {
             Layout.fillWidth: true
         }
 
-        RowLayout {
+        MySettingsLabel {
             Layout.row: 7
             Layout.column: 0
+            Layout.columnSpan: 1
+            Layout.topMargin: 15
+            id: isToolCallingLabel
+            text: qsTr("Is Tool Calling Model")
+            helpText: qsTr("Whether the model is capable of tool calling and has tool calling instructions in system prompt.")
+        }
+
+        MyCheckBox {
+            Layout.row: 7
+            Layout.column: 1
+            Layout.topMargin: 15
+            id: isToolCallingBox
+            checked: root.currentModelInfo.isToolCalling
+            onClicked: {
+                MySettings.setModelIsToolCalling(root.currentModelInfo, isToolCallingBox.checked);
+            }
+        }
+
+        RowLayout {
+            Layout.row: 8
+            Layout.column: 0
             Layout.columnSpan: 2
             Layout.topMargin: 15
             spacing: 10
@@ -182,7 +203,7 @@ MySettingsTab {
         Rectangle {
             id: systemPrompt
             visible: !root.currentModelInfo.isOnline
-            Layout.row: 8
+            Layout.row: 9
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.fillWidth: true
@@ -220,7 +241,7 @@ MySettingsTab {
         }
 
         RowLayout {
-            Layout.row: 9
+            Layout.row: 10
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.topMargin: 15
@@ -241,7 +262,7 @@ MySettingsTab {
 
         Rectangle {
             id: promptTemplate
-            Layout.row: 10
+            Layout.row: 11
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.fillWidth: true
@@ -276,18 +297,19 @@ MySettingsTab {
         }
 
         MySettingsLabel {
-            Layout.row: 11
+            Layout.row: 12
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.topMargin: 15
             id: toolTemplateLabel
             text: qsTr("Tool Template")
-            helpText: qsTr("The template that allows tool calls to inject information into the context.")
+            helpText: qsTr("The template that allows tool calls to inject information into the context. Only enabled for tool calling models.")
         }
 
         Rectangle {
             id: toolTemplate
-            Layout.row: 12
+            enabled: root.currentModelInfo.isToolCalling
+            Layout.row: 13
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.fillWidth: true
@@ -325,14 +347,14 @@ MySettingsTab {
             id: chatNamePromptLabel
             text: qsTr("Chat Name Prompt")
             helpText: qsTr("Prompt used to automatically generate chat names.")
-            Layout.row: 13
+            Layout.row: 14
             Layout.column: 0
             Layout.topMargin: 15
         }
 
         Rectangle {
             id: chatNamePrompt
-            Layout.row: 14
+            Layout.row: 15
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.fillWidth: true
@@ -368,14 +390,14 @@ MySettingsTab {
             id: suggestedFollowUpPromptLabel
             text: qsTr("Suggested FollowUp Prompt")
             helpText: qsTr("Prompt used to generate suggested follow-up questions.")
-            Layout.row: 15
+            Layout.row: 16
             Layout.column: 0
             Layout.topMargin: 15
         }
 
         Rectangle {
             id: suggestedFollowUpPrompt
-            Layout.row: 16
+            Layout.row: 17
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.fillWidth: true
@@ -408,7 +430,7 @@ MySettingsTab {
         }
 
         GridLayout {
-            Layout.row: 17
+            Layout.row: 18
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.topMargin: 15
@@ -904,7 +926,7 @@ MySettingsTab {
         }
 
         Rectangle {
-            Layout.row: 18
+            Layout.row: 19
             Layout.column: 0
             Layout.columnSpan: 2
             Layout.topMargin: 15

gpt4all-chat/qml/WebSearchSettings.qml

@@ -125,20 +125,21 @@ MySettingsTab {
             }
         }
 
-        RowLayout {
+// FIXME:
-            MySettingsLabel {
+//        RowLayout {
-                id: askBeforeRunningLabel
+//            MySettingsLabel {
-                text: qsTr("Ask before running")
+//                id: askBeforeRunningLabel
-                helpText: qsTr("The user is queried whether they want the tool to run in every instance")
+//                text: qsTr("Ask before running")
-            }
+//                helpText: qsTr("The user is queried whether they want the tool to run in every instance.")
-            MyCheckBox {
+//            }
-                id: askBeforeRunningBox
+//            MyCheckBox {
-                checked: MySettings.webSearchAskBeforeRunning
+//                id: askBeforeRunningBox
-                onClicked: {
+//                checked: MySettings.webSearchConfirmationMode
-                    MySettings.webSearchAskBeforeRunning = !MySettings.webSearchAskBeforeRunning
+//                onClicked: {
-                }
+//                    MySettings.webSearchConfirmationMode = !MySettings.webSearchAskBeforeRunning
-            }
+//                }
-        }
+//            }
+//        }
 
         Rectangle {
             Layout.topMargin: 15

gpt4all-chat/tool.h

@@ -23,6 +23,13 @@ namespace ToolEnums {
     };
     Q_ENUM_NS(UsageMode)
 
+    enum class ConfirmationMode {
+        NoConfirmation = 0,             // No confirmation required
+        AskBeforeRunning = 1,           // User is queried on every execution
+        AskBeforeRunningRecursive = 2,  // User is queried if the tool is invoked in a recursive tool call
+    };
+    Q_ENUM_NS(ConfirmationMode)
+
     // Ordered in increasing levels of privacy
     enum class PrivacyScope {
         None = 0,               // Tool call data does not have any privacy scope
@@ -43,7 +50,7 @@ class Tool : public QObject {
     Q_PROPERTY(QUrl url READ url CONSTANT)
     Q_PROPERTY(bool isBuiltin READ isBuiltin CONSTANT)
     Q_PROPERTY(ToolEnums::UsageMode usageMode READ usageMode NOTIFY usageModeChanged)
-    Q_PROPERTY(bool askBeforeRunning READ askBeforeRunning NOTIFY askBeforeRunningChanged)
+    Q_PROPERTY(ToolEnums::ConfirmationMode confirmationMode READ confirmationMode NOTIFY confirmationModeChanged)
     Q_PROPERTY(bool excerpts READ excerpts CONSTANT)
 
 public:
@@ -63,7 +70,7 @@ public:
     // [Required] Must be unique. Name of the function to invoke. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64.
     virtual QString function() const = 0;
 
-    // [Required] The privacy scope
+    // [Required] The privacy scope.
     virtual ToolEnums::PrivacyScope privacyScope() const = 0;
 
     // [Optional] Json schema describing the tool's parameters. An empty object specifies no parameters.
@@ -80,14 +87,14 @@ public:
     // [Optional] The local file or remote resource use to invoke the tool.
     virtual QUrl url() const { return QUrl(); }
 
-    // [Optional] Whether the tool is built-in
+    // [Optional] Whether the tool is built-in.
     virtual bool isBuiltin() const { return false; }
 
-    // [Optional] The current usage mode
+    // [Optional] The usage mode.
     virtual ToolEnums::UsageMode usageMode() const { return ToolEnums::UsageMode::Disabled; }
 
-    // [Optional] The user is queried whether they want the tool to run in every instance
+    // [Optional] The confirmation mode.
-    virtual bool askBeforeRunning() const { return false; }
+    virtual ToolEnums::ConfirmationMode confirmationMode() const { return ToolEnums::ConfirmationMode::NoConfirmation; }
 
     // [Optional] Whether json result produces source excerpts.
     virtual bool excerpts() const { return false; }
@@ -103,7 +110,7 @@ public:
 
 Q_SIGNALS:
     void usageModeChanged();
-    void askBeforeRunningChanged();
+    void confirmationModeChanged();
 };
 
 #endif // TOOL_H

gpt4all-chat/toolmodel.h

@@ -25,7 +25,7 @@ public:
         KeyRequiredRole,
         IsBuiltinRole,
         UsageModeRole,
-        AskBeforeRole,
+        ConfirmationModeRole,
         ExcerptsRole,
     };
 
@@ -58,8 +58,8 @@ public:
                 return item->isBuiltin();
             case UsageModeRole:
                 return QVariant::fromValue(item->usageMode());
-            case AskBeforeRole:
+            case ConfirmationModeRole:
-                return item->askBeforeRunning();
+                return QVariant::fromValue(item->confirmationMode());
             case ExcerptsRole:
                 return item->excerpts();
         }
@@ -80,7 +80,7 @@ public:
         roles[KeyRequiredRole] = "keyRequired";
         roles[IsBuiltinRole] = "isBuiltin";
         roles[UsageModeRole] = "usageMode";
-        roles[AskBeforeRole] = "askBeforeRunning";
+        roles[ConfirmationModeRole] = "confirmationMode";
         roles[ExcerptsRole] = "excerpts";
         return roles;
     }