Set version for release. (#3043)

Signed-off-by: Adam Treat <treat.adam@gmail.com>

.gitmodules

@@ -16,4 +16,4 @@
 	url = https://github.com/nomic-ai/DuckX.git
 [submodule "gpt4all-chat/deps/QXlsx"]
 	path = gpt4all-chat/deps/QXlsx
-	url = https://github.com/QtExcel/QXlsx.git
+	url = https://github.com/nomic-ai/QXlsx.git

README.md

@@ -15,7 +15,7 @@
   Read about what's new in <a href="https://www.nomic.ai/blog/tag/gpt4all">our blog</a>.
 </p>
 <p align="center">
-  <a href="https://nomic.ai/gpt4all">Subscribe to the newsletter</a>
+  <a href="https://nomic.ai/gpt4all/#newsletter-form">Subscribe to the newsletter</a>
 </p>
 
 https://github.com/nomic-ai/gpt4all/assets/70534565/513a0f15-4964-4109-89e4-4f9a9011f311

gpt4all-chat/CHANGELOG.md

@@ -4,24 +4,27 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
-## [Unreleased]
+## [3.4.0] - 2024-10-08
 
 ### Added
 - Add bm25 hybrid search to localdocs ([#2969](https://github.com/nomic-ai/gpt4all/pull/2969))
 - LocalDocs support for .docx files ([#2986](https://github.com/nomic-ai/gpt4all/pull/2986))
-- Add support for attaching Excel spreadsheet to chat ([#3007](https://github.com/nomic-ai/gpt4all/pull/3007))
+- Add support for attaching Excel spreadsheet to chat ([#3007](https://github.com/nomic-ai/gpt4all/pull/3007), [#3028](https://github.com/nomic-ai/gpt4all/pull/3028))
 
 ### Changed
 - Rebase llama.cpp on latest upstream as of September 26th ([#2998](https://github.com/nomic-ai/gpt4all/pull/2998))
 - Change the error message when a message is too long ([#3004](https://github.com/nomic-ai/gpt4all/pull/3004))
 - Simplify chatmodel to get rid of unnecessary field and bump chat version ([#3016](https://github.com/nomic-ai/gpt4all/pull/3016))
 - Allow ChatLLM to have direct access to ChatModel for restoring state from text ([#3018](https://github.com/nomic-ai/gpt4all/pull/3018))
+- Improvements to XLSX conversion and UI fix ([#3022](https://github.com/nomic-ai/gpt4all/pull/3022))
 
 ### Fixed
 - Fix a crash when attempting to continue a chat loaded from disk ([#2995](https://github.com/nomic-ai/gpt4all/pull/2995))
 - Fix the local server rejecting min\_p/top\_p less than 1 ([#2996](https://github.com/nomic-ai/gpt4all/pull/2996))
 - Fix "regenerate" always forgetting the most recent message ([#3011](https://github.com/nomic-ai/gpt4all/pull/3011))
 - Fix loaded chats forgetting context when there is a system prompt ([#3015](https://github.com/nomic-ai/gpt4all/pull/3015))
+- Make it possible to downgrade and keep some chats, and avoid crash for some model types ([#3030](https://github.com/nomic-ai/gpt4all/pull/3030))
+- Fix scroll positition being reset in model view, and attempt a better fix for the clone issue ([#3042](https://github.com/nomic-ai/gpt4all/pull/3042))
 
 ## [3.3.1] - 2024-09-27 ([v3.3.y](https://github.com/nomic-ai/gpt4all/tree/v3.3.y))
 
@@ -144,7 +147,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Fix several Vulkan resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
 - Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
 
-[Unreleased]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...HEAD
+[3.4.0]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.4.0
 [3.3.1]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.3.1
 [3.3.0]: https://github.com/nomic-ai/gpt4all/compare/v3.2.1...v3.3.0
 [3.2.1]: https://github.com/nomic-ai/gpt4all/compare/v3.2.0...v3.2.1

gpt4all-chat/CMakeLists.txt

@@ -4,7 +4,7 @@ set(APP_VERSION_MAJOR 3)
 set(APP_VERSION_MINOR 4)
 set(APP_VERSION_PATCH 0)
 set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
-set(APP_VERSION "${APP_VERSION_BASE}-dev0")
+set(APP_VERSION "${APP_VERSION_BASE}")
 
 project(gpt4all VERSION ${APP_VERSION_BASE} LANGUAGES CXX C)
 

gpt4all-chat/qml/ChatView.qml

@@ -939,6 +939,7 @@ Rectangle {
 
                                                     Text {
                                                         id: attachmentFileText
+                                                        width: 295
                                                         height: 40
                                                         text: modelData.file
                                                         color: theme.textColor
@@ -947,6 +948,7 @@ Rectangle {
                                                         font.pixelSize: theme.fontSizeMedium
                                                         font.bold: true
                                                         wrapMode: Text.WrapAnywhere
+                                                        elide: Qt.ElideRight
                                                     }
                                                 }
                                             }
@@ -1971,6 +1973,7 @@ Rectangle {
 
                                     Text {
                                         id: attachmentFileText2
+                                        width: 265
                                         height: 40
                                         text: model.file
                                         color: theme.textColor
@@ -1979,6 +1982,7 @@ Rectangle {
                                         font.pixelSize: theme.fontSizeMedium
                                         font.bold: true
                                         wrapMode: Text.WrapAnywhere
+                                        elide: Qt.ElideRight
                                     }
                                 }
 

gpt4all-chat/qml/HomeView.qml

@@ -281,7 +281,7 @@ Rectangle {
                 Layout.alignment: Qt.AlignCenter
                 text: qsTr("Subscribe to Newsletter")
                 imageSource: "qrc:/gpt4all/icons/email.svg"
-                onClicked: { Qt.openUrlExternally("https://nomic.ai/gpt4all") }
+                onClicked: { Qt.openUrlExternally("https://nomic.ai/gpt4all/#newsletter-form") }
             }
         }
     }

gpt4all-chat/src/chat.cpp

@@ -101,6 +101,7 @@ void Chat::reset()
     // is to allow switching models but throwing up a dialog warning users if we switch between types
     // of models that a long recalculation will ensue.
     m_chatModel->clear();
+    m_needsSave = true;
 }
 
 void Chat::processSystemPrompt()
@@ -163,6 +164,7 @@ void Chat::prompt(const QString &prompt)
 {
     resetResponseState();
     emit promptRequested(m_collections, prompt);
+    m_needsSave = true;
 }
 
 void Chat::regenerateResponse()
@@ -170,6 +172,7 @@ void Chat::regenerateResponse()
     const int index = m_chatModel->count() - 1;
     m_chatModel->updateSources(index, QList<ResultInfo>());
     emit regenerateResponseRequested();
+    m_needsSave = true;
 }
 
 void Chat::stopGenerating()
@@ -224,7 +227,7 @@ void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
 void Chat::promptProcessing()
 {
     m_responseState = !databaseResults().isEmpty() ? Chat::LocalDocsProcessing : Chat::PromptProcessing;
-     emit responseStateChanged();
+    emit responseStateChanged();
 }
 
 void Chat::generatingQuestions()
@@ -261,10 +264,12 @@ ModelInfo Chat::modelInfo() const
 
 void Chat::setModelInfo(const ModelInfo &modelInfo)
 {
-    if (m_modelInfo == modelInfo && isModelLoaded())
+    if (m_modelInfo != modelInfo) {
+        m_modelInfo = modelInfo;
+        m_needsSave = true;
+    } else if (isModelLoaded())
         return;
 
-    m_modelInfo = modelInfo;
     emit modelInfoChanged();
     emit modelChangeRequested(modelInfo);
 }
@@ -343,12 +348,14 @@ void Chat::generatedNameChanged(const QString &name)
     int wordCount = qMin(7, words.size());
     m_name = words.mid(0, wordCount).join(' ');
     emit nameChanged();
+    m_needsSave = true;
 }
 
 void Chat::generatedQuestionFinished(const QString &question)
 {
     m_generatedQuestions << question;
     emit generatedQuestionsChanged();
+    m_needsSave = true;
 }
 
 void Chat::handleRestoringFromText()
@@ -393,6 +400,7 @@ void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
     m_databaseResults = results;
     const int index = m_chatModel->count() - 1;
     m_chatModel->updateSources(index, m_databaseResults);
+    m_needsSave = true;
 }
 
 void Chat::handleModelInfoChanged(const ModelInfo &modelInfo)
@@ -402,6 +410,7 @@ void Chat::handleModelInfoChanged(const ModelInfo &modelInfo)
 
     m_modelInfo = modelInfo;
     emit modelInfoChanged();
+    m_needsSave = true;
 }
 
 void Chat::handleTrySwitchContextOfLoadedModelCompleted(int value)
@@ -416,15 +425,15 @@ bool Chat::serialize(QDataStream &stream, int version) const
     stream << m_id;
     stream << m_name;
     stream << m_userName;
-    if (version > 4)
+    if (version >= 5)
         stream << m_modelInfo.id();
     else
         stream << m_modelInfo.filename();
-    if (version > 2)
+    if (version >= 3)
         stream << m_collections;
 
     const bool serializeKV = MySettings::globalInstance()->saveChatsContext();
-    if (version > 5)
+    if (version >= 6)
         stream << serializeKV;
     if (!m_llmodel->serialize(stream, version, serializeKV))
         return false;
@@ -445,7 +454,7 @@ bool Chat::deserialize(QDataStream &stream, int version)
 
     QString modelId;
     stream >> modelId;
-    if (version > 4) {
+    if (version >= 5) {
         if (ModelList::globalInstance()->contains(modelId))
             m_modelInfo = ModelList::globalInstance()->modelInfo(modelId);
     } else {
@@ -457,13 +466,13 @@ bool Chat::deserialize(QDataStream &stream, int version)
 
     bool discardKV = m_modelInfo.id().isEmpty();
 
-    if (version > 2) {
+    if (version >= 3) {
         stream >> m_collections;
         emit collectionListChanged(m_collections);
     }
 
     bool deserializeKV = true;
-    if (version > 5)
+    if (version >= 6)
         stream >> deserializeKV;
 
     m_llmodel->setModelInfo(m_modelInfo);
@@ -473,7 +482,11 @@ bool Chat::deserialize(QDataStream &stream, int version)
         return false;
 
     emit chatModelChanged();
-    return stream.status() == QDataStream::Ok;
+    if (stream.status() != QDataStream::Ok)
+        return false;
+
+    m_needsSave = false;
+    return true;
 }
 
 QList<QString> Chat::collectionList() const
@@ -493,6 +506,7 @@ void Chat::addCollection(const QString &collection)
 
     m_collections.append(collection);
     emit collectionListChanged(m_collections);
+    m_needsSave = true;
 }
 
 void Chat::removeCollection(const QString &collection)
@@ -502,4 +516,5 @@ void Chat::removeCollection(const QString &collection)
 
     m_collections.removeAll(collection);
     emit collectionListChanged(m_collections);
+    m_needsSave = true;
 }

gpt4all-chat/src/chat.h

@@ -67,6 +67,7 @@ public:
     {
         m_userName = name;
         emit nameChanged();
+        m_needsSave = true;
     }
     ChatModel *chatModel() { return m_chatModel; }
 
@@ -124,6 +125,8 @@ public:
 
     QList<QString> generatedQuestions() const { return m_generatedQuestions; }
 
+    bool needsSave() const { return m_needsSave; }
+
 public Q_SLOTS:
     void serverNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments = {});
 
@@ -203,6 +206,10 @@ private:
     bool m_firstResponse = true;
     int m_trySwitchContextInProgress = 0;
     bool m_isCurrentlyLoading = false;
+    // True if we need to serialize the chat to disk, because of one of two reasons:
+    // - The chat was freshly created during this launch.
+    // - The chat was changed after loading it from disk.
+    bool m_needsSave = true;
 };
 
 #endif // CHAT_H

gpt4all-chat/src/chatlistmodel.cpp

@@ -99,7 +99,12 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
     QElapsedTimer timer;
     timer.start();
     const QString savePath = MySettings::globalInstance()->modelPath();
+    qsizetype nSavedChats = 0;
     for (Chat *chat : chats) {
+        if (!chat->needsSave())
+            continue;
+        ++nSavedChats;
+
         QString fileName = "gpt4all-" + chat->id() + ".chat";
         QString filePath = savePath + "/" + fileName;
         QFile originalFile(filePath);
@@ -129,7 +134,7 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
     }
 
     qint64 elapsedTime = timer.elapsed();
-    qDebug() << "serializing chats took:" << elapsedTime << "ms";
+    qDebug() << "serializing chats took" << elapsedTime << "ms, saved" << nSavedChats << "/" << chats.size() << "chats";
     emit saveChatsFinished();
 }
 
@@ -194,11 +199,16 @@ void ChatsRestoreThread::run()
             qint32 version;
             in >> version;
             if (version < 1) {
-                qWarning() << "ERROR: Chat file has non supported version:" << file.fileName();
+                qWarning() << "WARNING: Chat file version" << version << "is not supported:" << file.fileName();
+                continue;
+            }
+            if (version > CHAT_FORMAT_VERSION) {
+                qWarning().nospace() << "WARNING: Chat file is from a future version (have " << version << " want "
+                                     << CHAT_FORMAT_VERSION << "): " << file.fileName();
                 continue;
             }
 
-            if (version <= 1)
+            if (version < 2)
                 in.setVersion(QDataStream::Qt_6_2);
 
             FileInfo info;
@@ -239,7 +249,7 @@ void ChatsRestoreThread::run()
                 continue;
             }
 
-            if (version <= 1)
+            if (version < 2)
                 in.setVersion(QDataStream::Qt_6_2);
         }
 

gpt4all-chat/src/chatllm.cpp

@@ -42,8 +42,8 @@ using namespace Qt::Literals::StringLiterals;
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
-#define GPTJ_INTERNAL_STATE_VERSION  0 // GPT-J is gone but old chats still use this
-#define LLAMA_INTERNAL_STATE_VERSION 0
+static constexpr int LLAMA_INTERNAL_STATE_VERSION = 0;
+static constexpr int API_INTERNAL_STATE_VERSION   = 0;
 
 class LLModelStore {
 public:
@@ -105,6 +105,7 @@ void LLModelInfo::resetModel(ChatLLM *cllm, LLModel *model) {
 
 ChatLLM::ChatLLM(Chat *parent, bool isServer)
     : QObject{nullptr}
+    , m_chat(parent)
     , m_promptResponseTokens(0)
     , m_promptTokens(0)
     , m_restoringFromText(false)
@@ -355,7 +356,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     requestUrl = modelInfo.url();
                 }
             }
-            m_llModelType = LLModelType::API_;
+            m_llModelType = LLModelTypeV1::API;
             ChatAPI *model = new ChatAPI();
             model->setModelName(modelName);
             model->setRequestURL(requestUrl);
@@ -571,7 +572,7 @@ bool ChatLLM::loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadPro
     }
 
     switch (m_llModelInfo.model->implementation().modelType()[0]) {
-    case 'L': m_llModelType = LLModelType::LLAMA_; break;
+    case 'L': m_llModelType = LLModelTypeV1::LLAMA; break;
     default:
         {
             m_llModelInfo.resetModel(this);
@@ -624,7 +625,7 @@ void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
     // of n_past is of the number of prompt/response pairs, rather than for total tokens.
-    if (m_llModelType == LLModelType::API_)
+    if (m_llModelType == LLModelTypeV1::API)
         m_ctx.n_past -= 1;
     else
         m_ctx.n_past -= m_promptResponseTokens;
@@ -1045,12 +1046,18 @@ bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
 // we want to also serialize n_ctx, and read it at load time.
 bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
 {
-    if (version > 1) {
+    if (version >= 2) {
+        if (m_llModelType == LLModelTypeV1::NONE) {
+            qWarning() << "ChatLLM ERROR: attempted to serialize a null model for chat id" << m_chat->id()
+                       << "name" << m_chat->name();
+            return false;
+        }
+
         stream << m_llModelType;
         switch (m_llModelType) {
-        case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
-        case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        default: Q_UNREACHABLE();
+            case LLModelTypeV1::LLAMA: stream << LLAMA_INTERNAL_STATE_VERSION; break;
+            case LLModelTypeV1::API:   stream << API_INTERNAL_STATE_VERSION;   break;
+            default:                   stream << 0; // models removed in v2.5.0
         }
     }
     stream << response();
@@ -1064,7 +1071,7 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
         return stream.status() == QDataStream::Ok;
     }
 
-    if (version <= 3) {
+    if (version < 4) {
         int responseLogits = 0;
         stream << responseLogits;
     }
@@ -1085,10 +1092,20 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
 
 bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV)
 {
-    if (version > 1) {
-        int internalStateVersion;
-        stream >> m_llModelType;
-        stream >> internalStateVersion; // for future use
+    if (version >= 2) {
+        int llModelType;
+        stream >> llModelType;
+        m_llModelType = (version >= 6 ? parseLLModelTypeV1 : parseLLModelTypeV0)(llModelType);
+        if (m_llModelType == LLModelTypeV1::NONE) {
+            qWarning().nospace() << "error loading chat id " << m_chat->id() << ": unrecognized model type: "
+                                 << llModelType;
+            return false;
+        }
+
+        /* note: prior to chat version 10, API models and chats with models removed in v2.5.0 only wrote this because of
+         * undefined behavior in Release builds */
+        int internalStateVersion; // for future use
+        stream >> internalStateVersion;
     }
     QString response;
     stream >> response;
@@ -1114,7 +1131,7 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
         return stream.status() == QDataStream::Ok;
     }
 
-    if (version <= 3) {
+    if (version < 4) {
         int responseLogits;
         stream >> responseLogits;
     }
@@ -1144,7 +1161,7 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
         stream.skipRawData(tokensSize * sizeof(int));
     }
 
-    if (version > 0) {
+    if (version >= 1) {
         QByteArray compressed;
         stream >> compressed;
         if (!discardKV)
@@ -1169,7 +1186,7 @@ void ChatLLM::saveState()
     if (!isModelLoaded() || m_pristineLoadedState)
         return;
 
-    if (m_llModelType == LLModelType::API_) {
+    if (m_llModelType == LLModelTypeV1::API) {
         m_state.clear();
         QDataStream stream(&m_state, QIODeviceBase::WriteOnly);
         stream.setVersion(QDataStream::Qt_6_4);
@@ -1197,7 +1214,7 @@ void ChatLLM::restoreState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::API_) {
+    if (m_llModelType == LLModelTypeV1::API) {
         QDataStream stream(m_state);
         stream.setVersion(QDataStream::Qt_6_4);
         ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model.get());

gpt4all-chat/src/chatllm.h

@@ -28,12 +28,60 @@ using namespace Qt::Literals::StringLiterals;
 class QDataStream;
 
 // NOTE: values serialized to disk, do not change or reuse
-enum LLModelType {
-    GPTJ_  = 0, // no longer used
-    LLAMA_ = 1,
-    API_   = 2,
-    BERT_  = 3, // no longer used
+enum class LLModelTypeV0 { // chat versions 2-5
+    MPT       = 0,
+    GPTJ      = 1,
+    LLAMA     = 2,
+    CHATGPT   = 3,
+    REPLIT    = 4,
+    FALCON    = 5,
+    BERT      = 6, // not used
+    STARCODER = 7,
 };
+enum class LLModelTypeV1 { // since chat version 6 (v2.5.0)
+    GPTJ      = 0, // not for new chats
+    LLAMA     = 1,
+    API       = 2,
+    BERT      = 3, // not used
+    // none of the below are used in new chats
+    REPLIT    = 4,
+    FALCON    = 5,
+    MPT       = 6,
+    STARCODER = 7,
+    NONE      = -1, // no state
+};
+
+static LLModelTypeV1 parseLLModelTypeV1(int type)
+{
+    switch (LLModelTypeV1(type)) {
+    case LLModelTypeV1::GPTJ:
+    case LLModelTypeV1::LLAMA:
+    case LLModelTypeV1::API:
+    // case LLModelTypeV1::BERT: -- not used
+    case LLModelTypeV1::REPLIT:
+    case LLModelTypeV1::FALCON:
+    case LLModelTypeV1::MPT:
+    case LLModelTypeV1::STARCODER:
+        return LLModelTypeV1(type);
+    default:
+        return LLModelTypeV1::NONE;
+    }
+}
+
+static LLModelTypeV1 parseLLModelTypeV0(int v0)
+{
+    switch (LLModelTypeV0(v0)) {
+    case LLModelTypeV0::MPT:       return LLModelTypeV1::MPT;
+    case LLModelTypeV0::GPTJ:      return LLModelTypeV1::GPTJ;
+    case LLModelTypeV0::LLAMA:     return LLModelTypeV1::LLAMA;
+    case LLModelTypeV0::CHATGPT:   return LLModelTypeV1::API;
+    case LLModelTypeV0::REPLIT:    return LLModelTypeV1::REPLIT;
+    case LLModelTypeV0::FALCON:    return LLModelTypeV1::FALCON;
+    // case LLModelTypeV0::BERT: -- not used
+    case LLModelTypeV0::STARCODER: return LLModelTypeV1::STARCODER;
+    default:                       return LLModelTypeV1::NONE;
+    }
+}
 
 class ChatLLM;
 class ChatModel;
@@ -219,12 +267,13 @@ protected:
 private:
     bool loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadProps);
 
+    const Chat *m_chat;
     std::string m_response;
     std::string m_trimmedResponse;
     std::string m_nameResponse;
     QString m_questionResponse;
     LLModelInfo m_llModelInfo;
-    LLModelType m_llModelType;
+    LLModelTypeV1 m_llModelType = LLModelTypeV1::NONE;
     ModelInfo m_modelInfo;
     TokenTimer *m_timer;
     QByteArray m_state;

gpt4all-chat/src/chatmodel.h

@@ -45,7 +45,7 @@ public:
         buffer.open(QIODevice::ReadOnly);
         const QString md = XLSXToMD::toMarkdown(&buffer);
         buffer.close();
-        return md;
+        return u"## Attached: %1\n\n%2"_s.arg(file(), md);
     }
 
     bool operator==(const PromptAttachment &other) const { return url == other.url; }
@@ -386,7 +386,7 @@ public:
             stream << c.stopped;
             stream << c.thumbsUpState;
             stream << c.thumbsDownState;
-            if (version > 7) {
+            if (version >= 8) {
                 stream << c.sources.size();
                 for (const ResultInfo &info : c.sources) {
                     Q_ASSERT(!info.file.isEmpty());
@@ -401,7 +401,7 @@ public:
                     stream << info.from;
                     stream << info.to;
                 }
-            } else if (version > 2) {
+            } else if (version >= 3) {
                 QList<QString> references;
                 QList<QString> referencesContext;
                 int validReferenceNumber = 1;
@@ -468,7 +468,7 @@ public:
             stream >> c.stopped;
             stream >> c.thumbsUpState;
             stream >> c.thumbsDownState;
-            if (version > 7) {
+            if (version >= 8) {
                 qsizetype count;
                 stream >> count;
                 QList<ResultInfo> sources;
@@ -488,7 +488,7 @@ public:
                 }
                 c.sources = sources;
                 c.consolidatedSources = consolidateSources(sources);
-            } else if (version > 2) {
+            } else if (version >= 3) {
                 QString references;
                 QList<QString> referencesContext;
                 stream >> references;

gpt4all-chat/src/localdocsmodel.cpp

@@ -20,7 +20,6 @@ LocalDocsCollectionsModel::LocalDocsCollectionsModel(QObject *parent)
     connect(this, &LocalDocsCollectionsModel::rowsInserted, this, &LocalDocsCollectionsModel::countChanged);
     connect(this, &LocalDocsCollectionsModel::rowsRemoved, this, &LocalDocsCollectionsModel::countChanged);
     connect(this, &LocalDocsCollectionsModel::modelReset, this, &LocalDocsCollectionsModel::countChanged);
-    connect(this, &LocalDocsCollectionsModel::layoutChanged, this, &LocalDocsCollectionsModel::countChanged);
 }
 
 bool LocalDocsCollectionsModel::filterAcceptsRow(int sourceRow,
@@ -67,7 +66,6 @@ LocalDocsModel::LocalDocsModel(QObject *parent)
     connect(this, &LocalDocsModel::rowsInserted, this, &LocalDocsModel::countChanged);
     connect(this, &LocalDocsModel::rowsRemoved, this, &LocalDocsModel::countChanged);
     connect(this, &LocalDocsModel::modelReset, this, &LocalDocsModel::countChanged);
-    connect(this, &LocalDocsModel::layoutChanged, this, &LocalDocsModel::countChanged);
 }
 
 int LocalDocsModel::rowCount(const QModelIndex &parent) const

gpt4all-chat/src/modellist.cpp

@@ -398,7 +398,6 @@ InstalledModels::InstalledModels(QObject *parent, bool selectable)
     connect(this, &InstalledModels::rowsInserted, this, &InstalledModels::countChanged);
     connect(this, &InstalledModels::rowsRemoved, this, &InstalledModels::countChanged);
     connect(this, &InstalledModels::modelReset, this, &InstalledModels::countChanged);
-    connect(this, &InstalledModels::layoutChanged, this, &InstalledModels::countChanged);
 }
 
 bool InstalledModels::filterAcceptsRow(int sourceRow,
@@ -423,7 +422,6 @@ DownloadableModels::DownloadableModels(QObject *parent)
     connect(this, &DownloadableModels::rowsInserted, this, &DownloadableModels::countChanged);
     connect(this, &DownloadableModels::rowsRemoved, this, &DownloadableModels::countChanged);
     connect(this, &DownloadableModels::modelReset, this, &DownloadableModels::countChanged);
-    connect(this, &DownloadableModels::layoutChanged, this, &DownloadableModels::countChanged);
 }
 
 bool DownloadableModels::filterAcceptsRow(int sourceRow,
@@ -821,7 +819,11 @@ QVariant ModelList::data(const QModelIndex &index, int role) const
 
 void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>> &data)
 {
+    // We only sort when one of the fields used by the sorting algorithm actually changes that
+    // is implicated or used by the sorting algorithm
+    bool shouldSort = false;
     int index;
+
     {
         QMutexLocker locker(&m_mutex);
         if (!m_modelMap.contains(id)) {
@@ -836,10 +838,6 @@ void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>
             return;
         }
 
-        // We only sort when one of the fields used by the sorting algorithm actually changes that
-        // is implicated or used by the sorting algorithm
-        bool shouldSort = false;
-
         for (const auto &d : data) {
             const int role = d.first;
             const QVariant value = d.second;
@@ -1000,21 +998,12 @@ void ModelList::updateData(const QString &id, const QVector<QPair<int, QVariant>
             info->isEmbeddingModel = LLModel::Implementation::isEmbeddingModel(modelPath.toStdString());
             info->checkedEmbeddingModel = true;
         }
-
-        if (shouldSort) {
-            auto s = m_discoverSort;
-            auto d = m_discoverSortDirection;
-            std::stable_sort(m_models.begin(), m_models.end(), [s, d](const ModelInfo* lhs, const ModelInfo* rhs) {
-                return ModelList::lessThan(lhs, rhs, s, d);
-            });
-        }
     }
+
     emit dataChanged(createIndex(index, 0), createIndex(index, 0));
 
-    // FIXME(jared): for some reason these don't update correctly when the source model changes, so we explicitly invalidate them
-    m_selectableModels->invalidate();
-    m_installedModels->invalidate();
-    m_downloadableModels->invalidate();
+    if (shouldSort)
+        resortModel();
 
     emit selectableModelListChanged();
 }
@@ -1122,7 +1111,6 @@ void ModelList::removeClone(const ModelInfo &model)
         return;
 
     removeInternal(model);
-    emit layoutChanged();
 }
 
 void ModelList::removeInstalled(const ModelInfo &model)
@@ -1131,7 +1119,6 @@ void ModelList::removeInstalled(const ModelInfo &model)
     Q_ASSERT(!model.isClone());
     Q_ASSERT(model.isDiscovered() || model.isCompatibleApi || model.description() == "" /*indicates sideloaded*/);
     removeInternal(model);
-    emit layoutChanged();
 }
 
 void ModelList::removeInternal(const ModelInfo &model)
@@ -1928,7 +1915,6 @@ void ModelList::clearDiscoveredModels()
     }
     for (ModelInfo &info : infos)
         removeInternal(info);
-    emit layoutChanged();
 }
 
 float ModelList::discoverProgress() const
@@ -2176,7 +2162,6 @@ void ModelList::handleDiscoveryItemFinished()
     emit discoverProgressChanged();
 
     if (discoverProgress() >= 1.0) {
-        emit layoutChanged();
         m_discoverInProgress = false;
         emit discoverInProgressChanged();;
     }

gpt4all-chat/src/xlsxtomd.cpp

@@ -10,8 +10,10 @@
 #include <QDateTime>
 #include <QDebug>
 #include <QList>
+#include <QRegularExpression>
 #include <QString>
 #include <QStringList>
+#include <QStringView>
 #include <QVariant>
 #include <QtGlobal>
 #include <QtLogging>
@@ -30,10 +32,10 @@ static QString formatCellText(const QXlsx::Cell *cell)
     QString cellText;
 
     // Determine the cell type based on format
-    if (format.isDateTimeFormat()) {
+    if (cell->isDateTime()) {
         // Handle DateTime
-        QDateTime dateTime = value.toDateTime();
-        cellText = dateTime.isValid() ? dateTime.toString("yyyy-MM-dd") : value.toString();
+        QDateTime dateTime = cell->dateTime().toDateTime();
+        cellText = dateTime.isValid() ? dateTime.toString(QStringView(u"yyyy-MM-dd")) : value.toString();
     } else {
         cellText = value.toString();
     }
@@ -41,23 +43,32 @@ static QString formatCellText(const QXlsx::Cell *cell)
     if (cellText.isEmpty())
         return QString();
 
-    // Apply Markdown and HTML formatting based on font styles
-    QString formattedText = cellText;
-
-    if (format.fontBold() && format.fontItalic())
-        formattedText = "***" + formattedText + "***";
-    else if (format.fontBold())
-        formattedText = "**" + formattedText + "**";
-    else if (format.fontItalic())
-        formattedText = "*" + formattedText + "*";
+    // Escape special characters
+    static QRegularExpression special(
+        QStringLiteral(
+            R"(()([\\`*_[\]<>()!|])|)"    // special characters
+            R"(^(\s*)(#+(?:\s|$))|)"      // headings
+            R"(^(\s*[0-9])(\.(?:\s|$))|)" // ordered lists ("1. a")
+            R"(^(\s*)([+-](?:\s|$)))"     // unordered lists ("- a")
+        ),
+        QRegularExpression::MultilineOption
+    );
+    cellText.replace(special, uR"(\1\\2)"_s);
+    cellText.replace(u'&', "&amp;"_L1);
+    cellText.replace(u'<', "&lt;"_L1);
+    cellText.replace(u'>', "&gt;"_L1);
 
+    // Apply Markdown formatting based on font styles
+    if (format.fontUnderline())
+        cellText = u"_%1_"_s.arg(cellText);
+    if (format.fontBold())
+        cellText = u"**%1**"_s.arg(cellText);
+    if (format.fontItalic())
+        cellText = u"*%1*"_s.arg(cellText);
     if (format.fontStrikeOut())
-        formattedText = "~~" + formattedText + "~~";
+        cellText = u"~~%1~~"_s.arg(cellText);
 
-    // Escape pipe characters to prevent Markdown table issues
-    formattedText.replace("|", "\\|");
-
-    return formattedText;
+    return cellText;
 }
 
 static QString getCellValue(QXlsx::Worksheet *sheet, int row, int col)
@@ -113,7 +124,7 @@ QString XLSXToMD::toMarkdown(QIODevice *xlsxDevice)
             continue;
         }
 
-        markdown += u"## %1\n\n"_s.arg(sheetName);
+        markdown += u"### %1\n\n"_s.arg(sheetName);
 
         // Determine the used range
         QXlsx::CellRange range = sheet->dimension();
@@ -124,44 +135,35 @@ QString XLSXToMD::toMarkdown(QIODevice *xlsxDevice)
 
         if (firstRow > lastRow || firstCol > lastCol) {
             qWarning() << "Sheet" << sheetName << "is empty.";
-            markdown += "*No data available.*\n\n";
+            markdown += QStringView(u"*No data available.*\n\n");
             continue;
         }
 
-        // Assume the first row is the header
-        int headerRow = firstRow;
+        auto appendRow = [&markdown](auto &list) { markdown += u"|%1|\n"_s.arg(list.join(u'|')); };
 
-        // Collect headers
+        // Empty header
+        static QString header(u' ');
+        static QString separator(u'-');
         QStringList headers;
-        for (int col = firstCol; col <= lastCol; ++col) {
-            QString header = getCellValue(sheet, headerRow, col);
-            headers << header;
-        }
-
-        // Create Markdown header row
-        QString headerRowMarkdown = "|" + headers.join("|") + "|";
-        markdown += headerRowMarkdown + "\n";
-
-        // Create Markdown separator row
         QStringList separators;
-        for (int i = 0; i < headers.size(); ++i)
-            separators << "---";
-        QString separatorRow = "|" + separators.join("|") + "|";
-        markdown += separatorRow + "\n";
+        for (int col = firstCol; col <= lastCol; ++col) {
+            headers << header;
+            separators << separator;
+        }
+        appendRow(headers);
+        appendRow(separators);
 
-        // Iterate through data rows (starting from the row after header)
-        for (int row = headerRow + 1; row <= lastRow; ++row) {
+        // Iterate through data rows
+        for (int row = firstRow; row <= lastRow; ++row) {
             QStringList rowData;
             for (int col = firstCol; col <= lastCol; ++col) {
                 QString cellText = getCellValue(sheet, row, col);
-                rowData << cellText;
+                rowData << (cellText.isEmpty() ? u" "_s : cellText);
             }
-
-            QString dataRow = "|" + rowData.join("|") + "|";
-            markdown += dataRow + "\n";
+            appendRow(rowData);
         }
 
-        markdown += "\n"; // Add an empty line between sheets
+        markdown += u'\n'; // Add an empty line between sheets
     }
     return markdown;
 }