Support attaching an Excel spreadsheet to a chat message (#3007)

Signed-off-by: Adam Treat <treat.adam@gmail.com>
Signed-off-by: Jared Van Bortel <jared@nomic.ai>
Co-authored-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -321,7 +321,7 @@ jobs:
               libfreetype6 libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev
               libx11-6 libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
               libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
-              libxcb-xkb1 libxcb1 libxext6 libxfixes3 libxi6 libxkbcommon-x11-0 libxkbcommon0 libxrender1 patchelf
+              libxcb-xkb1 libxcb1 libxext6 libxfixes3 libxi6 libxkbcommon-dev libxkbcommon-x11-0 libxrender1 patchelf
               python3 vulkan-sdk
             )
             sudo apt-get update
@@ -397,7 +397,7 @@ jobs:
               libfreetype6 libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev
               libx11-6 libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
               libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
-              libxcb-xkb1 libxcb1 libxext6 libxfixes3 libxi6 libxkbcommon-x11-0 libxkbcommon0 libxrender1 patchelf
+              libxcb-xkb1 libxcb1 libxext6 libxfixes3 libxi6 libxkbcommon-dev libxkbcommon-x11-0 libxrender1 patchelf
               python3 vulkan-sdk
             )
             sudo apt-get update
@@ -728,7 +728,7 @@ jobs:
               libfreetype6 libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev
               libx11-6 libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
               libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
-              libxcb-xkb1 libxcb1 libxext6 libxfixes3 libxi6 libxkbcommon-x11-0 libxkbcommon0 libxrender1 python3
+              libxcb-xkb1 libxcb1 libxext6 libxfixes3 libxi6 libxkbcommon-dev libxkbcommon-x11-0 libxrender1 python3
               vulkan-sdk
             )
             sudo apt-get update

.gitmodules

@@ -14,3 +14,6 @@
 [submodule "gpt4all-chat/deps/DuckX"]
 	path = gpt4all-chat/deps/DuckX
 	url = https://github.com/nomic-ai/DuckX.git
+[submodule "gpt4all-chat/deps/QXlsx"]
+	path = gpt4all-chat/deps/QXlsx
+	url = https://github.com/QtExcel/QXlsx.git

gpt4all-chat/CMakeLists.txt

@@ -154,6 +154,7 @@ qt_add_executable(chat
     src/mysettings.cpp            src/mysettings.h
     src/network.cpp               src/network.h
     src/server.cpp                src/server.h
+    src/xlsxtomd.cpp              src/xlsxtomd.h
     ${CHAT_EXE_RESOURCES}
 )
 
@@ -190,6 +191,8 @@ qt_add_qml_module(chat
       qml/MyComboBox.qml
       qml/MyDialog.qml
       qml/MyDirectoryField.qml
+      qml/MyFileDialog.qml
+      qml/MyFolderDialog.qml
       qml/MyFancyLink.qml
       qml/MyMenu.qml
       qml/MyMenuItem.qml
@@ -222,9 +225,11 @@ qt_add_qml_module(chat
       icons/edit.svg
       icons/eject.svg
       icons/email.svg
+      icons/file-doc.svg
       icons/file-md.svg
       icons/file-pdf.svg
       icons/file-txt.svg
+      icons/file-xls.svg
       icons/file.svg
       icons/github.svg
       icons/globe.svg
@@ -242,7 +247,9 @@ qt_add_qml_module(chat
       icons/network.svg
       icons/nomic_logo.svg
       icons/notes.svg
+      icons/paperclip.svg
       icons/plus.svg
+      icons/plus_circle.svg
       icons/recycle.svg
       icons/regenerate.svg
       icons/search.svg
@@ -255,6 +262,7 @@ qt_add_qml_module(chat
       icons/trash.svg
       icons/twitter.svg
       icons/up_down.svg
+      icons/webpage.svg
       icons/you.svg
 )
 
@@ -327,7 +335,7 @@ target_include_directories(chat PRIVATE deps/usearch/include
 target_link_libraries(chat
     PRIVATE Qt6::Core Qt6::HttpServer Qt6::Pdf Qt6::Quick Qt6::Sql Qt6::Svg)
 target_link_libraries(chat
-    PRIVATE llmodel SingleApplication fmt::fmt duckx::duckx)
+    PRIVATE llmodel SingleApplication fmt::fmt duckx::duckx QXlsx)
 
 
 # -- install --

gpt4all-chat/build_and_run.md

@@ -21,12 +21,12 @@ sudo pacman -S --needed cmake gcc ninja qt6-5compat qt6-base qt6-declarative qt6
 
 On Ubuntu 23.04, this looks like:
 ```
-sudo apt install cmake g++ libgl-dev libqt6core5compat6 ninja-build qml6-module-qt5compat-graphicaleffects qt6-base-dev qt6-declarative-dev qt6-httpserver-dev qt6-svg-dev qtcreator
+sudo apt install cmake g++ libgl-dev libqt6core5compat6 ninja-build qml6-module-qt5compat-graphicaleffects qt6-base-private-dev qt6-declarative-dev qt6-httpserver-dev qt6-svg-dev qtcreator
 ```
 
 On Fedora 39, this looks like:
 ```
-sudo dnf install cmake gcc-c++ ninja-build qt-creator qt5-qtgraphicaleffects qt6-qt5compat qt6-qtbase-devel qt6-qtdeclarative-devel qt6-qthttpserver-devel qt6-qtsvg-devel
+sudo dnf install cmake gcc-c++ ninja-build qt-creator qt5-qtgraphicaleffects qt6-qt5compat qt6-qtbase-private-devel qt6-qtdeclarative-devel qt6-qthttpserver-devel qt6-qtsvg-devel
 ```
 
 ## Download Qt

gpt4all-chat/deps/CMakeLists.txt

@@ -8,3 +8,6 @@ add_subdirectory(SingleApplication)
 
 set(DUCKX_INSTALL OFF)
 add_subdirectory(DuckX)
+
+set(QT_VERSION_MAJOR 6)
+add_subdirectory(QXlsx/QXlsx)

gpt4all-chat/deps/QXlsx

@@ -0,0 +1 @@
+Subproject commit fda6b806e2ceebd81c01cdded07ae84c94f5879c

gpt4all-chat/icons/file-doc.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256"><rect width="256" height="256" fill="none"/><path d="M36,152v56H52a28,28,0,0,0,0-56Z" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M216,200.87A22.12,22.12,0,0,1,200,208c-13.26,0-24-12.54-24-28s10.74-28,24-28a22.12,22.12,0,0,1,16,7.13" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M48,112V40a8,8,0,0,1,8-8h96l56,56v24" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><polyline points="152 32 152 88 208 88" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><ellipse cx="128" cy="180" rx="24" ry="28" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/></svg>

gpt4all-chat/icons/file-xls.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256"><rect width="256" height="256" fill="none"/><polyline points="148 208 120 208 120 152" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M48,112V40a8,8,0,0,1,8-8h96l56,56v24" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><polyline points="152 32 152 88 208 88" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><line x1="48" y1="152" x2="88" y2="208" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><line x1="88" y1="152" x2="48" y2="208" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M203.9,153.6s-29.43-7.78-31.8,11,38.43,10.12,35.78,30.72c-2.47,19.16-31.78,11-31.78,11" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/></svg>

gpt4all-chat/icons/paperclip.svg

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   viewBox="0 0 256 256"
+   version="1.1"
+   id="svg6"
+   sodipodi:docname="paperclip-horizontal.svg"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs10" />
+  <sodipodi:namedview
+     id="namedview8"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="4.421875"
+     inkscape:cx="127.88693"
+     inkscape:cy="127.88693"
+     inkscape:window-width="2560"
+     inkscape:window-height="1495"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg6" />
+  <rect
+     width="256"
+     height="256"
+     fill="none"
+     id="rect2" />
+  <path
+     d="m 144,80 v 112 a -16,16 0 0 1 -32,0 V 48 a -32,32 0 0 1 64,0 v 144 a -48,48 0 0 1 -96,0 V 80"
+     fill="none"
+     stroke="currentColor"
+     stroke-linecap="round"
+     stroke-linejoin="round"
+     stroke-width="16"
+     id="path4" />
+</svg>

gpt4all-chat/icons/plus_circle.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="#000000" viewBox="0 0 256 256"><path d="M128,24A104,104,0,1,0,232,128,104.11,104.11,0,0,0,128,24Zm0,192a88,88,0,1,1,88-88A88.1,88.1,0,0,1,128,216Zm48-88a8,8,0,0,1-8,8H136v32a8,8,0,0,1-16,0V136H88a8,8,0,0,1,0-16h32V88a8,8,0,0,1,16,0v32h32A8,8,0,0,1,176,128Z"></path></svg>

gpt4all-chat/icons/webpage.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="#000000" viewBox="0 0 256 256"><path d="M216,40H40A16,16,0,0,0,24,56V200a16,16,0,0,0,16,16H216a16,16,0,0,0,16-16V56A16,16,0,0,0,216,40Zm0,16V88H40V56Zm0,144H40V104H216v96Z"></path></svg>

gpt4all-chat/qml/AddCollectionView.qml

@@ -89,15 +89,8 @@ Rectangle {
             property alias collection: collection.text
             property alias folder_path: folderEdit.text
 
-            FolderDialog {
+            MyFolderDialog {
                 id: folderDialog
-                title: qsTr("Please choose a directory")
-            }
-
-            function openFolderDialog(currentFolder, onAccepted) {
-                folderDialog.currentFolder = currentFolder;
-                folderDialog.accepted.connect(function() { onAccepted(folderDialog.selectedFolder); });
-                folderDialog.open();
             }
 
             Label {
@@ -170,7 +163,7 @@ Rectangle {
                     id: browseButton
                     text: qsTr("Browse")
                     onClicked: {
-                        root.openFolderDialog(StandardPaths.writableLocation(StandardPaths.HomeLocation), function(selectedFolder) {
+                        folderDialog.openFolderDialog(StandardPaths.writableLocation(StandardPaths.HomeLocation), function(selectedFolder) {
                             root.folder_path = selectedFolder
                         })
                     }

gpt4all-chat/qml/ApplicationSettings.qml

@@ -394,11 +394,14 @@ MySettingsTab {
                     }
                 }
             }
+            MyFolderDialog {
+                id: folderDialog
+            }
             MySettingsButton {
                 text: qsTr("Browse")
                 Accessible.description: qsTr("Choose where to save model files")
                 onClicked: {
-                    openFolderDialog("file://" + MySettings.modelPath, function(selectedFolder) {
+                    folderDialog.openFolderDialog("file://" + MySettings.modelPath, function(selectedFolder) {
                         MySettings.modelPath = selectedFolder
                     })
                 }

gpt4all-chat/qml/ChatView.qml

@@ -3,6 +3,7 @@ import QtCore
 import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
+import QtQuick.Dialogs
 import QtQuick.Layouts
 
 import chatlistmodel
@@ -893,6 +894,65 @@ Rectangle {
                                     Layout.row: 1
                                     Layout.column: 1
                                     Layout.fillWidth: true
+                                    spacing: 20
+                                    Flow {
+                                        id: attachedUrlsFlow
+                                        Layout.fillWidth: true
+                                        spacing: 10
+                                        visible: promptAttachments.length !== 0
+                                        Repeater {
+                                            model: promptAttachments
+
+                                            delegate: Rectangle {
+                                                width: 350
+                                                height: 50
+                                                radius: 5
+                                                color: theme.attachmentBackground
+                                                border.color: theme.controlBorder
+
+                                                Row {
+                                                    spacing: 5
+                                                    anchors.fill: parent
+                                                    anchors.margins: 5
+
+                                                    Item {
+                                                        id: attachmentFileIcon
+                                                        width: 40
+                                                        height: 40
+                                                        Image {
+                                                            id: fileIcon
+                                                            anchors.fill: parent
+                                                            visible: false
+                                                            sourceSize.width: 40
+                                                            sourceSize.height: 40
+                                                            mipmap: true
+                                                            source: {
+                                                                return "qrc:/gpt4all/icons/file-xls.svg"
+                                                            }
+                                                        }
+                                                        ColorOverlay {
+                                                            anchors.fill: fileIcon
+                                                            source: fileIcon
+                                                            color: theme.textColor
+                                                        }
+                                                    }
+
+                                                    Text {
+                                                        id: attachmentFileText
+                                                        height: 40
+                                                        text: modelData.file
+                                                        color: theme.textColor
+                                                        horizontalAlignment: Text.AlignHLeft
+                                                        verticalAlignment: Text.AlignVCenter
+                                                        font.pixelSize: theme.fontSizeMedium
+                                                        font.bold: true
+                                                        wrapMode: Text.WrapAnywhere
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+
                                     TextArea {
                                         id: myTextArea
                                         Layout.fillWidth: true
@@ -1434,17 +1494,7 @@ Rectangle {
                                                     var chat = window.currentChat
                                                     var followup = modelData
                                                     chat.stopGenerating()
-                                                    chat.newPromptResponsePair(followup);
+                                                    chat.newPromptResponsePair(followup)
-                                                    chat.prompt(followup,
-                                                                MySettings.promptTemplate,
-                                                                MySettings.maxLength,
-                                                                MySettings.topK,
-                                                                MySettings.topP,
-                                                                MySettings.minP,
-                                                                MySettings.temperature,
-                                                                MySettings.promptBatchSize,
-                                                                MySettings.repeatPenalty,
-                                                                MySettings.repeatPenaltyTokens)
                                                 }
                                             }
                                             Item {
@@ -1708,7 +1758,7 @@ Rectangle {
                             chatModel.updateThumbsUpState(responseIndex, false)
                             chatModel.updateThumbsDownState(responseIndex, false)
                             chatModel.updateNewResponse(responseIndex, "")
-                            currentChat.prompt(promptElement.value)
+                            currentChat.prompt(promptElement.promptPlusAttachments)
                         }
                         ToolTip.visible: regenerateButton.hovered
                         ToolTip.text: qsTr("Redo last chat response")
@@ -1827,23 +1877,163 @@ Rectangle {
                 opacity: 0.1
             }
 
-            ScrollView {
+            ListModel {
+                id: attachmentModel
+
+                function getAttachmentUrls() {
+                    var urls = [];
+                    for (var i = 0; i < attachmentModel.count; i++) {
+                        var item = attachmentModel.get(i);
+                        urls.push(item.url);
+                    }
+                    return urls;
+                }
+            }
+
+            Rectangle {
                 id: textInputView
+                color: theme.controlBackground
+                border.width: 1
+                border.color: theme.controlBorder
+                radius: 10
                 anchors.left: parent.left
                 anchors.right: parent.right
                 anchors.bottom: parent.bottom
                 anchors.margins: 30
                 anchors.leftMargin: Math.max((parent.width - 1310) / 2, 30)
                 anchors.rightMargin: Math.max((parent.width - 1310) / 2, 30)
-                height: Math.min(contentHeight, 200)
+                height: textInputViewLayout.implicitHeight
                 visible: !currentChat.isServer && ModelList.selectableModels.count !== 0
+
+                MouseArea {
+                    id: textInputViewMouseArea
+                    anchors.fill: parent
+                    onClicked: (mouse) => {
+                        if (textInput.enabled)
+                            textInput.forceActiveFocus();
+                    }
+                }
+
+                GridLayout {
+                    id: textInputViewLayout
+                    anchors.left: parent.left
+                    anchors.right: parent.right
+                    rows: 2
+                    columns: 3
+                    rowSpacing: 10
+                    columnSpacing: 0
+                    Flow {
+                        id: attachmentsFlow
+                        visible: attachmentModel.count
+                        Layout.row: 0
+                        Layout.column: 1
+                        Layout.topMargin: 15
+                        Layout.leftMargin: 5
+                        Layout.rightMargin: 15
+                        spacing: 10
+
+                        Repeater {
+                            model: attachmentModel
+
+                            Rectangle {
+                                width: 350
+                                height: 50
+                                radius: 5
+                                color: theme.attachmentBackground
+                                border.color: theme.controlBorder
+
+                                Row {
+                                    spacing: 5
+                                    anchors.fill: parent
+                                    anchors.margins: 5
+
+                                    Item {
+                                        id: attachmentFileIcon2
+                                        width: 40
+                                        height: 40
+                                        Image {
+                                            id: fileIcon2
+                                            anchors.fill: parent
+                                            visible: false
+                                            sourceSize.width: 40
+                                            sourceSize.height: 40
+                                            mipmap: true
+                                            source: {
+                                                return "qrc:/gpt4all/icons/file-xls.svg"
+                                            }
+                                        }
+                                        ColorOverlay {
+                                            anchors.fill: fileIcon2
+                                            source: fileIcon2
+                                            color: theme.textColor
+                                        }
+                                    }
+
+                                    Text {
+                                        id: attachmentFileText2
+                                        height: 40
+                                        text: model.file
+                                        color: theme.textColor
+                                        horizontalAlignment: Text.AlignHLeft
+                                        verticalAlignment: Text.AlignVCenter
+                                        font.pixelSize: theme.fontSizeMedium
+                                        font.bold: true
+                                        wrapMode: Text.WrapAnywhere
+                                    }
+                                }
+
+                                MyMiniButton {
+                                    id: removeAttachmentButton
+                                    anchors.top: parent.top
+                                    anchors.right: parent.right
+                                    backgroundColor: theme.textColor
+                                    backgroundColorHovered: theme.iconBackgroundDark
+                                    source: "qrc:/gpt4all/icons/close.svg"
+                                    onClicked: {
+                                        attachmentModel.remove(index)
+                                        if (textInput.enabled)
+                                            textInput.forceActiveFocus();
+                                    }
+                                }
+                            }
+                        }
+                    }
+
+                    MyToolButton {
+                        id: plusButton
+                        Layout.row: 1
+                        Layout.column: 0
+                        Layout.leftMargin: 15
+                        Layout.rightMargin: 15
+                        Layout.alignment: Qt.AlignCenter
+                        backgroundColor: theme.conversationInputButtonBackground
+                        backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
+                        imageWidth: theme.fontSizeLargest
+                        imageHeight: theme.fontSizeLargest
+                        visible: !currentChat.isServer && ModelList.selectableModels.count !== 0 && currentChat.isModelLoaded
+                        enabled: !currentChat.responseInProgress
+                        source: "qrc:/gpt4all/icons/paperclip.svg"
+                        Accessible.name: qsTr("Add media")
+                        Accessible.description: qsTr("Adds media to the prompt")
+
+                        onClicked: (mouse) => {
+                                       addMediaMenu.open()
+                                   }
+                    }
+
+                    ScrollView {
+                        id: textInputScrollView
+                        Layout.row: 1
+                        Layout.column: 1
+                        Layout.fillWidth: true
+                        Layout.leftMargin: plusButton.visible ? 5 : 15
+                        Layout.margins: 15
+                        height: Math.min(contentHeight, 200)
+
                         MyTextArea {
                             id: textInput
                             color: theme.textColor
-                    topPadding: 15
+                            padding: 0
-                    bottomPadding: 15
-                    leftPadding: 20
-                    rightPadding: 40
                             enabled: currentChat.isModelLoaded && !currentChat.isServer
                             onEnabledChanged: {
                                 if (textInput.enabled)
@@ -1863,21 +2053,12 @@ Rectangle {
                                                       }
                                                   }
                             function sendMessage() {
-                        if (textInput.text === "" || currentChat.responseInProgress || currentChat.restoringFromText)
+                                if ((textInput.text === "" && attachmentModel.count === 0) || currentChat.responseInProgress || currentChat.restoringFromText)
                                     return
 
                                 currentChat.stopGenerating()
-                        currentChat.newPromptResponsePair(textInput.text);
+                                currentChat.newPromptResponsePair(textInput.text, attachmentModel.getAttachmentUrls())
-                        currentChat.prompt(textInput.text,
+                                attachmentModel.clear();
-                                           MySettings.promptTemplate,
-                                           MySettings.maxLength,
-                                           MySettings.topK,
-                                           MySettings.topP,
-                                           MySettings.minP,
-                                           MySettings.temperature,
-                                           MySettings.promptBatchSize,
-                                           MySettings.repeatPenalty,
-                                           MySettings.repeatPenaltyTokens)
                                 textInput.text = ""
                             }
 
@@ -1895,6 +2076,11 @@ Rectangle {
                                            }
                             }
 
+                            background: Rectangle {
+                                implicitWidth: 150
+                                color: "transparent"
+                            }
+
                             MyMenu {
                                 id: textInputContextMenu
                                 MyMenuItem {
@@ -1921,14 +2107,16 @@ Rectangle {
                         }
                     }
 
+                    Row {
+                        Layout.row: 1
+                        Layout.column: 2
+                        Layout.rightMargin: 15
+                        Layout.alignment: Qt.AlignCenter
 
                         MyToolButton {
                             id: stopButton
                             backgroundColor: theme.conversationInputButtonBackground
                             backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
-                anchors.right: textInputView.right
-                anchors.verticalCenter: textInputView.verticalCenter
-                anchors.rightMargin: 15
                             visible: currentChat.responseInProgress && !currentChat.isServer
 
                             background: Item {
@@ -1977,9 +2165,6 @@ Rectangle {
                             id: sendButton
                             backgroundColor: theme.conversationInputButtonBackground
                             backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
-                anchors.right: textInputView.right
-                anchors.verticalCenter: textInputView.verticalCenter
-                anchors.rightMargin: 15
                             imageWidth: theme.fontSizeLargest
                             imageHeight: theme.fontSizeLargest
                             visible: !currentChat.responseInProgress && !currentChat.isServer && ModelList.selectableModels.count !== 0
@@ -1995,4 +2180,38 @@ Rectangle {
                         }
                     }
                 }
+            }
+
+            MyFileDialog {
+                id: fileDialog
+                nameFilters: ["Excel files (*.xlsx)"]
+            }
+
+            MyMenu {
+                id: addMediaMenu
+                x: textInputView.x
+                y: textInputView.y - addMediaMenu.height - 10;
+                title: qsTr("Attach")
+                MyMenuItem {
+                    text: qsTr("Single File")
+                    icon.source: "qrc:/gpt4all/icons/file.svg"
+                    icon.width: 24
+                    icon.height: 24
+                    onClicked: {
+                        fileDialog.openFileDialog(StandardPaths.writableLocation(StandardPaths.HomeLocation), function(selectedFile) {
+                            if (selectedFile) {
+                                var file = selectedFile.toString().split("/").pop()
+                                attachmentModel.append({
+                                    file: file,
+                                    url: selectedFile
+                                })
+                            }
+                            if (textInput.enabled)
+                                textInput.forceActiveFocus();
+                        })
+                    }
+                }
+            }
+        }
+    }
 }

gpt4all-chat/qml/MyFileDialog.qml

@@ -0,0 +1,19 @@
+import QtCore
+import QtQuick
+import QtQuick.Dialogs
+
+FileDialog {
+    id: fileDialog
+    title: qsTr("Please choose a file")
+    property var acceptedConnection: null
+
+    function openFileDialog(currentFolder, onAccepted) {
+        fileDialog.currentFolder = currentFolder;
+        if (acceptedConnection !== null) {
+            fileDialog.accepted.disconnect(acceptedConnection);
+        }
+        acceptedConnection = function() { onAccepted(fileDialog.selectedFile); };
+        fileDialog.accepted.connect(acceptedConnection);
+        fileDialog.open();
+    }
+}

gpt4all-chat/qml/MyFolderDialog.qml

@@ -0,0 +1,14 @@
+import QtCore
+import QtQuick
+import QtQuick.Dialogs
+
+FolderDialog {
+    id: folderDialog
+    title: qsTr("Please choose a directory")
+
+    function openFolderDialog(currentFolder, onAccepted) {
+        folderDialog.currentFolder = currentFolder;
+        folderDialog.accepted.connect(function() { onAccepted(folderDialog.selectedFolder); });
+        folderDialog.open();
+    }
+}

gpt4all-chat/qml/MyMenu.qml

@@ -22,12 +22,30 @@ Menu {
 
     contentItem: Rectangle {
         implicitWidth: myListView.contentWidth
-        implicitHeight: myListView.contentHeight
+        implicitHeight: (myTitle.visible ? myTitle.contentHeight + 10: 0) + myListView.contentHeight
         color: "transparent"
+
+        Text {
+            id: myTitle
+            visible: menu.title !== ""
+            text: menu.title
+            anchors.margins: 10
+            anchors.top: parent.top
+            anchors.right: parent.right
+            anchors.left: parent.left
+            leftPadding: 15
+            rightPadding: 10
+            padding: 5
+            color: theme.styledTextColor
+            font.pixelSize: theme.fontSizeSmall
+        }
         ListView {
             id: myListView
             anchors.margins: 10
-            anchors.fill: parent
+            anchors.top: title.bottom
+            anchors.bottom: parent.bottom
+            anchors.right: parent.right
+            anchors.left: parent.left
             implicitHeight: contentHeight
             model: menu.contentModel
             interactive: Window.window

gpt4all-chat/qml/MyMenuItem.qml

@@ -1,7 +1,9 @@
+import Qt5Compat.GraphicalEffects
 import QtCore
 import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
+import QtQuick.Layouts
 
 MenuItem {
     id: item
@@ -11,12 +13,40 @@ MenuItem {
         color: item.highlighted ? theme.menuHighlightColor : theme.menuBackgroundColor
     }
 
-    contentItem: Text {
+    contentItem: RowLayout {
-        leftPadding: 10
+        spacing: 0
-        rightPadding: 10
+        Item {
+            visible: item.icon.source.toString() !== ""
+            Layout.leftMargin: 6
+            Layout.preferredWidth: item.icon.width
+            Layout.preferredHeight: item.icon.height
+            Image {
+                id: image
+                anchors.centerIn: parent
+                visible: false
+                fillMode: Image.PreserveAspectFit
+                mipmap: true
+                sourceSize.width: item.icon.width
+                sourceSize.height: item.icon.height
+                source: item.icon.source
+            }
+            ColorOverlay {
+                anchors.fill: image
+                source: image
+                color: theme.textColor
+            }
+        }
+        Text {
+            Layout.alignment: Qt.AlignLeft
             padding: 5
             text: item.text
             color: theme.textColor
             font.pixelSize: theme.fontSizeLarge
         }
+        Rectangle {
+            color: "transparent"
+            Layout.fillWidth: true
+            height: 1
+        }
+    }
 }

gpt4all-chat/qml/MySettingsStack.qml

@@ -61,17 +61,6 @@ Item {
         color: theme.settingsDivider
     }
 
-    FolderDialog {
-        id: folderDialog
-        title: qsTr("Please choose a directory")
-    }
-
-    function openFolderDialog(currentFolder, onAccepted) {
-        folderDialog.currentFolder = currentFolder;
-        folderDialog.accepted.connect(function() { onAccepted(folderDialog.selectedFolder); });
-        folderDialog.open();
-    }
-
     StackLayout {
         id: stackLayout
         anchors.top: tabTitlesModel.count > 1 ? dividerTabBar.bottom : parent.top
@@ -88,7 +77,6 @@ Item {
                 sourceComponent: model.modelData
                 onLoaded: {
                     settingsStack.tabTitlesModel.append({ "title": loader.item.title });
-                    item.openFolderDialog = settingsStack.openFolderDialog;
                 }
             }
         }

gpt4all-chat/qml/MySettingsTab.qml

@@ -9,7 +9,6 @@ Item {
     property string title: ""
     property Item contentItem: null
     property bool showRestoreDefaultsButton: true
-    property var openFolderDialog
     signal restoreDefaultsClicked
 
     onContentItemChanged: function() {

gpt4all-chat/qml/Theme.qml

@@ -177,6 +177,17 @@ QtObject {
         }
     }
 
+    property color attachmentBackground: {
+        switch (MySettings.chatTheme) {
+            case MySettingsEnums.ChatTheme.LegacyDark:
+                return blue900
+            case MySettingsEnums.ChatTheme.Dark:
+                return darkgray200
+            default:
+                return gray0
+        }
+    }
+
     property color disabledControlBackground: {
         switch (MySettings.chatTheme) {
             case MySettingsEnums.ChatTheme.LegacyDark:

gpt4all-chat/src/chat.cpp

@@ -5,6 +5,7 @@
 #include "network.h"
 #include "server.h"
 
+#include <QBuffer>
 #include <QDataStream>
 #include <QDebug>
 #include <QLatin1String>
@@ -122,6 +123,42 @@ void Chat::resetResponseState()
     emit responseStateChanged();
 }
 
+void Chat::newPromptResponsePair(const QString &prompt, const QList<QUrl> &attachedUrls)
+{
+    QStringList attachedContexts;
+    QList<PromptAttachment> attachments;
+    for (const QUrl &url : attachedUrls) {
+        Q_ASSERT(url.isLocalFile());
+        const QString localFilePath = url.toLocalFile();
+        const QFileInfo info(localFilePath);
+        Q_ASSERT(info.suffix() == "xlsx"); // We only support excel right now
+
+        PromptAttachment attached;
+        attached.url = url;
+
+        QFile file(localFilePath);
+        if (file.open(QIODevice::ReadOnly)) {
+            attached.content = file.readAll();
+            file.close();
+        } else {
+            qWarning() << "ERROR: Failed to open the attachment:" << localFilePath;
+            continue;
+        }
+
+        attachments << attached;
+        attachedContexts << attached.processedContent();
+    }
+
+    QString promptPlusAttached = prompt;
+    if (!attachedContexts.isEmpty())
+        promptPlusAttached = attachedContexts.join("\n\n") + "\n\n" + prompt;
+
+    newPromptResponsePairInternal(prompt, attachments);
+    emit resetResponseRequested();
+
+    this->prompt(promptPlusAttached);
+}
+
 void Chat::prompt(const QString &prompt)
 {
     resetResponseState();
@@ -232,23 +269,17 @@ void Chat::setModelInfo(const ModelInfo &modelInfo)
     emit modelChangeRequested(modelInfo);
 }
 
-void Chat::newPromptResponsePair(const QString &prompt)
+// the server needs to block until response is reset, so it calls resetResponse on its own m_llmThread
+void Chat::serverNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments)
 {
-    resetResponseState();
+    newPromptResponsePairInternal(prompt, attachments);
-    m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
-    // the prompt is passed as the prompt item's value and the response item's prompt
-    m_chatModel->appendPrompt("Prompt: ", prompt);
-    m_chatModel->appendResponse("Response: ");
-    emit resetResponseRequested();
 }
 
-// the server needs to block until response is reset, so it calls resetResponse on its own m_llmThread
+void Chat::newPromptResponsePairInternal(const QString &prompt, const QList<PromptAttachment> &attachments)
-void Chat::serverNewPromptResponsePair(const QString &prompt)
 {
     resetResponseState();
     m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
-    // the prompt is passed as the prompt item's value and the response item's prompt
+    m_chatModel->appendPrompt("Prompt: ", prompt, attachments);
-    m_chatModel->appendPrompt("Prompt: ", prompt);
     m_chatModel->appendResponse("Response: ");
 }
 

gpt4all-chat/src/chat.h

@@ -77,10 +77,10 @@ public:
     bool  isModelLoaded()          const { return m_modelLoadingPercentage == 1.0f; }
     bool  isCurrentlyLoading()     const { return m_modelLoadingPercentage > 0.0f && m_modelLoadingPercentage < 1.0f; }
     float modelLoadingPercentage() const { return m_modelLoadingPercentage; }
+    Q_INVOKABLE void newPromptResponsePair(const QString &prompt, const QList<QUrl> &attachedUrls = {});
     Q_INVOKABLE void prompt(const QString &prompt);
     Q_INVOKABLE void regenerateResponse();
     Q_INVOKABLE void stopGenerating();
-    Q_INVOKABLE void newPromptResponsePair(const QString &prompt);
 
     QList<ResultInfo> databaseResults() const { return m_databaseResults; }
 
@@ -125,7 +125,7 @@ public:
     QList<QString> generatedQuestions() const { return m_generatedQuestions; }
 
 public Q_SLOTS:
-    void serverNewPromptResponsePair(const QString &prompt);
+    void serverNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments = {});
 
 Q_SIGNALS:
     void idChanged(const QString &id);
@@ -174,6 +174,9 @@ private Q_SLOTS:
     void handleModelInfoChanged(const ModelInfo &modelInfo);
     void handleTrySwitchContextOfLoadedModelCompleted(int value);
 
+private:
+    void newPromptResponsePairInternal(const QString &prompt, const QList<PromptAttachment> &attachments);
+
 private:
     QString m_id;
     QString m_name;

gpt4all-chat/src/chatllm.cpp

@@ -1333,7 +1333,7 @@ void ChatLLM::processRestoreStateFromText()
         // FIXME(jared): this doesn't work well with the "regenerate" button since we are not incrementing
         //               m_promptTokens or m_promptResponseTokens
         m_llModelInfo.model->prompt(
-            prompt.value.toStdString(), promptTemplate.toStdString(),
+            prompt.promptPlusAttachments().toStdString(), promptTemplate.toStdString(),
             promptFunc, /*responseFunc*/ [](auto &&...) { return true; },
             /*allowContextShift*/ true,
             m_ctx,

gpt4all-chat/src/chatmodel.h

@@ -2,8 +2,10 @@
 #define CHATMODEL_H
 
 #include "database.h"
+#include "xlsxtomd.h"
 
 #include <QAbstractListModel>
+#include <QBuffer>
 #include <QByteArray>
 #include <QDataStream>
 #include <QHash>
@@ -16,6 +18,40 @@
 #include <Qt>
 #include <QtGlobal>
 
+struct PromptAttachment {
+    Q_GADGET
+    Q_PROPERTY(QUrl url MEMBER url)
+    Q_PROPERTY(QByteArray content MEMBER content)
+    Q_PROPERTY(QString file READ file)
+    Q_PROPERTY(QString processedContent READ processedContent)
+
+public:
+    QUrl url;
+    QByteArray content;
+
+    QString file() const
+    {
+        if (!url.isLocalFile())
+            return QString();
+        const QString localFilePath = url.toLocalFile();
+        const QFileInfo info(localFilePath);
+        return info.fileName();
+    }
+
+    QString processedContent() const
+    {
+        QBuffer buffer;
+        buffer.setData(content);
+        buffer.open(QIODevice::ReadOnly);
+        const QString md = XLSXToMD::toMarkdown(&buffer);
+        buffer.close();
+        return md;
+    }
+
+    bool operator==(const PromptAttachment &other) const { return url == other.url; }
+};
+Q_DECLARE_METATYPE(PromptAttachment)
+
 struct ChatItem
 {
     Q_GADGET
@@ -29,8 +65,22 @@ struct ChatItem
     Q_PROPERTY(bool thumbsDownState MEMBER thumbsDownState)
     Q_PROPERTY(QList<ResultInfo> sources MEMBER sources)
     Q_PROPERTY(QList<ResultInfo> consolidatedSources MEMBER consolidatedSources)
+    Q_PROPERTY(QList<PromptAttachment> promptAttachments MEMBER promptAttachments);
+    Q_PROPERTY(QString promptPlusAttachments READ promptPlusAttachments);
 
 public:
+    QString promptPlusAttachments() const
+    {
+        QStringList attachedContexts;
+        for (auto attached : promptAttachments)
+            attachedContexts << attached.processedContent();
+
+        QString promptPlus = value;
+        if (!attachedContexts.isEmpty())
+            promptPlus = attachedContexts.join("\n\n") + "\n\n" + value;
+        return promptPlus;
+    }
+
     // TODO: Maybe we should include the model name here as well as timestamp?
     int id = 0;
     QString name;
@@ -38,6 +88,7 @@ public:
     QString newResponse;
     QList<ResultInfo> sources;
     QList<ResultInfo> consolidatedSources;
+    QList<PromptAttachment> promptAttachments;
     bool currentResponse = false;
     bool stopped = false;
     bool thumbsUpState = false;
@@ -65,7 +116,8 @@ public:
         ThumbsUpStateRole,
         ThumbsDownStateRole,
         SourcesRole,
-        ConsolidatedSourcesRole
+        ConsolidatedSourcesRole,
+        PromptAttachmentsRole
     };
 
     int rowCount(const QModelIndex &parent = QModelIndex()) const override
@@ -103,6 +155,8 @@ public:
                 return QVariant::fromValue(item.sources);
             case ConsolidatedSourcesRole:
                 return QVariant::fromValue(item.consolidatedSources);
+            case PromptAttachmentsRole:
+                return QVariant::fromValue(item.promptAttachments);
         }
 
         return QVariant();
@@ -121,14 +175,17 @@ public:
         roles[ThumbsDownStateRole] = "thumbsDownState";
         roles[SourcesRole] = "sources";
         roles[ConsolidatedSourcesRole] = "consolidatedSources";
+        roles[PromptAttachmentsRole] = "promptAttachments";
         return roles;
     }
 
-    void appendPrompt(const QString &name, const QString &value)
+    void appendPrompt(const QString &name, const QString &value, const QList<PromptAttachment> &attachments)
     {
         ChatItem item;
         item.name = name;
         item.value = value;
+        item.promptAttachments << attachments;
+
         m_mutex.lock();
         const int count = m_chatItems.count();
         m_mutex.unlock();
@@ -380,6 +437,14 @@ public:
                 stream << references.join("\n");
                 stream << referencesContext;
             }
+            if (version >= 10) {
+                stream << c.promptAttachments.size();
+                for (const PromptAttachment &a : c.promptAttachments) {
+                    Q_ASSERT(!a.url.isEmpty());
+                    stream << a.url;
+                    stream << a.content;
+                }
+            }
         }
         return stream.status() == QDataStream::Ok;
     }
@@ -423,7 +488,7 @@ public:
                 }
                 c.sources = sources;
                 c.consolidatedSources = consolidateSources(sources);
-            }else if (version > 2) {
+            } else if (version > 2) {
                 QString references;
                 QList<QString> referencesContext;
                 stream >> references;
@@ -507,6 +572,18 @@ public:
                     c.consolidatedSources = consolidateSources(sources);
                 }
             }
+            if (version >= 10) {
+                qsizetype count;
+                stream >> count;
+                QList<PromptAttachment> attachments;
+                for (int i = 0; i < count; ++i) {
+                    PromptAttachment a;
+                    stream >> a.url;
+                    stream >> a.content;
+                    attachments.append(a);
+                }
+                c.promptAttachments = attachments;
+            }
             m_mutex.lock();
             const int count = m_chatItems.size();
             m_mutex.unlock();

gpt4all-chat/src/server.h

@@ -2,6 +2,7 @@
 #define SERVER_H
 
 #include "chatllm.h"
+#include "chatmodel.h"
 #include "database.h"
 
 #include <QHttpServer>
@@ -32,7 +33,7 @@ public Q_SLOTS:
     void start();
 
 Q_SIGNALS:
-    void requestServerNewPromptResponsePair(const QString &prompt);
+    void requestServerNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments = {});
 
 private:
     auto handleCompletionRequest(const CompletionRequest &request) -> std::pair<QHttpServerResponse, std::optional<QJsonObject>>;

gpt4all-chat/src/xlsxtomd.cpp

@@ -0,0 +1,167 @@
+#include "xlsxtomd.h"
+
+#include <xlsxabstractsheet.h>
+#include <xlsxcell.h>
+#include <xlsxcellrange.h>
+#include <xlsxdocument.h>
+#include <xlsxformat.h>
+#include <xlsxworksheet.h>
+
+#include <QDateTime>
+#include <QDebug>
+#include <QList>
+#include <QString>
+#include <QStringList>
+#include <QVariant>
+#include <QtGlobal>
+#include <QtLogging>
+
+#include <memory>
+
+using namespace Qt::Literals::StringLiterals;
+
+
+static QString formatCellText(const QXlsx::Cell *cell)
+{
+    if (!cell) return QString();
+
+    QVariant value = cell->value();
+    QXlsx::Format format = cell->format();
+    QString cellText;
+
+    // Determine the cell type based on format
+    if (format.isDateTimeFormat()) {
+        // Handle DateTime
+        QDateTime dateTime = value.toDateTime();
+        cellText = dateTime.isValid() ? dateTime.toString("yyyy-MM-dd") : value.toString();
+    } else {
+        cellText = value.toString();
+    }
+
+    if (cellText.isEmpty())
+        return QString();
+
+    // Apply Markdown and HTML formatting based on font styles
+    QString formattedText = cellText;
+
+    if (format.fontBold() && format.fontItalic())
+        formattedText = "***" + formattedText + "***";
+    else if (format.fontBold())
+        formattedText = "**" + formattedText + "**";
+    else if (format.fontItalic())
+        formattedText = "*" + formattedText + "*";
+
+    if (format.fontStrikeOut())
+        formattedText = "~~" + formattedText + "~~";
+
+    // Escape pipe characters to prevent Markdown table issues
+    formattedText.replace("|", "\\|");
+
+    return formattedText;
+}
+
+static QString getCellValue(QXlsx::Worksheet *sheet, int row, int col)
+{
+    if (!sheet)
+        return QString();
+
+    // Attempt to retrieve the cell directly
+    std::shared_ptr<QXlsx::Cell> cell = sheet->cellAt(row, col);
+
+    // If the cell is part of a merged range and not directly available
+    if (!cell) {
+        for (const QXlsx::CellRange &range : sheet->mergedCells()) {
+            if (row >= range.firstRow() && row <= range.lastRow() &&
+                col >= range.firstColumn() && col <= range.lastColumn()) {
+                cell = sheet->cellAt(range.firstRow(), range.firstColumn());
+                break;
+            }
+        }
+    }
+
+    // Format and return the cell text if available
+    if (cell)
+        return formatCellText(cell.get());
+
+    // Return empty string if cell is not found
+    return QString();
+}
+
+QString XLSXToMD::toMarkdown(QIODevice *xlsxDevice)
+{
+    // Load the Excel document
+    QXlsx::Document xlsx(xlsxDevice);
+    if (!xlsx.load()) {
+        qCritical() << "Failed to load the Excel from device";
+        return QString();
+    }
+
+    QString markdown;
+
+    // Retrieve all sheet names
+    QStringList sheetNames = xlsx.sheetNames();
+    if (sheetNames.isEmpty()) {
+        qWarning() << "No sheets found in the Excel document.";
+        return QString();
+    }
+
+    // Iterate through each worksheet by name
+    for (const QString &sheetName : sheetNames) {
+        QXlsx::Worksheet *sheet = dynamic_cast<QXlsx::Worksheet *>(xlsx.sheet(sheetName));
+        if (!sheet) {
+            qWarning() << "Failed to load sheet:" << sheetName;
+            continue;
+        }
+
+        markdown += u"## %1\n\n"_s.arg(sheetName);
+
+        // Determine the used range
+        QXlsx::CellRange range = sheet->dimension();
+        int firstRow = range.firstRow();
+        int lastRow = range.lastRow();
+        int firstCol = range.firstColumn();
+        int lastCol = range.lastColumn();
+
+        if (firstRow > lastRow || firstCol > lastCol) {
+            qWarning() << "Sheet" << sheetName << "is empty.";
+            markdown += "*No data available.*\n\n";
+            continue;
+        }
+
+        // Assume the first row is the header
+        int headerRow = firstRow;
+
+        // Collect headers
+        QStringList headers;
+        for (int col = firstCol; col <= lastCol; ++col) {
+            QString header = getCellValue(sheet, headerRow, col);
+            headers << header;
+        }
+
+        // Create Markdown header row
+        QString headerRowMarkdown = "|" + headers.join("|") + "|";
+        markdown += headerRowMarkdown + "\n";
+
+        // Create Markdown separator row
+        QStringList separators;
+        for (int i = 0; i < headers.size(); ++i)
+            separators << "---";
+        QString separatorRow = "|" + separators.join("|") + "|";
+        markdown += separatorRow + "\n";
+
+        // Iterate through data rows (starting from the row after header)
+        for (int row = headerRow + 1; row <= lastRow; ++row) {
+            QStringList rowData;
+            for (int col = firstCol; col <= lastCol; ++col) {
+                QString cellText = getCellValue(sheet, row, col);
+                rowData << cellText;
+            }
+
+            QString dataRow = "|" + rowData.join("|") + "|";
+            markdown += dataRow + "\n";
+        }
+
+        markdown += "\n"; // Add an empty line between sheets
+    }
+    return markdown;
+}

gpt4all-chat/src/xlsxtomd.h

@@ -0,0 +1,13 @@
+#ifndef XLSXTOMD_H
+#define XLSXTOMD_H
+
+class QIODevice;
+class QString;
+
+class XLSXToMD
+{
+public:
+    static QString toMarkdown(QIODevice *xlsxDevice);
+};
+
+#endif // XLSXTOMD_H