download: make model downloads resumable

* save files as `incomplete-{filename}` in the dest folder
* rename into place after hash is confirmed or delete if hash is bad
* resume downloads using http `range`
* if DL is resumed from a different app session rewind a bit -
  this is to deal with the case where the file size changes before
  the content is fully flushed out
* flush dest file at end of readyRead, this mitigates the above
  and provides backpressure on the download if the destination disk
  is slower than the network connection

download.cpp

@@ -36,6 +36,7 @@ Download::Download()
     settings.sync();
     m_downloadLocalModelsPath = settings.value("modelPath",
         defaultLocalModelsPath()).toString();
+    m_startTime = QDateTime::currentDateTime();
 }
 
 bool operator==(const ModelInfo& lhs, const ModelInfo& rhs) {
@@ -143,6 +144,12 @@ bool Download::isFirstStart() const
     return first;
 }
 
+QString Download::incompleteDownloadPath(const QString &modelFile) {
+    QString downloadPath = downloadLocalModelsPath() + "incomplete-" +
+                           modelFile;
+    return downloadPath;
+}
+
 QString Download::defaultLocalModelsPath() const
 {
     QString localPath = QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation)
@@ -194,17 +201,31 @@ void Download::updateReleaseNotes()
 
 void Download::downloadModel(const QString &modelFile)
 {
-    QTemporaryFile *tempFile = new QTemporaryFile;
+    QFile *tempFile = new QFile(incompleteDownloadPath(modelFile));
-    bool success = tempFile->open();
+    QDateTime modTime = tempFile->fileTime(QFile::FileModificationTime);
+    bool success = tempFile->open(QIODevice::WriteOnly | QIODevice::Append);
     qWarning() << "Opening temp file for writing:" << tempFile->fileName();
     if (!success) {
         qWarning() << "ERROR: Could not open temp file:"
             << tempFile->fileName() << modelFile;
         return;
     }
+    size_t incomplete_size = tempFile->size();
+    if (incomplete_size > 0) {
+        if (modTime < m_startTime) {
+            qWarning() << "File last modified before app started, rewinding by 1MB";
+            if (incomplete_size >= 1024 * 1024) {
+                incomplete_size -= 1024 * 1024;
+            } else {
+                incomplete_size = 0;
+            }
+        }
+        tempFile->seek(incomplete_size);
+    }
 
     Network::globalInstance()->sendDownloadStarted(modelFile);
     QNetworkRequest request("http://gpt4all.io/models/" + modelFile);
+    request.setRawHeader("range", QString("bytes=%1-").arg(incomplete_size).toUtf8());
     QSslConfiguration conf = request.sslConfiguration();
     conf.setPeerVerifyMode(QSslSocket::VerifyNone);
     request.setSslConfiguration(conf);
@@ -230,7 +251,7 @@ void Download::cancelDownload(const QString &modelFile)
             modelReply->abort(); // Abort the download
             modelReply->deleteLater(); // Schedule the reply for deletion
 
-            QTemporaryFile *tempFile = m_activeDownloads.value(modelReply);
+            QFile *tempFile = m_activeDownloads.value(modelReply);
             tempFile->deleteLater();
             m_activeDownloads.remove(modelReply);
 
@@ -410,9 +431,17 @@ void Download::handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal)
     QNetworkReply *modelReply = qobject_cast<QNetworkReply *>(sender());
     if (!modelReply)
         return;
+    QFile *tempFile = m_activeDownloads.value(modelReply);
+    if (!tempFile)
+        return;
+    QString contentRange = modelReply->rawHeader("content-range");
+    if (contentRange.contains("/")) {
+        QString contentTotalSize = contentRange.split("/").last();
+        bytesTotal = contentTotalSize.toLongLong();
+    }
 
     QString modelFilename = modelReply->url().fileName();
-    emit downloadProgress(bytesReceived, bytesTotal, modelFilename);
+    emit downloadProgress(tempFile->pos(), bytesTotal, modelFilename);
 }
 
 HashAndSaveFile::HashAndSaveFile()
@@ -424,13 +453,13 @@ HashAndSaveFile::HashAndSaveFile()
 }
 
 void HashAndSaveFile::hashAndSave(const QString &expectedHash, const QString &saveFilePath,
-        QTemporaryFile *tempFile, QNetworkReply *modelReply)
+        QFile *tempFile, QNetworkReply *modelReply)
 {
     Q_ASSERT(!tempFile->isOpen());
     QString modelFilename = modelReply->url().fileName();
 
     // Reopen the tempFile for hashing
-    if (!tempFile->open()) {
+    if (!tempFile->open(QIODevice::ReadOnly)) {
         qWarning() << "ERROR: Could not open temp file for hashing:"
             << tempFile->fileName() << modelFilename;
         emit hashAndSaveFinished(false, tempFile, modelReply);
@@ -445,6 +474,7 @@ void HashAndSaveFile::hashAndSave(const QString &expectedHash, const QString &sa
         qWarning() << "ERROR: Download error MD5SUM did not match:"
             << hash.result().toHex()
             << "!=" << expectedHash << "for" << modelFilename;
+        tempFile->remove();
         emit hashAndSaveFinished(false, tempFile, modelReply);
         return;
     }
@@ -455,13 +485,12 @@ void HashAndSaveFile::hashAndSave(const QString &expectedHash, const QString &sa
     // Attempt to *move* the verified tempfile into place - this should be atomic
     // but will only work if the destination is on the same filesystem
     if (tempFile->rename(saveFilePath)) {
-        tempFile->setAutoRemove(false);
         emit hashAndSaveFinished(true, tempFile, modelReply);
         return;
     }
 
     // Reopen the tempFile for copying
-    if (!tempFile->open()) {
+    if (!tempFile->open(QIODevice::ReadOnly)) {
         qWarning() << "ERROR: Could not open temp file at finish:"
             << tempFile->fileName() << modelFilename;
         emit hashAndSaveFinished(false, tempFile, modelReply);
@@ -497,7 +526,7 @@ void Download::handleModelDownloadFinished()
         return;
 
     QString modelFilename = modelReply->url().fileName();
-    QTemporaryFile *tempFile = m_activeDownloads.value(modelReply);
+    QFile *tempFile = m_activeDownloads.value(modelReply);
     m_activeDownloads.remove(modelReply);
 
     if (modelReply->error()) {
@@ -522,7 +551,7 @@ void Download::handleModelDownloadFinished()
 }
 
 void Download::handleHashAndSaveFinished(bool success,
-        QTemporaryFile *tempFile, QNetworkReply *modelReply)
+        QFile *tempFile, QNetworkReply *modelReply)
 {
     // The hash and save should send back with tempfile closed
     Q_ASSERT(!tempFile->isOpen());
@@ -547,10 +576,11 @@ void Download::handleReadyRead()
         return;
 
     QString modelFilename = modelReply->url().fileName();
-    QTemporaryFile *tempFile = m_activeDownloads.value(modelReply);
+    QFile *tempFile = m_activeDownloads.value(modelReply);
     QByteArray buffer;
     while (!modelReply->atEnd()) {
         buffer = modelReply->read(16384);
         tempFile->write(buffer);
     }
+    tempFile->flush();
 }

download.h

@@ -56,11 +56,11 @@ public:
 
 public Q_SLOTS:
     void hashAndSave(const QString &hash, const QString &saveFilePath,
-        QTemporaryFile *tempFile, QNetworkReply *modelReply);
+        QFile *tempFile, QNetworkReply *modelReply);
 
 Q_SIGNALS:
     void hashAndSaveFinished(bool success,
-        QTemporaryFile *tempFile, QNetworkReply *modelReply);
+        QFile *tempFile, QNetworkReply *modelReply);
 
 private:
     QThread m_hashAndSaveThread;
@@ -99,7 +99,7 @@ private Q_SLOTS:
     void handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal);
     void handleModelDownloadFinished();
     void handleHashAndSaveFinished(bool success,
-        QTemporaryFile *tempFile, QNetworkReply *modelReply);
+        QFile *tempFile, QNetworkReply *modelReply);
     void handleReadyRead();
 
 Q_SIGNALS:
@@ -110,18 +110,20 @@ Q_SIGNALS:
     void hasNewerReleaseChanged();
     void downloadLocalModelsPathChanged();
     void requestHashAndSave(const QString &hash, const QString &saveFilePath,
-        QTemporaryFile *tempFile, QNetworkReply *modelReply);
+        QFile *tempFile, QNetworkReply *modelReply);
 
 private:
     void parseModelsJsonFile(const QByteArray &jsonData);
     void parseReleaseJsonFile(const QByteArray &jsonData);
+    QString incompleteDownloadPath(const QString &modelFile);
 
     HashAndSaveFile *m_hashAndSave;
     QMap<QString, ModelInfo> m_modelMap;
     QMap<QString, ReleaseInfo> m_releaseMap;
     QNetworkAccessManager m_networkManager;
-    QMap<QNetworkReply*, QTemporaryFile*> m_activeDownloads;
+    QMap<QNetworkReply*, QFile*> m_activeDownloads;
     QString m_downloadLocalModelsPath;
+    QDateTime m_startTime;
 
 private:
     explicit Download();