improve mixpanel usage statistics (#2238)

Other changes:
- Always display first start dialog if privacy options are unset (e.g. if the user closed GPT4All without selecting them)
- LocalDocs scanQueue is now always deferred
- Fix a potential crash in magic_match
- LocalDocs indexing is now started after the first start dialog is dismissed so usage stats are included

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

gpt4all-backend/gptj.cpp

@@ -785,7 +785,7 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
+const char *get_arch_name(gguf_context *ctx_gguf) {
     std::string arch_name;
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
@@ -814,21 +814,25 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
+DLL_EXPORT char *get_file_arch(const char *fname) {
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+    char *arch = nullptr;
+    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
+        arch = strdup(get_arch_name(ctx_gguf));
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return !strcmp(arch, "gptj");
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llamamodel.cpp

@@ -104,7 +104,7 @@ static int llama_sample_top_p_top_k(
     return llama_sample_token(ctx, &candidates_p);
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
+const char *get_arch_name(gguf_context *ctx_gguf) {
     std::string arch_name;
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
@@ -961,25 +961,23 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char *fname) {
-    auto * ctx = load_gguf(fname);
-    std::string arch = get_arch_name(ctx);
-
-    bool valid = true;
-
-    if (std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), arch) == KNOWN_ARCHES.end()) {
-        // not supported by this version of llama.cpp
-        if (arch != "gptj") { // we support this via another module
-            std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
+DLL_EXPORT char *get_file_arch(const char *fname) {
+    auto *ctx = load_gguf(fname);
+    char *arch = nullptr;
+    if (ctx) {
+        std::string archStr = get_arch_name(ctx);
+        if (is_embedding_arch(archStr) && gguf_find_key(ctx, (archStr + ".pooling_type").c_str()) < 0) {
+            // old bert.cpp embedding model
+        } else {
+            arch = strdup(archStr.c_str());
         }
-        valid = false;
     }
-
-    if (valid && is_embedding_arch(arch) && gguf_find_key(ctx, (arch + ".pooling_type").c_str()) < 0)
-        valid = false; // old pre-llama.cpp embedding model, e.g. all-MiniLM-L6-v2-f16.gguf
-
     gguf_free(ctx);
-    return valid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), std::string(arch)) < KNOWN_ARCHES.end();
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/llmodel.cpp

@@ -8,6 +8,7 @@
 #include <fstream>
 #include <iostream>
 #include <memory>
+#include <optional>
 #include <regex>
 #include <sstream>
 #include <string>
@@ -49,14 +50,17 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
-    assert(m_magicMatch);
+    m_getFileArch = m_dlhandle->get<char *(const char *)>("get_file_arch");
+    assert(m_getFileArch);
+    m_isArchSupported = m_dlhandle->get<bool(const char *)>("is_arch_supported");
+    assert(m_isArchSupported);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : m_magicMatch(o.m_magicMatch)
+    : m_getFileArch(o.m_getFileArch)
+    , m_isArchSupported(o.m_isArchSupported)
     , m_construct(o.m_construct)
     , m_modelType(o.m_modelType)
     , m_buildVariant(o.m_buildVariant)
@@ -123,18 +127,26 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
 
 const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
     bool buildVariantMatched = false;
+    std::optional<std::string> archName;
     for (const auto& i : implementationList()) {
         if (buildVariant != i.m_buildVariant) continue;
         buildVariantMatched = true;
 
-        if (!i.m_magicMatch(fname)) continue;
-        return &i;
+        char *arch = i.m_getFileArch(fname);
+        if (!arch) continue;
+        archName = arch;
+
+        bool archSupported = i.m_isArchSupported(arch);
+        free(arch);
+        if (archSupported) return &i;
     }
 
     if (!buildVariantMatched)
-        throw std::runtime_error("Could not find any implementations for build variant: " + buildVariant);
+        throw MissingImplementationError("Could not find any implementations for build variant: " + buildVariant);
+    if (!archName)
+        throw UnsupportedModelError("Unsupported file format");
 
-    return nullptr; // unsupported model format
+    throw BadArchError(std::move(*archName));
 }
 
 LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
@@ -144,7 +156,11 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
     #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
         if (buildVariant == "auto") {
             size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
+            try {
+                impl = implementation(modelPath.c_str(), "metal");
+            } catch (const std::exception &e) {
+                // fall back to CPU
+            }
             if(impl) {
                 LLModel* metalimpl = impl->m_construct();
                 metalimpl->m_implementation = impl;
@@ -177,7 +193,6 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
             }
         }
         impl = implementation(modelPath.c_str(), buildVariant);
-        if (!impl) return nullptr;
     }
 
     // Construct and return llmodel implementation

gpt4all-backend/llmodel.h

@@ -17,6 +17,29 @@ class LLModel {
 public:
     using Token = int32_t;
 
+    class BadArchError: public std::runtime_error {
+    public:
+        BadArchError(std::string arch)
+            : runtime_error("Unsupported model architecture: " + arch)
+            , m_arch(std::move(arch))
+            {}
+
+        const std::string &arch() const noexcept { return m_arch; }
+
+    private:
+        std::string m_arch;
+    };
+
+    class MissingImplementationError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    class UnsupportedModelError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
     struct GPUDevice {
         int index;
         int type;
@@ -53,7 +76,8 @@ public:
         static const Implementation *implementation(const char *fname, const std::string &buildVariant);
         static LLModel *constructDefaultLlama();
 
-        bool (*m_magicMatch)(const char *fname);
+        char *(*m_getFileArch)(const char *fname);
+        bool (*m_isArchSupported)(const char *arch);
         LLModel *(*m_construct)();
 
         std::string_view m_modelType;

gpt4all-backend/llmodel_c.cpp

@@ -40,11 +40,6 @@ llmodel_model llmodel_model_create2(const char *model_path, const char *build_va
         return nullptr;
     }
 
-    if (!llModel) {
-        llmodel_set_error(error, "Model format not supported (no matching implementation found)");
-        return nullptr;
-    }
-
     auto wrapper = new LLModelWrapper;
     wrapper->llModel = llModel;
     return wrapper;

gpt4all-chat/chat.cpp

@@ -179,7 +179,7 @@ void Chat::promptProcessing()
     emit responseStateChanged();
 }
 
-void Chat::responseStopped()
+void Chat::responseStopped(qint64 promptResponseMs)
 {
     m_tokenSpeed = QString();
     emit tokenSpeedChanged();
@@ -228,8 +228,13 @@ void Chat::responseStopped()
     emit responseStateChanged();
     if (m_generatedName.isEmpty())
         emit generateNameRequested();
-    if (chatModel()->count() < 3)
-        Network::globalInstance()->sendChatStarted();
+
+    Network::globalInstance()->trackChatEvent("response_complete", {
+        {"first", m_firstResponse},
+        {"message_count", chatModel()->count()},
+        {"$duration", promptResponseMs / 1000.},
+    });
+    m_firstResponse = false;
 }
 
 ModelInfo Chat::modelInfo() const
@@ -331,7 +336,7 @@ void Chat::generatedNameChanged(const QString &name)
 
 void Chat::handleRecalculating()
 {
-    Network::globalInstance()->sendRecalculatingContext(m_chatModel->count());
+    Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
     emit recalcChanged();
 }
 

gpt4all-chat/chat.h

@@ -143,7 +143,7 @@ private Q_SLOTS:
     void handleResponseChanged(const QString &response);
     void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
-    void responseStopped();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void handleRecalculating();
     void handleModelLoadingError(const QString &error);
@@ -175,6 +175,7 @@ private:
     bool m_shouldDeleteLater = false;
     float m_modelLoadingPercentage = 0.0f;
     LocalDocsCollectionsModel *m_collectionModel;
+    bool m_firstResponse = true;
 };
 
 #endif // CHAT_H

gpt4all-chat/chatllm.cpp

@@ -7,6 +7,8 @@
 #include "mysettings.h"
 #include "../gpt4all-backend/llmodel.h"
 
+#include <QElapsedTimer>
+
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
@@ -74,8 +76,6 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_restoreStateFromText(false)
 {
     moveToThread(&m_llmThread);
-    connect(this, &ChatLLM::sendStartup, Network::globalInstance(), &Network::sendStartup);
-    connect(this, &ChatLLM::sendModelLoaded, Network::globalInstance(), &Network::sendModelLoaded);
     connect(this, &ChatLLM::shouldBeLoadedChanged, this, &ChatLLM::handleShouldBeLoadedChanged,
         Qt::QueuedConnection); // explicitly queued
     connect(this, &ChatLLM::shouldTrySwitchContextChanged, this, &ChatLLM::handleShouldTrySwitchContextChanged,
@@ -278,6 +278,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     m_llModelInfo.fileInfo = fileInfo;
 
     if (fileInfo.exists()) {
+        QVariantMap modelLoadProps;
         if (modelInfo.isOnline) {
             QString apiKey;
             QString modelName;
@@ -298,6 +299,9 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             model->setAPIKey(apiKey);
             m_llModelInfo.model = model;
         } else {
+            QElapsedTimer modelLoadTimer;
+            modelLoadTimer.start();
+
             auto n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
             m_ctx.n_ctx = n_ctx;
             auto ngl = MySettings::globalInstance()->modelGpuLayers(modelInfo);
@@ -307,7 +311,21 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             if (m_forceMetal)
                 buildVariant = "metal";
 #endif
-            m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
+            QString constructError;
+            m_llModelInfo.model = nullptr;
+            try {
+                m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
+            } catch (const LLModel::MissingImplementationError &e) {
+                modelLoadProps.insert("error", "missing_model_impl");
+                constructError = e.what();
+            } catch (const LLModel::UnsupportedModelError &e) {
+                modelLoadProps.insert("error", "unsupported_model_file");
+                constructError = e.what();
+            } catch (const LLModel::BadArchError &e) {
+                constructError = e.what();
+                modelLoadProps.insert("error", "unsupported_model_arch");
+                modelLoadProps.insert("model_arch", QString::fromStdString(e.arch()));
+            }
 
             if (m_llModelInfo.model) {
                 if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
@@ -368,6 +386,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_load_failed");
                     success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, 0);
                 } else if (!m_llModelInfo.model->usingGPUDevice()) {
                     // ggml_vk_init was not called in llama.cpp
@@ -375,6 +394,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     // for instance if the quantization method is not supported on Vulkan yet
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>model or quant has no GPU support");
+                    modelLoadProps.insert("cpu_fallback_reason", "gpu_unsupported_model");
                 }
 
                 if (!success) {
@@ -384,6 +404,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                         LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
                     m_llModelInfo = LLModelInfo();
                     emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
+                    modelLoadProps.insert("error", "loadmodel_failed");
                 } else {
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
@@ -398,12 +419,14 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                             emit modelLoadingError(QString("Could not determine model type for %1").arg(modelInfo.filename()));
                         }
                     }
+
+                    modelLoadProps.insert("$duration", modelLoadTimer.elapsed() / 1000.);
                 }
             } else {
                 if (!m_isServer)
                     LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
                 m_llModelInfo = LLModelInfo();
-                emit modelLoadingError(QString("Could not load model due to invalid format for %1").arg(modelInfo.filename()));
+                emit modelLoadingError(QString("Error loading %1: %2").arg(modelInfo.filename()).arg(constructError));
             }
         }
 #if defined(DEBUG_MODEL_LOADING)
@@ -416,12 +439,9 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
         emit modelLoadingPercentageChanged(isModelLoaded() ? 1.0f : 0.0f);
 
-        static bool isFirstLoad = true;
-        if (isFirstLoad) {
-            emit sendStartup();
-            isFirstLoad = false;
-        } else
-            emit sendModelLoaded();
+        modelLoadProps.insert("requestedDevice", MySettings::globalInstance()->device());
+        modelLoadProps.insert("model", modelInfo.filename());
+        Network::globalInstance()->trackChatEvent("model_load", modelLoadProps);
     } else {
         if (!m_isServer)
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
@@ -632,6 +652,8 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     printf("%s", qPrintable(prompt));
     fflush(stdout);
 #endif
+    QElapsedTimer totalTime;
+    totalTime.start();
     m_timer->start();
     if (!docsContext.isEmpty()) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
@@ -644,12 +666,13 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     fflush(stdout);
 #endif
     m_timer->stop();
+    qint64 elapsed = totalTime.elapsed();
     std::string trimmed = trim_whitespace(m_response);
     if (trimmed != m_response) {
         m_response = trimmed;
         emit responseChanged(QString::fromStdString(m_response));
     }
-    emit responseStopped();
+    emit responseStopped(elapsed);
     return true;
 }
 

gpt4all-chat/chatllm.h

@@ -123,9 +123,7 @@ Q_SIGNALS:
     void modelLoadingWarning(const QString &warning);
     void responseChanged(const QString &response);
     void promptProcessing();
-    void responseStopped();
-    void sendStartup();
-    void sendModelLoaded();
+    void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void stateChanged();
     void threadStarted();

gpt4all-chat/database.cpp

@@ -1,7 +1,9 @@
 #include "database.h"
-#include "mysettings.h"
-#include "embllm.h"
+
 #include "embeddings.h"
+#include "embllm.h"
+#include "mysettings.h"
+#include "network.h"
 
 #include <QTimer>
 #include <QPdfDocument>
@@ -490,7 +492,7 @@ QSqlError initDb()
     i.collection = collection_name;
     i.folder_path = folder_path;
     i.folder_id = folder_id;
-    emit addCollectionItem(i);
+    emit addCollectionItem(i, false);
 
     // Add a document
     int document_time = 123456789;
@@ -535,13 +537,13 @@ QSqlError initDb()
 
 Database::Database(int chunkSize)
     : QObject(nullptr)
-    , m_watcher(new QFileSystemWatcher(this))
     , m_chunkSize(chunkSize)
+    , m_scanTimer(new QTimer(this))
+    , m_watcher(new QFileSystemWatcher(this))
     , m_embLLM(new EmbeddingLLM)
     , m_embeddings(new Embeddings(this))
 {
     moveToThread(&m_dbThread);
-    connect(&m_dbThread, &QThread::started, this, &Database::start);
     m_dbThread.setObjectName("database");
     m_dbThread.start();
 }
@@ -556,11 +558,13 @@ void Database::scheduleNext(int folder_id, size_t countForFolder)
 {
     emit updateCurrentDocsToIndex(folder_id, countForFolder);
     if (!countForFolder) {
-        emit updateIndexing(folder_id, false);
+        updateFolderStatus(folder_id, FolderStatus::Complete);
         emit updateInstalled(folder_id, true);
     }
-    if (!m_docsToScan.isEmpty())
-        QTimer::singleShot(0, this, &Database::scanQueue);
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
+    }
 }
 
 void Database::handleDocumentError(const QString &errorMessage,
@@ -721,7 +725,6 @@ void Database::removeFolderFromDocumentQueue(int folder_id)
         return;
     m_docsToScan.remove(folder_id);
     emit removeFolderById(folder_id);
-    emit docsToScanChanged();
 }
 
 void Database::enqueueDocumentInternal(const DocumentInfo &info, bool prepend)
@@ -745,13 +748,16 @@ void Database::enqueueDocuments(int folder_id, const QVector<DocumentInfo> &info
     const size_t bytes = countOfBytes(folder_id);
     emit updateCurrentBytesToIndex(folder_id, bytes);
     emit updateTotalBytesToIndex(folder_id, bytes);
-    emit docsToScanChanged();
+    m_scanTimer->start();
 }
 
 void Database::scanQueue()
 {
-    if (m_docsToScan.isEmpty())
+    if (m_docsToScan.isEmpty()) {
+        m_scanTimer->stop();
+        updateIndexingStatus();
         return;
+    }
 
     DocumentInfo info = dequeueDocument();
     const size_t countForFolder = countOfDocuments(info.folder);
@@ -818,6 +824,8 @@ void Database::scanQueue()
     QSqlDatabase::database().transaction();
     Q_ASSERT(document_id != -1);
     if (info.isPdf()) {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPage == 0);
+
         QPdfDocument doc;
         if (QPdfDocument::Error::None != doc.load(info.doc.canonicalFilePath())) {
             handleDocumentError("ERROR: Could not load pdf",
@@ -850,6 +858,8 @@ void Database::scanQueue()
             emit subtractCurrentBytesToIndex(info.folder, bytes - (bytesPerPage * doc.pageCount()));
         }
     } else {
+        updateFolderStatus(folder_id, FolderStatus::Embedding, -1, info.currentPosition == 0);
+
         QFile file(document_path);
         if (!file.open(QIODevice::ReadOnly)) {
             handleDocumentError("ERROR: Cannot open file for scanning",
@@ -884,7 +894,7 @@ void Database::scanQueue()
     return scheduleNext(folder_id, countForFolder);
 }
 
-void Database::scanDocuments(int folder_id, const QString &folder_path)
+void Database::scanDocuments(int folder_id, const QString &folder_path, bool isNew)
 {
 #if defined(DEBUG)
     qDebug() << "scanning folder for documents" << folder_path;
@@ -915,7 +925,7 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
     }
 
     if (!infos.isEmpty()) {
-        emit updateIndexing(folder_id, true);
+        updateFolderStatus(folder_id, FolderStatus::Started, infos.count(), false, isNew);
         enqueueDocuments(folder_id, infos);
     }
 }
@@ -925,7 +935,7 @@ void Database::start()
     connect(m_watcher, &QFileSystemWatcher::directoryChanged, this, &Database::directoryChanged);
     connect(m_embLLM, &EmbeddingLLM::embeddingsGenerated, this, &Database::handleEmbeddingsGenerated);
     connect(m_embLLM, &EmbeddingLLM::errorGenerated, this, &Database::handleErrorGenerated);
-    connect(this, &Database::docsToScanChanged, this, &Database::scanQueue);
+    m_scanTimer->callOnTimeout(this, &Database::scanQueue);
     if (!QSqlDatabase::drivers().contains("QSQLITE")) {
         qWarning() << "ERROR: missing sqllite driver";
     } else {
@@ -937,10 +947,11 @@ void Database::start()
     if (m_embeddings->fileExists() && !m_embeddings->load())
         qWarning() << "ERROR: Could not load embeddings";
 
-    addCurrentFolders();
+    int nAdded = addCurrentFolders();
+    Network::globalInstance()->trackEvent("localdocs_startup", { {"doc_collections_total", nAdded} });
 }
 
-void Database::addCurrentFolders()
+int Database::addCurrentFolders()
 {
 #if defined(DEBUG)
     qDebug() << "addCurrentFolders";
@@ -950,21 +961,26 @@ void Database::addCurrentFolders()
     QList<CollectionItem> collections;
     if (!selectAllFromCollections(q, &collections)) {
         qWarning() << "ERROR: Cannot select collections" << q.lastError();
-        return;
+        return 0;
     }
 
     emit collectionListUpdated(collections);
 
+    int nAdded = 0;
     for (const auto &i : collections)
-        addFolder(i.collection, i.folder_path);
+        nAdded += addFolder(i.collection, i.folder_path, true);
+
+    updateIndexingStatus();
+
+    return nAdded;
 }
 
-void Database::addFolder(const QString &collection, const QString &path)
+bool Database::addFolder(const QString &collection, const QString &path, bool fromDb)
 {
     QFileInfo info(path);
     if (!info.exists() || !info.isReadable()) {
         qWarning() << "ERROR: Cannot add folder that doesn't exist or not readable" << path;
-        return;
+        return false;
     }
 
     QSqlQuery q;
@@ -973,13 +989,13 @@ void Database::addFolder(const QString &collection, const QString &path)
     // See if the folder exists in the db
     if (!selectFolder(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
-        return;
+        return false;
     }
 
     // Add the folder
     if (folder_id == -1 && !addFolderToDB(q, path, &folder_id)) {
         qWarning() << "ERROR: Cannot add folder to db with path" << path << q.lastError();
-        return;
+        return false;
     }
 
     Q_ASSERT(folder_id != -1);
@@ -988,24 +1004,32 @@ void Database::addFolder(const QString &collection, const QString &path)
     QList<int> folders;
     if (!selectFoldersFromCollection(q, collection, &folders)) {
         qWarning() << "ERROR: Cannot select folders from collections" << collection << q.lastError();
-        return;
+        return false;
     }
 
+    bool added = false;
     if (!folders.contains(folder_id)) {
         if (!addCollection(q, collection, folder_id)) {
             qWarning() << "ERROR: Cannot add folder to collection" << collection << path << q.lastError();
-            return;
+            return false;
         }
 
         CollectionItem i;
         i.collection = collection;
         i.folder_path = path;
         i.folder_id = folder_id;
-        emit addCollectionItem(i);
+        emit addCollectionItem(i, fromDb);
+        added = true;
     }
 
     addFolderToWatch(path);
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, !fromDb);
+
+    if (!fromDb) {
+        updateIndexingStatus();
+    }
+
+    return added;
 }
 
 void Database::removeFolder(const QString &collection, const QString &path)
@@ -1285,5 +1309,69 @@ void Database::directoryChanged(const QString &path)
     cleanDB();
 
     // Rescan the documents associated with the folder
-    scanDocuments(folder_id, path);
+    scanDocuments(folder_id, path, false);
+    updateIndexingStatus();
+}
+
+void Database::updateIndexingStatus() {
+    Q_ASSERT(m_scanTimer->isActive() || m_docsToScan.isEmpty());
+    if (!m_indexingTimer.isValid() && m_scanTimer->isActive()) {
+        Network::globalInstance()->trackEvent("localdocs_indexing_start");
+        m_indexingTimer.start();
+    } else if (m_indexingTimer.isValid() && !m_scanTimer->isActive()) {
+        qint64 durationMs = m_indexingTimer.elapsed();
+        Network::globalInstance()->trackEvent("localdocs_indexing_complete", { {"$duration", durationMs / 1000.} });
+        m_indexingTimer.invalidate();
+    }
+}
+
+void Database::updateFolderStatus(int folder_id, Database::FolderStatus status, int numDocs, bool atStart, bool isNew) {
+    FolderStatusRecord *lastRecord = nullptr;
+    if (m_foldersBeingIndexed.contains(folder_id)) {
+        lastRecord = &m_foldersBeingIndexed[folder_id];
+    }
+    Q_ASSERT(lastRecord || status == FolderStatus::Started);
+
+    switch (status) {
+        case FolderStatus::Started:
+            if (lastRecord == nullptr) {
+                // record timestamp but don't send an event yet
+                m_foldersBeingIndexed.insert(folder_id, { QDateTime::currentMSecsSinceEpoch(), isNew, numDocs });
+                emit updateIndexing(folder_id, true);
+            }
+            break;
+        case FolderStatus::Embedding:
+            if (!lastRecord->docsChanged) {
+                Q_ASSERT(atStart);
+                // send start event with the original timestamp for folders that need updating
+                const auto *embeddingModels = ModelList::globalInstance()->installedEmbeddingModels();
+                Network::globalInstance()->trackEvent("localdocs_folder_indexing", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"document_count", lastRecord->numDocs},
+                    {"embedding_model", embeddingModels->defaultModelInfo().filename()},
+                    {"chunk_size", m_chunkSize},
+                    {"time", lastRecord->startTime},
+                });
+            }
+            lastRecord->docsChanged += atStart;
+            lastRecord->chunksRead++;
+            break;
+        case FolderStatus::Complete:
+            if (lastRecord->docsChanged) {
+                // send complete event for folders that were updated
+                qint64 durationMs = QDateTime::currentMSecsSinceEpoch() - lastRecord->startTime;
+                Network::globalInstance()->trackEvent("localdocs_folder_complete", {
+                    {"folder_id", folder_id},
+                    {"is_new_collection", lastRecord->isNew},
+                    {"documents_total", lastRecord->numDocs},
+                    {"documents_changed", lastRecord->docsChanged},
+                    {"chunks_read", lastRecord->chunksRead},
+                    {"$duration", durationMs / 1000.},
+                });
+            }
+            m_foldersBeingIndexed.remove(folder_id);
+            emit updateIndexing(folder_id, false);
+            break;
+    }
 }

gpt4all-chat/database.h

@@ -1,16 +1,19 @@
 #ifndef DATABASE_H
 #define DATABASE_H
 
+#include <QElapsedTimer>
+#include <QFileInfo>
+#include <QFileSystemWatcher>
 #include <QObject>
-#include <QtSql>
 #include <QQueue>
-#include <QFileInfo>
 #include <QThread>
-#include <QFileSystemWatcher>
+#include <QtSql>
 
 #include "embllm.h"
 
 class Embeddings;
+class QTimer;
+
 struct DocumentInfo
 {
     int folder;
@@ -58,9 +61,10 @@ public:
     virtual ~Database();
 
 public Q_SLOTS:
+    void start();
     void scanQueue();
-    void scanDocuments(int folder_id, const QString &folder_path);
-    void addFolder(const QString &collection, const QString &path);
+    void scanDocuments(int folder_id, const QString &folder_path, bool isNew);
+    bool addFolder(const QString &collection, const QString &path, bool fromDb);
     void removeFolder(const QString &collection, const QString &path);
     void retrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void cleanDB();
@@ -78,21 +82,22 @@ Q_SIGNALS:
     void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
     void updateCurrentEmbeddingsToIndex(int folder_id, size_t currentBytesToIndex);
     void updateTotalEmbeddingsToIndex(int folder_id, size_t totalBytesToIndex);
-    void addCollectionItem(const CollectionItem &item);
+    void addCollectionItem(const CollectionItem &item, bool fromDb);
     void removeFolderById(int folder_id);
-    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private Q_SLOTS:
-    void start();
     void directoryChanged(const QString &path);
     bool addFolderToWatch(const QString &path);
     bool removeFolderFromWatch(const QString &path);
-    void addCurrentFolders();
+    int addCurrentFolders();
     void handleEmbeddingsGenerated(const QVector<EmbeddingResult> &embeddings);
     void handleErrorGenerated(int folder_id, const QString &error);
 
 private:
+    enum class FolderStatus { Started, Embedding, Complete };
+    struct FolderStatusRecord { qint64 startTime; bool isNew; int numDocs, docsChanged, chunksRead; };
+
     void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
     size_t chunkStream(QTextStream &stream, int folder_id, int document_id, const QString &file,
         const QString &title, const QString &author, const QString &subject, const QString &keywords, int page,
@@ -107,10 +112,15 @@ private:
     void removeFolderFromDocumentQueue(int folder_id);
     void enqueueDocumentInternal(const DocumentInfo &info, bool prepend = false);
     void enqueueDocuments(int folder_id, const QVector<DocumentInfo> &infos);
+    void updateIndexingStatus();
+    void updateFolderStatus(int folder_id, FolderStatus status, int numDocs = -1, bool atStart = false, bool isNew = false);
 
 private:
     int m_chunkSize;
+    QTimer *m_scanTimer;
     QMap<int, QQueue<DocumentInfo>> m_docsToScan;
+    QElapsedTimer m_indexingTimer;
+    QMap<int, FolderStatusRecord> m_foldersBeingIndexed;
     QList<ResultInfo> m_retrieve;
     QThread m_dbThread;
     QFileSystemWatcher *m_watcher;

gpt4all-chat/download.cpp

@@ -75,15 +75,25 @@ bool Download::hasNewerRelease() const
     return compareVersions(versions.first(), currentVersion);
 }
 
-bool Download::isFirstStart() const
+bool Download::isFirstStart(bool writeVersion) const
 {
+    auto *mySettings = MySettings::globalInstance();
+
     QSettings settings;
     settings.sync();
     QString lastVersionStarted = settings.value("download/lastVersionStarted").toString();
     bool first = lastVersionStarted != QCoreApplication::applicationVersion();
-    settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
-    settings.sync();
-    return first;
+    if (first && writeVersion) {
+        settings.setValue("download/lastVersionStarted", QCoreApplication::applicationVersion());
+        // let the user select these again
+        settings.remove("network/usageStatsActive");
+        settings.remove("network/isActive");
+        settings.sync();
+        emit mySettings->networkUsageStatsActiveChanged();
+        emit mySettings->networkIsActiveChanged();
+    }
+
+    return first || !mySettings->isNetworkUsageStatsActiveSet() || !mySettings->isNetworkIsActiveSet();
 }
 
 void Download::updateReleaseNotes()
@@ -131,7 +141,7 @@ void Download::downloadModel(const QString &modelFile)
     ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::DownloadingRole, true }});
     ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
     QString url = !info.url().isEmpty() ? info.url() : "http://gpt4all.io/models/gguf/" + modelFile;
-    Network::globalInstance()->sendDownloadStarted(modelFile);
+    Network::globalInstance()->trackEvent("download_started", { {"model", modelFile} });
     QNetworkRequest request(url);
     request.setAttribute(QNetworkRequest::User, modelFile);
     request.setRawHeader("range", QString("bytes=%1-").arg(tempFile->pos()).toUtf8());
@@ -153,7 +163,7 @@ void Download::cancelDownload(const QString &modelFile)
         QNetworkReply *modelReply = m_activeDownloads.keys().at(i);
         QUrl url = modelReply->request().url();
         if (url.toString().endsWith(modelFile)) {
-            Network::globalInstance()->sendDownloadCanceled(modelFile);
+            Network::globalInstance()->trackEvent("download_canceled", { {"model", modelFile} });
 
             // Disconnect the signals
             disconnect(modelReply, &QNetworkReply::downloadProgress, this, &Download::handleDownloadProgress);
@@ -178,7 +188,8 @@ void Download::installModel(const QString &modelFile, const QString &apiKey)
     if (apiKey.isEmpty())
         return;
 
-    Network::globalInstance()->sendInstallModel(modelFile);
+    Network::globalInstance()->trackEvent("install_model", { {"model", modelFile} });
+
     QString filePath = MySettings::globalInstance()->modelPath() + modelFile;
     QFile file(filePath);
     if (file.open(QIODeviceBase::WriteOnly | QIODeviceBase::Text)) {
@@ -216,7 +227,7 @@ void Download::removeModel(const QString &modelFile)
         shouldRemoveInstalled = info.installed && !info.isClone() && (info.isDiscovered() || info.description() == "" /*indicates sideloaded*/);
         if (shouldRemoveInstalled)
             ModelList::globalInstance()->removeInstalled(info);
-        Network::globalInstance()->sendRemoveModel(modelFile);
+        Network::globalInstance()->trackEvent("remove_model", { {"model", modelFile} });
         file.remove();
     }
 
@@ -332,7 +343,11 @@ void Download::handleErrorOccurred(QNetworkReply::NetworkError code)
             .arg(modelReply->errorString());
     qWarning() << error;
     ModelList::globalInstance()->updateDataByFilename(modelFilename, {{ ModelList::DownloadErrorRole, error }});
-    Network::globalInstance()->sendDownloadError(modelFilename, (int)code, modelReply->errorString());
+    Network::globalInstance()->trackEvent("download_error", {
+        {"model", modelFilename},
+        {"code", (int)code},
+        {"error", modelReply->errorString()},
+    });
     cancelDownload(modelFilename);
 }
 
@@ -515,7 +530,7 @@ void Download::handleHashAndSaveFinished(bool success, const QString &error,
     // The hash and save should send back with tempfile closed
     Q_ASSERT(!tempFile->isOpen());
     QString modelFilename = modelReply->request().attribute(QNetworkRequest::User).toString();
-    Network::globalInstance()->sendDownloadFinished(modelFilename, success);
+    Network::globalInstance()->trackEvent("download_finished", { {"model", modelFilename}, {"success", success} });
 
     QVector<QPair<int, QVariant>> data {
         { ModelList::CalcHashRole, false },

gpt4all-chat/download.h

@@ -54,7 +54,7 @@ public:
     Q_INVOKABLE void cancelDownload(const QString &modelFile);
     Q_INVOKABLE void installModel(const QString &modelFile, const QString &apiKey);
     Q_INVOKABLE void removeModel(const QString &modelFile);
-    Q_INVOKABLE bool isFirstStart() const;
+    Q_INVOKABLE bool isFirstStart(bool writeVersion = false) const;
 
 public Q_SLOTS:
     void updateReleaseNotes();

gpt4all-chat/embllm.cpp

@@ -57,7 +57,14 @@ bool EmbeddingLLMWorker::loadModel()
         return true;
     }
 
-    m_model = LLModel::Implementation::construct(filePath.toStdString());
+    try {
+        m_model = LLModel::Implementation::construct(filePath.toStdString());
+    } catch (const std::exception &e) {
+        qWarning() << "WARNING: Could not load embedding model:" << e.what();
+        m_model = nullptr;
+        return false;
+    }
+
     // NOTE: explicitly loads model on CPU to avoid GPU OOM
     // TODO(cebtenzzre): support GPU-accelerated embeddings
     bool success = m_model->loadModel(filePath.toStdString(), 2048, 0);

gpt4all-chat/llm.cpp

@@ -49,7 +49,7 @@ bool LLM::checkForUpdates() const
     #pragma message "offline installer build will not check for updates!"
     return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
     #else
-    Network::globalInstance()->sendCheckForUpdates();
+    Network::globalInstance()->trackEvent("check_for_updates");
 
 #if defined(Q_OS_LINUX)
     QString tool("maintenancetool");

gpt4all-chat/localdocs.cpp

@@ -18,6 +18,8 @@ LocalDocs::LocalDocs()
     // Create the DB with the chunk size from settings
     m_database = new Database(MySettings::globalInstance()->localDocsChunkSize());
 
+    connect(this, &LocalDocs::requestStart, m_database,
+        &Database::start, Qt::QueuedConnection);
     connect(this, &LocalDocs::requestAddFolder, m_database,
         &Database::addFolder, Qt::QueuedConnection);
     connect(this, &LocalDocs::requestRemoveFolder, m_database,
@@ -50,8 +52,6 @@ LocalDocs::LocalDocs()
         m_localDocsModel, &LocalDocsModel::addCollectionItem, Qt::QueuedConnection);
     connect(m_database, &Database::removeFolderById,
         m_localDocsModel, &LocalDocsModel::removeFolderById, Qt::QueuedConnection);
-    connect(m_database, &Database::removeCollectionItem,
-        m_localDocsModel, &LocalDocsModel::removeCollectionItem, Qt::QueuedConnection);
     connect(m_database, &Database::collectionListUpdated,
         m_localDocsModel, &LocalDocsModel::collectionListUpdated, Qt::QueuedConnection);
 
@@ -68,7 +68,7 @@ void LocalDocs::addFolder(const QString &collection, const QString &path)
 {
     const QUrl url(path);
     const QString localPath = url.isLocalFile() ? url.toLocalFile() : path;
-    emit requestAddFolder(collection, localPath);
+    emit requestAddFolder(collection, localPath, false);
 }
 
 void LocalDocs::removeFolder(const QString &collection, const QString &path)

gpt4all-chat/localdocs.h

@@ -26,7 +26,8 @@ public Q_SLOTS:
     void aboutToQuit();
 
 Q_SIGNALS:
-    void requestAddFolder(const QString &collection, const QString &path);
+    void requestStart();
+    void requestAddFolder(const QString &collection, const QString &path, bool fromDb);
     void requestRemoveFolder(const QString &collection, const QString &path);
     void requestChunkSizeChange(int chunkSize);
     void localDocsModelChanged();

gpt4all-chat/localdocsdb.h

@@ -1,105 +0,0 @@
-#ifndef LOCALDOCS_H
-#define LOCALDOCS_H
-
-#include "localdocsmodel.h"
-
-#include <QObject>
-#include <QtSql>
-#include <QQueue>
-#include <QFileInfo>
-#include <QThread>
-#include <QFileSystemWatcher>
-
-struct DocumentInfo
-{
-    int folder;
-    QFileInfo doc;
-};
-
-struct CollectionItem {
-    QString collection;
-    QString folder_path;
-    int folder_id = -1;
-};
-Q_DECLARE_METATYPE(CollectionItem)
-
-class Database : public QObject
-{
-    Q_OBJECT
-public:
-    Database();
-
-public Q_SLOTS:
-    void scanQueue();
-    void scanDocuments(int folder_id, const QString &folder_path);
-    void addFolder(const QString &collection, const QString &path);
-    void removeFolder(const QString &collection, const QString &path);
-    void retrieveFromDB(const QList<QString> &collections, const QString &text);
-    void cleanDB();
-
-Q_SIGNALS:
-    void docsToScanChanged();
-    void retrieveResult(const QList<QString> &result);
-    void collectionListUpdated(const QList<CollectionItem> &collectionList);
-
-private Q_SLOTS:
-    void start();
-    void directoryChanged(const QString &path);
-    bool addFolderToWatch(const QString &path);
-    bool removeFolderFromWatch(const QString &path);
-    void addCurrentFolders();
-    void updateCollectionList();
-
-private:
-    void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
-    void chunkStream(QTextStream &stream, int document_id);
-    void handleDocumentErrorAndScheduleNext(const QString &errorMessage,
-        int document_id, const QString &document_path, const QSqlError &error);
-
-private:
-    QQueue<DocumentInfo> m_docsToScan;
-    QList<QString> m_retrieve;
-    QThread m_dbThread;
-    QFileSystemWatcher *m_watcher;
-};
-
-class LocalDocs : public QObject
-{
-    Q_OBJECT
-    Q_PROPERTY(LocalDocsModel *localDocsModel READ localDocsModel NOTIFY localDocsModelChanged)
-
-public:
-    static LocalDocs *globalInstance();
-
-    LocalDocsModel *localDocsModel() const { return m_localDocsModel; }
-
-    void addFolder(const QString &collection, const QString &path);
-    void removeFolder(const QString &collection, const QString &path);
-
-    QList<QString> result() const { return m_retrieveResult; }
-    void requestRetrieve(const QList<QString> &collections, const QString &text);
-
-Q_SIGNALS:
-    void requestAddFolder(const QString &collection, const QString &path);
-    void requestRemoveFolder(const QString &collection, const QString &path);
-    void requestRetrieveFromDB(const QList<QString> &collections, const QString &text);
-    void receivedResult();
-    void localDocsModelChanged();
-
-private Q_SLOTS:
-    void handleRetrieveResult(const QList<QString> &result);
-    void handleCollectionListUpdated(const QList<CollectionItem> &collectionList);
-
-private:
-    LocalDocsModel *m_localDocsModel;
-    Database *m_database;
-    QList<QString> m_retrieveResult;
-    QList<CollectionItem> m_collectionList;
-
-private:
-    explicit LocalDocs();
-    ~LocalDocs() {}
-    friend class MyLocalDocs;
-};
-
-#endif // LOCALDOCS_H

gpt4all-chat/localdocsmodel.cpp

@@ -1,6 +1,7 @@
 #include "localdocsmodel.h"
 
 #include "localdocs.h"
+#include "network.h"
 
 LocalDocsCollectionsModel::LocalDocsCollectionsModel(QObject *parent)
     : QSortFilterProxyModel(parent)
@@ -158,50 +159,43 @@ void LocalDocsModel::updateTotalEmbeddingsToIndex(int folder_id, size_t totalEmb
         [](CollectionItem& item, size_t val) { item.totalEmbeddingsToIndex += val; }, {TotalEmbeddingsToIndexRole});
 }
 
-void LocalDocsModel::addCollectionItem(const CollectionItem &item)
+void LocalDocsModel::addCollectionItem(const CollectionItem &item, bool fromDb)
 {
     beginInsertRows(QModelIndex(), m_collectionList.size(), m_collectionList.size());
     m_collectionList.append(item);
     endInsertRows();
+
+    if (!fromDb) {
+        Network::globalInstance()->trackEvent("doc_collection_add", {
+            {"collection_count", m_collectionList.count()},
+        });
+    }
 }
 
-void LocalDocsModel::removeFolderById(int folder_id)
-{
+void LocalDocsModel::removeCollectionIf(std::function<bool(CollectionItem)> const &predicate) {
     for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).folder_id == folder_id) {
+        if (predicate(m_collectionList.at(i))) {
             beginRemoveRows(QModelIndex(), i, i);
             m_collectionList.removeAt(i);
             endRemoveRows();
+
+            Network::globalInstance()->trackEvent("doc_collection_remove", {
+                {"collection_count", m_collectionList.count()},
+            });
         } else {
             ++i;
         }
     }
 }
 
-void LocalDocsModel::removeCollectionPath(const QString &name, const QString &path)
+void LocalDocsModel::removeFolderById(int folder_id)
 {
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).collection == name && m_collectionList.at(i).folder_path == path) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
+    removeCollectionIf([folder_id](const auto &c) { return c.folder_id == folder_id; });
 }
 
-void LocalDocsModel::removeCollectionItem(const QString &collectionName)
+void LocalDocsModel::removeCollectionPath(const QString &name, const QString &path)
 {
-    for (int i = 0; i < m_collectionList.size();) {
-        if (m_collectionList.at(i).collection == collectionName) {
-            beginRemoveRows(QModelIndex(), i, i);
-            m_collectionList.removeAt(i);
-            endRemoveRows();
-        } else {
-            ++i;
-        }
-    }
+    removeCollectionIf([&name, &path](const auto &c) { return c.collection == name && c.folder_path == path; });
 }
 
 void LocalDocsModel::collectionListUpdated(const QList<CollectionItem> &collectionList)

gpt4all-chat/localdocsmodel.h

@@ -55,10 +55,9 @@ public Q_SLOTS:
     void updateTotalBytesToIndex(int folder_id, size_t totalBytesToIndex);
     void updateCurrentEmbeddingsToIndex(int folder_id, size_t currentBytesToIndex);
     void updateTotalEmbeddingsToIndex(int folder_id, size_t totalBytesToIndex);
-    void addCollectionItem(const CollectionItem &item);
+    void addCollectionItem(const CollectionItem &item, bool fromDb);
     void removeFolderById(int folder_id);
     void removeCollectionPath(const QString &name, const QString &path);
-    void removeCollectionItem(const QString &collectionName);
     void collectionListUpdated(const QList<CollectionItem> &collectionList);
 
 private:
@@ -66,6 +65,7 @@ private:
     void updateField(int folder_id, T value,
         const std::function<void(CollectionItem&, T)>& updater,
         const QVector<int>& roles);
+    void removeCollectionIf(std::function<bool(CollectionItem)> const &predicate);
 
 private:
     QList<CollectionItem> m_collectionList;

gpt4all-chat/mysettings.cpp

@@ -910,15 +910,23 @@ bool MySettings::networkIsActive() const
     return setting.value("network/isActive", default_networkIsActive).toBool();
 }
 
-void MySettings::setNetworkIsActive(bool b)
+bool MySettings::isNetworkIsActiveSet() const
 {
-    if (networkIsActive() == b)
-        return;
+    QSettings setting;
+    setting.sync();
+    return setting.value("network/isActive").isValid();
+}
 
+void MySettings::setNetworkIsActive(bool b)
+{
     QSettings setting;
-    setting.setValue("network/isActive", b);
     setting.sync();
-    emit networkIsActiveChanged();
+    auto cur = setting.value("network/isActive");
+    if (!cur.isValid() || cur.toBool() != b) {
+        setting.setValue("network/isActive", b);
+        setting.sync();
+        emit networkIsActiveChanged();
+    }
 }
 
 bool MySettings::networkUsageStatsActive() const
@@ -928,13 +936,21 @@ bool MySettings::networkUsageStatsActive() const
     return setting.value("network/usageStatsActive", default_networkUsageStatsActive).toBool();
 }
 
-void MySettings::setNetworkUsageStatsActive(bool b)
+bool MySettings::isNetworkUsageStatsActiveSet() const
 {
-    if (networkUsageStatsActive() == b)
-        return;
+    QSettings setting;
+    setting.sync();
+    return setting.value("network/usageStatsActive").isValid();
+}
 
+void MySettings::setNetworkUsageStatsActive(bool b)
+{
     QSettings setting;
-    setting.setValue("network/usageStatsActive", b);
     setting.sync();
-    emit networkUsageStatsActiveChanged();
+    auto cur = setting.value("network/usageStatsActive");
+    if (!cur.isValid() || cur.toBool() != b) {
+        setting.setValue("network/usageStatsActive", b);
+        setting.sync();
+        emit networkUsageStatsActiveChanged();
+    }
 }

gpt4all-chat/mysettings.h

@@ -129,8 +129,10 @@ public:
     QString networkAttribution() const;
     void setNetworkAttribution(const QString &a);
     bool networkIsActive() const;
+    Q_INVOKABLE bool isNetworkIsActiveSet() const;
     void setNetworkIsActive(bool b);
     bool networkUsageStatsActive() const;
+    Q_INVOKABLE bool isNetworkUsageStatsActiveSet() const;
     void setNetworkUsageStatsActive(bool b);
     int networkPort() const;
     void setNetworkPort(int c);

gpt4all-chat/network.cpp

@@ -1,8 +1,13 @@
 #include "network.h"
-#include "llm.h"
+
 #include "chatlistmodel.h"
+#include "download.h"
+#include "llm.h"
+#include "localdocs.h"
 #include "mysettings.h"
 
+#include <cmath>
+
 #include <QCoreApplication>
 #include <QGuiApplication>
 #include <QUuid>
@@ -14,16 +19,49 @@
 
 //#define DEBUG
 
+static const char MIXPANEL_TOKEN[] = "ce362e568ddaee16ed243eaffb5860a2";
+
 #if defined(Q_OS_MAC)
+
 #include <sys/sysctl.h>
-std::string getCPUModel() {
+static QString getCPUModel() {
     char buffer[256];
     size_t bufferlen = sizeof(buffer);
     sysctlbyname("machdep.cpu.brand_string", &buffer, &bufferlen, NULL, 0);
-    return std::string(buffer);
+    return buffer;
 }
+
+#elif defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)
+
+#define get_cpuid(level, a, b, c, d) asm volatile("cpuid" : "=a" (a), "=b" (b), "=c" (c), "=d" (d) : "0" (level) : "memory")
+
+static QString getCPUModel() {
+    unsigned regs[12];
+
+    // EAX=800000000h: Get Highest Extended Function Implemented
+    get_cpuid(0x80000000, regs[0], regs[1], regs[2], regs[3]);
+    if (regs[0] < 0x80000004)
+        return "(unknown)";
+
+    // EAX=800000002h-800000004h: Processor Brand String
+    get_cpuid(0x80000002, regs[0], regs[1], regs[ 2], regs[ 3]);
+    get_cpuid(0x80000003, regs[4], regs[5], regs[ 6], regs[ 7]);
+    get_cpuid(0x80000004, regs[8], regs[9], regs[10], regs[11]);
+
+    char str[sizeof(regs) + 1];
+    memcpy(str, regs, sizeof(regs));
+    str[sizeof(regs)] = 0;
+
+    return QString(str).trimmed();
+}
+
+#else
+
+static QString getCPUModel() { return "(non-x86)"; }
+
 #endif
 
+
 class MyNetwork: public Network { };
 Q_GLOBAL_STATIC(MyNetwork, networkInstance)
 Network *Network::globalInstance()
@@ -33,38 +71,41 @@ Network *Network::globalInstance()
 
 Network::Network()
     : QObject{nullptr}
-    , m_shouldSendStartup(false)
 {
     QSettings settings;
     settings.sync();
     m_uniqueId = settings.value("uniqueId", generateUniqueId()).toString();
     settings.setValue("uniqueId", m_uniqueId);
     settings.sync();
-    connect(MySettings::globalInstance(), &MySettings::networkIsActiveChanged, this, &Network::handleIsActiveChanged);
-    connect(MySettings::globalInstance(), &MySettings::networkUsageStatsActiveChanged, this, &Network::handleUsageStatsActiveChanged);
-    if (MySettings::globalInstance()->networkIsActive())
+    m_sessionId = generateUniqueId();
+
+    // allow sendMixpanel to be called from any thread
+    connect(this, &Network::requestMixpanel, this, &Network::sendMixpanel, Qt::QueuedConnection);
+
+    const auto *mySettings = MySettings::globalInstance();
+    connect(mySettings, &MySettings::networkIsActiveChanged, this, &Network::handleIsActiveChanged);
+    connect(mySettings, &MySettings::networkUsageStatsActiveChanged, this, &Network::handleUsageStatsActiveChanged);
+
+    m_hasSentOptIn  = !Download::globalInstance()->isFirstStart() &&  mySettings->networkUsageStatsActive();
+    m_hasSentOptOut = !Download::globalInstance()->isFirstStart() && !mySettings->networkUsageStatsActive();
+
+    if (mySettings->networkIsActive())
         sendHealth();
-    if (MySettings::globalInstance()->networkUsageStatsActive())
-        sendIpify();
     connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this,
         &Network::handleSslErrors);
 }
 
-void Network::handleIsActiveChanged()
+// NOTE: this won't be useful until we make it possible to change this via the settings page
+void Network::handleUsageStatsActiveChanged()
 {
-    if (MySettings::globalInstance()->networkUsageStatsActive())
-        sendHealth();
+    if (!MySettings::globalInstance()->networkUsageStatsActive())
+        m_sendUsageStats = false;
 }
 
-void Network::handleUsageStatsActiveChanged()
+void Network::handleIsActiveChanged()
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        sendOptOut();
-    else {
-        // model might be loaded already when user opt-in for first time
-        sendStartup();
-        sendIpify();
-    }
+    if (MySettings::globalInstance()->networkUsageStatsActive())
+        sendHealth();
 }
 
 QString Network::generateUniqueId() const
@@ -167,8 +208,8 @@ void Network::handleSslErrors(QNetworkReply *reply, const QList<QSslError> &erro
 void Network::sendOptOut()
 {
     QJsonObject properties;
-    properties.insert("token", "ce362e568ddaee16ed243eaffb5860a2");
-    properties.insert("time", QDateTime::currentSecsSinceEpoch());
+    properties.insert("token", MIXPANEL_TOKEN);
+    properties.insert("time", QDateTime::currentMSecsSinceEpoch());
     properties.insert("distinct_id", m_uniqueId);
     properties.insert("$insert_id", generateUniqueId());
 
@@ -181,7 +222,7 @@ void Network::sendOptOut()
 
     QJsonDocument doc;
     doc.setArray(array);
-    sendMixpanel(doc.toJson(QJsonDocument::Compact), true /*isOptOut*/);
+    emit requestMixpanel(doc.toJson(QJsonDocument::Compact), true /*isOptOut*/);
 
 #if defined(DEBUG)
     printf("%s %s\n", qPrintable("opt_out"), qPrintable(doc.toJson(QJsonDocument::Indented)));
@@ -189,215 +230,76 @@ void Network::sendOptOut()
 #endif
 }
 
-void Network::sendModelLoaded()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("model_load");
-}
-
-void Network::sendResetContext(int conversationLength)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-
-    KeyValue kv;
-    kv.key = QString("length");
-    kv.value = QJsonValue(conversationLength);
-    sendMixpanelEvent("reset_context", QVector<KeyValue>{kv});
-}
-
 void Network::sendStartup()
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    m_shouldSendStartup = true;
-    if (m_ipify.isEmpty())
-        return; // when it completes it will send
-    sendMixpanelEvent("startup");
-}
-
-void Network::sendCheckForUpdates()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("check_for_updates");
-}
-
-void Network::sendModelDownloaderDialog()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("download_dialog");
-}
-
-void Network::sendInstallModel(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("install_model", QVector<KeyValue>{kv});
-}
-
-void Network::sendRemoveModel(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("remove_model", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadStarted(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("download_started", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadCanceled(const QString &model)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    sendMixpanelEvent("download_canceled", QVector<KeyValue>{kv});
-}
-
-void Network::sendDownloadError(const QString &model, int code, const QString &errorString)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    KeyValue kvCode;
-    kvCode.key = QString("code");
-    kvCode.value = QJsonValue(code);
-    KeyValue kvError;
-    kvError.key = QString("error");
-    kvError.value = QJsonValue(errorString);
-    sendMixpanelEvent("download_error", QVector<KeyValue>{kv, kvCode, kvError});
-}
-
-void Network::sendDownloadFinished(const QString &model, bool success)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("model");
-    kv.value = QJsonValue(model);
-    KeyValue kvSuccess;
-    kvSuccess.key = QString("success");
-    kvSuccess.value = QJsonValue(success);
-    sendMixpanelEvent("download_finished", QVector<KeyValue>{kv, kvSuccess});
-}
-
-void Network::sendSettingsDialog()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("settings_dialog");
-}
-
-void Network::sendNetworkToggled(bool isActive)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("isActive");
-    kv.value = QJsonValue(isActive);
-    sendMixpanelEvent("network_toggled", QVector<KeyValue>{kv});
-}
-
-void Network::sendNewChat(int count)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    KeyValue kv;
-    kv.key = QString("number_of_chats");
-    kv.value = QJsonValue(count);
-    sendMixpanelEvent("new_chat", QVector<KeyValue>{kv});
-}
-
-void Network::sendRemoveChat()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("remove_chat");
-}
-
-void Network::sendRenameChat()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("rename_chat");
-}
-
-void Network::sendChatStarted()
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("chat_started");
-}
-
-void Network::sendRecalculatingContext(int conversationLength)
-{
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
+    const auto *mySettings = MySettings::globalInstance();
+    Q_ASSERT(mySettings->isNetworkUsageStatsActiveSet());
+    if (!mySettings->networkUsageStatsActive()) {
+        // send a single opt-out per session after the user has made their selections,
+        // unless this is a normal start (same version) and the user was already opted out
+        if (!m_hasSentOptOut) {
+            sendOptOut();
+            m_hasSentOptOut = true;
+        }
         return;
+    }
 
-    KeyValue kv;
-    kv.key = QString("length");
-    kv.value = QJsonValue(conversationLength);
-    sendMixpanelEvent("recalc_context", QVector<KeyValue>{kv});
+    // only chance to enable usage stats is at the start of a new session
+    m_sendUsageStats = true;
+
+    const auto *display = QGuiApplication::primaryScreen();
+    trackEvent("startup", {
+        {"$screen_dpi", std::round(display->physicalDotsPerInch())},
+        {"display", QString("%1x%2").arg(display->size().width()).arg(display->size().height())},
+        {"ram", LLM::globalInstance()->systemTotalRAMInGB()},
+        {"cpu", getCPUModel()},
+        {"datalake_active", mySettings->networkIsActive()},
+    });
+    sendIpify();
+
+    // mirror opt-out logic so the ratio can be used to infer totals
+    if (!m_hasSentOptIn) {
+        trackEvent("opt_in");
+        m_hasSentOptIn = true;
+    }
 }
 
-void Network::sendNonCompatHardware()
+void Network::trackChatEvent(const QString &ev, QVariantMap props)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
-        return;
-    sendMixpanelEvent("noncompat_hardware");
+    const auto &curChat = ChatListModel::globalInstance()->currentChat();
+    if (!props.contains("model"))
+        props.insert("model", curChat->modelInfo().filename());
+    props.insert("actualDevice", curChat->device());
+    props.insert("doc_collections_enabled", curChat->collectionList().count());
+    props.insert("doc_collections_total", LocalDocs::globalInstance()->localDocsModel()->rowCount());
+    props.insert("datalake_active", MySettings::globalInstance()->networkIsActive());
+    props.insert("using_server", ChatListModel::globalInstance()->currentChat()->isServer());
+    trackEvent(ev, props);
 }
 
-void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &values)
+void Network::trackEvent(const QString &ev, const QVariantMap &props)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive())
+    if (!m_sendUsageStats)
         return;
 
     Q_ASSERT(ChatListModel::globalInstance()->currentChat());
     QJsonObject properties;
-    properties.insert("token", "ce362e568ddaee16ed243eaffb5860a2");
-    properties.insert("time", QDateTime::currentSecsSinceEpoch());
-    properties.insert("distinct_id", m_uniqueId);
+
+    properties.insert("token", MIXPANEL_TOKEN);
+    if (!props.contains("time"))
+        properties.insert("time", QDateTime::currentMSecsSinceEpoch());
+    properties.insert("distinct_id", m_uniqueId); // effectively a device ID
     properties.insert("$insert_id", generateUniqueId());
-    properties.insert("$os", QSysInfo::prettyProductName());
+
     if (!m_ipify.isEmpty())
         properties.insert("ip", m_ipify);
-    properties.insert("name", QCoreApplication::applicationName() + " v"
-        + QCoreApplication::applicationVersion());
-    properties.insert("model", ChatListModel::globalInstance()->currentChat()->modelInfo().filename());
-    properties.insert("requestedDevice", MySettings::globalInstance()->device());
-    properties.insert("actualDevice", ChatListModel::globalInstance()->currentChat()->device());
-
-    // Some additional startup information
-    if (ev == "startup") {
-        const QSize display = QGuiApplication::primaryScreen()->size();
-        properties.insert("display", QString("%1x%2").arg(display.width()).arg(display.height()));
-        properties.insert("ram", LLM::globalInstance()->systemTotalRAMInGB());
-#if defined(Q_OS_MAC)
-        properties.insert("cpu", QString::fromStdString(getCPUModel()));
-#endif
-    }
 
-    for (const auto& p : values)
-        properties.insert(p.key, p.value);
+    properties.insert("$os", QSysInfo::prettyProductName());
+    properties.insert("session_id", m_sessionId);
+    properties.insert("name", QCoreApplication::applicationName() + " v" + QCoreApplication::applicationVersion());
+
+    for (const auto &[key, value]: props.asKeyValueRange())
+        properties.insert(key, QJsonValue::fromVariant(value));
 
     QJsonObject event;
     event.insert("event", ev);
@@ -408,7 +310,7 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
 
     QJsonDocument doc;
     doc.setArray(array);
-    sendMixpanel(doc.toJson(QJsonDocument::Compact));
+    emit requestMixpanel(doc.toJson(QJsonDocument::Compact));
 
 #if defined(DEBUG)
     printf("%s %s\n", qPrintable(ev), qPrintable(doc.toJson(QJsonDocument::Indented)));
@@ -418,7 +320,7 @@ void Network::sendMixpanelEvent(const QString &ev, const QVector<KeyValue> &valu
 
 void Network::sendIpify()
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive() || !m_ipify.isEmpty())
+    if (!m_sendUsageStats || !m_ipify.isEmpty())
         return;
 
     QUrl ipifyUrl("https://api.ipify.org");
@@ -433,7 +335,7 @@ void Network::sendIpify()
 
 void Network::sendMixpanel(const QByteArray &json, bool isOptOut)
 {
-    if (!MySettings::globalInstance()->networkUsageStatsActive() && !isOptOut)
+    if (!m_sendUsageStats)
         return;
 
     QUrl trackUrl("https://api.mixpanel.com/track");
@@ -449,7 +351,6 @@ void Network::sendMixpanel(const QByteArray &json, bool isOptOut)
 
 void Network::handleIpifyFinished()
 {
-    Q_ASSERT(MySettings::globalInstance()->networkUsageStatsActive());
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply)
         return;
@@ -469,8 +370,7 @@ void Network::handleIpifyFinished()
 #endif
     reply->deleteLater();
 
-    if (m_shouldSendStartup)
-        sendStartup();
+    trackEvent("ipify_complete");
 }
 
 void Network::handleMixpanelFinished()

gpt4all-chat/network.h

@@ -19,31 +19,15 @@ public:
 
     Q_INVOKABLE QString generateUniqueId() const;
     Q_INVOKABLE bool sendConversation(const QString &ingestId, const QString &conversation);
+    Q_INVOKABLE void trackChatEvent(const QString &event, QVariantMap props = QVariantMap());
+    Q_INVOKABLE void trackEvent(const QString &event, const QVariantMap &props = QVariantMap());
 
 Q_SIGNALS:
     void healthCheckFailed(int code);
+    void requestMixpanel(const QByteArray &json, bool isOptOut = false);
 
 public Q_SLOTS:
-    void sendOptOut();
-    void sendModelLoaded();
     void sendStartup();
-    void sendCheckForUpdates();
-    Q_INVOKABLE void sendModelDownloaderDialog();
-    Q_INVOKABLE void sendResetContext(int conversationLength);
-    void sendInstallModel(const QString &model);
-    void sendRemoveModel(const QString &model);
-    void sendDownloadStarted(const QString &model);
-    void sendDownloadCanceled(const QString &model);
-    void sendDownloadError(const QString &model, int code, const QString &errorString);
-    void sendDownloadFinished(const QString &model, bool success);
-    Q_INVOKABLE void sendSettingsDialog();
-    Q_INVOKABLE void sendNetworkToggled(bool active);
-    Q_INVOKABLE void sendNewChat(int count);
-    Q_INVOKABLE void sendRemoveChat();
-    Q_INVOKABLE void sendRenameChat();
-    Q_INVOKABLE void sendNonCompatHardware();
-    void sendChatStarted();
-    void sendRecalculatingContext(int conversationLength);
 
 private Q_SLOTS:
     void handleIpifyFinished();
@@ -53,18 +37,21 @@ private Q_SLOTS:
     void handleMixpanelFinished();
     void handleIsActiveChanged();
     void handleUsageStatsActiveChanged();
+    void sendMixpanel(const QByteArray &json, bool isOptOut);
 
 private:
+    void sendOptOut();
     void sendHealth();
     void sendIpify();
-    void sendMixpanelEvent(const QString &event, const QVector<KeyValue> &values = QVector<KeyValue>());
-    void sendMixpanel(const QByteArray &json, bool isOptOut = false);
     bool packageAndSendJson(const QString &ingestId, const QString &json);
 
 private:
-    bool m_shouldSendStartup;
+    bool m_sendUsageStats = false;
+    bool m_hasSentOptIn;
+    bool m_hasSentOptOut;
     QString m_ipify;
     QString m_uniqueId;
+    QString m_sessionId;
     QNetworkAccessManager m_networkManager;
     QVector<QNetworkReply*> m_activeUploads;
 

gpt4all-chat/qml/ChatDrawer.qml

@@ -40,7 +40,7 @@ Rectangle {
             Accessible.description: qsTr("Create a new chat")
             onClicked: {
                 ChatListModel.addChat();
-                Network.sendNewChat(ChatListModel.count)
+                Network.trackEvent("new_chat", {"number_of_chats": ChatListModel.count})
             }
         }
 
@@ -110,8 +110,8 @@ Rectangle {
                             // having focus
                             if (chatName.readOnly)
                                 return;
+                            Network.trackChatEvent("rename_chat")
                             changeName();
-                            Network.sendRenameChat()
                         }
                         function changeName() {
                             ChatListModel.get(index).name = chatName.text
@@ -194,8 +194,8 @@ Rectangle {
                                     color: "transparent"
                                 }
                                 onClicked: {
+                                    Network.trackChatEvent("remove_chat")
                                     ChatListModel.removeChat(ChatListModel.get(index))
-                                    Network.sendRemoveChat()
                                 }
                                 Accessible.role: Accessible.Button
                                 Accessible.name: qsTr("Confirm chat deletion")

gpt4all-chat/qml/ChatView.qml

@@ -1,16 +1,18 @@
+import Qt5Compat.GraphicalEffects
 import QtCore
 import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
 import QtQuick.Layouts
-import Qt5Compat.GraphicalEffects
-import llm
+
 import chatlistmodel
 import download
-import modellist
-import network
 import gpt4all
+import llm
+import localdocs
+import modellist
 import mysettings
+import network
 
 Rectangle {
     id: window
@@ -29,6 +31,10 @@ Rectangle {
         startupDialogs();
     }
 
+    Component.onDestruction: {
+        Network.trackEvent("session_end")
+    }
+
     Connections {
         target: firstStartDialog
         function onClosed() {
@@ -66,12 +72,12 @@ Rectangle {
     }
 
     property bool hasShownModelDownload: false
-    property bool hasShownFirstStart: false
+    property bool hasCheckedFirstStart: false
     property bool hasShownSettingsAccess: false
 
     function startupDialogs() {
         if (!LLM.compatHardware()) {
-            Network.sendNonCompatHardware();
+            Network.trackEvent("noncompat_hardware")
             errorCompatHardware.open();
             return;
         }
@@ -84,10 +90,18 @@ Rectangle {
         }
 
         // check for first time start of this version
-        if (!hasShownFirstStart && Download.isFirstStart()) {
-            firstStartDialog.open();
-            hasShownFirstStart = true;
-            return;
+        if (!hasCheckedFirstStart) {
+            if (Download.isFirstStart(/*writeVersion*/ true)) {
+                firstStartDialog.open();
+                return;
+            }
+
+            // send startup or opt-out now that the user has made their choice
+            Network.sendStartup()
+            // start localdocs
+            LocalDocs.requestStart()
+
+            hasCheckedFirstStart = true
         }
 
         // check for any current models and if not, open download dialog once
@@ -547,7 +561,6 @@ Rectangle {
         onClicked: {
             if (MySettings.networkIsActive) {
                 MySettings.networkIsActive = false
-                Network.sendNetworkToggled(false);
             } else
                 networkDialog.open()
         }
@@ -733,7 +746,7 @@ Rectangle {
         Accessible.description: qsTr("Reset the context and erase current conversation")
 
         onClicked: {
-            Network.sendResetContext(chatModel.count)
+            Network.trackChatEvent("reset_context", { "length": chatModel.count })
             currentChat.reset();
             currentChat.processSystemPrompt();
         }
@@ -1288,9 +1301,11 @@ Rectangle {
                     var listElement = chatModel.get(index);
 
                     if (currentChat.responseInProgress) {
+                        Network.trackChatEvent("stop_generating_clicked")
                         listElement.stopped = true
                         currentChat.stopGenerating()
                     } else {
+                        Network.trackChatEvent("regenerate_clicked")
                         currentChat.regenerateResponse()
                         if (chatModel.count) {
                             if (listElement.name === qsTr("Response: ")) {
@@ -1405,6 +1420,7 @@ Rectangle {
                     if (textInput.text === "")
                         return
 
+                    Network.trackChatEvent("send_message")
                     currentChat.stopGenerating()
                     currentChat.newPromptResponsePair(textInput.text);
                     currentChat.prompt(textInput.text,

gpt4all-chat/qml/ModelDownloaderDialog.qml

@@ -19,7 +19,7 @@ MyDialog {
     property bool showEmbeddingModels: false
 
     onOpened: {
-        Network.sendModelDownloaderDialog();
+        Network.trackEvent("download_dialog")
 
         if (showEmbeddingModels) {
             ModelList.downloadableModels.expanded = true

gpt4all-chat/qml/NetworkDialog.qml

@@ -100,16 +100,10 @@ NOTE: By turning on this feature, you will be sending your data to the GPT4All O
     }
 
     onAccepted: {
-        if (MySettings.networkIsActive)
-            return
-        MySettings.networkIsActive = true;
-        Network.sendNetworkToggled(true);
+        MySettings.networkIsActive = true
     }
 
     onRejected: {
-        if (!MySettings.networkIsActive)
-            return
-        MySettings.networkIsActive = false;
-        Network.sendNetworkToggled(false);
+        MySettings.networkIsActive = false
     }
 }

gpt4all-chat/qml/SettingsDialog.qml

@@ -16,7 +16,7 @@ MyDialog {
     modal: true
     padding: 20
     onOpened: {
-        Network.sendSettingsDialog();
+        Network.trackEvent("settings_dialog")
     }
 
     signal downloadClicked

gpt4all-chat/qml/StartupDialog.qml

@@ -123,8 +123,6 @@ model release that uses your data!")
                 buttons: optInStatisticsRadio.children
                 onClicked: {
                     MySettings.networkUsageStatsActive = optInStatisticsRadio.checked
-                    if (!optInStatisticsRadio.checked)
-                        Network.sendOptOut();
                     if (optInNetworkRadio.choiceMade && optInStatisticsRadio.choiceMade)
                         startupDialog.close();
                 }
@@ -140,7 +138,7 @@ model release that uses your data!")
 
                 RadioButton {
                     id: optInStatisticsRadioYes
-                    checked: false
+                    checked: MySettings.networkUsageStatsActive
                     text: qsTr("Yes")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
@@ -182,6 +180,7 @@ model release that uses your data!")
                 }
                 RadioButton {
                     id: optInStatisticsRadioNo
+                    checked: MySettings.isNetworkUsageStatsActiveSet() && !MySettings.networkUsageStatsActive
                     text: qsTr("No")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
@@ -254,7 +253,7 @@ model release that uses your data!")
 
                 RadioButton {
                     id: optInNetworkRadioYes
-                    checked: false
+                    checked: MySettings.networkIsActive
                     text: qsTr("Yes")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton
@@ -296,6 +295,7 @@ model release that uses your data!")
                 }
                 RadioButton {
                     id: optInNetworkRadioNo
+                    checked: MySettings.isNetworkIsActiveSet() && !MySettings.networkIsActive
                     text: qsTr("No")
                     font.pixelSize: theme.fontSizeLarge
                     Accessible.role: Accessible.RadioButton