Dev

examples/KWS_ASR/KWS_ASR.ino

@@ -27,9 +27,12 @@ void setup()
     // language = "zh_CN";
 
     /* Init module serial port */
-    Serial2.begin(115200, SERIAL_8N1, 16, 17);  // Basic
-    // Serial2.begin(115200, SERIAL_8N1, 13, 14);  // Core2
-    // Serial2.begin(115200, SERIAL_8N1, 18, 17);  // CoreS3
+    // int rxd = 16, txd = 17;  // Basic
+    // int rxd = 13, txd = 14;  // Core2
+    // int rxd = 18, txd = 17;  // CoreS3
+    int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
+    int txd = M5.getPin(m5::pin_name_t::port_c_txd);
+    Serial2.begin(115200, SERIAL_8N1, rxd, txd);
 
     /* Init module */
     module_llm.begin(&Serial2);

examples/KWS_VAD_Whisper/KWS_VAD_Whisper.ino

@@ -0,0 +1,112 @@
+/*
+ * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#include <Arduino.h>
+#include <M5Unified.h>
+#include <M5ModuleLLM.h>
+
+M5ModuleLLM module_llm;
+
+/* Must be capitalized */
+String wake_up_keyword = "HELLO";
+// String wake_up_keyword = "你好你好";
+String kws_work_id;
+String vad_work_id;
+String whisper_work_id;
+String language;
+
+void setup()
+{
+    M5.begin();
+    M5.Display.setTextSize(2);
+    M5.Display.setTextScroll(true);
+    // M5.Display.setFont(&fonts::efontCN_12);  // Support Chinese display
+    // M5.Display.setFont(&fonts::efontJA_12);  // Support Japanese display
+
+    language = "en_US";
+    // language = "zh_CN";
+
+    /* Init module serial port */
+    // int rxd = 16, txd = 17;  // Basic
+    // int rxd = 13, txd = 14;  // Core2
+    // int rxd = 18, txd = 17;  // CoreS3
+    int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
+    int txd = M5.getPin(m5::pin_name_t::port_c_txd);
+    Serial2.begin(115200, SERIAL_8N1, rxd, txd);
+
+    /* Init module */
+    module_llm.begin(&Serial2);
+
+    /* Make sure module is connected */
+    M5.Display.printf(">> Check ModuleLLM connection..\n");
+    while (1) {
+        if (module_llm.checkConnection()) {
+            break;
+        }
+    }
+
+    /* Reset ModuleLLM */
+    M5.Display.printf(">> Reset ModuleLLM..\n");
+    module_llm.sys.reset();
+
+    /* Setup Audio module */
+    M5.Display.printf(">> Setup audio..\n");
+    module_llm.audio.setup();
+
+    /* Setup KWS module and save returned work id */
+    M5.Display.printf(">> Setup kws..\n");
+    m5_module_llm::ApiKwsSetupConfig_t kws_config;
+    kws_config.kws = wake_up_keyword;
+    kws_work_id    = module_llm.kws.setup(kws_config, "kws_setup", language);
+
+    /* Setup VAD module and save returned work id */
+    M5.Display.printf(">> Setup vad..\n");
+    m5_module_llm::ApiVadSetupConfig_t vad_config;
+    vad_config.input = {"sys.pcm", kws_work_id};
+    vad_work_id      = module_llm.vad.setup(vad_config, "vad_setup");
+
+    /* Setup Whisper module and save returned work id */
+    M5.Display.printf(">> Setup whisper..\n");
+    m5_module_llm::ApiWhisperSetupConfig_t whisper_config;
+    whisper_config.input    = {"sys.pcm", kws_work_id, vad_work_id};
+    whisper_config.language = "en";
+    // whisper_config.language = "zh";
+    // whisper_config.language = "ja";
+    whisper_work_id = module_llm.whisper.setup(whisper_config, "whisper_setup");
+
+    M5.Display.printf(">> Setup ok\n>> Say \"%s\" to wakeup\n", wake_up_keyword.c_str());
+}
+
+void loop()
+{
+    /* Update ModuleLLM */
+    module_llm.update();
+
+    /* Handle module response messages */
+    for (auto& msg : module_llm.msg.responseMsgList) {
+        /* If KWS module message */
+        if (msg.work_id == kws_work_id) {
+            M5.Display.setTextColor(TFT_GREENYELLOW);
+            M5.Display.printf(">> Keyword detected\n");
+        }
+
+        /* If ASR module message */
+        if (msg.work_id == whisper_work_id) {
+            /* Check message object type */
+            if (msg.object == "asr.utf-8") {
+                /* Parse message json and get ASR result */
+                JsonDocument doc;
+                deserializeJson(doc, msg.raw_msg);
+                String asr_result = doc["data"].as<String>();
+
+                M5.Display.setTextColor(TFT_YELLOW);
+                M5.Display.printf(">> %s\n", asr_result.c_str());
+            }
+        }
+    }
+
+    /* Clear handled messages */
+    module_llm.msg.responseMsgList.clear();
+}

examples/SerialTextAssistant/SerialTextAssistant.ino

@@ -25,9 +25,12 @@ void setup()
     CommSerialPort.begin(115200);
 
     /* Init module serial port */
-    Serial2.begin(115200, SERIAL_8N1, 16, 17);  // Basic
-    // Serial2.begin(115200, SERIAL_8N1, 13, 14);  // Core2
-    // Serial2.begin(115200, SERIAL_8N1, 18, 17);  // CoreS3
+    // int rxd = 16, txd = 17;  // Basic
+    // int rxd = 13, txd = 14;  // Core2
+    // int rxd = 18, txd = 17;  // CoreS3
+    int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
+    int txd = M5.getPin(m5::pin_name_t::port_c_txd);
+    Serial2.begin(115200, SERIAL_8N1, rxd, txd);
 
     /* Init module */
     module_llm.begin(&Serial2);

examples/TTS/TTS.ino

@@ -22,9 +22,12 @@ void setup()
     // language = "zh_CN";
 
     /* Init module serial port */
-    Serial2.begin(115200, SERIAL_8N1, 16, 17);  // Basic
-    // Serial2.begin(115200, SERIAL_8N1, 13, 14);  // Core2
-    // Serial2.begin(115200, SERIAL_8N1, 18, 17);  // CoreS3
+    // int rxd = 16, txd = 17;  // Basic
+    // int rxd = 13, txd = 14;  // Core2
+    // int rxd = 18, txd = 17;  // CoreS3
+    int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
+    int txd = M5.getPin(m5::pin_name_t::port_c_txd);
+    Serial2.begin(115200, SERIAL_8N1, rxd, txd);
 
     /* Init module */
     module_llm.begin(&Serial2);

examples/TextAssistant/TextAssistant.ino

@@ -17,9 +17,12 @@ void setup()
     M5.Display.setTextScroll(true);
 
     /* Init module serial port */
-    Serial2.begin(115200, SERIAL_8N1, 16, 17);  // Basic
-    // Serial2.begin(115200, SERIAL_8N1, 13, 14);  // Core2
-    // Serial2.begin(115200, SERIAL_8N1, 18, 17);  // CoreS3
+    // int rxd = 16, txd = 17;  // Basic
+    // int rxd = 13, txd = 14;  // Core2
+    // int rxd = 18, txd = 17;  // CoreS3
+    int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
+    int txd = M5.getPin(m5::pin_name_t::port_c_txd);
+    Serial2.begin(115200, SERIAL_8N1, rxd, txd);
 
     /* Init module */
     module_llm.begin(&Serial2);

examples/VoiceAssistant/VoiceAssistant.ino

@@ -42,9 +42,12 @@ void setup()
     M5.Display.setTextScroll(true);
 
     /* Init module serial port */
-    Serial2.begin(115200, SERIAL_8N1, 16, 17);  // Basic
-    // Serial2.begin(115200, SERIAL_8N1, 13, 14);  // Core2
-    // Serial2.begin(115200, SERIAL_8N1, 18, 17);  // CoreS3
+    // int rxd = 16, txd = 17;  // Basic
+    // int rxd = 13, txd = 14;  // Core2
+    // int rxd = 18, txd = 17;  // CoreS3
+    int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
+    int txd = M5.getPin(m5::pin_name_t::port_c_txd);
+    Serial2.begin(115200, SERIAL_8N1, rxd, txd);
 
     /* Init module */
     module_llm.begin(&Serial2);

examples/YOLO/YOLO.ino

@@ -28,9 +28,12 @@ void setup()
     M5.Display.setTextScroll(true);
 
     /* Init module serial port */
-    Serial2.begin(115200, SERIAL_8N1, 16, 17);  // Basic
-    // Serial2.begin(115200, SERIAL_8N1, 13, 14);  // Core2
-    // Serial2.begin(115200, SERIAL_8N1, 18, 17);  // CoreS3
+    // int rxd = 16, txd = 17;  // Basic
+    // int rxd = 13, txd = 14;  // Core2
+    // int rxd = 18, txd = 17;  // CoreS3
+    int rxd = M5.getPin(m5::pin_name_t::port_c_rxd);
+    int txd = M5.getPin(m5::pin_name_t::port_c_txd);
+    Serial2.begin(115200, SERIAL_8N1, rxd, txd);
 
     /* Init module */
     module_llm.begin(&Serial2);
@@ -119,5 +122,8 @@ void loop()
     }
 
     /* Clear handled messages */
+    module_llm.msg.clearMsg("yolo_setup");
     module_llm.msg.responseMsgList.clear();
+
+    usleep(500000);
 }

library.json

@@ -14,7 +14,7 @@
         "M5GFX": "*",
         "ArduinoJson": "*"
     },
-    "version": "1.3.0",
+    "version": "1.5.0",
     "frameworks": "arduino",
     "platforms": "espressif32"
-}
+}

library.properties

@@ -1,5 +1,5 @@
 name=M5ModuleLLM
-version=1.3.0
+version=1.5.0
 author=M5Stack
 maintainer=M5Stack
 sentence=M5ModuleLLM is a library for M5ModuleLLM
@@ -8,4 +8,4 @@ category=Device Control
 url=https://github.com/m5stack/M5Module-LLM.git
 architectures=esp32
 includes=M5ModuleLLM.h
-depends=M5Unified,ArduinoJson
+depends=M5Unified,ArduinoJson

src/M5ModuleLLM.cpp

@@ -12,13 +12,17 @@ bool M5ModuleLLM::begin(Stream* serialPort)
     msg.init(&comm);
     sys.init(&msg);
     llm.init(&msg);
+    vlm.init(&msg);
     audio.init(&msg);
     tts.init(&msg);
     melotts.init(&msg);
     kws.init(&msg);
     asr.init(&msg);
     yolo.init(&msg);
     camera.init(&msg);
+    vad.init(&msg);
+    whisper.init(&msg);
+    depthanything.init(&msg);
     return true;
 }
 

src/M5ModuleLLM.h

@@ -9,13 +9,17 @@
 #include "utils/msg.h"
 #include "api/api_sys.h"
 #include "api/api_llm.h"
+#include "api/api_vlm.h"
 #include "api/api_audio.h"
 #include "api/api_tts.h"
 #include "api/api_melotts.h"
 #include "api/api_kws.h"
 #include "api/api_asr.h"
 #include "api/api_yolo.h"
+#include "api/api_depth_anything.h"
 #include "api/api_camera.h"
+#include "api/api_vad.h"
+#include "api/api_whisper.h"
 #include "api/api_version.h"
 
 class M5ModuleLLM {
@@ -55,6 +59,12 @@ class M5ModuleLLM {
      */
     m5_module_llm::ApiLlm llm;
 
+    /**
+     * @brief VLM module api set
+     *
+     */
+    m5_module_llm::ApiVlm vlm;
+
     /**
      * @brief Audio module api set
      *
@@ -97,6 +107,24 @@ class M5ModuleLLM {
      */
     m5_module_llm::ApiYolo yolo;
 
+    /**
+     * @brief VAD module api set
+     *
+     */
+    m5_module_llm::ApiVad vad;
+
+    /**
+     * @brief Whisper module api set
+     *
+     */
+    m5_module_llm::ApiWhisper whisper;
+
+    /**
+     * @brief DepthAnything module api set
+     *
+     */
+    m5_module_llm::ApiDepthAnything depthanything;
+
     /**
      * @brief MSG module to handle module response message
      *

src/api/api_asr.cpp

@@ -51,3 +51,24 @@ String ApiAsr::setup(ApiAsrSetupConfig_t config, String request_id, String langu
         10000);
     return work_id;
 }
+
+String ApiAsr::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_asr.h

@@ -34,6 +34,15 @@ class ApiAsr {
     String setup(ApiAsrSetupConfig_t config = ApiAsrSetupConfig_t(), String request_id = "asr_setup",
                  String language = "en_US");
 
+    /**
+     * @brief Exit module ASR, return ASR work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "asr_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };

src/api/api_audio.cpp

@@ -40,3 +40,24 @@ String ApiAudio::setup(ApiAudioSetupConfig_t config, String request_id)
         5000);
     return work_id;
 }
+
+String ApiAudio::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}