feat(genai): live samples part1

genai/live/live-code-exec-with-txt.js

@@ -0,0 +1,101 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START googlegenaisdk_live_code_exec_with_txt]
+
+'use strict';
+
+const {GoogleGenAI, Modality} = require('@google/genai');
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
+
+async function generateLiveCodeExec(
+  projectId = GOOGLE_CLOUD_PROJECT,
+  location = GOOGLE_CLOUD_LOCATION
+) {
+  const client = new GoogleGenAI({
+    vertexai: true,
+    project: projectId,
+    location: location,
+  });
+
+  const modelId = 'gemini-2.0-flash-live-preview-04-09';
+  const config = {
+    responseModalities: [Modality.TEXT],
+    tools: [
+      {
+        codeExecution: {},
+      },
+    ],
+  };
+
+  const responseQueue = [];
+
+  async function waitMessage() {
+    while (responseQueue.length === 0) {
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    return responseQueue.shift();
+  }
+
+  async function handleTurn() {
+    const turns = [];
+    let done = false;
+    while (!done) {
+      const message = await waitMessage();
+      turns.push(message);
+      if (message.serverContent && message.serverContent.turnComplete) {
+        done = true;
+      }
+    }
+    return turns;
+  }
+
+  const session = await client.live.connect({
+    model: modelId,
+    config: config,
+    callbacks: {
+      onmessage: msg => responseQueue.push(msg),
+      onerror: e => console.error('Error:', e.message),
+    },
+  });
+
+  const textInput = 'Compute the largest prime palindrome under 10';
+  console.log('> ', textInput, '\n');
+
+  await session.sendClientContent({
+    turns: [{role: 'user', parts: [{text: textInput}]}],
+  });
+
+  const turns = await handleTurn();
+  for (const turn of turns) {
+    if (turn.text) {
+      console.log('Received text:', turn.text);
+    }
+  }
+
+  // Example output:
+  // > Compute the largest prime palindrome under 10
+  // The largest prime palindrome under 10 is 7.
+
+  session.close();
+  return turns;
+}
+
+// [END googlegenaisdk_live_code_exec_with_txt]
+
+module.exports = {
+  generateLiveCodeExec,
+};

genai/live/live-conversation-audio-with-audio.js

@@ -0,0 +1,170 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START googlegenaisdk_live_conversation_audio_with_audio]
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const {GoogleGenAI, Modality} = require('@google/genai');
+
+const MODEL = 'gemini-2.0-flash-live-preview-04-09';
+const INPUT_RATE = 16000;
+const OUTPUT_RATE = 24000;
+const SAMPLE_WIDTH = 2; // 16-bit
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
+
+function readWavefile(filepath) {
+  const buffer = fs.readFileSync(filepath);
+  const audioBytes = buffer.subarray(44);
+  const base64Data = audioBytes.toString('base64');
+  const mimeType = `audio/pcm;rate=${INPUT_RATE}`;
+  return {base64Data, mimeType};
+}
+
+// Utility: write bytes -> .wav file
+function writeWavefile(filepath, audioFrames, rate = OUTPUT_RATE) {
+  const rawAudioBytes = Buffer.concat(audioFrames);
+  const header = Buffer.alloc(44);
+  header.write('RIFF', 0);
+  header.writeUInt32LE(36 + rawAudioBytes.length, 4);
+  header.write('WAVE', 8);
+  header.write('fmt ', 12);
+  header.writeUInt32LE(16, 16);
+  header.writeUInt16LE(1, 20);
+  header.writeUInt16LE(1, 22);
+  header.writeUInt32LE(rate, 24);
+  header.writeUInt32LE(rate * SAMPLE_WIDTH, 28);
+  header.writeUInt16LE(SAMPLE_WIDTH, 32);
+  header.writeUInt16LE(16, 34);
+  header.write('data', 36);
+  header.writeUInt32LE(rawAudioBytes.length, 40);
+
+  fs.writeFileSync(filepath, Buffer.concat([header, rawAudioBytes]));
+  console.log(`Model response saved to ${filepath}`);
+}
+
+async function generateLiveConversation(
+  projectId = GOOGLE_CLOUD_PROJECT,
+  location = GOOGLE_CLOUD_LOCATION
+) {
+  console.log('Starting audio conversation sample...');
+  console.log(`Project: ${projectId}, Location: ${location}`);
+
+  const client = new GoogleGenAI({
+    vertexai: true,
+    project: projectId,
+    location: location,
+  });
+
+  const responseQueue = [];
+
+  async function waitMessage(timeoutMs = 60 * 1000) {
+    const startTime = Date.now();
+
+    while (responseQueue.length === 0) {
+      if (Date.now() - startTime > timeoutMs) {
+        console.warn('No messages received within timeout. Exiting...');
+        return null; // timeout occurred
+      }
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+
+    return responseQueue.shift();
+  }
+
+  async function handleTurn() {
+    const audioFrames = [];
+    let done = false;
+
+    while (!done) {
+      const message = await waitMessage();
+      const serverContent = message.serverContent;
+
+      if (serverContent && serverContent.inputTranscription) {
+        console.log('Input transcription', serverContent.inputTranscription);
+      }
+      if (serverContent && serverContent.outputTranscription) {
+        console.log('Output transcription', serverContent.outputTranscription);
+      }
+      if (
+        serverContent &&
+        serverContent.modelTurn &&
+        serverContent.modelTurn.parts
+      ) {
+        for (const part of serverContent.modelTurn.parts) {
+          if (part && part.inlineData && part.inlineData.data) {
+            const audioData = Buffer.from(part.inlineData.data, 'base64');
+            audioFrames.push(audioData);
+          }
+        }
+      }
+      if (serverContent && serverContent.turnComplete) {
+        done = true;
+      }
+    }
+
+    return audioFrames;
+  }
+
+  const session = await client.live.connect({
+    model: MODEL,
+    config: {
+      responseModalities: [Modality.AUDIO],
+      inputAudioTranscription: {},
+      outputAudioTranscription: {},
+    },
+    callbacks: {
+      onmessage: msg => responseQueue.push(msg),
+      onerror: e => console.error(e.message),
+      onclose: () => console.log('Closed'),
+    },
+  });
+
+  const wavFilePath = path.join(__dirname, 'hello_gemini_are_you_there.wav');
+  console.log('Reading file:', wavFilePath);
+
+  const {base64Data, mimeType} = readWavefile(wavFilePath);
+  const audioBytes = Buffer.from(base64Data, 'base64');
+
+  await session.sendRealtimeInput({
+    media: {
+      data: audioBytes.toString('base64'),
+      mimeType: mimeType,
+    },
+  });
+
+  console.log('Audio sent, waiting for response...');
+
+  const audioFrames = await handleTurn();
+  if (audioFrames.length > 0) {
+    writeWavefile(
+      path.join(__dirname, 'example_model_response.wav'),
+      audioFrames,
+      OUTPUT_RATE
+    );
+  }
+
+  await session.close();
+  return audioFrames;
+}
+
+// [END googlegenaisdk_live_conversation_audio_with_audio]
+
+module.exports = {
+  generateLiveConversation,
+};

genai/live/live-func-call-with-txt.js

@@ -0,0 +1,125 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START googlegenaisdk_live_func_call_with_txt]
+
+'use strict';
+
+const {GoogleGenAI, Modality} = require('@google/genai');
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
+
+async function generateLiveFunctionCall(
+  projectId = GOOGLE_CLOUD_PROJECT,
+  location = GOOGLE_CLOUD_LOCATION
+) {
+  const client = new GoogleGenAI({
+    vertexai: true,
+    project: projectId,
+    location: location,
+  });
+
+  const modelId = 'gemini-2.0-flash-live-preview-04-09';
+
+  const config = {
+    responseModalities: [Modality.TEXT],
+    tools: [
+      {
+        functionDeclarations: [
+          {name: 'turn_on_the_lights'},
+          {name: 'turn_off_the_lights'},
+        ],
+      },
+    ],
+  };
+
+  const responseQueue = [];
+
+  async function waitMessage() {
+    while (responseQueue.length === 0) {
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    return responseQueue.shift();
+  }
+
+  async function handleTurn() {
+    const turns = [];
+    let done = false;
+    while (!done) {
+      const message = await waitMessage();
+      turns.push(message);
+
+      if (message.toolCall) {
+        for (const fc of message.toolCall.functionCalls) {
+          console.log(`Model requested function call: ${fc.name}`);
+
+          await session.sendToolResponse({
+            functionResponses: [
+              {
+                id: fc.id,
+                name: fc.name,
+                response: {result: 'ok'},
+              },
+            ],
+          });
+          console.log(`Sent tool response for ${fc.name}:`, {result: 'ok'});
+        }
+      }
+
+      if (message.serverContent && message.serverContent.turnComplete) {
+        done = true;
+      }
+    }
+    return turns;
+  }
+
+  const session = await client.live.connect({
+    model: modelId,
+    config: config,
+    callbacks: {
+      onmessage: msg => responseQueue.push(msg),
+      onerror: e => console.error('Error:', e.message),
+    },
+  });
+
+  const textInput = 'Turn on the lights please';
+  console.log('> ', textInput, '\n');
+
+  await session.sendClientContent({
+    turns: [{role: 'user', parts: [{text: textInput}]}],
+  });
+
+  const turns = await handleTurn();
+
+  for (const turn of turns) {
+    if (turn.text) {
+      console.log('Received text:', turn.text);
+    }
+  }
+
+  // Example output:
+  //>> Turn on the lights please
+  // Model requested function call: turn_on_the_lights
+  // Sent tool response for turn_on_the_lights: { result: 'ok' }
+
+  session.close();
+  return turns;
+}
+
+// [END googlegenaisdk_live_func_call_with_txt]
+
+module.exports = {
+  generateLiveFunctionCall,
+};