not working on Android ...

[zoldaten]

hi. i build libs 'libllama.so' and so on. put `em to jniLibs\arm64-v8a
and use this example with file_picker to load 'model by hands' to avoid path issues:

import 'package:flutter/material.dart';
import 'package:file_picker/file_picker.dart';
import 'package:llama_cpp_dart/llama_cpp_dart.dart';

void main() {
  WidgetsFlutterBinding.ensureInitialized();
  runApp(const MyApp());
}

class MyApp extends StatefulWidget {
  const MyApp({Key? key}) : super(key: key);
  @override
  State<MyApp> createState() => _MyAppState();
}

class _MyAppState extends State<MyApp> {
  String modelOutput = "";
  bool isLoading = false;

  LlamaParent? llamaParent;

  @override
  void dispose() {
    llamaParent?.dispose();
    super.dispose();
  }

  Future<void> loadModelAndRun() async {
    setState(() {
      isLoading = true;
      modelOutput = "";
    });

    String? modelPath = await pickModelFile();
    if (modelPath == null) {
      setState(() {
        isLoading = false;
        modelOutput = "Модель не выбрана.";
      });
      return;
    }

    Llama.libraryPath = "libllama.so";

    ModelParams modelParams = ModelParams();
    modelParams.mainGpu = -1;

    ContextParams contextParams = ContextParams();
    contextParams.nPredict = -1;
    contextParams.nCtx = 8192;

    final samplerParams = SamplerParams();
    samplerParams.temp = 0.7;
    samplerParams.topK = 64;
    samplerParams.topP = 0.95;
    samplerParams.penaltyRepeat = 1.1;

    final loadCommand = LlamaLoad(
      path: modelPath,
      modelParams: modelParams,
      contextParams: contextParams,
      samplingParams: samplerParams,
    );

    llamaParent = LlamaParent(loadCommand);

    try {
      await llamaParent!.init();

      List<String> prompts = [
        getPrompt("What is 2 * 4?"),
        //getPrompt("What is 4 * 4?"),
        //getPrompt("hey what is your name?")
      ];

      int promptIndex = 0;

      // Подписываемся на поток текста модели и складываем в строку
      llamaParent!.stream.listen((response) {
        setState(() {
          modelOutput += response;
        });
      }, onError: (e) {
        setState(() {
          modelOutput += "\nОшибка в потоке: $e";
        });
      });

      // Слушаем завершение генерации одного prompt и запускаем следующий
      llamaParent!.completions.listen((event) {
        if (!event.success) {
          setState(() {
            modelOutput += "\nОшибка генерации: ${event.errorDetails}";
          });
          return;
        }

        promptIndex++;
        if (promptIndex >= prompts.length) {
          // Все запросы завершены
          llamaParent!.dispose();
          setState(() {
            isLoading = false;
          });
        } else {
          setState(() {
            modelOutput += "\n----- Следующий запрос -----\n";
          });
          llamaParent!.sendPrompt(prompts[promptIndex]);
        }
      }, onError: (e) {
        setState(() {
          modelOutput += "\nОшибка в completions: $e";
          isLoading = false;
        });
      });

      // Стартуем с первого prompt
      llamaParent!.sendPrompt(prompts[0]);
    } catch (e) {
      setState(() {
        modelOutput = "Ошибка: $e";
        isLoading = false;
      });
      await llamaParent!.dispose();
    }
  }

  Future<String?> pickModelFile() async {
    final result = await FilePicker.platform.pickFiles(
      type: FileType.custom,
      allowedExtensions: ['gguf'],
    );

    if (result == null || result.files.isEmpty) return null;
    return result.files.single.path;
  }

  String getPrompt(String content) {
    ChatHistory history = ChatHistory()
      ..addMessage(role: Role.user, content: content)
      ..addMessage(role: Role.assistant, content: "");
    return history.exportFormat(ChatFormat.gemma, leaveLastAssistantOpen: true);
  }

  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      home: Scaffold(
        appBar: AppBar(title: const Text("Llama Output on Canvas")),
        body: Center(
          child: Column(
            children: [
              Expanded(
                child: CustomPaint(
                  painter: TextPainterOnCanvas(modelOutput),
                  child: Container(),
                ),
              ),
              if (isLoading)
                const Padding(
                  padding: EdgeInsets.all(8.0),
                  child: CircularProgressIndicator(),
                ),
              Padding(
                padding: const EdgeInsets.all(8.0),
                child: ElevatedButton(
                  onPressed: isLoading ? null : loadModelAndRun,
                  child: const Text("Выбрать модель и запустить"),
                ),
              )
            ],
          ),
        ),
      ),
    );
  }
}

class TextPainterOnCanvas extends CustomPainter {
  final String text;
  TextPainterOnCanvas(this.text);

  @override
  void paint(Canvas canvas, Size size) {
    final paint = Paint();

    final textSpan = TextSpan(
      text: text,
      style: const TextStyle(color: Colors.black, fontSize: 16),
    );

    final textPainter = TextPainter(
      text: textSpan,
      textDirection: TextDirection.ltr,
      maxLines: null,
    );

    textPainter.layout(maxWidth: size.width);

    textPainter.paint(canvas, const Offset(10, 10));
  }

  @override
  bool shouldRepaint(covariant TextPainterOnCanvas oldDelegate) {
    return oldDelegate.text != text;
  }
}

it seems libllama.so and model loads successfully as i do not see any issues. but after that app holds on. just freezes up.

i tried to add NOT to use gpu as in code above:

   modelParams.mainGpu = -1;

but that is not working ?

what could be done more ?

[netdur]

can you look at this app here https://github.com/netdur/plant-pathology-ai
it is fully working on android, it may help

[zoldaten]

@netdur thanks for app. i built it but it cant load model )
perhaps i choose wrong model for CLIP model - Qwen3VL-8B-Instruct-Q4_K_M.gguf
could you provide which would work ?