mirror of
https://github.com/block/buzz.git
synced 2026-11-10 11:21:05 +00:00
showing how to use nascent sdk approach
This commit is contained in:
4724
desktop/src-tauri/Cargo.lock
generated
4724
desktop/src-tauri/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -27,6 +27,7 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
|
||||
|
||||
[dependencies]
|
||||
atomic-write-file = "0.3"
|
||||
anyhow = "1"
|
||||
dirs = "6"
|
||||
tauri = { version = "2", features = [] }
|
||||
tauri-plugin-deep-link = "2"
|
||||
@@ -56,6 +57,7 @@ sprout-auth = { path = "../../crates/sprout-auth" }
|
||||
sprout-core = { path = "../../crates/sprout-core" }
|
||||
sprout-persona = { path = "../../crates/sprout-persona" }
|
||||
sprout-sdk = { path = "../../crates/sprout-sdk" }
|
||||
mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "6818bc779e33aef9b8599c30129b3853a0248c5a", package = "mesh-llm-host-runtime", default-features = false }
|
||||
iroh-base = { version = "=1.0.0-rc.0", features = ["key"] }
|
||||
base64 = "0.22"
|
||||
sha2 = "0.11"
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
io::Write,
|
||||
sync::{atomic::AtomicU16, Arc, Mutex},
|
||||
sync::{Arc, Mutex, atomic::AtomicU16},
|
||||
};
|
||||
|
||||
use nostr::{Keys, ToBech32};
|
||||
use tauri::{AppHandle, Manager};
|
||||
use tokio::sync::Mutex as AsyncMutex;
|
||||
|
||||
use crate::huddle::HuddleState;
|
||||
use crate::managed_agents::ManagedAgentProcess;
|
||||
@@ -33,6 +34,9 @@ pub struct AppState {
|
||||
pub media_proxy_port: AtomicU16,
|
||||
/// IOKit power assertion state — prevents idle sleep while agents run.
|
||||
pub prevent_sleep: Arc<Mutex<crate::prevent_sleep::PreventSleepState>>,
|
||||
/// In-process mesh-llm runtime started by Sprout. This is deliberately not
|
||||
/// a sidecar process; the handle owns the embedded serve task and local API.
|
||||
pub mesh_llm_runtime: AsyncMutex<Option<crate::mesh_llm::runtime::SproutMeshRuntime>>,
|
||||
}
|
||||
|
||||
pub fn build_app_state() -> AppState {
|
||||
@@ -78,6 +82,7 @@ pub fn build_app_state() -> AppState {
|
||||
prevent_sleep: Arc::new(Mutex::new(
|
||||
crate::prevent_sleep::PreventSleepState::default(),
|
||||
)),
|
||||
mesh_llm_runtime: AsyncMutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -75,6 +75,59 @@ pub async fn mesh_relay_iroh_url(
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// Start a local mesh-llm node inside the Sprout process.
|
||||
///
|
||||
/// This exposes the same localhost OpenAI-compatible API shape as
|
||||
/// `mesh-llm serve`, but it calls the mesh-llm Rust SDK directly instead of
|
||||
/// launching a sidecar binary.
|
||||
#[tauri::command]
|
||||
pub async fn mesh_start_node(
|
||||
app: AppHandle,
|
||||
state: State<'_, AppState>,
|
||||
request: mesh_llm::runtime::StartMeshNodeRequest,
|
||||
) -> CmdResult<mesh_llm::runtime::MeshNodeStatus> {
|
||||
let prefs = mesh_llm::offer::load_prefs(&app).map_err(|e| e.to_string())?;
|
||||
let mut runtime = state.mesh_llm_runtime.lock().await;
|
||||
if runtime.is_some() {
|
||||
return Err("mesh-llm node is already running".to_string());
|
||||
}
|
||||
|
||||
let started = mesh_llm::runtime::SproutMeshRuntime::start(request, &prefs)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let status = match started.status().await {
|
||||
Ok(status) => status,
|
||||
Err(error) => {
|
||||
let _ = started.stop().await;
|
||||
return Err(error.to_string());
|
||||
}
|
||||
};
|
||||
*runtime = Some(started);
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
/// Stop the in-process mesh-llm node if Sprout started one.
|
||||
#[tauri::command]
|
||||
pub async fn mesh_stop_node(state: State<'_, AppState>) -> CmdResult<()> {
|
||||
let runtime = state.mesh_llm_runtime.lock().await.take();
|
||||
if let Some(runtime) = runtime {
|
||||
runtime.stop().await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the embedded node status, including the local OpenAI API URL.
|
||||
#[tauri::command]
|
||||
pub async fn mesh_node_status(
|
||||
state: State<'_, AppState>,
|
||||
) -> CmdResult<mesh_llm::runtime::MeshNodeStatus> {
|
||||
let runtime = state.mesh_llm_runtime.lock().await;
|
||||
match runtime.as_ref() {
|
||||
Some(runtime) => runtime.status().await.map_err(|e| e.to_string()),
|
||||
None => Ok(mesh_llm::runtime::stopped_status()),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Publisher ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// Result of `mesh_publish_offer` — surface enough state so the frontend
|
||||
@@ -111,8 +164,7 @@ pub async fn mesh_publish_offer(
|
||||
) -> CmdResult<PublishOfferResult> {
|
||||
// Load prefs + endpoint key. These are sync; complete before any await.
|
||||
let prefs = mesh_llm::offer::load_prefs(&app).map_err(|e| e.to_string())?;
|
||||
let endpoint_key =
|
||||
mesh_llm::load_or_create_endpoint_key(&app).map_err(|e| e.to_string())?;
|
||||
let endpoint_key = mesh_llm::load_or_create_endpoint_key(&app).map_err(|e| e.to_string())?;
|
||||
let endpoint_id_str = endpoint_key.public().to_string();
|
||||
|
||||
let d_tag = prefs.d_tag.clone();
|
||||
|
||||
@@ -459,6 +459,9 @@ pub fn run() {
|
||||
mesh_get_sharing_prefs,
|
||||
mesh_set_sharing_prefs,
|
||||
mesh_relay_iroh_url,
|
||||
mesh_start_node,
|
||||
mesh_stop_node,
|
||||
mesh_node_status,
|
||||
mesh_publish_offer,
|
||||
discover_acp_providers,
|
||||
discover_managed_agent_prereqs,
|
||||
|
||||
@@ -19,6 +19,7 @@ pub mod endpoint;
|
||||
pub mod nip11;
|
||||
pub mod nip98;
|
||||
pub mod offer;
|
||||
pub mod runtime;
|
||||
|
||||
// Wildcard re-exports are deliberately avoided so that adding an
|
||||
// unused-by-design helper to a submodule (e.g. a publisher that hasn't been
|
||||
|
||||
489
desktop/src-tauri/src/mesh_llm/runtime.rs
Normal file
489
desktop/src-tauri/src/mesh_llm/runtime.rs
Normal file
@@ -0,0 +1,489 @@
|
||||
//! In-process mesh-llm runtime adapter.
|
||||
//!
|
||||
//! Sprout uses this module to stand up the same localhost OpenAI surface that
|
||||
//! `mesh-llm serve` would expose, without spawning a sidecar binary.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::net::IpAddr;
|
||||
use std::time::Duration;
|
||||
|
||||
use mesh_llm_host_runtime::sdk::{
|
||||
EmbeddedMeshDiscoveryMode, EmbeddedMeshNodeConfig, EmbeddedMeshNodeHandle,
|
||||
EmbeddedMeshNodeMode, start_embedded_node,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::ComputeSharingPrefs;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct StartMeshNodeRequest {
|
||||
pub mode: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub join_token: Option<String>,
|
||||
pub auto: Option<bool>,
|
||||
pub discovery_mode: Option<String>,
|
||||
pub api_port: Option<u16>,
|
||||
pub console_port: Option<u16>,
|
||||
pub mesh_name: Option<String>,
|
||||
pub region: Option<String>,
|
||||
pub node_name: Option<String>,
|
||||
pub max_vram_gb: Option<f64>,
|
||||
pub publish: Option<bool>,
|
||||
pub relay_urls: Option<Vec<String>>,
|
||||
pub relay_auth: Option<BTreeMap<String, String>>,
|
||||
pub nostr_relays: Option<Vec<String>>,
|
||||
pub bind_ip: Option<String>,
|
||||
pub bind_port: Option<u16>,
|
||||
pub listen_all: Option<bool>,
|
||||
pub enumerate_host: Option<bool>,
|
||||
pub console_ui: Option<bool>,
|
||||
pub startup_timeout_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MeshNodeStatus {
|
||||
pub running: bool,
|
||||
pub api_base_url: Option<String>,
|
||||
pub console_url: Option<String>,
|
||||
pub invite_token: Option<String>,
|
||||
pub status: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
pub struct SproutMeshRuntime {
|
||||
handle: EmbeddedMeshNodeHandle,
|
||||
}
|
||||
|
||||
impl SproutMeshRuntime {
|
||||
pub async fn start(
|
||||
request: StartMeshNodeRequest,
|
||||
prefs: &ComputeSharingPrefs,
|
||||
) -> anyhow::Result<Self> {
|
||||
let handle = start_embedded_node(build_config(request, prefs)?).await?;
|
||||
Ok(Self { handle })
|
||||
}
|
||||
|
||||
pub async fn status(&self) -> anyhow::Result<MeshNodeStatus> {
|
||||
let status = self.handle.status().await?;
|
||||
Ok(MeshNodeStatus {
|
||||
running: true,
|
||||
api_base_url: Some(status.api_base_url),
|
||||
console_url: Some(status.console_url),
|
||||
invite_token: status.invite_token,
|
||||
status: Some(status.payload),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn stop(self) -> anyhow::Result<()> {
|
||||
self.handle.stop().await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stopped_status() -> MeshNodeStatus {
|
||||
MeshNodeStatus {
|
||||
running: false,
|
||||
api_base_url: None,
|
||||
console_url: None,
|
||||
invite_token: None,
|
||||
status: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_config(
|
||||
request: StartMeshNodeRequest,
|
||||
prefs: &ComputeSharingPrefs,
|
||||
) -> anyhow::Result<EmbeddedMeshNodeConfig> {
|
||||
let mode = match request.mode.as_deref().unwrap_or("serve") {
|
||||
"serve" => EmbeddedMeshNodeMode::Serve,
|
||||
"client" => EmbeddedMeshNodeMode::Client,
|
||||
other => anyhow::bail!("unsupported mesh-llm mode: {other}"),
|
||||
};
|
||||
|
||||
let mut models = Vec::new();
|
||||
if let Some(model) = request.model.filter(|model| !model.trim().is_empty()) {
|
||||
models.push(model);
|
||||
} else if mode == EmbeddedMeshNodeMode::Serve {
|
||||
models.extend(prefs.models.iter().map(|model| model.id.clone()));
|
||||
}
|
||||
|
||||
let mut join = Vec::new();
|
||||
if let Some(token) = request.join_token.filter(|token| !token.trim().is_empty()) {
|
||||
join.push(token);
|
||||
}
|
||||
|
||||
let mut builder = EmbeddedMeshNodeConfig::builder()
|
||||
.mode(mode)
|
||||
.models(models)
|
||||
.join_tokens(join)
|
||||
.auto_join(request.auto.unwrap_or(false))
|
||||
.api_port(request.api_port.unwrap_or(9337))
|
||||
.console_port(request.console_port.unwrap_or(3131))
|
||||
.publish(request.publish.unwrap_or(false))
|
||||
.isolated_config(true)
|
||||
.startup_timeout(Duration::from_millis(
|
||||
request.startup_timeout_ms.unwrap_or(30_000),
|
||||
))
|
||||
.console_ui(request.console_ui.unwrap_or(false))
|
||||
.listen_all(request.listen_all.unwrap_or(false))
|
||||
.enumerate_host(request.enumerate_host.unwrap_or(true));
|
||||
|
||||
if let Some(mode) = parse_discovery_mode(request.discovery_mode.as_deref())? {
|
||||
builder = builder.discovery_mode(mode);
|
||||
}
|
||||
if let Some(mesh_name) = request.mesh_name.or_else(|| Some("sprout".to_string())) {
|
||||
builder = builder.mesh_name(mesh_name);
|
||||
}
|
||||
if let Some(region) = request.region.filter(|value| !value.trim().is_empty()) {
|
||||
builder = builder.region(region);
|
||||
}
|
||||
if let Some(node_name) = request.node_name.filter(|value| !value.trim().is_empty()) {
|
||||
builder = builder.node_name(node_name);
|
||||
}
|
||||
if let Some(max_vram_gb) = request.max_vram_gb.or_else(|| {
|
||||
prefs
|
||||
.caps
|
||||
.max_vram_mb
|
||||
.map(|max_vram_mb| max_vram_mb as f64 / 1024.0)
|
||||
}) {
|
||||
builder = builder.max_vram_gb(max_vram_gb);
|
||||
}
|
||||
for relay in request.relay_urls.unwrap_or_default() {
|
||||
if !relay.trim().is_empty() {
|
||||
builder = builder.iroh_relay(relay);
|
||||
}
|
||||
}
|
||||
for (relay, token) in request.relay_auth.unwrap_or_default() {
|
||||
if !relay.trim().is_empty() && !token.trim().is_empty() {
|
||||
builder = builder.iroh_relay_auth(relay, token);
|
||||
}
|
||||
}
|
||||
for relay in request.nostr_relays.unwrap_or_default() {
|
||||
if !relay.trim().is_empty() {
|
||||
builder = builder.nostr_relay(relay);
|
||||
}
|
||||
}
|
||||
if let Some(ip) = request.bind_ip.filter(|value| !value.trim().is_empty()) {
|
||||
builder = builder.bind_ip(ip.parse::<IpAddr>()?);
|
||||
}
|
||||
if let Some(port) = request.bind_port {
|
||||
builder = builder.bind_port(port);
|
||||
}
|
||||
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
fn parse_discovery_mode(value: Option<&str>) -> anyhow::Result<Option<EmbeddedMeshDiscoveryMode>> {
|
||||
match value {
|
||||
None => Ok(None),
|
||||
Some("nostr") => Ok(Some(EmbeddedMeshDiscoveryMode::Nostr)),
|
||||
Some("mdns") => Ok(Some(EmbeddedMeshDiscoveryMode::Mdns)),
|
||||
Some(other) => anyhow::bail!("unsupported mesh discovery mode: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::TcpListener;
|
||||
use std::time::Duration;
|
||||
|
||||
use sprout_core::mesh_llm::{ModelOffer, ResourceCaps};
|
||||
|
||||
#[test]
|
||||
fn serve_config_uses_request_model_as_model_ref() {
|
||||
let prefs = ComputeSharingPrefs::default();
|
||||
let config = build_config(
|
||||
StartMeshNodeRequest {
|
||||
mode: None,
|
||||
model: Some("unsloth/Qwen3-4B-128K-GGUF:Q4_K_M".to_string()),
|
||||
join_token: Some("join-token".to_string()),
|
||||
auto: Some(true),
|
||||
api_port: Some(19337),
|
||||
console_port: Some(13131),
|
||||
mesh_name: None,
|
||||
max_vram_gb: Some(3.0),
|
||||
publish: None,
|
||||
startup_timeout_ms: Some(1_000),
|
||||
..start_request()
|
||||
},
|
||||
&prefs,
|
||||
)
|
||||
.expect("config");
|
||||
|
||||
assert_eq!(config.mode, EmbeddedMeshNodeMode::Serve);
|
||||
assert_eq!(
|
||||
config.serving.models,
|
||||
vec!["unsloth/Qwen3-4B-128K-GGUF:Q4_K_M".to_string()]
|
||||
);
|
||||
assert_eq!(config.network.join_tokens, vec!["join-token".to_string()]);
|
||||
assert!(config.network.auto_join);
|
||||
assert_eq!(config.network.mesh_name.as_deref(), Some("sprout"));
|
||||
assert_eq!(config.serving.max_vram_gb, Some(3.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_config_can_fall_back_to_prefs_models_and_caps() {
|
||||
let prefs = ComputeSharingPrefs {
|
||||
enabled: true,
|
||||
caps: ResourceCaps {
|
||||
max_vram_mb: Some(3072),
|
||||
max_ram_mb: None,
|
||||
max_concurrency: Some(1),
|
||||
},
|
||||
models: vec![ModelOffer {
|
||||
id: "meshllm/Qwen3-8B-Q4_K_M-layers".to_string(),
|
||||
label: None,
|
||||
context_tokens: None,
|
||||
}],
|
||||
d_tag: "default".to_string(),
|
||||
};
|
||||
|
||||
let config = build_config(
|
||||
StartMeshNodeRequest {
|
||||
mode: Some("serve".to_string()),
|
||||
model: None,
|
||||
join_token: None,
|
||||
auto: None,
|
||||
api_port: None,
|
||||
console_port: None,
|
||||
mesh_name: Some("local-sprout".to_string()),
|
||||
max_vram_gb: None,
|
||||
publish: Some(true),
|
||||
startup_timeout_ms: None,
|
||||
..start_request()
|
||||
},
|
||||
&prefs,
|
||||
)
|
||||
.expect("config");
|
||||
|
||||
assert_eq!(
|
||||
config.serving.models,
|
||||
vec!["meshllm/Qwen3-8B-Q4_K_M-layers".to_string()]
|
||||
);
|
||||
assert_eq!(config.serving.max_vram_gb, Some(3.0));
|
||||
assert_eq!(config.network.mesh_name.as_deref(), Some("local-sprout"));
|
||||
assert!(config.network.publish);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_accepts_relay_and_discovery_options() {
|
||||
let mut relay_auth = BTreeMap::new();
|
||||
relay_auth.insert("https://relay.example".to_string(), "secret".to_string());
|
||||
|
||||
let config = build_config(
|
||||
StartMeshNodeRequest {
|
||||
mode: Some("client".to_string()),
|
||||
auto: Some(true),
|
||||
discovery_mode: Some("mdns".to_string()),
|
||||
relay_urls: Some(vec!["https://relay.example".to_string()]),
|
||||
relay_auth: Some(relay_auth),
|
||||
nostr_relays: Some(vec!["wss://nostr.example".to_string()]),
|
||||
bind_ip: Some("127.0.0.1".to_string()),
|
||||
bind_port: Some(17777),
|
||||
listen_all: Some(true),
|
||||
enumerate_host: Some(false),
|
||||
console_ui: Some(true),
|
||||
region: Some("AU".to_string()),
|
||||
node_name: Some("sprout-node".to_string()),
|
||||
..start_request()
|
||||
},
|
||||
&ComputeSharingPrefs::default(),
|
||||
)
|
||||
.expect("config");
|
||||
|
||||
assert_eq!(config.mode, EmbeddedMeshNodeMode::Client);
|
||||
assert_eq!(
|
||||
config.network.discovery_mode,
|
||||
EmbeddedMeshDiscoveryMode::Mdns
|
||||
);
|
||||
assert_eq!(
|
||||
config.network.iroh_relays,
|
||||
vec!["https://relay.example".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
config.network.iroh_relay_auth.get("https://relay.example"),
|
||||
Some(&"secret".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
config.network.nostr_relays,
|
||||
vec!["wss://nostr.example".to_string()]
|
||||
);
|
||||
assert_eq!(config.network.bind_ip, Some("127.0.0.1".parse().unwrap()));
|
||||
assert_eq!(config.network.bind_port, Some(17777));
|
||||
assert!(config.network.listen_all);
|
||||
assert!(!config.network.enumerate_host);
|
||||
assert!(config.http.console_ui);
|
||||
assert_eq!(config.network.region.as_deref(), Some("AU"));
|
||||
assert_eq!(config.network.node_name.as_deref(), Some("sprout-node"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[ignore = "opens localhost mesh runtime sockets"]
|
||||
async fn client_runtime_start_stop_exposes_local_status() {
|
||||
let runtime = SproutMeshRuntime::start(
|
||||
StartMeshNodeRequest {
|
||||
mode: Some("client".to_string()),
|
||||
model: None,
|
||||
join_token: None,
|
||||
auto: None,
|
||||
api_port: Some(unused_loopback_port()),
|
||||
console_port: Some(unused_loopback_port()),
|
||||
mesh_name: Some("sprout-sdk-smoke".to_string()),
|
||||
max_vram_gb: None,
|
||||
publish: Some(false),
|
||||
startup_timeout_ms: Some(45_000),
|
||||
..start_request()
|
||||
},
|
||||
&ComputeSharingPrefs::default(),
|
||||
)
|
||||
.await
|
||||
.expect("start embedded mesh runtime");
|
||||
|
||||
let status = runtime.status().await.expect("status");
|
||||
assert!(status.running);
|
||||
assert!(status.api_base_url.as_deref().is_some_and(|url| {
|
||||
url.starts_with("http://127.0.0.1:") || url.starts_with("http://localhost:")
|
||||
}));
|
||||
assert!(status.console_url.as_deref().is_some_and(|url| {
|
||||
url.starts_with("http://127.0.0.1:") || url.starts_with("http://localhost:")
|
||||
}));
|
||||
|
||||
runtime.stop().await.expect("stop embedded mesh runtime");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[ignore = "joins the public mesh and sends a real inference request"]
|
||||
async fn client_auto_runtime_can_infer_through_public_mesh() {
|
||||
let runtime = SproutMeshRuntime::start(
|
||||
StartMeshNodeRequest {
|
||||
mode: Some("client".to_string()),
|
||||
model: None,
|
||||
join_token: None,
|
||||
auto: Some(true),
|
||||
api_port: Some(unused_loopback_port()),
|
||||
console_port: Some(unused_loopback_port()),
|
||||
mesh_name: None,
|
||||
max_vram_gb: None,
|
||||
publish: Some(false),
|
||||
startup_timeout_ms: Some(90_000),
|
||||
..start_request()
|
||||
},
|
||||
&ComputeSharingPrefs::default(),
|
||||
)
|
||||
.await
|
||||
.expect("start public auto mesh client");
|
||||
|
||||
let result = chat_completion(
|
||||
&runtime,
|
||||
"mesh",
|
||||
"Reply with exactly: OK",
|
||||
Duration::from_secs(120),
|
||||
)
|
||||
.await;
|
||||
let _ = runtime.stop().await;
|
||||
|
||||
let content = result.expect("public mesh inference response");
|
||||
assert!(!content.trim().is_empty(), "empty mesh response");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[ignore = "loads a local/cached small GGUF through mesh and sends a real inference request"]
|
||||
async fn serve_small_cached_model_can_infer_locally() {
|
||||
let model = "unsloth/Qwen3-0.6B-GGUF:Q4_K_M";
|
||||
let runtime = SproutMeshRuntime::start(
|
||||
StartMeshNodeRequest {
|
||||
mode: Some("serve".to_string()),
|
||||
model: Some(model.to_string()),
|
||||
join_token: None,
|
||||
auto: None,
|
||||
api_port: Some(unused_loopback_port()),
|
||||
console_port: Some(unused_loopback_port()),
|
||||
mesh_name: Some("sprout-local-smoke".to_string()),
|
||||
max_vram_gb: Some(6.0),
|
||||
publish: Some(false),
|
||||
startup_timeout_ms: Some(180_000),
|
||||
..start_request()
|
||||
},
|
||||
&ComputeSharingPrefs::default(),
|
||||
)
|
||||
.await
|
||||
.expect("start local model mesh runtime");
|
||||
|
||||
let result = chat_completion(
|
||||
&runtime,
|
||||
model,
|
||||
"Reply with exactly: OK",
|
||||
Duration::from_secs(120),
|
||||
)
|
||||
.await;
|
||||
let _ = runtime.stop().await;
|
||||
|
||||
let content = result.expect("local model inference response");
|
||||
assert!(!content.trim().is_empty(), "empty local model response");
|
||||
}
|
||||
|
||||
async fn chat_completion(
|
||||
runtime: &SproutMeshRuntime,
|
||||
model: &str,
|
||||
prompt: &str,
|
||||
timeout: Duration,
|
||||
) -> anyhow::Result<String> {
|
||||
let status = runtime.status().await?;
|
||||
let api_base_url = status
|
||||
.api_base_url
|
||||
.ok_or_else(|| anyhow::anyhow!("missing API base URL"))?;
|
||||
let response = reqwest::Client::builder()
|
||||
.timeout(timeout)
|
||||
.build()?
|
||||
.post(format!("{api_base_url}/chat/completions"))
|
||||
.json(&serde_json::json!({
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 16,
|
||||
"stream": false,
|
||||
}))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
let payload = response.json::<serde_json::Value>().await?;
|
||||
payload["choices"][0]["message"]["content"]
|
||||
.as_str()
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| anyhow::anyhow!("missing completion content: {payload}"))
|
||||
}
|
||||
|
||||
fn unused_loopback_port() -> u16 {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral loopback port");
|
||||
listener
|
||||
.local_addr()
|
||||
.expect("loopback socket address")
|
||||
.port()
|
||||
}
|
||||
|
||||
fn start_request() -> StartMeshNodeRequest {
|
||||
StartMeshNodeRequest {
|
||||
mode: None,
|
||||
model: None,
|
||||
join_token: None,
|
||||
auto: None,
|
||||
discovery_mode: None,
|
||||
api_port: None,
|
||||
console_port: None,
|
||||
mesh_name: None,
|
||||
region: None,
|
||||
node_name: None,
|
||||
max_vram_gb: None,
|
||||
publish: None,
|
||||
relay_urls: None,
|
||||
relay_auth: None,
|
||||
nostr_relays: None,
|
||||
bind_ip: None,
|
||||
bind_port: None,
|
||||
listen_all: None,
|
||||
enumerate_host: None,
|
||||
console_ui: None,
|
||||
startup_timeout_ms: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
import * as React from "react";
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
import { Cpu, Users } from "lucide-react";
|
||||
import { Cpu, Play, Square, Users } from "lucide-react";
|
||||
|
||||
import { useMeshLlmOffers } from "@/features/settings/hooks/useMeshLlmOffers";
|
||||
import { useMeshOfferHeartbeat } from "@/features/settings/hooks/useMeshOfferHeartbeat";
|
||||
import { Button } from "@/shared/ui/button";
|
||||
import { Switch } from "@/shared/ui/switch";
|
||||
import { Input } from "@/shared/ui/input";
|
||||
|
||||
@@ -34,6 +35,14 @@ interface MeshEndpointInfo {
|
||||
endpoint_id: string;
|
||||
}
|
||||
|
||||
interface MeshNodeStatus {
|
||||
running: boolean;
|
||||
apiBaseUrl: string | null;
|
||||
consoleUrl: string | null;
|
||||
inviteToken: string | null;
|
||||
status: unknown | null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -58,9 +67,11 @@ function formatCap(value: number | null): string {
|
||||
export function MeshComputeSettingsCard() {
|
||||
const [prefs, setPrefs] = React.useState<ComputeSharingPrefs | null>(null);
|
||||
const [endpoint, setEndpoint] = React.useState<MeshEndpointInfo | null>(null);
|
||||
const [meshNode, setMeshNode] = React.useState<MeshNodeStatus | null>(null);
|
||||
const [irohRelayUrl, setIrohRelayUrl] = React.useState<string | null>(null);
|
||||
const [error, setError] = React.useState<string | null>(null);
|
||||
const [saving, setSaving] = React.useState(false);
|
||||
const [meshBusy, setMeshBusy] = React.useState(false);
|
||||
const { offers, error: offersError } = useMeshLlmOffers();
|
||||
|
||||
// While the card is mounted AND the user has compute-sharing enabled,
|
||||
@@ -80,13 +91,15 @@ export function MeshComputeSettingsCard() {
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
try {
|
||||
const [p, e] = await Promise.all([
|
||||
const [p, e, node] = await Promise.all([
|
||||
invoke<ComputeSharingPrefs>("mesh_get_sharing_prefs"),
|
||||
invoke<MeshEndpointInfo>("mesh_get_endpoint_id"),
|
||||
invoke<MeshNodeStatus>("mesh_node_status"),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setPrefs(p);
|
||||
setEndpoint(e);
|
||||
setMeshNode(node);
|
||||
// Probe the relay's iroh_relay_url so the heartbeat hook has
|
||||
// something to publish against. Failures here are non-fatal —
|
||||
// the user can still see/edit prefs; we just won't heartbeat.
|
||||
@@ -161,6 +174,47 @@ export function MeshComputeSettingsCard() {
|
||||
});
|
||||
};
|
||||
|
||||
const startMeshNode = async () => {
|
||||
setMeshBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
const model = prefs.models[0]?.id ?? null;
|
||||
const maxVramGb =
|
||||
prefs.caps.max_vram_mb == null ? null : prefs.caps.max_vram_mb / 1024;
|
||||
const node = await invoke<MeshNodeStatus>("mesh_start_node", {
|
||||
request: {
|
||||
apiPort: 9337,
|
||||
auto: !model,
|
||||
consolePort: 3131,
|
||||
maxVramGb,
|
||||
meshName: "sprout",
|
||||
mode: model ? "serve" : "client",
|
||||
model,
|
||||
publish: false,
|
||||
},
|
||||
});
|
||||
setMeshNode(node);
|
||||
} catch (e) {
|
||||
setError(String(e));
|
||||
} finally {
|
||||
setMeshBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const stopMeshNode = async () => {
|
||||
setMeshBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
await invoke("mesh_stop_node");
|
||||
const node = await invoke<MeshNodeStatus>("mesh_node_status");
|
||||
setMeshNode(node);
|
||||
} catch (e) {
|
||||
setError(String(e));
|
||||
} finally {
|
||||
setMeshBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<section className="min-w-0" data-testid="settings-compute">
|
||||
<div className="mb-3 min-w-0">
|
||||
@@ -179,6 +233,47 @@ export function MeshComputeSettingsCard() {
|
||||
) : null}
|
||||
|
||||
<div className="flex flex-col gap-4">
|
||||
{/* ── Local node ─────────────────────────────────────────── */}
|
||||
<section className="rounded border border-border/60 bg-muted/20 px-3 py-2">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<h3 className="text-sm font-medium">Local mesh node</h3>
|
||||
<p className="truncate text-xs text-muted-foreground">
|
||||
{meshNode?.running
|
||||
? `OpenAI endpoint: ${meshNode.apiBaseUrl ?? "starting"}`
|
||||
: "Stopped"}
|
||||
</p>
|
||||
</div>
|
||||
{meshNode?.running ? (
|
||||
<Button
|
||||
disabled={meshBusy}
|
||||
onClick={stopMeshNode}
|
||||
size="sm"
|
||||
type="button"
|
||||
variant="outline"
|
||||
>
|
||||
<Square className="h-4 w-4" />
|
||||
Stop
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
disabled={meshBusy}
|
||||
onClick={startMeshNode}
|
||||
size="sm"
|
||||
type="button"
|
||||
>
|
||||
<Play className="h-4 w-4" />
|
||||
Start
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
{meshNode?.running && meshNode.consoleUrl ? (
|
||||
<div className="mt-1 text-[10px] text-muted-foreground">
|
||||
Console: <code>{meshNode.consoleUrl}</code>
|
||||
</div>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
{/* ── Master toggle ────────────────────────────────────────── */}
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<div className="min-w-0">
|
||||
|
||||
Reference in New Issue
Block a user