1322	1322	"minijinja-embed",
1323	1323	"ocrs",
1324	1324	"qrcode",
	1325	+ "reqwest",
1325	1326	"rten",
1326	1327	"rten-imageproc",
1327	1328	"sea-orm",

1708	1709	checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
1709	1710	dependencies = [
1710	1711	"cfg-if",
	1712	+ "js-sys",
1711	1713	"libc",
1712	1714	"wasi 0.11.1+wasi-snapshot-preview1",
	1715	+ "wasm-bindgen",
1713	1716	]
1714	1717
1715	1718	[[package]]

1719	1722	checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
1720	1723	dependencies = [
1721	1724	"cfg-if",
	1725	+ "js-sys",
1722	1726	"libc",
1723	1727	"r-efi",
1724	1728	"wasi 0.14.7+wasi-0.2.4",
	1729	+ "wasm-bindgen",
1725	1730	]
1726	1731
1727	1732	[[package]]

1941	1946	"pin-utils",
1942	1947	"smallvec",
1943	1948	"tokio",
	1949	+ "want",
	1950	+]
	1951	+
	1952	+[[package]]
	1953	+name = "hyper-rustls"
	1954	+version = "0.27.9"
	1955	+source = "registry+https://github.com/rust-lang/crates.io-index"
	1956	+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
	1957	+dependencies = [
	1958	+ "http",
	1959	+ "hyper",
	1960	+ "hyper-util",
	1961	+ "rustls",
	1962	+ "tokio",
	1963	+ "tokio-rustls",
	1964	+ "tower-service",
	1965	+ "webpki-roots 1.0.2",
1944	1966	]
1945	1967
1946	1968	[[package]]

1949	1971	source = "registry+https://github.com/rust-lang/crates.io-index"
1950	1972	checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8"
1951	1973	dependencies = [
	1974	+ "base64",
1952	1975	"bytes",
	1976	+ "futures-channel",
1953	1977	"futures-core",
	1978	+ "futures-util",
1954	1979	"http",
1955	1980	"http-body",
1956	1981	"hyper",
	1982	+ "ipnet",
	1983	+ "libc",
	1984	+ "percent-encoding",
1957	1985	"pin-project-lite",
	1986	+ "socket2 0.6.0",
1958	1987	"tokio",
1959	1988	"tower-service",
	1989	+ "tracing",
1960	1990	]
1961	1991
1962	1992	[[package]]

2209	2239	"libc",
2210	2240	]
2211	2241
	2242	+[[package]]
	2243	+name = "ipnet"
	2244	+version = "2.12.0"
	2245	+source = "registry+https://github.com/rust-lang/crates.io-index"
	2246	+checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
	2247	+
	2248	+[[package]]
	2249	+name = "iri-string"
	2250	+version = "0.7.12"
	2251	+source = "registry+https://github.com/rust-lang/crates.io-index"
	2252	+checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20"
	2253	+dependencies = [
	2254	+ "memchr",
	2255	+ "serde",
	2256	+]
	2257	+
2212	2258	[[package]]
2213	2259	name = "is_terminal_polyfill"
2214	2260	version = "1.70.1"

2421	2467	"imgref",
2422	2468	]
2423	2469
	2470	+[[package]]
	2471	+name = "lru-slab"
	2472	+version = "0.1.2"
	2473	+source = "registry+https://github.com/rust-lang/crates.io-index"
	2474	+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
	2475	+
2424	2476	[[package]]
2425	2477	name = "mach2"
2426	2478	version = "0.4.3"

3287	3339	source = "registry+https://github.com/rust-lang/crates.io-index"
3288	3340	checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
3289	3341
	3342	+[[package]]
	3343	+name = "quinn"
	3344	+version = "0.11.9"
	3345	+source = "registry+https://github.com/rust-lang/crates.io-index"
	3346	+checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
	3347	+dependencies = [
	3348	+ "bytes",
	3349	+ "cfg_aliases",
	3350	+ "pin-project-lite",
	3351	+ "quinn-proto",
	3352	+ "quinn-udp",
	3353	+ "rustc-hash 2.1.2",
	3354	+ "rustls",
	3355	+ "socket2 0.6.0",
	3356	+ "thiserror",
	3357	+ "tokio",
	3358	+ "tracing",
	3359	+ "web-time",
	3360	+]
	3361	+
	3362	+[[package]]
	3363	+name = "quinn-proto"
	3364	+version = "0.11.14"
	3365	+source = "registry+https://github.com/rust-lang/crates.io-index"
	3366	+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
	3367	+dependencies = [
	3368	+ "bytes",
	3369	+ "getrandom 0.3.3",
	3370	+ "lru-slab",
	3371	+ "rand 0.9.2",
	3372	+ "ring",
	3373	+ "rustc-hash 2.1.2",
	3374	+ "rustls",
	3375	+ "rustls-pki-types",
	3376	+ "slab",
	3377	+ "thiserror",
	3378	+ "tinyvec",
	3379	+ "tracing",
	3380	+ "web-time",
	3381	+]
	3382	+
	3383	+[[package]]
	3384	+name = "quinn-udp"
	3385	+version = "0.5.14"
	3386	+source = "registry+https://github.com/rust-lang/crates.io-index"
	3387	+checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
	3388	+dependencies = [
	3389	+ "cfg_aliases",
	3390	+ "libc",
	3391	+ "once_cell",
	3392	+ "socket2 0.6.0",
	3393	+ "tracing",
	3394	+ "windows-sys 0.60.2",
	3395	+]
	3396	+
3290	3397	[[package]]
3291	3398	name = "quote"
3292	3399	version = "1.0.41"

3522	3629	"bytecheck",
3523	3630	]
3524	3631
	3632	+[[package]]
	3633	+name = "reqwest"
	3634	+version = "0.12.24"
	3635	+source = "registry+https://github.com/rust-lang/crates.io-index"
	3636	+checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f"
	3637	+dependencies = [
	3638	+ "base64",
	3639	+ "bytes",
	3640	+ "futures-core",
	3641	+ "futures-util",
	3642	+ "http",
	3643	+ "http-body",
	3644	+ "http-body-util",
	3645	+ "hyper",
	3646	+ "hyper-rustls",
	3647	+ "hyper-util",
	3648	+ "js-sys",
	3649	+ "log",
	3650	+ "mime_guess",
	3651	+ "percent-encoding",
	3652	+ "pin-project-lite",
	3653	+ "quinn",
	3654	+ "rustls",
	3655	+ "rustls-pki-types",
	3656	+ "serde",
	3657	+ "serde_json",
	3658	+ "serde_urlencoded",
	3659	+ "sync_wrapper",
	3660	+ "tokio",
	3661	+ "tokio-rustls",
	3662	+ "tower",
	3663	+ "tower-http",
	3664	+ "tower-service",
	3665	+ "url",
	3666	+ "wasm-bindgen",
	3667	+ "wasm-bindgen-futures",
	3668	+ "web-sys",
	3669	+ "webpki-roots 1.0.2",
	3670	+]
	3671	+
3525	3672	[[package]]
3526	3673	name = "rgb"
3527	3674	version = "0.8.53"

3823	3970	source = "registry+https://github.com/rust-lang/crates.io-index"
3824	3971	checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
3825	3972	dependencies = [
	3973	+ "web-time",
3826	3974	"zeroize",
3827	3975	]
3828	3976

4570	4718	version = "1.0.2"
4571	4719	source = "registry+https://github.com/rust-lang/crates.io-index"
4572	4720	checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
	4721	+dependencies = [
	4722	+ "futures-core",
	4723	+]
4573	4724
4574	4725	[[package]]
4575	4726	name = "synstructure"

4728	4879	"syn 2.0.106",
4729	4880	]
4730	4881
	4882	+[[package]]
	4883	+name = "tokio-rustls"
	4884	+version = "0.26.4"
	4885	+source = "registry+https://github.com/rust-lang/crates.io-index"
	4886	+checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
	4887	+dependencies = [
	4888	+ "rustls",
	4889	+ "tokio",
	4890	+]
	4891	+
4731	4892	[[package]]
4732	4893	name = "tokio-stream"
4733	4894	version = "0.1.17"

4813	4974	"http-body-util",
4814	4975	"http-range-header",
4815	4976	"httpdate",
	4977	+ "iri-string",
4816	4978	"mime",
4817	4979	"mime_guess",
4818	4980	"percent-encoding",
4819	4981	"pin-project-lite",
4820	4982	"tokio",
4821	4983	"tokio-util",
	4984	+ "tower",
4822	4985	"tower-layer",
4823	4986	"tower-service",
4824	4987	"tracing",

4898	5061	"tracing-log",
4899	5062	]
4900	5063
	5064	+[[package]]
	5065	+name = "try-lock"
	5066	+version = "0.2.5"
	5067	+source = "registry+https://github.com/rust-lang/crates.io-index"
	5068	+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
	5069	+
4901	5070	[[package]]
4902	5071	name = "typeid"
4903	5072	version = "1.0.3"

5026	5195	source = "registry+https://github.com/rust-lang/crates.io-index"
5027	5196	checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
5028	5197
	5198	+[[package]]
	5199	+name = "want"
	5200	+version = "0.3.1"
	5201	+source = "registry+https://github.com/rust-lang/crates.io-index"
	5202	+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
	5203	+dependencies = [
	5204	+ "try-lock",
	5205	+]
	5206	+
5029	5207	[[package]]
5030	5208	name = "wasi"
5031	5209	version = "0.11.1+wasi-snapshot-preview1"

5138	5316	"wasm-bindgen",
5139	5317	]
5140	5318
	5319	+[[package]]
	5320	+name = "web-time"
	5321	+version = "1.1.0"
	5322	+source = "registry+https://github.com/rust-lang/crates.io-index"
	5323	+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
	5324	+dependencies = [
	5325	+ "js-sys",
	5326	+ "wasm-bindgen",
	5327	+]
	5328	+
5141	5329	[[package]]
5142	5330	name = "webpki-roots"
5143	5331	version = "0.26.11"

57	57	rten = "0.21"
58	58	rten-imageproc = "0.21"
59	59	qrcode = { version = "0.14", default-features = false, features = ["svg"] }
	60	+reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls"] }
60	61
61	62	[build-dependencies]
62	63	minijinja-embed = "2.12"

13	13	pub max_upload_bytes: usize,
14	14	pub auto_migrate: bool,
15	15	pub registration_enabled: bool,
	16	+ pub describe: DescribeConfig,
	17	+}
	18	+
	19	+#[derive(Debug, Clone)]
	20	+pub enum DescribeBackendKind {
	21	+ Noop,
	22	+ Rknn,
	23	+}
	24	+
	25	+#[derive(Debug, Clone)]
	26	+pub struct DescribeConfig {
	27	+ pub backend: DescribeBackendKind,
	28	+ pub endpoint: String,
	29	+ pub model: String,
	30	+ pub prompt: String,
	31	+ pub timeout_secs: u64,
	32	+ pub max_new_tokens: u32,
16	33	}
17	34
18	35	impl Config {

48	65	registration_enabled: env::var("REGISTRATION_ENABLED")
49	66	.map(|v| matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on"))
50	67	.unwrap_or(false),
	68	+ describe: DescribeConfig::from_env(),
51	69	})
52	70	}
53	71	}
54	72
	73	+impl DescribeConfig {
	74	+ fn from_env() -> Self {
	75	+ let backend = match env::var("DESCRIBE_BACKEND")
	76	+ .unwrap_or_else(|_| "noop".to_string())
	77	+ .to_ascii_lowercase()
	78	+ .as_str()
	79	+ {
	80	+ "rknn" | "rknn-llm" | "rknnllm" => DescribeBackendKind::Rknn,
	81	+ _ => DescribeBackendKind::Noop,
	82	+ };
	83	+ DescribeConfig {
	84	+ backend,
	85	+ endpoint: env::var("RKNN_LLM_ENDPOINT")
	86	+ .unwrap_or_else(|_| "http://r5bp-0:8080/describe".to_string()),
	87	+ model: env::var("RKNN_LLM_MODEL").unwrap_or_else(|_| "qwen2-vl-2b".to_string()),
	88	+ prompt: env::var("RKNN_LLM_PROMPT").unwrap_or_else(|_| {
	89	+ "Describe this image in one or two concise sentences. \
	90	+ List the salient objects, setting, and any visible text."
	91	+ .to_string()
	92	+ }),
	93	+ timeout_secs: env::var("RKNN_LLM_TIMEOUT_SECS")
	94	+ .ok()
	95	+ .and_then(|v| v.parse().ok())
	96	+ .unwrap_or(120),
	97	+ max_new_tokens: env::var("RKNN_LLM_MAX_NEW_TOKENS")
	98	+ .ok()
	99	+ .and_then(|v| v.parse().ok())
	100	+ .unwrap_or(256),
	101	+ }
	102	+ }
	103	+}
	104	+
55	105	pub async fn create_db_connection(database_url: &str) -> Result<DatabaseConnection, sea_orm::DbErr> {
56	106	sea_orm::Database::connect(database_url).await
57	107	}

1	1	use async_trait::async_trait;
2	2	use futures_util::future::BoxFuture;
3	3	use sea_orm::*;
	4	+use serde::{Deserialize, Serialize};
	5	+use std::sync::Arc;
	6	+use std::time::Duration;
	7	+use tokio::sync::OnceCell;
4	8	use tracing::{error, info, warn};
5	9
	10	+use crate::config::{Config, DescribeBackendKind};
6	11	use crate::entities::{ai_descriptions, uploads};
7	12	use crate::jobs;
8	13	use crate::queue::{publish_job, JobKind, JobMessage, JobOutcome};
9	14	use crate::workers::WorkerContext;
10	15
11	16	/// Pluggable backend for image description. Real implementations call out to
12		−/// a VLM (Anthropic, OpenAI, Ollama, etc); the noop variant just records a
13		−/// placeholder so the rest of the pipeline can be exercised in dev.
	17	+/// a VLM (Anthropic, OpenAI, Ollama, rknn-llm); the noop variant just records
	18	+/// a placeholder so the rest of the pipeline can be exercised in dev.
14	19	#[async_trait]
15	20	pub trait DescribeBackend: Send + Sync {
16		− async fn describe(&self, image_bytes: &[u8], content_type: Option<&str>) -> Result<String, String>;
17		− fn model_name(&self) -> &'static str;
	21	+ async fn describe(
	22	+ &self,
	23	+ image_bytes: &[u8],
	24	+ content_type: Option<&str>,
	25	+ filename: Option<&str>,
	26	+ ) -> Result<String, String>;
	27	+ fn model_name(&self) -> String;
18	28	}
19	29
20	30	pub struct NoopBackend;
21	31
22	32	#[async_trait]
23	33	impl DescribeBackend for NoopBackend {
24		− async fn describe(&self, _image_bytes: &[u8], _content_type: Option<&str>) -> Result<String, String> {
	34	+ async fn describe(
	35	+ &self,
	36	+ _image_bytes: &[u8],
	37	+ _content_type: Option<&str>,
	38	+ _filename: Option<&str>,
	39	+ ) -> Result<String, String> {
25	40	Ok(String::new())
26	41	}
27		− fn model_name(&self) -> &'static str {
28		− "noop"
	42	+ fn model_name(&self) -> String {
	43	+ "noop".to_string()
29	44	}
30	45	}
31	46
	47	+/// HTTP client for the rknn-llm multimodal server (see
	48	+/// `tools/rknn-describe-server/` for the reference implementation).
	49	+///
	50	+/// Wire protocol — POST `endpoint`, multipart/form-data:
	51	+/// - `image` binary file part
	52	+/// - `prompt` optional text part (overrides server default)
	53	+/// - `model` optional text part (selects loaded model on the server)
	54	+/// - `max_new_tokens` optional integer text part
	55	+///
	56	+/// Response — JSON: `{ "description": "...", "model": "qwen2-vl-2b", "tokens": 123 }`
	57	+pub struct RknnBackend {
	58	+ client: reqwest::Client,
	59	+ endpoint: String,
	60	+ model: String,
	61	+ prompt: String,
	62	+ max_new_tokens: u32,
	63	+}
	64	+
	65	+impl RknnBackend {
	66	+ pub fn new(cfg: &crate::config::DescribeConfig) -> Result<Self, String> {
	67	+ let client = reqwest::Client::builder()
	68	+ .timeout(Duration::from_secs(cfg.timeout_secs))
	69	+ .build()
	70	+ .map_err(|e| format!("build http client: {e}"))?;
	71	+ Ok(RknnBackend {
	72	+ client,
	73	+ endpoint: cfg.endpoint.clone(),
	74	+ model: cfg.model.clone(),
	75	+ prompt: cfg.prompt.clone(),
	76	+ max_new_tokens: cfg.max_new_tokens,
	77	+ })
	78	+ }
	79	+}
	80	+
	81	+#[derive(Serialize, Deserialize)]
	82	+struct DescribeResponse {
	83	+ description: String,
	84	+ #[serde(default)]
	85	+ model: Option<String>,
	86	+}
	87	+
	88	+#[async_trait]
	89	+impl DescribeBackend for RknnBackend {
	90	+ async fn describe(
	91	+ &self,
	92	+ image_bytes: &[u8],
	93	+ content_type: Option<&str>,
	94	+ filename: Option<&str>,
	95	+ ) -> Result<String, String> {
	96	+ let mut part = reqwest::multipart::Part::bytes(image_bytes.to_vec())
	97	+ .file_name(filename.unwrap_or("upload.bin").to_string());
	98	+ if let Some(ct) = content_type {
	99	+ part = part.mime_str(ct).map_err(|e| format!("invalid content-type: {e}"))?;
	100	+ }
	101	+ let form = reqwest::multipart::Form::new()
	102	+ .part("image", part)
	103	+ .text("prompt", self.prompt.clone())
	104	+ .text("model", self.model.clone())
	105	+ .text("max_new_tokens", self.max_new_tokens.to_string());
	106	+
	107	+ let resp = self
	108	+ .client
	109	+ .post(&self.endpoint)
	110	+ .multipart(form)
	111	+ .send()
	112	+ .await
	113	+ .map_err(|e| format!("rknn-llm POST {}: {e}", self.endpoint))?;
	114	+ let status = resp.status();
	115	+ if !status.is_success() {
	116	+ let body = resp.text().await.unwrap_or_default();
	117	+ return Err(format!("rknn-llm http {status}: {body}"));
	118	+ }
	119	+ let parsed: DescribeResponse = resp
	120	+ .json()
	121	+ .await
	122	+ .map_err(|e| format!("decode rknn-llm response: {e}"))?;
	123	+ Ok(parsed.description)
	124	+ }
	125	+ fn model_name(&self) -> String {
	126	+ format!("rknn-llm:{}", self.model)
	127	+ }
	128	+}
	129	+
	130	+static BACKEND: OnceCell<Arc<dyn DescribeBackend>> = OnceCell::const_new();
	131	+
	132	+async fn backend_for(cfg: &Config) -> Arc<dyn DescribeBackend> {
	133	+ BACKEND
	134	+ .get_or_init(|| async {
	135	+ match cfg.describe.backend {
	136	+ DescribeBackendKind::Rknn => match RknnBackend::new(&cfg.describe) {
	137	+ Ok(b) => {
	138	+ info!(
	139	+ endpoint = %cfg.describe.endpoint,
	140	+ model = %cfg.describe.model,
	141	+ "describe backend: rknn-llm"
	142	+ );
	143	+ Arc::new(b) as Arc<dyn DescribeBackend>
	144	+ }
	145	+ Err(e) => {
	146	+ error!(error = %e, "rknn backend init failed; falling back to noop");
	147	+ Arc::new(NoopBackend) as Arc<dyn DescribeBackend>
	148	+ }
	149	+ },
	150	+ DescribeBackendKind::Noop => {
	151	+ info!("describe backend: noop");
	152	+ Arc::new(NoopBackend) as Arc<dyn DescribeBackend>
	153	+ }
	154	+ }
	155	+ })
	156	+ .await
	157	+ .clone()
	158	+}
	159	+
32	160	pub fn run(ctx: WorkerContext, msg: JobMessage) -> BoxFuture<'static, JobOutcome> {
33	161	Box::pin(async move {
34	162	if matches!(jobs::is_complete(ctx.db.as_ref(), msg.upload_id, JobKind::Describe).await, Ok(true)) {

37	165	}
38	166	let _ = jobs::mark_processing(ctx.db.as_ref(), msg.upload_id, JobKind::Describe).await;
39	167
40		− // For now use the noop backend; swap in a real VLM client once chosen.
41		− let backend: Box<dyn DescribeBackend> = Box::new(NoopBackend);
	168	+ let backend = backend_for(ctx.config.as_ref()).await;
42	169
43	170	let upload = match uploads::Entity::find_by_id(msg.upload_id)
44	171	.one(ctx.db.as_ref())

65	192	}
66	193	};
67	194
68		− let description = match backend.describe(&bytes, upload.content_type.as_deref()).await {
	195	+ let filename = std::path::Path::new(&upload.item_path)
	196	+ .file_name()
	197	+ .and_then(|s| s.to_str());
	198	+
	199	+ let description = match backend
	200	+ .describe(&bytes, upload.content_type.as_deref(), filename)
	201	+ .await
	202	+ {
69	203	Ok(d) => d,
70	204	Err(e) => {
71	205	warn!(error = %e, "describe backend failed");
	206	+ // Network/transport errors are retryable; persistent ones get
	207	+ // surfaced via the dead-letter queue when retries exhaust.
72	208	let _ = jobs::mark_failed(ctx.db.as_ref(), upload.id, JobKind::Describe, &e).await;
73		− return JobOutcome::Reject;
	209	+ return JobOutcome::Retry;
74	210	}
75	211	};
76	212

78	214	let active = ai_descriptions::ActiveModel {
79	215	upload_id: Set(upload.id),
80	216	description: Set(description.clone()),
81		− model_used: Set(Some(backend.model_name().to_string())),
	217	+ model_used: Set(Some(backend.model_name())),
82	218	confidence: Set(None),
83	219	..Default::default()
84	220	};

96	232	}
97	233	info!(upload_id = upload.id, "description recorded");
98	234	} else {
99		− info!(upload_id = upload.id, "noop describe; marking complete with empty result");
	235	+ info!(upload_id = upload.id, "empty describe result; marking complete");
100	236	let _ = jobs::mark_complete(ctx.db.as_ref(), upload.id, JobKind::Describe).await;
101	237	}
102	238	JobOutcome::Ack

	1	+# rknn-describe-server
	2	+
	3	+A small HTTP server that runs **on the Rockchip board** (RK3588 / RK3576 — e.g.
	4	+`r5bp-0`) and exposes the [airockchip/rknn-llm](https://github.com/airockchip/rknn-llm)
	5	+multimodal demo over a simple multipart endpoint that the DMS describe worker
	6	+can call.
	7	+
	8	+The DMS Rust worker stays on x86; this server is the only thing that needs to
	9	+live on the NPU host.
	10	+
	11	+## Wire protocol
	12	+
	13	+`POST /describe` — multipart/form-data:
	14	+
	15	+| field | required | notes |
	16	+| ---------------- | -------- | --------------------------------------------------------- |
	17	+| `image` | yes | binary image (jpg/png/webp/...) |
	18	+| `prompt` | no | overrides default prompt |
	19	+| `model` | no | name registered in `models.yaml` (default: first entry) |
	20	+| `max_new_tokens` | no | integer, default 256 |
	21	+
	22	+Response:
	23	+
	24	+```json
	25	+{ "description": "A red barn at dusk with a tractor in front.",
	26	+ "model": "qwen2-vl-2b",
	27	+ "tokens": 27 }
	28	+```
	29	+
	30	+`GET /healthz` — `200 ok` once the model is loaded.
	31	+
	32	+## Approach
	33	+
	34	+The reference implementation here just shells out to the upstream
	35	+`multimodal_demo` binary per request. That keeps the integration minimal but
	36	+**reloads the model every call** — fine for a smoke test, far too slow for
	37	+production.
	38	+
	39	+For sustained throughput, replace `subprocess` with a long-lived ctypes
	40	+wrapper around `librkllmrt.so` (see `rkllm.h` in the upstream repo) that
	41	+keeps the language model warm across requests. The request/response shape
	42	+above does not need to change.
	43	+
	44	+## Setup on r5bp-0 (RK3588, librknnrt 2.3.0 already shipped by Armbian)
	45	+
	46	+The bring-up below is what we ran on `r5bp-0` and should reproduce on any
	47	+RK3588 board with the same OS image.
	48	+
	49	+```bash
	50	+sudo apt install -y git cmake g++ python3-venv python3-pip
	51	+
	52	+# 1. clone the upstream toolkit and build the multimodal demo (the cmake
	53	+# build-linux.sh assumes a cross-compiler — on-device just call cmake
	54	+# directly with the system toolchain).
	55	+mkdir -p ~/dms-vlm && cd ~/dms-vlm
	56	+git clone --depth 1 https://github.com/airockchip/rknn-llm.git
	57	+cd rknn-llm/examples/multimodal_model_demo/deploy
	58	+mkdir build && cd build
	59	+cmake .. -DCMAKE_BUILD_TYPE=Release \
	60	+ -DCMAKE_SYSTEM_NAME=Linux \
	61	+ -DCMAKE_SYSTEM_PROCESSOR=aarch64
	62	+make -j8 install
	63	+# ⇒ ./install/demo_Linux_aarch64/{demo,imgenc,lib/}
	64	+
	65	+# 2. fetch a converted VLM. The Qwen2-VL-2B build below totals ~3.3 GiB.
	66	+mkdir -p ~/dms-vlm/models/qwen2-vl-2b && cd ~/dms-vlm/models/qwen2-vl-2b
	67	+wget -O Qwen2-VL-2B-Instruct.rkllm \
	68	+ https://huggingface.co/thanhtantran/Qwen2-VL-2B-RKLLM/resolve/main/Qwen2-VL-2B-Instruct.rkllm
	69	+wget -O Qwen2-VL-2B-Instruct.rknn \
	70	+ https://huggingface.co/thanhtantran/Qwen2-VL-2B-RKLLM/resolve/main/Qwen2-VL-2B-Instruct.rknn
	71	+
	72	+# 3. drop server.py + models.yaml into ~/dms-vlm, create a venv, install deps
	73	+cd ~/dms-vlm
	74	+python3 -m venv venv
	75	+./venv/bin/pip install fastapi "uvicorn[standard]" python-multipart pyyaml
	76	+
	77	+# 4. install the user-level systemd unit and enable linger so it keeps
	78	+# running across reboots without an active login session.
	79	+mkdir -p ~/.config/systemd/user
	80	+install -m 644 dms-vlm.service ~/.config/systemd/user/
	81	+systemctl --user daemon-reload
	82	+systemctl --user enable --now dms-vlm.service
	83	+sudo loginctl enable-linger "$USER"
	84	+```
	85	+
	86	+Smoke test from another host:
	87	+
	88	+```bash
	89	+curl -s http://r5bp-0:8080/healthz
	90	+curl -F image=@/path/to/sample.jpg http://r5bp-0:8080/describe
	91	+```
	92	+
	93	+First request takes ~10–15 s because the demo binary reloads the model
	94	+every call. That's *the* thing to fix when going to production — replace
	95	+`subprocess` with a long-lived ctypes wrapper around `librkllmrt.so` so
	96	+the LLM stays warm. The wire contract above does not change.
	97	+
	98	+`models.yaml` example:
	99	+
	100	+```yaml
	101	+qwen2-vl-2b:
	102	+ vision: /opt/rknn-llm/models/qwen2-vl-2b/vision.rknn
	103	+ llm: /opt/rknn-llm/models/qwen2-vl-2b/llm.rkllm
	104	+ max_context_len: 2048
	105	+ rknn_core_num: 3
	106	+ img_start: "<|vision_start|>"
	107	+ img_end: "<|vision_end|>"
	108	+ img_content: "<|image_pad|>"
	109	+minicpm-v-2_6:
	110	+ vision: /opt/rknn-llm/models/minicpm-v-2_6/vision.rknn
	111	+ llm: /opt/rknn-llm/models/minicpm-v-2_6/llm.rkllm
	112	+ max_context_len: 2048
	113	+ rknn_core_num: 3
	114	+internvl2-1b:
	115	+ vision: /opt/rknn-llm/models/internvl2-1b/vision.rknn
	116	+ llm: /opt/rknn-llm/models/internvl2-1b/llm.rkllm
	117	+ max_context_len: 2048
	118	+ rknn_core_num: 3
	119	+```
	120	+
	121	+The DMS worker then talks to `http://r5bp-0:8080/describe` — see
	122	+`workers.yml` for the env-var wiring.

	1	+[Unit]
	2	+Description=DMS rknn-llm describe server
	3	+After=network-online.target
	4	+Wants=network-online.target
	5	+
	6	+[Service]
	7	+Type=simple
	8	+WorkingDirectory=%h/dms-vlm
	9	+ExecStart=%h/dms-vlm/venv/bin/python server.py \
	10	+ --models-config %h/dms-vlm/models.yaml \
	11	+ --demo-bin %h/dms-vlm/rknn-llm/examples/multimodal_model_demo/deploy/install/demo_Linux_aarch64/demo \
	12	+ --bind 0.0.0.0:8080
	13	+Restart=on-failure
	14	+RestartSec=5
	15	+StandardOutput=journal
	16	+StandardError=journal
	17	+
	18	+[Install]
	19	+WantedBy=default.target

	1	+#!/usr/bin/env python3
	2	+"""HTTP wrapper around the rknn-llm multimodal_demo binary.
	3	+
	4	+Wire contract documented in README.md. Each /describe request:
	5	+ 1. saves the uploaded image to a tmp file
	6	+ 2. spawns ./demo with that image; demo loads vision encoder + LLM, encodes
	7	+ the image once, then enters an interactive REPL on stdin
	8	+ 3. we feed it `<image><prompt>\\n` and `exit\\n`, capture stdout, and
	9	+ extract the text after the "robot: " marker
	10	+
	11	+This reloads the model per request -- fine for first-cut validation, slow
	12	+for production. Replace with a long-lived ctypes wrapper around
	13	+librkllmrt.so once the integration is proven end-to-end.
	14	+"""
	15	+from __future__ import annotations
	16	+
	17	+import argparse
	18	+import os
	19	+import re
	20	+import shutil
	21	+import subprocess
	22	+import tempfile
	23	+import time
	24	+from pathlib import Path
	25	+
	26	+import yaml
	27	+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
	28	+from fastapi.responses import JSONResponse
	29	+import uvicorn
	30	+
	31	+
	32	+def load_models(path: str) -> dict:
	33	+ with open(path) as f:
	34	+ cfg = yaml.safe_load(f) or {}
	35	+ if not cfg:
	36	+ raise SystemExit(f"no models defined in {path}")
	37	+ for name, m in cfg.items():
	38	+ for key in ("vision", "llm"):
	39	+ if not Path(m[key]).exists():
	40	+ raise SystemExit(f"model {name}: {key} missing: {m[key]}")
	41	+ return cfg
	42	+
	43	+
	44	+# multimodal_demo prints `robot: <text>\n` after each prompt; we want the text
	45	+# from the *last* `robot:` (after our own prompt, not the startup banner).
	46	+ROBOT_RE = re.compile(r"robot:\s*(.*?)(?=\nuser:|\Z)", re.DOTALL)
	47	+
	48	+
	49	+def run_demo(demo_bin: str, demo_dir: str, image_path: str, model_cfg: dict,
	50	+ prompt: str, max_new_tokens: int, max_context_len: int,
	51	+ timeout: int) -> tuple[str, str]:
	52	+ abs_demo = os.path.abspath(demo_bin)
	53	+ cmd = [
	54	+ abs_demo,
	55	+ image_path,
	56	+ model_cfg["vision"],
	57	+ model_cfg["llm"],
	58	+ str(max_new_tokens),
	59	+ str(max_context_len),
	60	+ str(model_cfg.get("rknn_core_num", 3)),
	61	+ model_cfg.get("img_start", "<|vision_start|>"),
	62	+ model_cfg.get("img_end", "<|vision_end|>"),
	63	+ model_cfg.get("img_content", "<|image_pad|>"),
	64	+ ]
	65	+ env = dict(os.environ)
	66	+ # cwd is changed before exec, so use absolute lib path or the dynamic
	67	+ # linker can't find librkllmrt.so / librknnrt.so.
	68	+ lib_dir = str(Path(abs_demo).parent / "lib")
	69	+ env["LD_LIBRARY_PATH"] = lib_dir + ":" + env.get("LD_LIBRARY_PATH", "")
	70	+
	71	+ # Demo's REPL: send `<image>{prompt}\n`, then `exit\n` to break the loop.
	72	+ # The pre-canned numeric shortcuts (`0`, `1`) are ignored because our
	73	+ # input is plain text starting with `<image>`.
	74	+ stdin_payload = f"<image>{prompt}\nexit\n"
	75	+ proc = subprocess.run(
	76	+ cmd,
	77	+ input=stdin_payload,
	78	+ capture_output=True,
	79	+ text=True,
	80	+ cwd=demo_dir,
	81	+ env=env,
	82	+ timeout=timeout,
	83	+ )
	84	+ return proc.stdout, proc.stderr
	85	+
	86	+
	87	+def extract_robot_reply(stdout: str) -> str:
	88	+ # The interactive prompt prints "user: " before each line, then "robot: "
	89	+ # followed by streamed tokens, then a newline at finish. Take the LAST
	90	+ # robot block to ignore any startup chatter.
	91	+ matches = ROBOT_RE.findall(stdout)
	92	+ if not matches:
	93	+ return ""
	94	+ return matches[-1].strip()
	95	+
	96	+
	97	+def make_app(models: dict, demo_bin: str, default_prompt: str) -> FastAPI:
	98	+ app = FastAPI()
	99	+ default_model = next(iter(models))
	100	+ demo_dir = str(Path(demo_bin).parent)
	101	+
	102	+ @app.get("/healthz")
	103	+ def healthz():
	104	+ return {"ok": True, "models": list(models.keys()), "default": default_model}
	105	+
	106	+ @app.post("/describe")
	107	+ async def describe(
	108	+ image: UploadFile = File(...),
	109	+ prompt: str = Form(default_prompt),
	110	+ model: str = Form(default_model),
	111	+ max_new_tokens: int = Form(256),
	112	+ ):
	113	+ if model not in models:
	114	+ raise HTTPException(404, f"unknown model {model!r}; have {list(models)}")
	115	+ m = models[model]
	116	+ max_context_len = int(m.get("max_context_len", 4096))
	117	+
	118	+ with tempfile.TemporaryDirectory() as td:
	119	+ ext = Path(image.filename or "in.bin").suffix or ".bin"
	120	+ img_path = Path(td) / f"input{ext}"
	121	+ with img_path.open("wb") as f:
	122	+ shutil.copyfileobj(image.file, f)
	123	+
	124	+ t0 = time.monotonic()
	125	+ try:
	126	+ stdout, stderr = run_demo(
	127	+ demo_bin, demo_dir, str(img_path), m,
	128	+ prompt, max_new_tokens, max_context_len,
	129	+ timeout=int(os.environ.get("RKNN_DEMO_TIMEOUT", "300")),
	130	+ )
	131	+ except subprocess.TimeoutExpired:
	132	+ return JSONResponse({"error": "demo timeout"}, status_code=504)
	133	+ elapsed = time.monotonic() - t0
	134	+
	135	+ text = extract_robot_reply(stdout)
	136	+ if not text:
	137	+ return JSONResponse(
	138	+ {"error": "no reply extracted",
	139	+ "stdout_tail": stdout[-2000:],
	140	+ "stderr_tail": stderr[-2000:]},
	141	+ status_code=500,
	142	+ )
	143	+ return {
	144	+ "description": text,
	145	+ "model": model,
	146	+ "elapsed_seconds": round(elapsed, 2),
	147	+ }
	148	+
	149	+ return app
	150	+
	151	+
	152	+def main():
	153	+ ap = argparse.ArgumentParser()
	154	+ ap.add_argument("--models-config", default="models.yaml")
	155	+ ap.add_argument("--demo-bin", required=True,
	156	+ help="path to the built rknn-llm multimodal demo binary")
	157	+ ap.add_argument("--bind", default="0.0.0.0:8080")
	158	+ ap.add_argument("--default-prompt", default=(
	159	+ "Describe this image in one or two concise sentences. "
	160	+ "List the salient objects, setting, and any visible text."))
	161	+ args = ap.parse_args()
	162	+
	163	+ models = load_models(args.models_config)
	164	+ app = make_app(models, args.demo_bin, args.default_prompt)
	165	+ host, port = args.bind.split(":")
	166	+ uvicorn.run(app, host=host, port=int(port))
	167	+
	168	+
	169	+if __name__ == "__main__":
	170	+ main()