neiam /dms

public · Issues · Pulls · Labels · Forks · Compare · Actions failure · Badges · Packages

Deploy rknn-describe-server to the CM5, fix the demo invocation

45647d3 · Gabriel Morell · 2026-08-25 16:21

6 files +70 -18

Message

{commit_body(@commit)}

Files changed

modified tools/rknn-describe-server/README.md

+37 −18

@@ -41,7 +41,11 @@ wrapper around `librkllmrt.so` (see `rkllm.h` in the upstream repo) that

41	41	keeps the language model warm across requests. The request/response shape
42	42	above does not need to change.
43	43
44		−## Setup on r5bp-0 (RK3588, librknnrt 2.3.0 already shipped by Armbian)
	44	+## Setup on an RK3588 board
	45	+
	46	+`librknnrt.so` and `librkllmrt.so` both ship inside the upstream repo
	47	+(`3rdparty/librknnrt/`, `rkllm-runtime/`) and get installed next to the demo
	48	+binary, so no system-wide RKNN runtime is required.
45	49
46	50	The bring-up below is what we ran on `r5bp-0` and should reproduce on any
47	51	RK3588 board with the same OS image.

@@ -54,7 +58,12 @@ sudo apt install -y git cmake g++ python3-venv python3-pip

54	58	# directly with the system toolchain).
55	59	mkdir -p ~/dms-vlm && cd ~/dms-vlm
56	60	git clone --depth 1 https://github.com/airockchip/rknn-llm.git
57		−cd rknn-llm/examples/multimodal_model_demo/deploy
	61	+cd rknn-llm
	62	+# rkllm's default CPU mask assumes 8 online cores. Required on boards with
	63	+# fewer (the CM5 at 172.16.15.136 has 6: 4x A55 + 2x A76) -- without it
	64	+# rkllm_init dies with "Mismatch between enabled CPUs mask and expected count".
	65	+patch -p1 < /path/to/dms/tools/rknn-describe-server/patches/0001-demo-cpu-mask-env.patch
	66	+cd examples/multimodal_model_demo/deploy
58	67	mkdir build && cd build
59	68	cmake .. -DCMAKE_BUILD_TYPE=Release \
60	69	-DCMAKE_SYSTEM_NAME=Linux \

@@ -72,7 +81,9 @@ wget -O Qwen2-VL-2B-Instruct.rknn \

72	81	# 3. drop server.py + models.yaml into ~/dms-vlm, create a venv, install deps
73	82	cd ~/dms-vlm
74	83	python3 -m venv venv
75		−./venv/bin/pip install fastapi "uvicorn[standard]" python-multipart pyyaml
	84	+./venv/bin/pip install fastapi "uvicorn[standard]" python-multipart pyyaml pillow
	85	+# optional: HEIC/AVIF uploads (server.py registers it if importable)
	86	+./venv/bin/pip install pillow-heif
76	87
77	88	# 4. install the user-level systemd unit and enable linger so it keeps
78	89	# running across reboots without an active login session.

@@ -86,8 +97,8 @@ sudo loginctl enable-linger "$USER"

86	97	Smoke test from another host:
87	98
88	99	```bash
89		−curl -s http://r5bp-0:8080/healthz
90		−curl -F image=@/path/to/sample.jpg http://r5bp-0:8080/describe
	100	+curl -s http://172.16.15.136:8080/healthz
	101	+curl -F image=@/path/to/sample.jpg http://172.16.15.136:8080/describe
91	102	```
92	103
93	104	First request takes ~10–15 s because the demo binary reloads the model

@@ -95,28 +106,36 @@ every call. That's *the* thing to fix when going to production — replace

95	106	`subprocess` with a long-lived ctypes wrapper around `librkllmrt.so` so
96	107	the LLM stays warm. The wire contract above does not change.
97	108
98		−`models.yaml` example:
	109	+`models.yaml` — paths must point at the files as downloaded in step 2; the
	110	+Qwen2-VL build ships one `.rknn` (vision) and one `.rkllm` (LLM), not
	111	+`vision.rknn` / `llm.rkllm`:
99	112
100	113	```yaml
101	114	qwen2-vl-2b:
102		− vision: /opt/rknn-llm/models/qwen2-vl-2b/vision.rknn
103		− llm: /opt/rknn-llm/models/qwen2-vl-2b/llm.rkllm
	115	+ vision: /home/gmorell/dms-vlm/models/qwen2-vl-2b/Qwen2-VL-2B-Instruct.rknn
	116	+ llm: /home/gmorell/dms-vlm/models/qwen2-vl-2b/Qwen2-VL-2B-Instruct.rkllm
104	117	max_context_len: 2048
105	118	rknn_core_num: 3
	119	+ platform: rk3588 # rk3588 \| rk3576 \| rk3562 \| rv1126b
	120	+ enabled_cpus_num: 4 # must be >= rknn_core_num and match the mask popcount
	121	+ enabled_cpus_mask: "0x3C" # 6-core CM5: both A76 (cpu4,5) + two A55 (cpu2,3)
106	122	img_start: "<\|vision_start\|>"
107	123	img_end: "<\|vision_end\|>"
108	124	img_content: "<\|image_pad\|>"
109		−minicpm-v-2_6:
110		− vision: /opt/rknn-llm/models/minicpm-v-2_6/vision.rknn
111		− llm: /opt/rknn-llm/models/minicpm-v-2_6/llm.rkllm
112		− max_context_len: 2048
113		− rknn_core_num: 3
114		−internvl2-1b:
115		− vision: /opt/rknn-llm/models/internvl2-1b/vision.rknn
116		− llm: /opt/rknn-llm/models/internvl2-1b/llm.rkllm
117		− max_context_len: 2048
118		− rknn_core_num: 3
119	125	```
120	126
	127	+`platform` is mandatory — the demo takes it as argv[7] and refuses to start
	128	+without a value it recognises. `enabled_cpus_num` / `enabled_cpus_mask` are
	129	+only honoured by a demo built with `patches/0001-demo-cpu-mask-env.patch`;
	130	+omit them on a stock 8-core board.
	131	+
	132	+### Known limitation: short descriptions
	133	+
	134	+The `thanhtantran/Qwen2-VL-2B-RKLLM` W8A8 conversion emits EOS after a single
	135	+clause, e.g. `"The image depicts an astronaut on the moon,"`. This is the
	136	+model, not the wiring: raising `max_new_tokens` from 256 to 512 returns
	137	+byte-identical output, and enabling the demo's chat template changes nothing.
	138	+Try a different conversion or a larger VLM if fuller captions are needed.
	139	+
121	140	The DMS worker then talks to `http://r5bp-0:8080/describe` — see
122	141	`workers.yml` for the env-var wiring.

added tools/rknn-describe-server/patches/0001-demo-cpu-mask-env.patch

+21 −0

@@ -0,0 +1,21 @@

	1	+--- a/examples/multimodal_model_demo/deploy/src/main.cpp 2026-08-25 11:41:05.799461096 -0400
	2	++++ b/examples/multimodal_model_demo/deploy/src/main.cpp 2026-08-25 11:41:05.829457680 -0400
	3	+@@ -119,6 +119,18 @@
	4	+ return -1;
	5	+ }
	6	+
	7	++ // DMS: let the caller pin inference to specific CPUs. rkllm's default
	8	++ // mask assumes an 8-core RK3588; boards with fewer online cores (the
	9	++ // 6-core CM5 here: 4x A55 + 2x A76) fail rkllm_init with "Mismatch
	10	++ // between enabled CPUs mask and expected count". The runtime also
	11	++ // requires enabled CPUs >= rknn_core_num.
	12	++ if (const char* env_num = getenv("RKLLM_ENABLED_CPUS_NUM")) {
	13	++ param.extend_param.enabled_cpus_num = (int8_t)atoi(env_num);
	14	++ }
	15	++ if (const char* env_mask = getenv("RKLLM_ENABLED_CPUS_MASK")) {
	16	++ param.extend_param.enabled_cpus_mask = (uint32_t)strtoul(env_mask, NULL, 0);
	17	++ }
	18	++
	19	+ const char* img_start = "<\|vision_start\|>";
	20	+ const char* img_end = "<\|vision_end\|>";
	21	+ const char* img_content = "<\|image_pad\|>";

modified tools/rknn-describe-server/server.py

+12 −0

@@ -69,6 +69,10 @@ def run_demo(demo_bin: str, demo_dir: str, image_path: str, model_cfg: dict,

69	69	str(max_new_tokens),
70	70	str(max_context_len),
71	71	str(model_cfg.get("rknn_core_num", 3)),
	72	+ # argv[7]: demo selects rkllm base_domain_id from the SoC. Must sit
	73	+ # between rknn_core_num and img_start or the token strings shift into
	74	+ # the platform slot and the demo exits before loading anything.
	75	+ model_cfg.get("platform", "rk3588"),
72	76	model_cfg.get("img_start", "<\|vision_start\|>"),
73	77	model_cfg.get("img_end", "<\|vision_end\|>"),
74	78	model_cfg.get("img_content", "<\|image_pad\|>"),

@@ -79,6 +83,14 @@ def run_demo(demo_bin: str, demo_dir: str, image_path: str, model_cfg: dict,

79	83	lib_dir = str(Path(abs_demo).parent / "lib")
80	84	env["LD_LIBRARY_PATH"] = lib_dir + ":" + env.get("LD_LIBRARY_PATH", "")
81	85
	86	+ # rkllm's default CPU mask assumes 8 online cores and rejects init on
	87	+ # boards with fewer. Read by the locally patched demo (see
	88	+ # patches/0001-demo-cpu-mask-env.patch); ignored by a stock build.
	89	+ for cfg_key, env_key in (("enabled_cpus_num", "RKLLM_ENABLED_CPUS_NUM"),
	90	+ ("enabled_cpus_mask", "RKLLM_ENABLED_CPUS_MASK")):
	91	+ if model_cfg.get(cfg_key) is not None:
	92	+ env[env_key] = str(model_cfg[cfg_key])
	93	+
82	94	# Demo's REPL: send `<image>{prompt}\n`, then `exit\n` to break the loop.
83	95	# The pre-canned numeric shortcuts (`0`, `1`) are ignored because our
84	96	# input is plain text starting with `<image>`.

Parents: 8b74183