Spool zip uploads to disk so a multi-gigabyte archive doesn't cost RAM

6fa3ef9 · gmorell · 2026-08-30 00:03

13 files +256 -66
Message
{commit_body(@commit)}

Files changed

modified .gitignore
+2 −0
@@ -1 +1,3 @@
1 1 /target
2 +# In-flight upload spools — transient, never part of the archive.
3 +/storage/tmp/
modified README.md
+22 −4
@@ -103,7 +103,7 @@ All env-driven — defaults make sense for local dev.
103 103 | `AMQP_URL` | `amqp://dms_user:dms_password@localhost:5674/%2f` | RabbitMQ. |
104 104 | `BIND_ADDRESS` | `0.0.0.0:3007` | Webservice listener. |
105 105 | `STORAGE_ROOT` | `./storage` | Content-addressed file root. |
106 −| `MAX_UPLOAD_BYTES` | `104857600` (100 MiB) | Per-upload cap. Bigger than this and the user gets a friendly redirect; the body limit layer matches. |
106 +| `MAX_UPLOAD_BYTES` | `104857600` (100 MiB) | Per-upload cap; `compose.yml` raises it to 5 GiB. Bigger than this and the user gets a friendly redirect; the body limit layer matches. Keep the reverse proxy's own body limit at least as large, or it rejects first. |
107 107 | `MAX_ZIP_ENTRIES` | `1000` | Most images read out of one zip. Past it the import stops and says so. |
108 108 | `MAX_ZIP_TOTAL_BYTES` | `1073741824` (1 GiB) | Total *decompressed* bytes read from one zip. A zip is bounded on the wire by `MAX_UPLOAD_BYTES`, but its contents are not — this is the zip-bomb guard. |
109 109 | `SESSION_DURATION_HOURS` | `24` | Browser session cookie lifetime. |
@@ -174,9 +174,15 @@ Pasting anywhere on the site works too: an image on the clipboard attaches to
174 174 the quick-upload form (or uploads directly), and a pasted link is fetched
175 175 server-side so animated GIFs and videos keep their original bytes.
176 176
177 −Zip import walks the archive on a blocking thread and streams each image
178 −through a small channel, so peak memory is the zip plus a few images rather
179 −than the whole decompressed archive. Per entry:
177 +Zip import spools the request body to `STORAGE_ROOT/tmp` as it arrives, then
178 +walks the archive from disk on a blocking thread, streaming each image through
179 +a small channel. Peak memory is a handful of images regardless of archive size
180 +— a 673 MiB zip of 120 photos imports at ~65 MiB RSS. The spool needs as much
181 +free disk as the upload and is removed however the request ends. It lives
182 +under `STORAGE_ROOT` rather than `/tmp` deliberately: `/tmp` is tmpfs on most
183 +hosts, which would put those gigabytes back in RAM.
184 +
185 +Per entry:
180 186
181 187 - Only entries whose name resolves to an image type are stored. Everything
182 188 else is counted and skipped.
@@ -192,6 +198,18 @@ AMQP connection dispatches jobs for the entire batch. A failing entry is
192 198 counted, not fatal — the response reports imported / duplicate / skipped /
193 199 failed counts, and says explicitly when a limit truncated the read.
194 200
201 +Two caveats worth knowing:
202 +
203 +- The single-file paths (`POST /uploads`, `POST /uploads/url`) still buffer the
204 + whole body in memory, so a request there costs its own size in RSS. Only the
205 + zip path spools. Raising `MAX_UPLOAD_BYTES` raises that exposure.
206 +- An upload over the limit is rejected as soon as the server knows, which
207 + leaves the rest of the body in flight; a streaming reverse proxy reports that
208 + to the browser as a 502 rather than showing the redirect. The upload forms
209 + therefore check the file size client-side and refuse before sending a byte,
210 + and the proxy should carry its own `client_max_body_size` so it can answer
211 + with a proper 413.
212 +
195 213 ## Search
196 214
197 215 Two surfaces, same engine.
modified assets/f/js/src.js
+33 −0
@@ -75,6 +75,8 @@
75 75
76 76 // Path 2: anywhere else on the site — POST it directly and surface the result.
77 77 const uploadDirect = async (file) => {
78 + const limitInput = document.querySelector("input[type=file][data-max-bytes]");
79 + if (limitInput && overLimit(file, limitInput)) return;
78 80 const fd = new FormData();
79 81 fd.append("file", file, file.name);
80 82 const status = showToast(`Uploading ${file.name}…`, { kind: "info", ttl: 0 });
@@ -151,6 +153,37 @@
151 153 }
152 154 });
153 155
156 + const humanBytes = (n) => {
157 + const mib = 1024 * 1024;
158 + if (n >= mib) return `${(n / mib).toFixed(1)} MiB`;
159 + if (n >= 1024) return `${(n / 1024).toFixed(1)} KiB`;
160 + return `${n} B`;
161 + };
162 +
163 + // Block oversize files in the browser rather than discovering the limit
164 + // mid-upload. The server answers as soon as it has read the headers, which
165 + // leaves the rest of the body in flight and reads to a proxy as a 502 — so
166 + // the friendly "too large" page never reaches the user. Checking here means
167 + // the bytes are never sent at all.
168 + const overLimit = (file, input) => {
169 + const max = parseInt(input.dataset.maxBytes || "0", 10);
170 + if (!max || !file || file.size <= max) return false;
171 + showToast(
172 + `${file.name} is ${humanBytes(file.size)} — the limit is ${humanBytes(max)}.`,
173 + { kind: "error", ttl: 8000 }
174 + );
175 + return true;
176 + };
177 +
178 + document.querySelectorAll("input[type=file][data-max-bytes]").forEach((input) => {
179 + const form = input.closest("form");
180 + if (!form) return;
181 + form.addEventListener("submit", (e) => {
182 + const file = input.files && input.files[0];
183 + if (overLimit(file, input)) e.preventDefault();
184 + });
185 + });
186 +
154 187 // When the quick-upload form is on the page, also preview manual file picks.
155 188 const quickFile = document.getElementById("quick-upload-file");
156 189 const quickPreview = document.getElementById("quick-upload-preview");
modified compose.yml
+6 −0
@@ -48,6 +48,12 @@ services:
48 48 AMQP_URL: amqp://dms_user:dms_password@rabbitmq:5672/%2f
49 49 BIND_ADDRESS: 0.0.0.0:3007
50 50 STORAGE_ROOT: /data/storage
51 + # 5 GiB on the wire, 8 GiB of decompressed zip contents. NOTE: an upload
52 + # is buffered in memory end to end, so a request this size costs that
53 + # much RSS while it runs — size the container accordingly, and keep the
54 + # reverse proxy's own body limit at least as large or it rejects first.
55 + MAX_UPLOAD_BYTES: "5368709120"
56 + MAX_ZIP_TOTAL_BYTES: "8589934592"
51 57 AUTO_MIGRATE: "true"
52 58 REGISTRATION_ENABLED: "false"
53 59 # Set this to the public origin so the QR pairing URL works from a
modified src/handlers/uploads.rs
+191 −60
@@ -54,6 +54,18 @@ fn urlencode(s: &str) -> String {
54 54 .collect()
55 55 }
56 56
57 +/// The user-facing "too large" sentence, with or without a known size.
58 +fn too_large_message(noun: &str, actual: Option<usize>, limit: usize) -> String {
59 + match actual {
60 + Some(n) => format!(
61 + "{noun} too large ({}). Maximum is {}.",
62 + human_bytes(n),
63 + human_bytes(limit)
64 + ),
65 + None => format!("{noun} exceeds the maximum of {}.", human_bytes(limit)),
66 + }
67 +}
68 +
57 69 fn upload_redirect_error(msg: &str) -> Redirect {
58 70 Redirect::to(&format!("/uploads?error={}", urlencode(msg)))
59 71 }
@@ -94,6 +106,9 @@ pub async fn list_uploads(
94 106 filter_q => qs.q.clone().unwrap_or_default(),
95 107 error => qs.error.clone(),
96 108 notice => qs.notice.clone(),
109 + // Surfaced to the upload forms so the browser can reject an
110 + // oversize file before sending a byte of it.
111 + max_upload_bytes => state.config.max_upload_bytes,
97 112 })
98 113 .map_err(|e| {
99 114 error!(error = %e, "list_uploads render failed");
@@ -117,10 +132,10 @@ pub async fn upload_file(
117 132 .and_then(|s| s.parse::<usize>().ok())
118 133 {
119 134 if len > state.config.max_upload_bytes {
120 − return upload_redirect_error(&format!(
121 − "Upload too large ({}). Maximum is {}.",
122 − human_bytes(len),
123 − human_bytes(state.config.max_upload_bytes)
135 + return upload_redirect_error(&too_large_message(
136 + "Upload",
137 + Some(len),
138 + state.config.max_upload_bytes,
124 139 ))
125 140 .into_response();
126 141 }
@@ -128,12 +143,9 @@ pub async fn upload_file(
128 143
129 144 match ingest_multipart(&state, &user, multipart).await {
130 145 Ok(id) => Redirect::to(&format!("/uploads/{}", id)).into_response(),
131 − Err(IngestError::TooLarge { actual, limit }) => upload_redirect_error(&format!(
132 − "Upload too large ({}). Maximum is {}.",
133 − human_bytes(actual),
134 − human_bytes(limit)
135 − ))
136 − .into_response(),
146 + Err(IngestError::TooLarge { actual, limit }) => {
147 + upload_redirect_error(&too_large_message("Upload", actual, limit)).into_response()
148 + }
137 149 Err(IngestError::BadRequest(msg)) => upload_redirect_error(&msg).into_response(),
138 150 Err(IngestError::Status(code)) => code.into_response(),
139 151 }
@@ -157,12 +169,9 @@ pub async fn upload_from_url(
157 169 ) -> Response {
158 170 match ingest_url(&state, &user, form).await {
159 171 Ok(id) => Redirect::to(&format!("/uploads/{}", id)).into_response(),
160 − Err(IngestError::TooLarge { actual, limit }) => upload_redirect_error(&format!(
161 − "Upload too large ({}). Maximum is {}.",
162 − human_bytes(actual),
163 − human_bytes(limit)
164 − ))
165 − .into_response(),
172 + Err(IngestError::TooLarge { actual, limit }) => {
173 + upload_redirect_error(&too_large_message("Upload", actual, limit)).into_response()
174 + }
166 175 Err(IngestError::BadRequest(msg)) => upload_redirect_error(&msg).into_response(),
167 176 Err(IngestError::Status(code)) => code.into_response(),
168 177 }
@@ -183,10 +192,10 @@ pub async fn upload_zip(
183 192 .and_then(|s| s.parse::<usize>().ok())
184 193 {
185 194 if len > state.config.max_upload_bytes {
186 − return upload_redirect_error(&format!(
187 − "Zip too large ({}). Maximum is {}.",
188 − human_bytes(len),
189 − human_bytes(state.config.max_upload_bytes)
195 + return upload_redirect_error(&too_large_message(
196 + "Zip",
197 + Some(len),
198 + state.config.max_upload_bytes,
190 199 ))
191 200 .into_response();
192 201 }
@@ -194,12 +203,9 @@ pub async fn upload_zip(
194 203
195 204 match ingest_zip(&state, &user, multipart).await {
196 205 Ok(summary) => upload_redirect_notice(&summary.message()).into_response(),
197 − Err(IngestError::TooLarge { actual, limit }) => upload_redirect_error(&format!(
198 − "Zip too large ({}). Maximum is {}.",
199 − human_bytes(actual),
200 − human_bytes(limit)
201 − ))
202 − .into_response(),
206 + Err(IngestError::TooLarge { actual, limit }) => {
207 + upload_redirect_error(&too_large_message("Zip", actual, limit)).into_response()
208 + }
203 209 Err(IngestError::BadRequest(msg)) => upload_redirect_error(&msg).into_response(),
204 210 Err(IngestError::Status(code)) => code.into_response(),
205 211 }
@@ -454,6 +460,7 @@ pub async fn api_upload_file(
454 460 "error": "payload_too_large",
455 461 "max_bytes": limit,
456 462 "received_bytes": actual,
463 + "message": too_large_message("Upload", actual, limit),
457 464 })),
458 465 )
459 466 .into_response(),
@@ -481,6 +488,7 @@ pub async fn api_upload_from_url(
481 488 "error": "payload_too_large",
482 489 "max_bytes": limit,
483 490 "received_bytes": actual,
491 + "message": too_large_message("Upload", actual, limit),
484 492 })),
485 493 )
486 494 .into_response(),
@@ -537,6 +545,7 @@ pub async fn api_upload_zip(
537 545 "error": "payload_too_large",
538 546 "max_bytes": limit,
539 547 "received_bytes": actual,
548 + "message": too_large_message("Upload", actual, limit),
540 549 })),
541 550 )
542 551 .into_response(),
@@ -584,7 +593,10 @@ pub async fn api_show_upload(
584 593 }
585 594
586 595 pub enum IngestError {
587 − TooLarge { actual: usize, limit: usize },
596 + /// `actual` is None when the limit tripped mid-stream and the real size is
597 + /// unknown — reporting a synthetic "limit + 1" would read as nonsense
598 + /// ("400.0 MiB. Maximum is 400.0 MiB.").
599 + TooLarge { actual: Option<usize>, limit: usize },
588 600 BadRequest(String),
589 601 Status(StatusCode),
590 602 }
@@ -593,15 +605,32 @@ impl From<StatusCode> for IngestError {
593 605 fn from(c: StatusCode) -> Self { IngestError::Status(c) }
594 606 }
595 607
596 −/// Map a multipart read failure onto our friendly variants. Axum surfaces the
597 −/// body-limit rejection as an opaque string, so match on it to turn "body too
598 −/// large" into the same TooLarge the explicit byte checks produce.
608 +/// Flatten an error and everything it wraps into one string.
609 +///
610 +/// Axum's `MultipartError` displays as a bland "Error parsing multipart/form-data
611 +/// request" and keeps the interesting part — including the body-limit rejection —
612 +/// in its source chain, so matching on the top-level message alone misses it.
613 +fn error_chain(err: &dyn std::error::Error) -> String {
614 + let mut out = err.to_string();
615 + let mut source = err.source();
616 + while let Some(cause) = source {
617 + out.push_str(": ");
618 + out.push_str(&cause.to_string());
619 + source = cause.source();
620 + }
621 + out
622 +}
623 +
624 +/// Map a multipart read failure onto our friendly variants. The body-limit
625 +/// rejection arrives as a wrapped error rather than a distinct type, so match
626 +/// the flattened chain to turn "body too large" into the same TooLarge the
627 +/// explicit byte checks produce.
599 628 fn multipart_err(state: &AppState, err: String, message: &str) -> IngestError {
600 629 error!(error = %err, context = message, "multipart read failed");
601 630 let lower = err.to_ascii_lowercase();
602 − if lower.contains("length limit") || lower.contains("too large") {
631 + if lower.contains("length limit") || lower.contains("too large") || lower.contains("body limit") {
603 632 IngestError::TooLarge {
604 − actual: state.config.max_upload_bytes + 1,
633 + actual: None,
605 634 limit: state.config.max_upload_bytes,
606 635 }
607 636 } else {
@@ -626,7 +655,7 @@ async fn ingest_multipart(
626 655 Ok(Some(f)) => f,
627 656 Ok(None) => break,
628 657 Err(e) => {
629 − return Err(multipart_err(state, e.to_string(), "Malformed multipart upload."));
658 + return Err(multipart_err(state, error_chain(&e), "Malformed multipart upload."));
630 659 }
631 660 };
632 661 let name = field.name().unwrap_or("").to_string();
@@ -637,10 +666,10 @@ async fn ingest_multipart(
637 666 let bytes = field
638 667 .bytes()
639 668 .await
640 − .map_err(|e| multipart_err(state, e.to_string(), "Could not read uploaded file."))?;
669 + .map_err(|e| multipart_err(state, error_chain(&e), "Could not read uploaded file."))?;
641 670 if bytes.len() > state.config.max_upload_bytes {
642 671 return Err(IngestError::TooLarge {
643 − actual: bytes.len(),
672 + actual: Some(bytes.len()),
644 673 limit: state.config.max_upload_bytes,
645 674 });
646 675 }
@@ -830,7 +859,7 @@ async fn ingest_url(
830 859 if let Some(len) = resp.content_length() {
831 860 if len as usize > state.config.max_upload_bytes {
832 861 return Err(IngestError::TooLarge {
833 − actual: len as usize,
862 + actual: Some(len as usize),
834 863 limit: state.config.max_upload_bytes,
835 864 });
836 865 }
@@ -851,7 +880,7 @@ async fn ingest_url(
851 880 .to_vec();
852 881 if bytes.len() > state.config.max_upload_bytes {
853 882 return Err(IngestError::TooLarge {
854 − actual: bytes.len(),
883 + actual: Some(bytes.len()),
855 884 limit: state.config.max_upload_bytes,
856 885 });
857 886 }
@@ -899,6 +928,70 @@ async fn ingest_url(
899 928 // Zip ingest — one archive of images in, many uploads out
900 929 // ============================================================================
901 930
931 +/// A spooled upload that deletes itself when it goes out of scope, whichever
932 +/// way the handler exits.
933 +struct SpooledFile(std::path::PathBuf);
934 +
935 +impl Drop for SpooledFile {
936 + fn drop(&mut self) {
937 + match std::fs::remove_file(&self.0) {
938 + Ok(()) => {}
939 + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
940 + Err(e) => error!(error = %e, path = %self.0.display(), "spooled upload cleanup failed"),
941 + }
942 + }
943 +}
944 +
945 +/// Stream a multipart field to disk, enforcing `limit` as the bytes arrive.
946 +///
947 +/// Spooling rather than buffering is what keeps a multi-gigabyte archive off
948 +/// the heap — `Field::bytes()` would hold the whole thing in memory at once.
949 +/// The spool lives under STORAGE_ROOT, not `/tmp`, because `/tmp` is tmpfs on
950 +/// most hosts and would put those gigabytes straight back into RAM.
951 +async fn spool_field(
952 + state: &AppState,
953 + field: &mut axum::extract::multipart::Field<'_>,
954 + limit: usize,
955 +) -> Result<(SpooledFile, usize), IngestError> {
956 + use tokio::io::AsyncWriteExt;
957 +
958 + let dir = std::path::Path::new(&state.config.storage_root).join("tmp");
959 + tokio::fs::create_dir_all(&dir).await.map_err(|e| {
960 + error!(error = %e, "create spool dir failed");
961 + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR)
962 + })?;
963 +
964 + let spooled = SpooledFile(dir.join(format!("upload-{}.part", Uuid::now_v7())));
965 + let mut file = tokio::fs::File::create(&spooled.0).await.map_err(|e| {
966 + error!(error = %e, "create spool file failed");
967 + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR)
968 + })?;
969 +
970 + let mut written: usize = 0;
971 + while let Some(chunk) = field
972 + .chunk()
973 + .await
974 + .map_err(|e| multipart_err(state, error_chain(&e), "Could not read uploaded file."))?
975 + {
976 + written += chunk.len();
977 + // Bail the moment the running total crosses the line — no reason to
978 + // keep writing bytes we have already decided to reject.
979 + if written > limit {
980 + return Err(IngestError::TooLarge { actual: Some(written), limit });
981 + }
982 + file.write_all(&chunk).await.map_err(|e| {
983 + error!(error = %e, "write to spool failed");
984 + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR)
985 + })?;
986 + }
987 + file.flush().await.map_err(|e| {
988 + error!(error = %e, "flush spool failed");
989 + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR)
990 + })?;
991 +
992 + Ok((spooled, written))
993 +}
994 +
902 995 /// Tally of a zip import. Everything the import chose *not* to store is
903 996 /// counted so the user gets an honest report instead of a silent truncation.
904 997 #[derive(Debug, Default)]
@@ -963,11 +1056,11 @@ struct ZipScan {
963 1056 }
964 1057
965 1058 /// Walk the archive on a blocking thread, handing each image to `tx` as it is
966 −/// decompressed. Streaming through a small channel — rather than extracting up
967 −/// front — keeps peak memory at the zip plus a handful of images, not the
968 −/// whole decompressed archive.
1059 +/// decompressed. The archive is read from its spool on disk and entries stream
1060 +/// through a small channel, so peak memory is a handful of images regardless of
1061 +/// how large the zip is.
969 1062 fn scan_zip_blocking(
970 − zip_bytes: Vec<u8>,
1063 + zip_path: std::path::PathBuf,
971 1064 max_entry_bytes: usize,
972 1065 max_total_bytes: usize,
973 1066 max_entries: usize,
@@ -975,7 +1068,13 @@ fn scan_zip_blocking(
975 1068 ) -> Result<ZipScan, String> {
976 1069 use std::io::Read;
977 1070
978 − let mut archive = zip::ZipArchive::new(std::io::Cursor::new(zip_bytes))
1071 + let file = std::fs::File::open(&zip_path).map_err(|e| {
1072 + error!(error = %e, path = %zip_path.display(), "open spooled zip failed");
1073 + "Could not read the uploaded zip file.".to_string()
1074 + })?;
1075 + // Seeking through the file on disk rather than a heap buffer is the whole
1076 + // point — peak memory is one entry, not the whole archive.
1077 + let mut archive = zip::ZipArchive::new(std::io::BufReader::new(file))
979 1078 .map_err(|e| format!("Could not read that zip file: {e}"))?;
980 1079
981 1080 let mut scan = ZipScan::default();
@@ -1073,29 +1172,22 @@ async fn ingest_zip(
1073 1172 user: &UserContext,
1074 1173 mut multipart: Multipart,
1075 1174 ) -> Result<ZipImportSummary, IngestError> {
1076 − let mut zip_bytes: Option<Vec<u8>> = None;
1175 + let mut spooled_zip: Option<SpooledFile> = None;
1077 1176 let mut item_description: Option<String> = None;
1078 1177 let mut tags_raw: Option<String> = None;
1079 1178
1080 1179 loop {
1081 − let field = match multipart.next_field().await {
1180 + let mut field = match multipart.next_field().await {
1082 1181 Ok(Some(f)) => f,
1083 1182 Ok(None) => break,
1084 − Err(e) => return Err(multipart_err(state, e.to_string(), "Malformed multipart upload.")),
1183 + Err(e) => return Err(multipart_err(state, error_chain(&e), "Malformed multipart upload.")),
1085 1184 };
1086 1185 match field.name().unwrap_or("") {
1087 1186 "file" => {
1088 − let bytes = field
1089 − .bytes()
1090 − .await
1091 − .map_err(|e| multipart_err(state, e.to_string(), "Could not read uploaded file."))?;
1092 − if bytes.len() > state.config.max_upload_bytes {
1093 − return Err(IngestError::TooLarge {
1094 − actual: bytes.len(),
1095 − limit: state.config.max_upload_bytes,
1096 − });
1097 − }
1098 − zip_bytes = Some(bytes.to_vec());
1187 + let (spooled, size) =
1188 + spool_field(state, &mut field, state.config.max_upload_bytes).await?;
1189 + debug!(bytes = size, path = %spooled.0.display(), "zip spooled to disk");
1190 + spooled_zip = Some(spooled);
1099 1191 }
1100 1192 "item_description" => {
1101 1193 item_description = field.text().await.ok().filter(|s| !s.trim().is_empty());
@@ -1109,14 +1201,18 @@ async fn ingest_zip(
1109 1201 }
1110 1202 }
1111 1203
1112 − let zip_bytes = zip_bytes.ok_or_else(|| IngestError::BadRequest("No zip file selected.".into()))?;
1204 + // Held to the end of the function: dropping it deletes the spool, and the
1205 + // blocking scan below is reading from that exact path.
1206 + let spooled_zip =
1207 + spooled_zip.ok_or_else(|| IngestError::BadRequest("No zip file selected.".into()))?;
1113 1208
1114 1209 let (tx, mut rx) = tokio::sync::mpsc::channel::<ZipEntryFile>(4);
1210 + let zip_path = spooled_zip.0.clone();
1115 1211 let max_entry_bytes = state.config.max_upload_bytes;
1116 1212 let max_total_bytes = state.config.max_zip_total_bytes;
1117 1213 let max_entries = state.config.max_zip_entries;
1118 1214 let scanner = tokio::task::spawn_blocking(move || {
1119 − scan_zip_blocking(zip_bytes, max_entry_bytes, max_total_bytes, max_entries, tx)
1215 + scan_zip_blocking(zip_path, max_entry_bytes, max_total_bytes, max_entries, tx)
1120 1216 });
1121 1217
1122 1218 let mut summary = ZipImportSummary::default();
@@ -1176,6 +1272,7 @@ async fn ingest_zip(
1176 1272 truncated = summary.truncated,
1177 1273 "zip import finished"
1178 1274 );
1275 + drop(spooled_zip);
1179 1276 Ok(summary)
1180 1277 }
1181 1278
@@ -1262,13 +1359,22 @@ mod zip_unit_tests {
1262 1359 w.finish().expect("finish").into_inner()
1263 1360 }
1264 1361
1362 + /// The scan reads from disk, so spool the bytes the way the handler does.
1363 + /// SpooledFile cleans the path up when the test drops it.
1364 + fn spool(zip_bytes: &[u8]) -> SpooledFile {
1365 + let path = std::env::temp_dir().join(format!("dms-test-{}.zip", Uuid::now_v7()));
1366 + std::fs::write(&path, zip_bytes).expect("write test zip");
1367 + SpooledFile(path)
1368 + }
1369 +
1265 1370 /// Run the scan and drain the channel. Capacity comfortably exceeds the
1266 1371 /// entry count so the blocking sends never park.
1267 1372 fn scan(zip_bytes: Vec<u8>, max_entry: usize, max_total: usize, max_entries: usize)
1268 1373 -> (Vec<ZipEntryFile>, ZipScan)
1269 1374 {
1375 + let spooled = spool(&zip_bytes);
1270 1376 let (tx, mut rx) = tokio::sync::mpsc::channel::<ZipEntryFile>(64);
1271 − let scan = scan_zip_blocking(zip_bytes, max_entry, max_total, max_entries, tx)
1377 + let scan = scan_zip_blocking(spooled.0.clone(), max_entry, max_total, max_entries, tx)
1272 1378 .expect("scan should succeed");
1273 1379 let mut got = Vec::new();
1274 1380 while let Ok(entry) = rx.try_recv() {
@@ -1329,12 +1435,37 @@ mod zip_unit_tests {
1329 1435
1330 1436 #[test]
1331 1437 fn garbage_is_not_a_zip() {
1438 + let spooled = spool(b"definitely not a zip");
1332 1439 let (tx, _rx) = tokio::sync::mpsc::channel::<ZipEntryFile>(4);
1333 − let err = scan_zip_blocking(b"definitely not a zip".to_vec(), 1024, 1024, 10, tx)
1440 + let err = scan_zip_blocking(spooled.0.clone(), 1024, 1024, 10, tx)
1334 1441 .expect_err("garbage must be rejected");
1335 1442 assert!(err.starts_with("Could not read that zip file"), "got: {err}");
1336 1443 }
1337 1444
1445 + #[test]
1446 + fn spool_is_removed_on_drop() {
1447 + let path = {
1448 + let spooled = spool(b"transient");
1449 + let p = spooled.0.clone();
1450 + assert!(p.exists());
1451 + p
1452 + };
1453 + assert!(!path.exists(), "the spool must not outlive its guard");
1454 + }
1455 +
1456 + #[test]
1457 + fn too_large_message_omits_an_unknown_size() {
1458 + // The mid-stream limit trip knows only that the request went over.
1459 + assert_eq!(
1460 + too_large_message("Zip", None, 50 * 1024 * 1024),
1461 + "Zip exceeds the maximum of 50.0 MiB."
1462 + );
1463 + assert_eq!(
1464 + too_large_message("Upload", Some(200 * 1024 * 1024), 100 * 1024 * 1024),
1465 + "Upload too large (200.0 MiB). Maximum is 100.0 MiB."
1466 + );
1467 + }
1468 +
1338 1469 #[test]
1339 1470 fn summary_message_reports_every_bucket() {
1340 1471 let summary = ZipImportSummary {
modified templates/uploads/list.html
+2 −2
@@ -43,7 +43,7 @@
43 43 Paste an image (<kbd class="kbd kbd-xs">Ctrl</kbd>+<kbd class="kbd kbd-xs">V</kbd>) to attach it
44 44 </span>
45 45 </div>
46 − <input id="quick-upload-file" class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept="image/*" required/>
46 + <input id="quick-upload-file" class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept="image/*" data-max-bytes="{{ max_upload_bytes }}" required/>
47 47 <div id="quick-upload-preview" class="hidden items-center gap-2 text-xs">
48 48 <img class="h-16 w-16 object-cover rounded border border-base-300" alt="pasted preview"/>
49 49 <span class="opacity-70" data-name></span>
@@ -59,7 +59,7 @@
59 59 <h3 class="font-bold">Bulk upload (zip)</h3>
60 60 <span class="text-xs opacity-60">Every image inside is ingested and processed</span>
61 61 </div>
62 − <input class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept=".zip,application/zip,application/x-zip-compressed" required/>
62 + <input class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept=".zip,application/zip,application/x-zip-compressed" data-max-bytes="{{ max_upload_bytes }}" required/>
63 63 <textarea class="textarea textarea-bordered textarea-sm w-full" name="item_description" placeholder="Optional description — applied to every image"></textarea>
64 64 <input class="input input-bordered input-sm w-full" type="text" name="tags" placeholder="Optional tags — applied to every image"/>
65 65 <button class="btn btn-primary btn-sm" type="submit">Upload zip</button>

Parents: 5a7f679