⭐
Log in to mark this repository.
Spool zip uploads to disk so a multi-gigabyte archive doesn't cost RAM
6fa3ef9 · gmorell · 2026-08-30 00:03
Message
{commit_body(@commit)}
Files changed
modified
.gitignore
+2
−0
@@ -1 +1,3 @@
| 1 | 1 | /target |
| 2 | +# In-flight upload spools — transient, never part of the archive. | |
| 3 | +/storage/tmp/ |
modified
README.md
+22
−4
@@ -103,7 +103,7 @@ All env-driven — defaults make sense for local dev.
| 103 | 103 | | `AMQP_URL` | `amqp://dms_user:dms_password@localhost:5674/%2f` | RabbitMQ. | |
| 104 | 104 | | `BIND_ADDRESS` | `0.0.0.0:3007` | Webservice listener. | |
| 105 | 105 | | `STORAGE_ROOT` | `./storage` | Content-addressed file root. | |
| 106 | −| `MAX_UPLOAD_BYTES` | `104857600` (100 MiB) | Per-upload cap. Bigger than this and the user gets a friendly redirect; the body limit layer matches. | | |
| 106 | +| `MAX_UPLOAD_BYTES` | `104857600` (100 MiB) | Per-upload cap; `compose.yml` raises it to 5 GiB. Bigger than this and the user gets a friendly redirect; the body limit layer matches. Keep the reverse proxy's own body limit at least as large, or it rejects first. | | |
| 107 | 107 | | `MAX_ZIP_ENTRIES` | `1000` | Most images read out of one zip. Past it the import stops and says so. | |
| 108 | 108 | | `MAX_ZIP_TOTAL_BYTES` | `1073741824` (1 GiB) | Total *decompressed* bytes read from one zip. A zip is bounded on the wire by `MAX_UPLOAD_BYTES`, but its contents are not — this is the zip-bomb guard. | |
| 109 | 109 | | `SESSION_DURATION_HOURS` | `24` | Browser session cookie lifetime. | |
@@ -174,9 +174,15 @@ Pasting anywhere on the site works too: an image on the clipboard attaches to
| 174 | 174 | the quick-upload form (or uploads directly), and a pasted link is fetched |
| 175 | 175 | server-side so animated GIFs and videos keep their original bytes. |
| 176 | 176 | |
| 177 | −Zip import walks the archive on a blocking thread and streams each image | |
| 178 | −through a small channel, so peak memory is the zip plus a few images rather | |
| 179 | −than the whole decompressed archive. Per entry: | |
| 177 | +Zip import spools the request body to `STORAGE_ROOT/tmp` as it arrives, then | |
| 178 | +walks the archive from disk on a blocking thread, streaming each image through | |
| 179 | +a small channel. Peak memory is a handful of images regardless of archive size | |
| 180 | +— a 673 MiB zip of 120 photos imports at ~65 MiB RSS. The spool needs as much | |
| 181 | +free disk as the upload and is removed however the request ends. It lives | |
| 182 | +under `STORAGE_ROOT` rather than `/tmp` deliberately: `/tmp` is tmpfs on most | |
| 183 | +hosts, which would put those gigabytes back in RAM. | |
| 184 | + | |
| 185 | +Per entry: | |
| 180 | 186 | |
| 181 | 187 | - Only entries whose name resolves to an image type are stored. Everything |
| 182 | 188 | else is counted and skipped. |
@@ -192,6 +198,18 @@ AMQP connection dispatches jobs for the entire batch. A failing entry is
| 192 | 198 | counted, not fatal — the response reports imported / duplicate / skipped / |
| 193 | 199 | failed counts, and says explicitly when a limit truncated the read. |
| 194 | 200 | |
| 201 | +Two caveats worth knowing: | |
| 202 | + | |
| 203 | +- The single-file paths (`POST /uploads`, `POST /uploads/url`) still buffer the | |
| 204 | + whole body in memory, so a request there costs its own size in RSS. Only the | |
| 205 | + zip path spools. Raising `MAX_UPLOAD_BYTES` raises that exposure. | |
| 206 | +- An upload over the limit is rejected as soon as the server knows, which | |
| 207 | + leaves the rest of the body in flight; a streaming reverse proxy reports that | |
| 208 | + to the browser as a 502 rather than showing the redirect. The upload forms | |
| 209 | + therefore check the file size client-side and refuse before sending a byte, | |
| 210 | + and the proxy should carry its own `client_max_body_size` so it can answer | |
| 211 | + with a proper 413. | |
| 212 | + | |
| 195 | 213 | ## Search |
| 196 | 214 | |
| 197 | 215 | Two surfaces, same engine. |
modified
assets/f/js/src.js
+33
−0
@@ -75,6 +75,8 @@
| 75 | 75 | |
| 76 | 76 | // Path 2: anywhere else on the site — POST it directly and surface the result. |
| 77 | 77 | const uploadDirect = async (file) => { |
| 78 | + const limitInput = document.querySelector("input[type=file][data-max-bytes]"); | |
| 79 | + if (limitInput && overLimit(file, limitInput)) return; | |
| 78 | 80 | const fd = new FormData(); |
| 79 | 81 | fd.append("file", file, file.name); |
| 80 | 82 | const status = showToast(`Uploading ${file.name}…`, { kind: "info", ttl: 0 }); |
@@ -151,6 +153,37 @@
| 151 | 153 | } |
| 152 | 154 | }); |
| 153 | 155 | |
| 156 | + const humanBytes = (n) => { | |
| 157 | + const mib = 1024 * 1024; | |
| 158 | + if (n >= mib) return `${(n / mib).toFixed(1)} MiB`; | |
| 159 | + if (n >= 1024) return `${(n / 1024).toFixed(1)} KiB`; | |
| 160 | + return `${n} B`; | |
| 161 | + }; | |
| 162 | + | |
| 163 | + // Block oversize files in the browser rather than discovering the limit | |
| 164 | + // mid-upload. The server answers as soon as it has read the headers, which | |
| 165 | + // leaves the rest of the body in flight and reads to a proxy as a 502 — so | |
| 166 | + // the friendly "too large" page never reaches the user. Checking here means | |
| 167 | + // the bytes are never sent at all. | |
| 168 | + const overLimit = (file, input) => { | |
| 169 | + const max = parseInt(input.dataset.maxBytes || "0", 10); | |
| 170 | + if (!max || !file || file.size <= max) return false; | |
| 171 | + showToast( | |
| 172 | + `${file.name} is ${humanBytes(file.size)} — the limit is ${humanBytes(max)}.`, | |
| 173 | + { kind: "error", ttl: 8000 } | |
| 174 | + ); | |
| 175 | + return true; | |
| 176 | + }; | |
| 177 | + | |
| 178 | + document.querySelectorAll("input[type=file][data-max-bytes]").forEach((input) => { | |
| 179 | + const form = input.closest("form"); | |
| 180 | + if (!form) return; | |
| 181 | + form.addEventListener("submit", (e) => { | |
| 182 | + const file = input.files && input.files[0]; | |
| 183 | + if (overLimit(file, input)) e.preventDefault(); | |
| 184 | + }); | |
| 185 | + }); | |
| 186 | + | |
| 154 | 187 | // When the quick-upload form is on the page, also preview manual file picks. |
| 155 | 188 | const quickFile = document.getElementById("quick-upload-file"); |
| 156 | 189 | const quickPreview = document.getElementById("quick-upload-preview"); |
modified
compose.yml
+6
−0
@@ -48,6 +48,12 @@ services:
| 48 | 48 | AMQP_URL: amqp://dms_user:dms_password@rabbitmq:5672/%2f |
| 49 | 49 | BIND_ADDRESS: 0.0.0.0:3007 |
| 50 | 50 | STORAGE_ROOT: /data/storage |
| 51 | + # 5 GiB on the wire, 8 GiB of decompressed zip contents. NOTE: an upload | |
| 52 | + # is buffered in memory end to end, so a request this size costs that | |
| 53 | + # much RSS while it runs — size the container accordingly, and keep the | |
| 54 | + # reverse proxy's own body limit at least as large or it rejects first. | |
| 55 | + MAX_UPLOAD_BYTES: "5368709120" | |
| 56 | + MAX_ZIP_TOTAL_BYTES: "8589934592" | |
| 51 | 57 | AUTO_MIGRATE: "true" |
| 52 | 58 | REGISTRATION_ENABLED: "false" |
| 53 | 59 | # Set this to the public origin so the QR pairing URL works from a |
modified
src/handlers/uploads.rs
+191
−60
@@ -54,6 +54,18 @@ fn urlencode(s: &str) -> String {
| 54 | 54 | .collect() |
| 55 | 55 | } |
| 56 | 56 | |
| 57 | +/// The user-facing "too large" sentence, with or without a known size. | |
| 58 | +fn too_large_message(noun: &str, actual: Option<usize>, limit: usize) -> String { | |
| 59 | + match actual { | |
| 60 | + Some(n) => format!( | |
| 61 | + "{noun} too large ({}). Maximum is {}.", | |
| 62 | + human_bytes(n), | |
| 63 | + human_bytes(limit) | |
| 64 | + ), | |
| 65 | + None => format!("{noun} exceeds the maximum of {}.", human_bytes(limit)), | |
| 66 | + } | |
| 67 | +} | |
| 68 | + | |
| 57 | 69 | fn upload_redirect_error(msg: &str) -> Redirect { |
| 58 | 70 | Redirect::to(&format!("/uploads?error={}", urlencode(msg))) |
| 59 | 71 | } |
@@ -94,6 +106,9 @@ pub async fn list_uploads(
| 94 | 106 | filter_q => qs.q.clone().unwrap_or_default(), |
| 95 | 107 | error => qs.error.clone(), |
| 96 | 108 | notice => qs.notice.clone(), |
| 109 | + // Surfaced to the upload forms so the browser can reject an | |
| 110 | + // oversize file before sending a byte of it. | |
| 111 | + max_upload_bytes => state.config.max_upload_bytes, | |
| 97 | 112 | }) |
| 98 | 113 | .map_err(|e| { |
| 99 | 114 | error!(error = %e, "list_uploads render failed"); |
@@ -117,10 +132,10 @@ pub async fn upload_file(
| 117 | 132 | .and_then(|s| s.parse::<usize>().ok()) |
| 118 | 133 | { |
| 119 | 134 | if len > state.config.max_upload_bytes { |
| 120 | − return upload_redirect_error(&format!( | |
| 121 | − "Upload too large ({}). Maximum is {}.", | |
| 122 | − human_bytes(len), | |
| 123 | − human_bytes(state.config.max_upload_bytes) | |
| 135 | + return upload_redirect_error(&too_large_message( | |
| 136 | + "Upload", | |
| 137 | + Some(len), | |
| 138 | + state.config.max_upload_bytes, | |
| 124 | 139 | )) |
| 125 | 140 | .into_response(); |
| 126 | 141 | } |
@@ -128,12 +143,9 @@ pub async fn upload_file(
| 128 | 143 | |
| 129 | 144 | match ingest_multipart(&state, &user, multipart).await { |
| 130 | 145 | Ok(id) => Redirect::to(&format!("/uploads/{}", id)).into_response(), |
| 131 | − Err(IngestError::TooLarge { actual, limit }) => upload_redirect_error(&format!( | |
| 132 | − "Upload too large ({}). Maximum is {}.", | |
| 133 | − human_bytes(actual), | |
| 134 | − human_bytes(limit) | |
| 135 | − )) | |
| 136 | − .into_response(), | |
| 146 | + Err(IngestError::TooLarge { actual, limit }) => { | |
| 147 | + upload_redirect_error(&too_large_message("Upload", actual, limit)).into_response() | |
| 148 | + } | |
| 137 | 149 | Err(IngestError::BadRequest(msg)) => upload_redirect_error(&msg).into_response(), |
| 138 | 150 | Err(IngestError::Status(code)) => code.into_response(), |
| 139 | 151 | } |
@@ -157,12 +169,9 @@ pub async fn upload_from_url(
| 157 | 169 | ) -> Response { |
| 158 | 170 | match ingest_url(&state, &user, form).await { |
| 159 | 171 | Ok(id) => Redirect::to(&format!("/uploads/{}", id)).into_response(), |
| 160 | − Err(IngestError::TooLarge { actual, limit }) => upload_redirect_error(&format!( | |
| 161 | − "Upload too large ({}). Maximum is {}.", | |
| 162 | − human_bytes(actual), | |
| 163 | − human_bytes(limit) | |
| 164 | − )) | |
| 165 | − .into_response(), | |
| 172 | + Err(IngestError::TooLarge { actual, limit }) => { | |
| 173 | + upload_redirect_error(&too_large_message("Upload", actual, limit)).into_response() | |
| 174 | + } | |
| 166 | 175 | Err(IngestError::BadRequest(msg)) => upload_redirect_error(&msg).into_response(), |
| 167 | 176 | Err(IngestError::Status(code)) => code.into_response(), |
| 168 | 177 | } |
@@ -183,10 +192,10 @@ pub async fn upload_zip(
| 183 | 192 | .and_then(|s| s.parse::<usize>().ok()) |
| 184 | 193 | { |
| 185 | 194 | if len > state.config.max_upload_bytes { |
| 186 | − return upload_redirect_error(&format!( | |
| 187 | − "Zip too large ({}). Maximum is {}.", | |
| 188 | − human_bytes(len), | |
| 189 | − human_bytes(state.config.max_upload_bytes) | |
| 195 | + return upload_redirect_error(&too_large_message( | |
| 196 | + "Zip", | |
| 197 | + Some(len), | |
| 198 | + state.config.max_upload_bytes, | |
| 190 | 199 | )) |
| 191 | 200 | .into_response(); |
| 192 | 201 | } |
@@ -194,12 +203,9 @@ pub async fn upload_zip(
| 194 | 203 | |
| 195 | 204 | match ingest_zip(&state, &user, multipart).await { |
| 196 | 205 | Ok(summary) => upload_redirect_notice(&summary.message()).into_response(), |
| 197 | − Err(IngestError::TooLarge { actual, limit }) => upload_redirect_error(&format!( | |
| 198 | − "Zip too large ({}). Maximum is {}.", | |
| 199 | − human_bytes(actual), | |
| 200 | − human_bytes(limit) | |
| 201 | − )) | |
| 202 | − .into_response(), | |
| 206 | + Err(IngestError::TooLarge { actual, limit }) => { | |
| 207 | + upload_redirect_error(&too_large_message("Zip", actual, limit)).into_response() | |
| 208 | + } | |
| 203 | 209 | Err(IngestError::BadRequest(msg)) => upload_redirect_error(&msg).into_response(), |
| 204 | 210 | Err(IngestError::Status(code)) => code.into_response(), |
| 205 | 211 | } |
@@ -454,6 +460,7 @@ pub async fn api_upload_file(
| 454 | 460 | "error": "payload_too_large", |
| 455 | 461 | "max_bytes": limit, |
| 456 | 462 | "received_bytes": actual, |
| 463 | + "message": too_large_message("Upload", actual, limit), | |
| 457 | 464 | })), |
| 458 | 465 | ) |
| 459 | 466 | .into_response(), |
@@ -481,6 +488,7 @@ pub async fn api_upload_from_url(
| 481 | 488 | "error": "payload_too_large", |
| 482 | 489 | "max_bytes": limit, |
| 483 | 490 | "received_bytes": actual, |
| 491 | + "message": too_large_message("Upload", actual, limit), | |
| 484 | 492 | })), |
| 485 | 493 | ) |
| 486 | 494 | .into_response(), |
@@ -537,6 +545,7 @@ pub async fn api_upload_zip(
| 537 | 545 | "error": "payload_too_large", |
| 538 | 546 | "max_bytes": limit, |
| 539 | 547 | "received_bytes": actual, |
| 548 | + "message": too_large_message("Upload", actual, limit), | |
| 540 | 549 | })), |
| 541 | 550 | ) |
| 542 | 551 | .into_response(), |
@@ -584,7 +593,10 @@ pub async fn api_show_upload(
| 584 | 593 | } |
| 585 | 594 | |
| 586 | 595 | pub enum IngestError { |
| 587 | − TooLarge { actual: usize, limit: usize }, | |
| 596 | + /// `actual` is None when the limit tripped mid-stream and the real size is | |
| 597 | + /// unknown — reporting a synthetic "limit + 1" would read as nonsense | |
| 598 | + /// ("400.0 MiB. Maximum is 400.0 MiB."). | |
| 599 | + TooLarge { actual: Option<usize>, limit: usize }, | |
| 588 | 600 | BadRequest(String), |
| 589 | 601 | Status(StatusCode), |
| 590 | 602 | } |
@@ -593,15 +605,32 @@ impl From<StatusCode> for IngestError {
| 593 | 605 | fn from(c: StatusCode) -> Self { IngestError::Status(c) } |
| 594 | 606 | } |
| 595 | 607 | |
| 596 | −/// Map a multipart read failure onto our friendly variants. Axum surfaces the | |
| 597 | −/// body-limit rejection as an opaque string, so match on it to turn "body too | |
| 598 | −/// large" into the same TooLarge the explicit byte checks produce. | |
| 608 | +/// Flatten an error and everything it wraps into one string. | |
| 609 | +/// | |
| 610 | +/// Axum's `MultipartError` displays as a bland "Error parsing multipart/form-data | |
| 611 | +/// request" and keeps the interesting part — including the body-limit rejection — | |
| 612 | +/// in its source chain, so matching on the top-level message alone misses it. | |
| 613 | +fn error_chain(err: &dyn std::error::Error) -> String { | |
| 614 | + let mut out = err.to_string(); | |
| 615 | + let mut source = err.source(); | |
| 616 | + while let Some(cause) = source { | |
| 617 | + out.push_str(": "); | |
| 618 | + out.push_str(&cause.to_string()); | |
| 619 | + source = cause.source(); | |
| 620 | + } | |
| 621 | + out | |
| 622 | +} | |
| 623 | + | |
| 624 | +/// Map a multipart read failure onto our friendly variants. The body-limit | |
| 625 | +/// rejection arrives as a wrapped error rather than a distinct type, so match | |
| 626 | +/// the flattened chain to turn "body too large" into the same TooLarge the | |
| 627 | +/// explicit byte checks produce. | |
| 599 | 628 | fn multipart_err(state: &AppState, err: String, message: &str) -> IngestError { |
| 600 | 629 | error!(error = %err, context = message, "multipart read failed"); |
| 601 | 630 | let lower = err.to_ascii_lowercase(); |
| 602 | − if lower.contains("length limit") || lower.contains("too large") { | |
| 631 | + if lower.contains("length limit") || lower.contains("too large") || lower.contains("body limit") { | |
| 603 | 632 | IngestError::TooLarge { |
| 604 | − actual: state.config.max_upload_bytes + 1, | |
| 633 | + actual: None, | |
| 605 | 634 | limit: state.config.max_upload_bytes, |
| 606 | 635 | } |
| 607 | 636 | } else { |
@@ -626,7 +655,7 @@ async fn ingest_multipart(
| 626 | 655 | Ok(Some(f)) => f, |
| 627 | 656 | Ok(None) => break, |
| 628 | 657 | Err(e) => { |
| 629 | − return Err(multipart_err(state, e.to_string(), "Malformed multipart upload.")); | |
| 658 | + return Err(multipart_err(state, error_chain(&e), "Malformed multipart upload.")); | |
| 630 | 659 | } |
| 631 | 660 | }; |
| 632 | 661 | let name = field.name().unwrap_or("").to_string(); |
@@ -637,10 +666,10 @@ async fn ingest_multipart(
| 637 | 666 | let bytes = field |
| 638 | 667 | .bytes() |
| 639 | 668 | .await |
| 640 | − .map_err(|e| multipart_err(state, e.to_string(), "Could not read uploaded file."))?; | |
| 669 | + .map_err(|e| multipart_err(state, error_chain(&e), "Could not read uploaded file."))?; | |
| 641 | 670 | if bytes.len() > state.config.max_upload_bytes { |
| 642 | 671 | return Err(IngestError::TooLarge { |
| 643 | − actual: bytes.len(), | |
| 672 | + actual: Some(bytes.len()), | |
| 644 | 673 | limit: state.config.max_upload_bytes, |
| 645 | 674 | }); |
| 646 | 675 | } |
@@ -830,7 +859,7 @@ async fn ingest_url(
| 830 | 859 | if let Some(len) = resp.content_length() { |
| 831 | 860 | if len as usize > state.config.max_upload_bytes { |
| 832 | 861 | return Err(IngestError::TooLarge { |
| 833 | − actual: len as usize, | |
| 862 | + actual: Some(len as usize), | |
| 834 | 863 | limit: state.config.max_upload_bytes, |
| 835 | 864 | }); |
| 836 | 865 | } |
@@ -851,7 +880,7 @@ async fn ingest_url(
| 851 | 880 | .to_vec(); |
| 852 | 881 | if bytes.len() > state.config.max_upload_bytes { |
| 853 | 882 | return Err(IngestError::TooLarge { |
| 854 | − actual: bytes.len(), | |
| 883 | + actual: Some(bytes.len()), | |
| 855 | 884 | limit: state.config.max_upload_bytes, |
| 856 | 885 | }); |
| 857 | 886 | } |
@@ -899,6 +928,70 @@ async fn ingest_url(
| 899 | 928 | // Zip ingest — one archive of images in, many uploads out |
| 900 | 929 | // ============================================================================ |
| 901 | 930 | |
| 931 | +/// A spooled upload that deletes itself when it goes out of scope, whichever | |
| 932 | +/// way the handler exits. | |
| 933 | +struct SpooledFile(std::path::PathBuf); | |
| 934 | + | |
| 935 | +impl Drop for SpooledFile { | |
| 936 | + fn drop(&mut self) { | |
| 937 | + match std::fs::remove_file(&self.0) { | |
| 938 | + Ok(()) => {} | |
| 939 | + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} | |
| 940 | + Err(e) => error!(error = %e, path = %self.0.display(), "spooled upload cleanup failed"), | |
| 941 | + } | |
| 942 | + } | |
| 943 | +} | |
| 944 | + | |
| 945 | +/// Stream a multipart field to disk, enforcing `limit` as the bytes arrive. | |
| 946 | +/// | |
| 947 | +/// Spooling rather than buffering is what keeps a multi-gigabyte archive off | |
| 948 | +/// the heap — `Field::bytes()` would hold the whole thing in memory at once. | |
| 949 | +/// The spool lives under STORAGE_ROOT, not `/tmp`, because `/tmp` is tmpfs on | |
| 950 | +/// most hosts and would put those gigabytes straight back into RAM. | |
| 951 | +async fn spool_field( | |
| 952 | + state: &AppState, | |
| 953 | + field: &mut axum::extract::multipart::Field<'_>, | |
| 954 | + limit: usize, | |
| 955 | +) -> Result<(SpooledFile, usize), IngestError> { | |
| 956 | + use tokio::io::AsyncWriteExt; | |
| 957 | + | |
| 958 | + let dir = std::path::Path::new(&state.config.storage_root).join("tmp"); | |
| 959 | + tokio::fs::create_dir_all(&dir).await.map_err(|e| { | |
| 960 | + error!(error = %e, "create spool dir failed"); | |
| 961 | + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR) | |
| 962 | + })?; | |
| 963 | + | |
| 964 | + let spooled = SpooledFile(dir.join(format!("upload-{}.part", Uuid::now_v7()))); | |
| 965 | + let mut file = tokio::fs::File::create(&spooled.0).await.map_err(|e| { | |
| 966 | + error!(error = %e, "create spool file failed"); | |
| 967 | + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR) | |
| 968 | + })?; | |
| 969 | + | |
| 970 | + let mut written: usize = 0; | |
| 971 | + while let Some(chunk) = field | |
| 972 | + .chunk() | |
| 973 | + .await | |
| 974 | + .map_err(|e| multipart_err(state, error_chain(&e), "Could not read uploaded file."))? | |
| 975 | + { | |
| 976 | + written += chunk.len(); | |
| 977 | + // Bail the moment the running total crosses the line — no reason to | |
| 978 | + // keep writing bytes we have already decided to reject. | |
| 979 | + if written > limit { | |
| 980 | + return Err(IngestError::TooLarge { actual: Some(written), limit }); | |
| 981 | + } | |
| 982 | + file.write_all(&chunk).await.map_err(|e| { | |
| 983 | + error!(error = %e, "write to spool failed"); | |
| 984 | + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR) | |
| 985 | + })?; | |
| 986 | + } | |
| 987 | + file.flush().await.map_err(|e| { | |
| 988 | + error!(error = %e, "flush spool failed"); | |
| 989 | + IngestError::Status(StatusCode::INTERNAL_SERVER_ERROR) | |
| 990 | + })?; | |
| 991 | + | |
| 992 | + Ok((spooled, written)) | |
| 993 | +} | |
| 994 | + | |
| 902 | 995 | /// Tally of a zip import. Everything the import chose *not* to store is |
| 903 | 996 | /// counted so the user gets an honest report instead of a silent truncation. |
| 904 | 997 | #[derive(Debug, Default)] |
@@ -963,11 +1056,11 @@ struct ZipScan {
| 963 | 1056 | } |
| 964 | 1057 | |
| 965 | 1058 | /// Walk the archive on a blocking thread, handing each image to `tx` as it is |
| 966 | −/// decompressed. Streaming through a small channel — rather than extracting up | |
| 967 | −/// front — keeps peak memory at the zip plus a handful of images, not the | |
| 968 | −/// whole decompressed archive. | |
| 1059 | +/// decompressed. The archive is read from its spool on disk and entries stream | |
| 1060 | +/// through a small channel, so peak memory is a handful of images regardless of | |
| 1061 | +/// how large the zip is. | |
| 969 | 1062 | fn scan_zip_blocking( |
| 970 | − zip_bytes: Vec<u8>, | |
| 1063 | + zip_path: std::path::PathBuf, | |
| 971 | 1064 | max_entry_bytes: usize, |
| 972 | 1065 | max_total_bytes: usize, |
| 973 | 1066 | max_entries: usize, |
@@ -975,7 +1068,13 @@ fn scan_zip_blocking(
| 975 | 1068 | ) -> Result<ZipScan, String> { |
| 976 | 1069 | use std::io::Read; |
| 977 | 1070 | |
| 978 | − let mut archive = zip::ZipArchive::new(std::io::Cursor::new(zip_bytes)) | |
| 1071 | + let file = std::fs::File::open(&zip_path).map_err(|e| { | |
| 1072 | + error!(error = %e, path = %zip_path.display(), "open spooled zip failed"); | |
| 1073 | + "Could not read the uploaded zip file.".to_string() | |
| 1074 | + })?; | |
| 1075 | + // Seeking through the file on disk rather than a heap buffer is the whole | |
| 1076 | + // point — peak memory is one entry, not the whole archive. | |
| 1077 | + let mut archive = zip::ZipArchive::new(std::io::BufReader::new(file)) | |
| 979 | 1078 | .map_err(|e| format!("Could not read that zip file: {e}"))?; |
| 980 | 1079 | |
| 981 | 1080 | let mut scan = ZipScan::default(); |
@@ -1073,29 +1172,22 @@ async fn ingest_zip(
| 1073 | 1172 | user: &UserContext, |
| 1074 | 1173 | mut multipart: Multipart, |
| 1075 | 1174 | ) -> Result<ZipImportSummary, IngestError> { |
| 1076 | − let mut zip_bytes: Option<Vec<u8>> = None; | |
| 1175 | + let mut spooled_zip: Option<SpooledFile> = None; | |
| 1077 | 1176 | let mut item_description: Option<String> = None; |
| 1078 | 1177 | let mut tags_raw: Option<String> = None; |
| 1079 | 1178 | |
| 1080 | 1179 | loop { |
| 1081 | − let field = match multipart.next_field().await { | |
| 1180 | + let mut field = match multipart.next_field().await { | |
| 1082 | 1181 | Ok(Some(f)) => f, |
| 1083 | 1182 | Ok(None) => break, |
| 1084 | − Err(e) => return Err(multipart_err(state, e.to_string(), "Malformed multipart upload.")), | |
| 1183 | + Err(e) => return Err(multipart_err(state, error_chain(&e), "Malformed multipart upload.")), | |
| 1085 | 1184 | }; |
| 1086 | 1185 | match field.name().unwrap_or("") { |
| 1087 | 1186 | "file" => { |
| 1088 | − let bytes = field | |
| 1089 | − .bytes() | |
| 1090 | − .await | |
| 1091 | − .map_err(|e| multipart_err(state, e.to_string(), "Could not read uploaded file."))?; | |
| 1092 | − if bytes.len() > state.config.max_upload_bytes { | |
| 1093 | − return Err(IngestError::TooLarge { | |
| 1094 | − actual: bytes.len(), | |
| 1095 | − limit: state.config.max_upload_bytes, | |
| 1096 | − }); | |
| 1097 | − } | |
| 1098 | − zip_bytes = Some(bytes.to_vec()); | |
| 1187 | + let (spooled, size) = | |
| 1188 | + spool_field(state, &mut field, state.config.max_upload_bytes).await?; | |
| 1189 | + debug!(bytes = size, path = %spooled.0.display(), "zip spooled to disk"); | |
| 1190 | + spooled_zip = Some(spooled); | |
| 1099 | 1191 | } |
| 1100 | 1192 | "item_description" => { |
| 1101 | 1193 | item_description = field.text().await.ok().filter(|s| !s.trim().is_empty()); |
@@ -1109,14 +1201,18 @@ async fn ingest_zip(
| 1109 | 1201 | } |
| 1110 | 1202 | } |
| 1111 | 1203 | |
| 1112 | − let zip_bytes = zip_bytes.ok_or_else(|| IngestError::BadRequest("No zip file selected.".into()))?; | |
| 1204 | + // Held to the end of the function: dropping it deletes the spool, and the | |
| 1205 | + // blocking scan below is reading from that exact path. | |
| 1206 | + let spooled_zip = | |
| 1207 | + spooled_zip.ok_or_else(|| IngestError::BadRequest("No zip file selected.".into()))?; | |
| 1113 | 1208 | |
| 1114 | 1209 | let (tx, mut rx) = tokio::sync::mpsc::channel::<ZipEntryFile>(4); |
| 1210 | + let zip_path = spooled_zip.0.clone(); | |
| 1115 | 1211 | let max_entry_bytes = state.config.max_upload_bytes; |
| 1116 | 1212 | let max_total_bytes = state.config.max_zip_total_bytes; |
| 1117 | 1213 | let max_entries = state.config.max_zip_entries; |
| 1118 | 1214 | let scanner = tokio::task::spawn_blocking(move || { |
| 1119 | − scan_zip_blocking(zip_bytes, max_entry_bytes, max_total_bytes, max_entries, tx) | |
| 1215 | + scan_zip_blocking(zip_path, max_entry_bytes, max_total_bytes, max_entries, tx) | |
| 1120 | 1216 | }); |
| 1121 | 1217 | |
| 1122 | 1218 | let mut summary = ZipImportSummary::default(); |
@@ -1176,6 +1272,7 @@ async fn ingest_zip(
| 1176 | 1272 | truncated = summary.truncated, |
| 1177 | 1273 | "zip import finished" |
| 1178 | 1274 | ); |
| 1275 | + drop(spooled_zip); | |
| 1179 | 1276 | Ok(summary) |
| 1180 | 1277 | } |
| 1181 | 1278 |
@@ -1262,13 +1359,22 @@ mod zip_unit_tests {
| 1262 | 1359 | w.finish().expect("finish").into_inner() |
| 1263 | 1360 | } |
| 1264 | 1361 | |
| 1362 | + /// The scan reads from disk, so spool the bytes the way the handler does. | |
| 1363 | + /// SpooledFile cleans the path up when the test drops it. | |
| 1364 | + fn spool(zip_bytes: &[u8]) -> SpooledFile { | |
| 1365 | + let path = std::env::temp_dir().join(format!("dms-test-{}.zip", Uuid::now_v7())); | |
| 1366 | + std::fs::write(&path, zip_bytes).expect("write test zip"); | |
| 1367 | + SpooledFile(path) | |
| 1368 | + } | |
| 1369 | + | |
| 1265 | 1370 | /// Run the scan and drain the channel. Capacity comfortably exceeds the |
| 1266 | 1371 | /// entry count so the blocking sends never park. |
| 1267 | 1372 | fn scan(zip_bytes: Vec<u8>, max_entry: usize, max_total: usize, max_entries: usize) |
| 1268 | 1373 | -> (Vec<ZipEntryFile>, ZipScan) |
| 1269 | 1374 | { |
| 1375 | + let spooled = spool(&zip_bytes); | |
| 1270 | 1376 | let (tx, mut rx) = tokio::sync::mpsc::channel::<ZipEntryFile>(64); |
| 1271 | − let scan = scan_zip_blocking(zip_bytes, max_entry, max_total, max_entries, tx) | |
| 1377 | + let scan = scan_zip_blocking(spooled.0.clone(), max_entry, max_total, max_entries, tx) | |
| 1272 | 1378 | .expect("scan should succeed"); |
| 1273 | 1379 | let mut got = Vec::new(); |
| 1274 | 1380 | while let Ok(entry) = rx.try_recv() { |
@@ -1329,12 +1435,37 @@ mod zip_unit_tests {
| 1329 | 1435 | |
| 1330 | 1436 | #[test] |
| 1331 | 1437 | fn garbage_is_not_a_zip() { |
| 1438 | + let spooled = spool(b"definitely not a zip"); | |
| 1332 | 1439 | let (tx, _rx) = tokio::sync::mpsc::channel::<ZipEntryFile>(4); |
| 1333 | − let err = scan_zip_blocking(b"definitely not a zip".to_vec(), 1024, 1024, 10, tx) | |
| 1440 | + let err = scan_zip_blocking(spooled.0.clone(), 1024, 1024, 10, tx) | |
| 1334 | 1441 | .expect_err("garbage must be rejected"); |
| 1335 | 1442 | assert!(err.starts_with("Could not read that zip file"), "got: {err}"); |
| 1336 | 1443 | } |
| 1337 | 1444 | |
| 1445 | + #[test] | |
| 1446 | + fn spool_is_removed_on_drop() { | |
| 1447 | + let path = { | |
| 1448 | + let spooled = spool(b"transient"); | |
| 1449 | + let p = spooled.0.clone(); | |
| 1450 | + assert!(p.exists()); | |
| 1451 | + p | |
| 1452 | + }; | |
| 1453 | + assert!(!path.exists(), "the spool must not outlive its guard"); | |
| 1454 | + } | |
| 1455 | + | |
| 1456 | + #[test] | |
| 1457 | + fn too_large_message_omits_an_unknown_size() { | |
| 1458 | + // The mid-stream limit trip knows only that the request went over. | |
| 1459 | + assert_eq!( | |
| 1460 | + too_large_message("Zip", None, 50 * 1024 * 1024), | |
| 1461 | + "Zip exceeds the maximum of 50.0 MiB." | |
| 1462 | + ); | |
| 1463 | + assert_eq!( | |
| 1464 | + too_large_message("Upload", Some(200 * 1024 * 1024), 100 * 1024 * 1024), | |
| 1465 | + "Upload too large (200.0 MiB). Maximum is 100.0 MiB." | |
| 1466 | + ); | |
| 1467 | + } | |
| 1468 | + | |
| 1338 | 1469 | #[test] |
| 1339 | 1470 | fn summary_message_reports_every_bucket() { |
| 1340 | 1471 | let summary = ZipImportSummary { |
modified
templates/uploads/list.html
+2
−2
@@ -43,7 +43,7 @@
| 43 | 43 | Paste an image (<kbd class="kbd kbd-xs">Ctrl</kbd>+<kbd class="kbd kbd-xs">V</kbd>) to attach it |
| 44 | 44 | </span> |
| 45 | 45 | </div> |
| 46 | − <input id="quick-upload-file" class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept="image/*" required/> | |
| 46 | + <input id="quick-upload-file" class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept="image/*" data-max-bytes="{{ max_upload_bytes }}" required/> | |
| 47 | 47 | <div id="quick-upload-preview" class="hidden items-center gap-2 text-xs"> |
| 48 | 48 | <img class="h-16 w-16 object-cover rounded border border-base-300" alt="pasted preview"/> |
| 49 | 49 | <span class="opacity-70" data-name></span> |
@@ -59,7 +59,7 @@
| 59 | 59 | <h3 class="font-bold">Bulk upload (zip)</h3> |
| 60 | 60 | <span class="text-xs opacity-60">Every image inside is ingested and processed</span> |
| 61 | 61 | </div> |
| 62 | − <input class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept=".zip,application/zip,application/x-zip-compressed" required/> | |
| 62 | + <input class="file-input file-input-bordered file-input-sm w-full" type="file" name="file" accept=".zip,application/zip,application/x-zip-compressed" data-max-bytes="{{ max_upload_bytes }}" required/> | |
| 63 | 63 | <textarea class="textarea textarea-bordered textarea-sm w-full" name="item_description" placeholder="Optional description — applied to every image"></textarea> |
| 64 | 64 | <input class="input input-bordered input-sm w-full" type="text" name="tags" placeholder="Optional tags — applied to every image"/> |
| 65 | 65 | <button class="btn btn-primary btn-sm" type="submit">Upload zip</button> |
Parents: 5a7f679