From b7c1ceb2b35b806bce35c860b855a25d52ee8b48 Mon Sep 17 00:00:00 2001 From: Khairul Hidayat Date: Sat, 16 Mar 2024 21:29:29 +0700 Subject: [PATCH] fix: memory leak on file web stream fix --- backend/lib/utils.ts | 16 ++++ backend/routes/files.ts | 144 +++++++++++++++-------------------- src/app/apps/files/index.tsx | 12 +-- src/app/apps/files/utils.ts | 15 ++++ src/app/apps/terminal.tsx | 6 +- src/app/auth/login.tsx | 3 +- src/lib/api.ts | 2 +- src/lib/constants.ts | 4 +- yarn.lock | 48 +++++++++++- 9 files changed, 148 insertions(+), 102 deletions(-) create mode 100644 src/app/apps/files/utils.ts diff --git a/backend/lib/utils.ts b/backend/lib/utils.ts index 7f2e7dd..5d8cd22 100644 --- a/backend/lib/utils.ts +++ b/backend/lib/utils.ts @@ -1,3 +1,7 @@ +import { Mime } from "mime/lite"; +import standardTypes from "mime/types/standard.js"; +import otherTypes from "mime/types/other.js"; + export const formatBytes = (bytes: number) => { const sizes = ["Bytes", "KB", "MB", "GB", "TB"]; if (bytes === 0) return "n/a"; @@ -13,3 +17,15 @@ export const secondsToTime = (seconds: number) => { // const s = Math.floor(seconds % 60); return `${d}d ${h}h ${m}m`; }; + +export const mime = new Mime(standardTypes, otherTypes); +mime.define( + { + "video/webm": ["mkv"], + }, + true +); + +export const getMimeType = (path: string) => { + return mime.getType(path) || "application/octet-stream"; +}; diff --git a/backend/routes/files.ts b/backend/routes/files.ts index 70fe9a1..6d48c78 100644 --- a/backend/routes/files.ts +++ b/backend/routes/files.ts @@ -3,9 +3,8 @@ import { Hono } from "hono"; import { z } from "zod"; import fs from "node:fs/promises"; import { HTTPException } from "hono/http-exception"; -import { ReadStream, createReadStream } from "node:fs"; -import { ReadableStream } from "stream/web"; -import mime from "mime"; +import { createReadStream } from "node:fs"; +import { getMimeType } from "../lib/utils"; const getFilesSchema = z .object({ @@ -69,89 +68,68 @@ const route = new Hono() return c.json([]); }) - .get( - "/download", - zValidator("query", z.object({ path: z.string().min(1) })), - async (c) => { - const pathname = (c.req.query("path") || "").split("/"); - const path = "/" + pathname.slice(2).join("/"); - const baseName = pathname[1]; + .get("/download/*", async (c) => { + const dlFile = c.req.query("dl") === "true"; + const url = new URL(c.req.url, `http://${c.req.header("host")}`); + const pathname = decodeURI(url.pathname).split("/"); + const pathSlice = pathname.slice(pathname.indexOf("download") + 1); + const baseName = pathSlice[0]; + const path = "/" + pathSlice.slice(1).join("/"); - try { - if (!baseName?.length) { - throw new Error(); - } - - const baseDir = filesDirList.find((i) => i.name === baseName)?.path; - if (!baseDir) { - throw new Error(); - } - - const filepath = baseDir + path; - const stat = await fs.stat(filepath); - const size = stat.size; - - // c.header("Content-Type", "application/octet-stream"); - // c.header("Content-Disposition", `attachment; filename="${path}"`); - - c.header( - "Content-Type", - mime.getType(filepath) || "application/octet-stream" - ); - - if (c.req.method == "HEAD" || c.req.method == "OPTIONS") { - c.header("Content-Length", size.toString()); - c.status(200); - return c.body(null); - } - - const range = c.req.header("range") || ""; - - if (!range) { - c.header("Content-Length", size.toString()); - return c.body(createStreamBody(createReadStream(filepath)), 200); - } - - c.header("Accept-Ranges", "bytes"); - c.header("Date", stat.birthtime.toUTCString()); - - const parts = range.replace(/bytes=/, "").split("-", 2); - const start = parts[0] ? parseInt(parts[0], 10) : 0; - let end = parts[1] ? parseInt(parts[1], 10) : stat.size - 1; - if (size < end - start + 1) { - end = size - 1; - } - - const chunksize = end - start + 1; - const stream = createReadStream(filepath, { start, end }); - - c.header("Content-Length", chunksize.toString()); - c.header("Content-Range", `bytes ${start}-${end}/${stat.size}`); - - return c.body(createStreamBody(stream), 206); - } catch (err) { - console.error(err); - throw new HTTPException(404, { message: "Not Found!" }); + try { + if (!baseName?.length) { + throw new Error(); } + + const baseDir = filesDirList.find((i) => i.name === baseName)?.path; + if (!baseDir) { + throw new Error(); + } + + const filepath = baseDir + path; + const stat = await fs.stat(filepath); + const size = stat.size; + + if (dlFile) { + c.header("Content-Type", "application/octet-stream"); + c.header("Content-Disposition", `attachment; filename="${path}"`); + } else { + c.header("Content-Type", getMimeType(filepath)); + } + + if (c.req.method == "HEAD" || c.req.method == "OPTIONS") { + c.header("Content-Length", size.toString()); + c.status(200); + return c.body(null); + } + + const range = c.req.header("range") || ""; + + if (!range || dlFile) { + c.header("Content-Length", size.toString()); + return c.body(createReadStream(filepath), 200); + } + + c.header("Accept-Ranges", "bytes"); + c.header("Date", stat.birthtime.toUTCString()); + + const parts = range.replace(/bytes=/, "").split("-", 2); + const start = parts[0] ? parseInt(parts[0], 10) : 0; + let end = parts[1] ? parseInt(parts[1], 10) : stat.size - 1; + if (size < end - start + 1) { + end = size - 1; + } + + const chunksize = end - start + 1; + const stream = createReadStream(filepath, { start, end }); + + c.header("Content-Length", chunksize.toString()); + c.header("Content-Range", `bytes ${start}-${end}/${stat.size}`); + + return c.body(stream, 206); + } catch (err) { + throw new HTTPException(404, { message: "Not Found!" }); } - ); - -const createStreamBody = (stream: ReadStream) => { - const body = new ReadableStream({ - start(controller) { - stream.on("data", (chunk) => { - controller.enqueue(chunk); - }); - stream.on("end", () => { - controller.close(); - }); - }, - - cancel() { - stream.destroy(); - }, }); - return body; -}; export default route; diff --git a/src/app/apps/files/index.tsx b/src/app/apps/files/index.tsx index 2258f72..5c821b7 100644 --- a/src/app/apps/files/index.tsx +++ b/src/app/apps/files/index.tsx @@ -1,13 +1,14 @@ import FileList, { FileItem } from "@/components/pages/files/FileList"; import { useAsyncStorage } from "@/hooks/useAsyncStorage"; import api from "@/lib/api"; -import authStore, { useAuth } from "@/stores/authStore"; +import { useAuth } from "@/stores/authStore"; import BackButton from "@ui/BackButton"; import Box from "@ui/Box"; import Input from "@ui/Input"; import { Stack } from "expo-router"; import React from "react"; import { useQuery } from "react-query"; +import { openFile } from "./utils"; const FilesPage = () => { const { isLoggedIn } = useAuth(); @@ -49,7 +50,7 @@ const FilesPage = () => { return setParams({ ...params, path: file.path }); } - downloadFile(file); + openFile(file); }} canGoBack={parentPath != null} /> @@ -57,11 +58,4 @@ const FilesPage = () => { ); }; -async function downloadFile(file: FileItem) { - const url = api.files.download.$url(); - url.searchParams.set("path", file.path); - url.searchParams.set("token", authStore.getState().token); - window.open(url.toString(), "_blank"); -} - export default FilesPage; diff --git a/src/app/apps/files/utils.ts b/src/app/apps/files/utils.ts new file mode 100644 index 0000000..05ce887 --- /dev/null +++ b/src/app/apps/files/utils.ts @@ -0,0 +1,15 @@ +import { FileItem } from "@/components/pages/files/FileList"; +import { API_BASEURL } from "@/lib/constants"; +import authStore from "@/stores/authStore"; + +export function openFile(file: FileItem, dl = false) { + const url = getFileUrl(file, dl); + window.open(url, "_blank"); +} + +export function getFileUrl(file: FileItem, dl = false) { + const url = new URL(API_BASEURL + "/files/download" + file.path); + url.searchParams.set("token", authStore.getState().token); + dl && url.searchParams.set("dlmode", "true"); + return url.toString(); +} diff --git a/src/app/apps/terminal.tsx b/src/app/apps/terminal.tsx index 7ce5c9f..c1f6cdc 100644 --- a/src/app/apps/terminal.tsx +++ b/src/app/apps/terminal.tsx @@ -7,8 +7,8 @@ import Box from "@ui/Box"; import Text from "@ui/Text"; import React, { useEffect, useRef } from "react"; import "xterm/css/xterm.css"; -import { API_BASEURL } from "@/lib/constants"; -import authStore, { useAuth } from "@/stores/authStore"; +import { BASEURL } from "@/lib/constants"; +import { useAuth } from "@/stores/authStore"; import { Stack } from "expo-router"; import BackButton from "@ui/BackButton"; @@ -28,7 +28,7 @@ const TerminalPage = () => { const fitAddon = new FitAddon(); term.loadAddon(fitAddon); - const baseUrl = API_BASEURL.replace("https://", "wss://").replace( + const baseUrl = BASEURL.replace("https://", "wss://").replace( "http://", "ws://" ); diff --git a/src/app/auth/login.tsx b/src/app/auth/login.tsx index 31982b3..f0d70b6 100644 --- a/src/app/auth/login.tsx +++ b/src/app/auth/login.tsx @@ -13,7 +13,7 @@ import { useMutation } from "react-query"; import api from "@/lib/api"; import Alert from "@ui/Alert"; import { setAuthToken } from "@/stores/authStore"; -import { router } from "expo-router"; +import { Stack, router } from "expo-router"; const schema = z.object({ username: z.string().min(1), @@ -44,6 +44,7 @@ const LoginPage = () => { + Login diff --git a/src/lib/api.ts b/src/lib/api.ts index ae2b93e..4986e8f 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -10,7 +10,7 @@ import type { import type { AppType } from "../../backend/routes/_routes"; import authStore, { logout } from "@/stores/authStore"; -const api: ReturnType> = hc(API_BASEURL + "/api", { +const api: ReturnType> = hc(API_BASEURL, { fetch: fetchHandler, }); diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 2c5cf7e..50a42a0 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -1,3 +1,5 @@ -export const API_BASEURL = __DEV__ +export const BASEURL = __DEV__ ? "http://localhost:3000" : location.protocol + "//" + location.host; + +export const API_BASEURL = BASEURL + "/api"; diff --git a/yarn.lock b/yarn.lock index 3669b80..ecca994 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6247,7 +6247,7 @@ prompts@^2.3.2, prompts@^2.4.2: kleur "^3.0.3" sisteransi "^1.0.5" -prop-types@^15.7.2, prop-types@^15.8.1: +prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: version "15.8.1" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== @@ -6368,6 +6368,13 @@ react-is@^17.0.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== +react-native-animatable@1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/react-native-animatable/-/react-native-animatable-1.3.3.tgz#a13a4af8258e3bb14d0a9d839917e9bb9274ec8a" + integrity sha512-2ckIxZQAsvWn25Ho+DK3d1mXIgj7tITkrS4pYDvx96WyOttSvzzFeQnM2od0+FUMzILbdHDsDEqZvnz1DYNQ1w== + dependencies: + prop-types "^15.7.2" + react-native-circular-progress@^1.3.9: version "1.3.9" resolved "https://registry.yarnpkg.com/react-native-circular-progress/-/react-native-circular-progress-1.3.9.tgz#9552c26d6b2e6bb447a47da201ef2ae9a453e273" @@ -6375,6 +6382,14 @@ react-native-circular-progress@^1.3.9: dependencies: prop-types "^15.8.1" +react-native-modal@^13.0.1: + version "13.0.1" + resolved "https://registry.yarnpkg.com/react-native-modal/-/react-native-modal-13.0.1.tgz#691f1e646abb96fa82c1788bf18a16d585da37cd" + integrity sha512-UB+mjmUtf+miaG/sDhOikRfBOv0gJdBU2ZE1HtFWp6UixW9jCk/bhGdHUgmZljbPpp0RaO/6YiMmQSSK3kkMaw== + dependencies: + prop-types "^15.6.2" + react-native-animatable "1.3.3" + react-native-safe-area-context@4.8.2: version "4.8.2" resolved "https://registry.yarnpkg.com/react-native-safe-area-context/-/react-native-safe-area-context-4.8.2.tgz#e6b3d8acf3c6afcb4b5db03a97f9c37df7668f65" @@ -7067,7 +7082,16 @@ strict-uri-encode@^2.0.0: resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546" integrity sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ== -"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -7099,7 +7123,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -7113,6 +7137,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -7738,7 +7769,7 @@ wonka@^6.3.2: resolved "https://registry.yarnpkg.com/wonka/-/wonka-6.3.4.tgz#76eb9316e3d67d7febf4945202b5bdb2db534594" integrity sha512-CjpbqNtBGNAeyNS/9W6q3kSkKE52+FjIj7AkFlLr11s/VWGUu6a2CdYSdGxocIhIVjaW/zchesBQUKPVU69Cqg== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -7756,6 +7787,15 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"