Handling fs
This commit is contained in:
+7
-2
@@ -16,9 +16,14 @@ STORAGE_DIR=./storage
|
|||||||
# case-insensitive). A request larger than this is rejected with 413.
|
# case-insensitive). A request larger than this is rejected with 413.
|
||||||
MAX_FILE_SIZE=10GB
|
MAX_FILE_SIZE=10GB
|
||||||
|
|
||||||
|
# Maximum fraction of the storage filesystem that may be used, from 0.0 to 1.0.
|
||||||
|
# When usage exceeds this ratio uploads are rejected with 503 "try again later".
|
||||||
|
# Default 0.5 = block uploads once more than 50% of the disk is used.
|
||||||
|
MAX_STORAGE_USAGE=0.5
|
||||||
|
|
||||||
# How long an uploaded file lives before it expires and is deleted.
|
# How long an uploaded file lives before it expires and is deleted.
|
||||||
# In milliseconds. Default is 1 week (604800000).
|
# In milliseconds. Default and hard maximum is 48 hours (172800000).
|
||||||
FILE_TTL_MS=604800000
|
FILE_TTL_MS=172800000
|
||||||
|
|
||||||
# Brute-force protection: rate limit per client IP on download and
|
# Brute-force protection: rate limit per client IP on download and
|
||||||
# deletion endpoints. Set RATE_LIMIT_ENABLED=0 to disable.
|
# deletion endpoints. Set RATE_LIMIT_ENABLED=0 to disable.
|
||||||
|
|||||||
+6
-3
@@ -63,7 +63,8 @@ HOST=127.0.0.1 # bind loopback only; the proxy talks to it
|
|||||||
PORT=3000
|
PORT=3000
|
||||||
STORAGE_DIR=/opt/filehost/storage
|
STORAGE_DIR=/opt/filehost/storage
|
||||||
MAX_FILE_SIZE=10GB # must be <= proxy upload limit
|
MAX_FILE_SIZE=10GB # must be <= proxy upload limit
|
||||||
FILE_TTL_MS=604800000 # expire files after 1 week
|
FILE_TTL_MS=172800000 # expire files after 48 hours (hard maximum)
|
||||||
|
MAX_STORAGE_USAGE=0.5 # block uploads when >50% of the disk is used
|
||||||
RATE_LIMIT_ENABLED=1
|
RATE_LIMIT_ENABLED=1
|
||||||
TRUST_PROXY=1 # proxy sets X-Forwarded-For
|
TRUST_PROXY=1 # proxy sets X-Forwarded-For
|
||||||
```
|
```
|
||||||
@@ -247,6 +248,8 @@ Files in `storage/` are untouched by updates and keep working.
|
|||||||
short-lived HMAC-signed cookie (server secret in `storage/.secret`).
|
short-lived HMAC-signed cookie (server secret in `storage/.secret`).
|
||||||
- Rate limiting protects the keyed endpoints; `TRUST_PROXY=1` must only be set
|
- Rate limiting protects the keyed endpoints; `TRUST_PROXY=1` must only be set
|
||||||
when a proxy really sets `X-Forwarded-For`/`CF-Connecting-IP`.
|
when a proxy really sets `X-Forwarded-For`/`CF-Connecting-IP`.
|
||||||
- Files auto-expire after one week (configurable via `FILE_TTL_MS`) and are
|
- Files auto-expire after 48 hours (configurable via `FILE_TTL_MS`, hard-capped
|
||||||
swept hourly. Deletion also requires possession of the key
|
at 48h) and are swept hourly. Deletion also requires possession of the key
|
||||||
(`DELETE /api/files/:id` with `X-Delete-Key`).
|
(`DELETE /api/files/:id` with `X-Delete-Key`).
|
||||||
|
- Uploads are rejected with `503 try again later` when the storage filesystem is
|
||||||
|
more than 50% full (configurable via `MAX_STORAGE_USAGE`).
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
A minimal, self-hosted private file hosting service built with **Bun** and
|
A minimal, self-hosted private file hosting service built with **Bun** and
|
||||||
**TypeScript**. Upload a file, get a secret, unguessable download link. Files are
|
**TypeScript**. Upload a file, get a secret, unguessable download link. Files are
|
||||||
stored on the local filesystem and expire after one week.
|
stored on the local filesystem and expire after 48 hours.
|
||||||
|
|
||||||
Everything lives on disk — the filesystem is the source of truth. There is no
|
Everything lives on disk — the filesystem is the source of truth. There is no
|
||||||
database, no object storage, and no queue. Expired files are swept automatically.
|
database, no object storage, and no queue. Expired files are swept automatically.
|
||||||
@@ -15,7 +15,7 @@ database, no object storage, and no queue. Expired files are swept automatically
|
|||||||
- A cryptographically secure, unguessable download key for every file.
|
- A cryptographically secure, unguessable download key for every file.
|
||||||
- Streaming uploads and downloads — no full buffering in RAM, large files work.
|
- Streaming uploads and downloads — no full buffering in RAM, large files work.
|
||||||
- Configurable maximum upload size (`413` when exceeded, default 10GB).
|
- Configurable maximum upload size (`413` when exceeded, default 10GB).
|
||||||
- Files **auto-expire after one week** and are removed automatically.
|
- Files **auto-expire after 48 hours** and are removed automatically.
|
||||||
- Authenticated deletion (possession of the file's secret key is required).
|
- Authenticated deletion (possession of the file's secret key is required).
|
||||||
- Rate-limited download/delete endpoints to blunt brute force.
|
- Rate-limited download/delete endpoints to blunt brute force.
|
||||||
- Zero runtime dependencies (Bun is the only runtime).
|
- Zero runtime dependencies (Bun is the only runtime).
|
||||||
@@ -49,7 +49,8 @@ cp .env.example .env
|
|||||||
| `PORT` | `3000` | Bind port. |
|
| `PORT` | `3000` | Bind port. |
|
||||||
| `STORAGE_DIR` | `./storage` | Directory where files + metadata are stored (relative to CWD). |
|
| `STORAGE_DIR` | `./storage` | Directory where files + metadata are stored (relative to CWD). |
|
||||||
| `MAX_FILE_SIZE` | `10GB` | Max upload size. Units: `B`, `KB`, `MB`, `GB`, `TB` (or plain int). |
|
| `MAX_FILE_SIZE` | `10GB` | Max upload size. Units: `B`, `KB`, `MB`, `GB`, `TB` (or plain int). |
|
||||||
| `FILE_TTL_MS` | `604800000` | Milliseconds before an upload expires and is deleted (1 week). |
|
| `MAX_STORAGE_USAGE` | `0.5` | Max fraction (0.0–1.0) of the storage disk that may be used; uploads are rejected with `503 try again later` above this. |
|
||||||
|
| `FILE_TTL_MS` | `172800000` | Milliseconds before an upload expires and is deleted (48 hours, hard maximum). |
|
||||||
| `RATE_LIMIT_ENABLED` | `1` | Enable per-IP rate limiting on `/(key)` and `/api/files/`. |
|
| `RATE_LIMIT_ENABLED` | `1` | Enable per-IP rate limiting on `/(key)` and `/api/files/`. |
|
||||||
| `RATE_LIMIT_MAX` | `300` | Max requests per IP per window. |
|
| `RATE_LIMIT_MAX` | `300` | Max requests per IP per window. |
|
||||||
| `RATE_LIMIT_WINDOW_MS` | `60000` | Rate-limit window length in ms. |
|
| `RATE_LIMIT_WINDOW_MS` | `60000` | Rate-limit window length in ms. |
|
||||||
@@ -73,7 +74,7 @@ Then open http://localhost:3000.
|
|||||||
4. Copy the URL. Anyone possessing it can download the file — **after entering
|
4. Copy the URL. Anyone possessing it can download the file — **after entering
|
||||||
the password you set at upload**. Nobody without the secret URL and the
|
the password you set at upload**. Nobody without the secret URL and the
|
||||||
password can download it.
|
password can download it.
|
||||||
5. The file is removed automatically after one week, or earlier by explicit
|
5. The file is removed automatically after 48 hours, or earlier by explicit
|
||||||
deletion.
|
deletion.
|
||||||
|
|
||||||
## Filesystem layout
|
## Filesystem layout
|
||||||
|
|||||||
@@ -48,8 +48,14 @@ in
|
|||||||
|
|
||||||
fileTtlMs = lib.mkOption {
|
fileTtlMs = lib.mkOption {
|
||||||
type = lib.types.ints.positive;
|
type = lib.types.ints.positive;
|
||||||
default = 7 * 24 * 60 * 60 * 1000;
|
default = 48 * 60 * 60 * 1000;
|
||||||
description = "Milliseconds before an uploaded file expires and is deleted.";
|
description = "Milliseconds before an uploaded file expires and is deleted (hard-capped at 48 hours).";
|
||||||
|
};
|
||||||
|
|
||||||
|
maxStorageUsage = lib.mkOption {
|
||||||
|
type = lib.types.float;
|
||||||
|
default = 0.5;
|
||||||
|
description = "Maximum fraction (0.0-1.0) of the storage filesystem that may be used; uploads are rejected above this.";
|
||||||
};
|
};
|
||||||
|
|
||||||
publicBaseUrl = lib.mkOption {
|
publicBaseUrl = lib.mkOption {
|
||||||
@@ -127,6 +133,7 @@ in
|
|||||||
"STORAGE_DIR=${cfg.storageDir}"
|
"STORAGE_DIR=${cfg.storageDir}"
|
||||||
"MAX_FILE_SIZE=${cfg.maxFileSize}"
|
"MAX_FILE_SIZE=${cfg.maxFileSize}"
|
||||||
"FILE_TTL_MS=${toString cfg.fileTtlMs}"
|
"FILE_TTL_MS=${toString cfg.fileTtlMs}"
|
||||||
|
"MAX_STORAGE_USAGE=${toString cfg.maxStorageUsage}"
|
||||||
]
|
]
|
||||||
++ lib.optionals (cfg.publicBaseUrl != null) [ "PUBLIC_BASE_URL=${cfg.publicBaseUrl}" ]
|
++ lib.optionals (cfg.publicBaseUrl != null) [ "PUBLIC_BASE_URL=${cfg.publicBaseUrl}" ]
|
||||||
++ lib.optionals cfg.trustProxy [ "TRUST_PROXY=1" ];
|
++ lib.optionals cfg.trustProxy [ "TRUST_PROXY=1" ];
|
||||||
|
|||||||
+12
-3
@@ -6,6 +6,7 @@ export interface Config {
|
|||||||
storageDir: string;
|
storageDir: string;
|
||||||
maxFileSize: number;
|
maxFileSize: number;
|
||||||
fileTtlMs: number;
|
fileTtlMs: number;
|
||||||
|
maxStorageUsage: number;
|
||||||
publicBaseUrl?: string;
|
publicBaseUrl?: string;
|
||||||
rateLimitEnabled: boolean;
|
rateLimitEnabled: boolean;
|
||||||
rateLimitMax: number;
|
rateLimitMax: number;
|
||||||
@@ -13,7 +14,8 @@ export interface Config {
|
|||||||
trustProxy: boolean;
|
trustProxy: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 1 week
|
const MAX_TTL_MS = 48 * 60 * 60 * 1000; // 48 hours hard maximum
|
||||||
|
const DEFAULT_TTL_MS = MAX_TTL_MS;
|
||||||
|
|
||||||
const SIZE_UNITS: Record<string, number> = {
|
const SIZE_UNITS: Record<string, number> = {
|
||||||
B: 1,
|
B: 1,
|
||||||
@@ -56,10 +58,16 @@ export function loadConfig(env: Record<string, string | undefined> = process.env
|
|||||||
const maxFileSize = parseSize(env.MAX_FILE_SIZE ?? "10GB");
|
const maxFileSize = parseSize(env.MAX_FILE_SIZE ?? "10GB");
|
||||||
if (maxFileSize < 1024) throw new Error("MAX_FILE_SIZE must be at least 1KB");
|
if (maxFileSize < 1024) throw new Error("MAX_FILE_SIZE must be at least 1KB");
|
||||||
|
|
||||||
const fileTtlMs = Number(env.FILE_TTL_MS ?? String(DEFAULT_TTL_MS));
|
const fileTtlMsRaw = Number(env.FILE_TTL_MS ?? String(DEFAULT_TTL_MS));
|
||||||
if (!Number.isInteger(fileTtlMs) || fileTtlMs < 1) {
|
if (!Number.isInteger(fileTtlMsRaw) || fileTtlMsRaw < 1) {
|
||||||
throw new Error(`Invalid FILE_TTL_MS: ${env.FILE_TTL_MS}`);
|
throw new Error(`Invalid FILE_TTL_MS: ${env.FILE_TTL_MS}`);
|
||||||
}
|
}
|
||||||
|
const fileTtlMs = Math.min(fileTtlMsRaw, MAX_TTL_MS);
|
||||||
|
|
||||||
|
const maxStorageUsage = Number(env.MAX_STORAGE_USAGE ?? "0.5");
|
||||||
|
if (!Number.isFinite(maxStorageUsage) || maxStorageUsage < 0 || maxStorageUsage > 1) {
|
||||||
|
throw new Error(`Invalid MAX_STORAGE_USAGE: ${env.MAX_STORAGE_USAGE}`);
|
||||||
|
}
|
||||||
|
|
||||||
const rateLimitMax = Number(env.RATE_LIMIT_MAX ?? "300");
|
const rateLimitMax = Number(env.RATE_LIMIT_MAX ?? "300");
|
||||||
if (!Number.isInteger(rateLimitMax) || rateLimitMax < 1) {
|
if (!Number.isInteger(rateLimitMax) || rateLimitMax < 1) {
|
||||||
@@ -88,6 +96,7 @@ export function loadConfig(env: Record<string, string | undefined> = process.env
|
|||||||
storageDir,
|
storageDir,
|
||||||
maxFileSize,
|
maxFileSize,
|
||||||
fileTtlMs,
|
fileTtlMs,
|
||||||
|
maxStorageUsage,
|
||||||
publicBaseUrl,
|
publicBaseUrl,
|
||||||
rateLimitEnabled: boolFromEnv(env.RATE_LIMIT_ENABLED, true),
|
rateLimitEnabled: boolFromEnv(env.RATE_LIMIT_ENABLED, true),
|
||||||
rateLimitMax,
|
rateLimitMax,
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ export async function handleUpload(
|
|||||||
maxFileSize: number,
|
maxFileSize: number,
|
||||||
ttlMs: number,
|
ttlMs: number,
|
||||||
publicBaseUrl?: string,
|
publicBaseUrl?: string,
|
||||||
|
maxStorageUsage = 0.5,
|
||||||
): Promise<Response> {
|
): Promise<Response> {
|
||||||
const contentType = req.headers.get("content-type");
|
const contentType = req.headers.get("content-type");
|
||||||
const boundary = contentType ? getBoundary(contentType) : null;
|
const boundary = contentType ? getBoundary(contentType) : null;
|
||||||
@@ -36,6 +37,15 @@ export async function handleUpload(
|
|||||||
const key = id;
|
const key = id;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const usage = await storage.storageUsageRatio();
|
||||||
|
if (usage > maxStorageUsage) {
|
||||||
|
log(
|
||||||
|
"warn",
|
||||||
|
`upload rejected id=${id}: storage usage ${(usage * 100).toFixed(1)}% exceeds limit ${(maxStorageUsage * 100).toFixed(0)}%`,
|
||||||
|
);
|
||||||
|
return jsonError(503, "storage is full, try again later");
|
||||||
|
}
|
||||||
|
|
||||||
let filename = "";
|
let filename = "";
|
||||||
let mime = "application/octet-stream";
|
let mime = "application/octet-stream";
|
||||||
let size = 0;
|
let size = 0;
|
||||||
|
|||||||
+1
-1
@@ -108,7 +108,7 @@ async function route(req: Request, srv: Server<undefined>): Promise<Response> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (path === "/api/upload" && method === "POST") {
|
if (path === "/api/upload" && method === "POST") {
|
||||||
return handleUpload(req, storage, cfg.maxFileSize, cfg.fileTtlMs, cfg.publicBaseUrl);
|
return handleUpload(req, storage, cfg.maxFileSize, cfg.fileTtlMs, cfg.publicBaseUrl, cfg.maxStorageUsage);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (path.startsWith("/api/auth/") && method === "POST") {
|
if (path.startsWith("/api/auth/") && method === "POST") {
|
||||||
|
|||||||
+13
-1
@@ -1,4 +1,4 @@
|
|||||||
import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
import { mkdir, readdir, readFile, rename, rm, stat, statfs, writeFile } from "node:fs/promises";
|
||||||
import { randomBytes } from "node:crypto";
|
import { randomBytes } from "node:crypto";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { ID_PATTERN } from "./security";
|
import { ID_PATTERN } from "./security";
|
||||||
@@ -62,6 +62,18 @@ export class Storage {
|
|||||||
return join(this.filesDir, id);
|
return join(this.filesDir, id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async storageUsageRatio(): Promise<number> {
|
||||||
|
try {
|
||||||
|
const stats = await statfs(this.root);
|
||||||
|
const total = stats.blocks * stats.bsize;
|
||||||
|
const avail = stats.bavail * stats.bsize;
|
||||||
|
if (total <= 0) return 0;
|
||||||
|
return Math.min(1, Math.max(0, 1 - avail / total));
|
||||||
|
} catch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private metaPath(id: string): string {
|
private metaPath(id: string): string {
|
||||||
if (!ID_PATTERN.test(id)) throw new Error("invalid file id");
|
if (!ID_PATTERN.test(id)) throw new Error("invalid file id");
|
||||||
return join(this.metadataDir, `${id}.json`);
|
return join(this.metadataDir, `${id}.json`);
|
||||||
|
|||||||
Reference in New Issue
Block a user