Handling fs
This commit is contained in:
+7
-2
@@ -16,9 +16,14 @@ STORAGE_DIR=./storage
|
||||
# case-insensitive). A request larger than this is rejected with 413.
|
||||
MAX_FILE_SIZE=10GB
|
||||
|
||||
# Maximum fraction of the storage filesystem that may be used, from 0.0 to 1.0.
|
||||
# When usage exceeds this ratio uploads are rejected with 503 "try again later".
|
||||
# Default 0.5 = block uploads once more than 50% of the disk is used.
|
||||
MAX_STORAGE_USAGE=0.5
|
||||
|
||||
# How long an uploaded file lives before it expires and is deleted.
|
||||
# In milliseconds. Default is 1 week (604800000).
|
||||
FILE_TTL_MS=604800000
|
||||
# In milliseconds. Default and hard maximum is 48 hours (172800000).
|
||||
FILE_TTL_MS=172800000
|
||||
|
||||
# Brute-force protection: rate limit per client IP on download and
|
||||
# deletion endpoints. Set RATE_LIMIT_ENABLED=0 to disable.
|
||||
|
||||
+6
-3
@@ -63,7 +63,8 @@ HOST=127.0.0.1 # bind loopback only; the proxy talks to it
|
||||
PORT=3000
|
||||
STORAGE_DIR=/opt/filehost/storage
|
||||
MAX_FILE_SIZE=10GB # must be <= proxy upload limit
|
||||
FILE_TTL_MS=604800000 # expire files after 1 week
|
||||
FILE_TTL_MS=172800000 # expire files after 48 hours (hard maximum)
|
||||
MAX_STORAGE_USAGE=0.5 # block uploads when >50% of the disk is used
|
||||
RATE_LIMIT_ENABLED=1
|
||||
TRUST_PROXY=1 # proxy sets X-Forwarded-For
|
||||
```
|
||||
@@ -247,6 +248,8 @@ Files in `storage/` are untouched by updates and keep working.
|
||||
short-lived HMAC-signed cookie (server secret in `storage/.secret`).
|
||||
- Rate limiting protects the keyed endpoints; `TRUST_PROXY=1` must only be set
|
||||
when a proxy really sets `X-Forwarded-For`/`CF-Connecting-IP`.
|
||||
- Files auto-expire after one week (configurable via `FILE_TTL_MS`) and are
|
||||
swept hourly. Deletion also requires possession of the key
|
||||
- Files auto-expire after 48 hours (configurable via `FILE_TTL_MS`, hard-capped
|
||||
at 48h) and are swept hourly. Deletion also requires possession of the key
|
||||
(`DELETE /api/files/:id` with `X-Delete-Key`).
|
||||
- Uploads are rejected with `503 try again later` when the storage filesystem is
|
||||
more than 50% full (configurable via `MAX_STORAGE_USAGE`).
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
A minimal, self-hosted private file hosting service built with **Bun** and
|
||||
**TypeScript**. Upload a file, get a secret, unguessable download link. Files are
|
||||
stored on the local filesystem and expire after one week.
|
||||
stored on the local filesystem and expire after 48 hours.
|
||||
|
||||
Everything lives on disk — the filesystem is the source of truth. There is no
|
||||
database, no object storage, and no queue. Expired files are swept automatically.
|
||||
@@ -15,7 +15,7 @@ database, no object storage, and no queue. Expired files are swept automatically
|
||||
- A cryptographically secure, unguessable download key for every file.
|
||||
- Streaming uploads and downloads — no full buffering in RAM, large files work.
|
||||
- Configurable maximum upload size (`413` when exceeded, default 10GB).
|
||||
- Files **auto-expire after one week** and are removed automatically.
|
||||
- Files **auto-expire after 48 hours** and are removed automatically.
|
||||
- Authenticated deletion (possession of the file's secret key is required).
|
||||
- Rate-limited download/delete endpoints to blunt brute force.
|
||||
- Zero runtime dependencies (Bun is the only runtime).
|
||||
@@ -49,7 +49,8 @@ cp .env.example .env
|
||||
| `PORT` | `3000` | Bind port. |
|
||||
| `STORAGE_DIR` | `./storage` | Directory where files + metadata are stored (relative to CWD). |
|
||||
| `MAX_FILE_SIZE` | `10GB` | Max upload size. Units: `B`, `KB`, `MB`, `GB`, `TB` (or plain int). |
|
||||
| `FILE_TTL_MS` | `604800000` | Milliseconds before an upload expires and is deleted (1 week). |
|
||||
| `MAX_STORAGE_USAGE` | `0.5` | Max fraction (0.0–1.0) of the storage disk that may be used; uploads are rejected with `503 try again later` above this. |
|
||||
| `FILE_TTL_MS` | `172800000` | Milliseconds before an upload expires and is deleted (48 hours, hard maximum). |
|
||||
| `RATE_LIMIT_ENABLED` | `1` | Enable per-IP rate limiting on `/(key)` and `/api/files/`. |
|
||||
| `RATE_LIMIT_MAX` | `300` | Max requests per IP per window. |
|
||||
| `RATE_LIMIT_WINDOW_MS` | `60000` | Rate-limit window length in ms. |
|
||||
@@ -73,7 +74,7 @@ Then open http://localhost:3000.
|
||||
4. Copy the URL. Anyone possessing it can download the file — **after entering
|
||||
the password you set at upload**. Nobody without the secret URL and the
|
||||
password can download it.
|
||||
5. The file is removed automatically after one week, or earlier by explicit
|
||||
5. The file is removed automatically after 48 hours, or earlier by explicit
|
||||
deletion.
|
||||
|
||||
## Filesystem layout
|
||||
|
||||
@@ -48,8 +48,14 @@ in
|
||||
|
||||
fileTtlMs = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 7 * 24 * 60 * 60 * 1000;
|
||||
description = "Milliseconds before an uploaded file expires and is deleted.";
|
||||
default = 48 * 60 * 60 * 1000;
|
||||
description = "Milliseconds before an uploaded file expires and is deleted (hard-capped at 48 hours).";
|
||||
};
|
||||
|
||||
maxStorageUsage = lib.mkOption {
|
||||
type = lib.types.float;
|
||||
default = 0.5;
|
||||
description = "Maximum fraction (0.0-1.0) of the storage filesystem that may be used; uploads are rejected above this.";
|
||||
};
|
||||
|
||||
publicBaseUrl = lib.mkOption {
|
||||
@@ -127,6 +133,7 @@ in
|
||||
"STORAGE_DIR=${cfg.storageDir}"
|
||||
"MAX_FILE_SIZE=${cfg.maxFileSize}"
|
||||
"FILE_TTL_MS=${toString cfg.fileTtlMs}"
|
||||
"MAX_STORAGE_USAGE=${toString cfg.maxStorageUsage}"
|
||||
]
|
||||
++ lib.optionals (cfg.publicBaseUrl != null) [ "PUBLIC_BASE_URL=${cfg.publicBaseUrl}" ]
|
||||
++ lib.optionals cfg.trustProxy [ "TRUST_PROXY=1" ];
|
||||
|
||||
+12
-3
@@ -6,6 +6,7 @@ export interface Config {
|
||||
storageDir: string;
|
||||
maxFileSize: number;
|
||||
fileTtlMs: number;
|
||||
maxStorageUsage: number;
|
||||
publicBaseUrl?: string;
|
||||
rateLimitEnabled: boolean;
|
||||
rateLimitMax: number;
|
||||
@@ -13,7 +14,8 @@ export interface Config {
|
||||
trustProxy: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 1 week
|
||||
const MAX_TTL_MS = 48 * 60 * 60 * 1000; // 48 hours hard maximum
|
||||
const DEFAULT_TTL_MS = MAX_TTL_MS;
|
||||
|
||||
const SIZE_UNITS: Record<string, number> = {
|
||||
B: 1,
|
||||
@@ -56,10 +58,16 @@ export function loadConfig(env: Record<string, string | undefined> = process.env
|
||||
const maxFileSize = parseSize(env.MAX_FILE_SIZE ?? "10GB");
|
||||
if (maxFileSize < 1024) throw new Error("MAX_FILE_SIZE must be at least 1KB");
|
||||
|
||||
const fileTtlMs = Number(env.FILE_TTL_MS ?? String(DEFAULT_TTL_MS));
|
||||
if (!Number.isInteger(fileTtlMs) || fileTtlMs < 1) {
|
||||
const fileTtlMsRaw = Number(env.FILE_TTL_MS ?? String(DEFAULT_TTL_MS));
|
||||
if (!Number.isInteger(fileTtlMsRaw) || fileTtlMsRaw < 1) {
|
||||
throw new Error(`Invalid FILE_TTL_MS: ${env.FILE_TTL_MS}`);
|
||||
}
|
||||
const fileTtlMs = Math.min(fileTtlMsRaw, MAX_TTL_MS);
|
||||
|
||||
const maxStorageUsage = Number(env.MAX_STORAGE_USAGE ?? "0.5");
|
||||
if (!Number.isFinite(maxStorageUsage) || maxStorageUsage < 0 || maxStorageUsage > 1) {
|
||||
throw new Error(`Invalid MAX_STORAGE_USAGE: ${env.MAX_STORAGE_USAGE}`);
|
||||
}
|
||||
|
||||
const rateLimitMax = Number(env.RATE_LIMIT_MAX ?? "300");
|
||||
if (!Number.isInteger(rateLimitMax) || rateLimitMax < 1) {
|
||||
@@ -88,6 +96,7 @@ export function loadConfig(env: Record<string, string | undefined> = process.env
|
||||
storageDir,
|
||||
maxFileSize,
|
||||
fileTtlMs,
|
||||
maxStorageUsage,
|
||||
publicBaseUrl,
|
||||
rateLimitEnabled: boolFromEnv(env.RATE_LIMIT_ENABLED, true),
|
||||
rateLimitMax,
|
||||
|
||||
@@ -27,6 +27,7 @@ export async function handleUpload(
|
||||
maxFileSize: number,
|
||||
ttlMs: number,
|
||||
publicBaseUrl?: string,
|
||||
maxStorageUsage = 0.5,
|
||||
): Promise<Response> {
|
||||
const contentType = req.headers.get("content-type");
|
||||
const boundary = contentType ? getBoundary(contentType) : null;
|
||||
@@ -36,6 +37,15 @@ export async function handleUpload(
|
||||
const key = id;
|
||||
|
||||
try {
|
||||
const usage = await storage.storageUsageRatio();
|
||||
if (usage > maxStorageUsage) {
|
||||
log(
|
||||
"warn",
|
||||
`upload rejected id=${id}: storage usage ${(usage * 100).toFixed(1)}% exceeds limit ${(maxStorageUsage * 100).toFixed(0)}%`,
|
||||
);
|
||||
return jsonError(503, "storage is full, try again later");
|
||||
}
|
||||
|
||||
let filename = "";
|
||||
let mime = "application/octet-stream";
|
||||
let size = 0;
|
||||
|
||||
+1
-1
@@ -108,7 +108,7 @@ async function route(req: Request, srv: Server<undefined>): Promise<Response> {
|
||||
}
|
||||
|
||||
if (path === "/api/upload" && method === "POST") {
|
||||
return handleUpload(req, storage, cfg.maxFileSize, cfg.fileTtlMs, cfg.publicBaseUrl);
|
||||
return handleUpload(req, storage, cfg.maxFileSize, cfg.fileTtlMs, cfg.publicBaseUrl, cfg.maxStorageUsage);
|
||||
}
|
||||
|
||||
if (path.startsWith("/api/auth/") && method === "POST") {
|
||||
|
||||
+13
-1
@@ -1,4 +1,4 @@
|
||||
import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { mkdir, readdir, readFile, rename, rm, stat, statfs, writeFile } from "node:fs/promises";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { ID_PATTERN } from "./security";
|
||||
@@ -62,6 +62,18 @@ export class Storage {
|
||||
return join(this.filesDir, id);
|
||||
}
|
||||
|
||||
async storageUsageRatio(): Promise<number> {
|
||||
try {
|
||||
const stats = await statfs(this.root);
|
||||
const total = stats.blocks * stats.bsize;
|
||||
const avail = stats.bavail * stats.bsize;
|
||||
if (total <= 0) return 0;
|
||||
return Math.min(1, Math.max(0, 1 - avail / total));
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private metaPath(id: string): string {
|
||||
if (!ID_PATTERN.test(id)) throw new Error("invalid file id");
|
||||
return join(this.metadataDir, `${id}.json`);
|
||||
|
||||
Reference in New Issue
Block a user