feat: add cluster & bucket management

backend/.env.example

@@ -0,0 +1,3 @@
+# App
+API_BASE_URL=http://localhost:3903
+CONFIG_PATH=/app/garage/garage.toml

backend/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

backend/README.md

@@ -0,0 +1,15 @@
+# backend
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.18. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

backend/lib/api.ts

@@ -0,0 +1,83 @@
+import { config } from "./garage";
+
+type FetchOptions = Omit<RequestInit, "headers" | "body"> & {
+  params?: Record<string, any>;
+  headers?: Record<string, string>;
+  body?: any;
+};
+
+const adminPort = config?.admin?.api_bind_addr?.split(":").pop();
+const adminAddr =
+  import.meta.env.API_BASE_URL ||
+  config?.rpc_public_addr?.split(":")[0] + ":" + adminPort ||
+  "";
+
+export const API_BASE_URL =
+  !adminAddr.startsWith("http") && !adminAddr.startsWith("https")
+    ? `http://${adminAddr}`
+    : adminAddr;
+
+export const API_ADMIN_KEY =
+  import.meta.env.API_ADMIN_KEY || config?.admin?.admin_token;
+
+const api = {
+  async fetch<T = any>(url: string, options?: Partial<FetchOptions>) {
+    const headers: Record<string, string> = {
+      Authorization: `Bearer ${API_ADMIN_KEY}`,
+    };
+    const _url = new URL(API_BASE_URL + url);
+
+    if (options?.params) {
+      Object.entries(options.params).forEach(([key, value]) => {
+        _url.searchParams.set(key, String(value));
+      });
+    }
+
+    if (
+      typeof options?.body === "object" &&
+      !(options.body instanceof FormData)
+    ) {
+      options.body = JSON.stringify(options.body);
+      headers["Content-Type"] = "application/json";
+    }
+
+    const res = await fetch(_url, {
+      ...options,
+      headers: { ...headers, ...(options?.headers || {}) },
+    });
+
+    if (!res.ok) {
+      const err = new Error(res.statusText);
+      (err as any).status = res.status;
+      throw err;
+    }
+
+    const isJson = res.headers
+      .get("Content-Type")
+      ?.includes("application/json");
+
+    if (isJson) {
+      const json = (await res.json()) as T;
+      return json;
+    }
+
+    const text = await res.text();
+    return text as unknown as T;
+  },
+
+  async get<T = any>(url: string, options?: Partial<FetchOptions>) {
+    return this.fetch<T>(url, {
+      ...options,
+      method: "GET",
+    });
+  },
+
+  async post<T = any>(url: string, options?: Partial<FetchOptions>) {
+    return this.fetch<T>(url, {
+      ...options,
+      method: "POST",
+    });
+  },
+};
+
+export default api;

backend/lib/garage.ts

@@ -0,0 +1,4 @@
+import type { Config } from "../types/garage";
+import { readTomlFile } from "./utils";
+
+export const config = readTomlFile<Config>(process.env.CONFIG_PATH);

backend/lib/proxy-api.ts

@@ -0,0 +1,35 @@
+import type { Context } from "hono";
+import { API_ADMIN_KEY, API_BASE_URL } from "./api";
+
+export const proxyApi = async (c: Context) => {
+  const url = new URL(c.req.url);
+  const reqUrl = new URL(API_BASE_URL + url.pathname + url.search);
+
+  try {
+    const headers = c.req.raw.headers;
+    let body: BodyInit | ReadableStream<Uint8Array> | null = c.req.raw.body;
+
+    headers.set("authorization", `Bearer ${API_ADMIN_KEY}`);
+
+    if (headers.get("content-type")?.includes("application/json")) {
+      const json = await c.req.json();
+      body = JSON.stringify(json);
+    }
+
+    const res = await fetch(reqUrl, {
+      ...c.req.raw,
+      method: c.req.method,
+      headers,
+      body,
+    });
+    return res;
+  } catch (err) {
+    return c.json(
+      {
+        success: false,
+        error: (err as Error)?.message || "Server error",
+      },
+      500
+    );
+  }
+};

backend/lib/utils.ts

@@ -0,0 +1,9 @@
+import fs from "node:fs";
+import toml from "toml";
+
+export const readTomlFile = <T = any>(path?: string | null) => {
+  if (!path || !fs.existsSync(path)) {
+    return undefined;
+  }
+  return toml.parse(fs.readFileSync(path, "utf8")) as T;
+};

backend/main.ts

@@ -0,0 +1,23 @@
+import { Hono } from "hono";
+import { logger } from "hono/logger";
+import router from "./routes";
+import { proxyApi } from "./lib/proxy-api";
+
+const HOST = import.meta.env.HOST || "0.0.0.0";
+const PORT = Number(import.meta.env.PORT) || 3909;
+
+const app = new Hono();
+
+app.use(logger());
+
+// API router
+app.route("/", router);
+
+// Proxy to garage admin API
+app.all("*", proxyApi);
+
+export default {
+  fetch: app.fetch,
+  hostname: HOST,
+  port: PORT,
+};

backend/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "backend",
+  "module": "main.ts",
+  "type": "module",
+  "scripts": {
+    "dev": "bun --watch main.ts"
+  },
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "hono": "^4.5.5",
+    "toml": "^3.0.0"
+  }
+}

backend/pnpm-lock.yaml

@@ -0,0 +1,69 @@
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .:
+    dependencies:
+      hono:
+        specifier: ^4.5.5
+        version: 4.5.5
+      toml:
+        specifier: ^3.0.0
+        version: 3.0.0
+    devDependencies:
+      '@types/bun':
+        specifier: latest
+        version: 1.1.6
+
+packages:
+
+  '@types/[email protected]':
+    resolution: {integrity: sha512-uJgKjTdX0GkWEHZzQzFsJkWp5+43ZS7HC8sZPFnOwnSo1AsNl2q9o2bFeS23disNDqbggEgyFkKCHl/w8iZsMA==}
+
+  '@types/[email protected]':
+    resolution: {integrity: sha512-scnD59RpYD91xngrQQLGkE+6UrHUPzeKZWhhjBSa3HSkwjbQc38+q3RoIVEwxQGRw3M+j5hpNAM+lgV3cVormg==}
+
+  '@types/[email protected]':
+    resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==}
+
+  [email protected]:
+    resolution: {integrity: sha512-Z4+OplcSd/YZq7ZsrfD00DKJeCwuNY96a1IDJyR73+cTBaFIS7SC6LhpY/W3AMEXO9iYq5NJ58WAwnwL1p5vKg==}
+
+  [email protected]:
+    resolution: {integrity: sha512-fXBXHqaVfimWofbelLXci8pZyIwBMkDIwCa4OwZvK+xVbEyYLELVP4DfbGaj1aEM6ZY3hHgs4qLvCO2ChkhgQw==}
+    engines: {node: '>=16.0.0'}
+
+  [email protected]:
+    resolution: {integrity: sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==}
+
+  [email protected]:
+    resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
+
+snapshots:
+
+  '@types/[email protected]':
+    dependencies:
+      bun-types: 1.1.17
+
+  '@types/[email protected]':
+    dependencies:
+      undici-types: 5.26.5
+
+  '@types/[email protected]':
+    dependencies:
+      '@types/node': 20.12.14
+
+  [email protected]:
+    dependencies:
+      '@types/node': 20.12.14
+      '@types/ws': 8.5.12
+
+  [email protected]: {}
+
+  [email protected]: {}
+
+  [email protected]: {}

backend/routes/buckets.ts

@@ -0,0 +1,19 @@
+import { Hono } from "hono";
+import api from "../lib/api";
+
+export const buckets = new Hono()
+
+  /**
+   * Get all buckets
+   */
+  .get("/", async (c) => {
+    const data = await api.get("/v1/bucket?list");
+
+    const buckets = await Promise.all(
+      data.map(async (bucket: any) => {
+        return api.get("/v1/bucket", { params: { id: bucket.id } });
+      })
+    );
+
+    return c.json(buckets);
+  });

backend/routes/config.ts

@@ -0,0 +1,21 @@
+import { Hono } from "hono";
+import { config } from "../lib/garage";
+
+export const configRoute = new Hono()
+
+  /**
+   * Get garage config
+   */
+  .get("/", async (c) => {
+    const data = {
+      ...(config || {}),
+      rpc_secret: undefined,
+      admin: {
+        ...(config?.admin || {}),
+        admin_token: undefined,
+        metrics_token: undefined,
+      },
+    };
+
+    return c.json(data);
+  });

backend/routes/index.ts

@@ -0,0 +1,10 @@
+import { Hono } from "hono";
+import { buckets } from "./buckets";
+import { configRoute } from "./config";
+
+const router = new Hono()
+  //
+  .route("/config", configRoute)
+  .route("/buckets", buckets);
+
+export default router;

backend/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

backend/types/garage.ts

@@ -0,0 +1,32 @@
+export type Config = {
+  metadata_dir: string;
+  data_dir: string;
+  db_engine: string;
+  metadata_auto_snapshot_interval: string;
+  replication_factor: number;
+  compression_level: number;
+  rpc_bind_addr: string;
+  rpc_public_addr: string;
+  rpc_secret: string;
+  s3_api?: S3API;
+  s3_web?: S3Web;
+  admin?: Admin;
+};
+
+export type Admin = {
+  api_bind_addr: string;
+  admin_token: string;
+  metrics_token: string;
+};
+
+export type S3API = {
+  s3_region: string;
+  api_bind_addr: string;
+  root_domain: string;
+};
+
+export type S3Web = {
+  bind_addr: string;
+  root_domain: string;
+  index: string;
+};