khairul169/github-leaderboard
1
0
Fork 0
mirror of https://github.com/khairul169/github-leaderboard.git synced 2026-09-17 09:53:21 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: migrate db to postgres

Browse Source
This commit is contained in:
khairul169
2024-08-09 22:17:09 +07:00
parent 0142224f60
commit 8ff59ba6b3
18 changed files with 462 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/index.ts
+8 -5
View File
@@ -1,10 +1,13 @@
import { drizzle } from "drizzle-orm/bun-sqlite";
import { Database } from "bun:sqlite";
import { drizzle } from "drizzle-orm/postgres-js";
import postgres from "postgres";
import * as schema from "../models";
const DATABASE_PATH = import.meta.env.DATABASE_PATH || "./data.db";
const DATABASE_PATH = import.meta.env.DATABASE_PATH;
if (!DATABASE_PATH) {
throw new Error("DATABASE_PATH is not set");
}
const sqlite = new Database(DATABASE_PATH);
const db = drizzle(sqlite, { schema });
const queryClient = postgres(DATABASE_PATH);
const db = drizzle(queryClient, { schema });
export default db;
server/db/migrations/0000_round_hellion.sql
+59
View File
@@ -0,0 +1,59 @@
CREATE TABLE IF NOT EXISTS "repository_languages" (
"id" serial PRIMARY KEY NOT NULL,
"repo_id" integer NOT NULL,
"name" varchar NOT NULL,
"percentage" double precision NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "repositories" (
"id" serial PRIMARY KEY NOT NULL,
"user_id" integer NOT NULL,
"name" varchar NOT NULL,
"uri" varchar NOT NULL,
"language" varchar NOT NULL,
"stars" integer NOT NULL,
"forks" integer NOT NULL,
"last_update" varchar NOT NULL,
"contributors" jsonb,
"is_pending" boolean DEFAULT false NOT NULL,
"is_error" boolean DEFAULT false NOT NULL,
"created_at" timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
"updated_at" timestamp NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "users" (
"id" serial PRIMARY KEY NOT NULL,
"username" varchar NOT NULL,
"name" varchar NOT NULL,
"avatar" varchar,
"location" varchar,
"followers" integer DEFAULT 0 NOT NULL,
"following" integer DEFAULT 0 NOT NULL,
"achievements" jsonb DEFAULT '[]'::jsonb,
"points" integer DEFAULT 0 NOT NULL,
"commits" integer DEFAULT 0 NOT NULL,
"line_of_codes" integer DEFAULT 0 NOT NULL,
"github_id" integer,
"access_token" varchar,
"created_at" timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
"updated_at" timestamp NOT NULL,
CONSTRAINT "users_username_unique" UNIQUE("username"),
CONSTRAINT "users_github_id_unique" UNIQUE("github_id")
);
--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "repository_languages" ADD CONSTRAINT "repository_languages_repo_id_repositories_id_fk" FOREIGN KEY ("repo_id") REFERENCES "public"."repositories"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "repositories" ADD CONSTRAINT "repositories_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
--> statement-breakpoint
CREATE INDEX IF NOT EXISTS "repository_languages_name_idx" ON "repository_languages" USING btree ("name");--> statement-breakpoint
CREATE INDEX IF NOT EXISTS "repositories_name_idx" ON "repositories" USING btree ("name");--> statement-breakpoint
CREATE INDEX IF NOT EXISTS "repositories_uri_idx" ON "repositories" USING btree ("uri");--> statement-breakpoint
CREATE INDEX IF NOT EXISTS "repositories_language_idx" ON "repositories" USING btree ("language");
server/db/migrations/0000_tough_jubilee.sql
-49
View File
@@ -1,49 +0,0 @@
CREATE TABLE `repository_languages` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`repo_id` integer NOT NULL,
`name` text NOT NULL,
`percentage` real NOT NULL,
FOREIGN KEY (`repo_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `repositories` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`user_id` integer NOT NULL,
`name` text NOT NULL,
`uri` text NOT NULL,
`language` text NOT NULL,
`stars` integer NOT NULL,
`forks` integer NOT NULL,
`last_update` text NOT NULL,
`contributors` text,
`is_pending` integer DEFAULT false NOT NULL,
`is_error` integer DEFAULT false NOT NULL,
`created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL,
`updated_at` text NOT NULL,
FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action
);
--> statement-breakpoint
CREATE TABLE `users` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`username` text NOT NULL,
`name` text NOT NULL,
`avatar` text,
`location` text,
`followers` integer DEFAULT 0 NOT NULL,
`following` integer DEFAULT 0 NOT NULL,
`achievements` text DEFAULT '[]',
`points` integer DEFAULT 0 NOT NULL,
`commits` integer DEFAULT 0 NOT NULL,
`line_of_codes` integer DEFAULT 0 NOT NULL,
`github_id` integer,
`access_token` text,
`created_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL,
`updated_at` text NOT NULL
);
--> statement-breakpoint
CREATE INDEX `repository_languages_name_idx` ON `repository_languages` (`name`);--> statement-breakpoint
CREATE INDEX `repositories_name_idx` ON `repositories` (`name`);--> statement-breakpoint
CREATE INDEX `repositories_uri_idx` ON `repositories` (`uri`);--> statement-breakpoint
CREATE INDEX `repositories_language_idx` ON `repositories` (`language`);--> statement-breakpoint
CREATE UNIQUE INDEX `users_username_unique` ON `users` (`username`);--> statement-breakpoint
CREATE UNIQUE INDEX `users_github_id_unique` ON `users` (`github_id`);
server/db/migrations/meta/0000_snapshot.json
+107 -105
View File
@@ -1,48 +1,53 @@
{
"version": "6",
"dialect": "sqlite",
"id": "a268ec23-239a-4d86-8830-2f3c1c2110df",
"id": "5131e61a-98fe-40ef-b206-62b860d39639",
"prevId": "00000000-0000-0000-0000-000000000000",
"version": "7",
"dialect": "postgresql",
"tables": {
"repository_languages": {
"public.repository_languages": {
"name": "repository_languages",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "integer",
"type": "serial",
"primaryKey": true,
"notNull": true,
"autoincrement": true
"notNull": true
},
"repo_id": {
"name": "repo_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"name": {
"name": "name",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"percentage": {
"name": "percentage",
"type": "real",
"type": "double precision",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
}
},
"indexes": {
"repository_languages_name_idx": {
"name": "repository_languages_name_idx",
"columns": [
"name"
{
"expression": "name",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -63,125 +68,137 @@
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"repositories": {
"public.repositories": {
"name": "repositories",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "integer",
"type": "serial",
"primaryKey": true,
"notNull": true,
"autoincrement": true
"notNull": true
},
"user_id": {
"name": "user_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"name": {
"name": "name",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"uri": {
"name": "uri",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"language": {
"name": "language",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"stars": {
"name": "stars",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"forks": {
"name": "forks",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"last_update": {
"name": "last_update",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"contributors": {
"name": "contributors",
"type": "text",
"type": "jsonb",
"primaryKey": false,
"notNull": false,
"autoincrement": false
"notNull": false
},
"is_pending": {
"name": "is_pending",
"type": "integer",
"type": "boolean",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"is_error": {
"name": "is_error",
"type": "integer",
"type": "boolean",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"created_at": {
"name": "created_at",
"type": "text",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "CURRENT_TIMESTAMP"
},
"updated_at": {
"name": "updated_at",
"type": "text",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
}
},
"indexes": {
"repositories_name_idx": {
"name": "repositories_name_idx",
"columns": [
"name"
{
"expression": "name",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"repositories_uri_idx": {
"name": "repositories_uri_idx",
"columns": [
"uri"
{
"expression": "uri",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"repositories_language_idx": {
"name": "repositories_language_idx",
"columns": [
"language"
{
"expression": "language",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {
@@ -202,50 +219,45 @@
"compositePrimaryKeys": {},
"uniqueConstraints": {}
},
"users": {
"public.users": {
"name": "users",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "integer",
"type": "serial",
"primaryKey": true,
"notNull": true,
"autoincrement": true
"notNull": true
},
"username": {
"name": "username",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"name": {
"name": "name",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
},
"avatar": {
"name": "avatar",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": false,
"autoincrement": false
"notNull": false
},
"location": {
"name": "location",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": false,
"autoincrement": false
"notNull": false
},
"followers": {
"name": "followers",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"following": {
@@ -253,23 +265,20 @@
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"achievements": {
"name": "achievements",
"type": "text",
"type": "jsonb",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "'[]'"
"default": "'[]'::jsonb"
},
"points": {
"name": "points",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"commits": {
@@ -277,7 +286,6 @@
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"line_of_codes": {
@@ -285,67 +293,61 @@
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"github_id": {
"name": "github_id",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
"notNull": false
},
"access_token": {
"name": "access_token",
"type": "text",
"type": "varchar",
"primaryKey": false,
"notNull": false,
"autoincrement": false
"notNull": false
},
"created_at": {
"name": "created_at",
"type": "text",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "CURRENT_TIMESTAMP"
},
"updated_at": {
"name": "updated_at",
"type": "text",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"autoincrement": false
"notNull": true
}
},
"indexes": {
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {
"users_username_unique": {
"name": "users_username_unique",
"nullsNotDistinct": false,
"columns": [
"username"
],
"isUnique": true
]
},
"users_github_id_unique": {
"name": "users_github_id_unique",
"nullsNotDistinct": false,
"columns": [
"github_id"
],
"isUnique": true
]
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
}
},
"enums": {},
"schemas": {},
"sequences": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
"tables": {}
}
}
server/db/migrations/meta/_journal.json
+3 -3
View File
@@ -4,9 +4,9 @@
"entries": [
{
"idx": 0,
"version": "6",
"when": 1723211354217,
"tag": "0000_tough_jubilee",
"version": "7",
"when": 1723216591610,
"tag": "0000_round_hellion",
"breakpoints": true
}
]
server/drizzle.config.ts
+5 -2
View File
@@ -1,11 +1,14 @@
import { defineConfig } from "drizzle-kit";
const DATABASE_PATH = process.env.DATABASE_PATH || "./data.db";
const DATABASE_PATH = process.env.DATABASE_PATH;
if (!DATABASE_PATH) {
throw new Error("DATABASE_PATH is not set");
}
export default defineConfig({
schema: "./server/models/index.ts",
out: "./server/db/migrations",
dialect: "sqlite",
dialect: "postgresql",
dbCredentials: { url: DATABASE_PATH },
verbose: true,
});
server/jobs/fetch-repo-contributors.ts
+9 -3
View File
@@ -22,13 +22,15 @@ export const fetchRepoContributors = async (
throw new Error("Repository not found!");
}
if (!repo.userAccessToken) {
const accessToken =
repo.userAccessToken || import.meta.env.GITHUB_DEFAULT_TOKEN;
if (!accessToken) {
throw new Error("User access token not found!");
}
const contributors = await github.getRepoContributors(data.uri, {
headers: {
Authorization: `Bearer ${repo.userAccessToken}`,
Authorization: `Bearer ${accessToken}`,
},
});
@@ -42,7 +44,11 @@ export const fetchRepoContributors = async (
throw new Error("Cannot update repository!");
}
await queue.add("calculateUserPoints", { userId: result.userId });
await queue.add(
"calculateUserPoints",
{ userId: result.userId },
{ jobId: `calculateUserPoints:${result.userId}` }
);
};
export const onFetchRepoContribFailed = async (
server/jobs/fetch-repo-data.ts
+5 -1
View File
@@ -61,5 +61,9 @@ export const fetchRepoData = async (data: FetchRepoDataJobType) => {
}
});
await queue.add("calculateUserPoints", { userId: repository.userId });
await queue.add(
"calculateUserPoints",
{ userId: repository.userId },
{ jobId: `calculateUserPoints:${repository.userId}` }
);
};
server/jobs/fetch-user-profile.ts
+5 -1
View File
@@ -27,5 +27,9 @@ export const fetchUserProfile = async (data: FetchUserProfileType) => {
})
.where(eq(users.id, user.id));
await queue.add("calculateUserPoints", { userId: user.id });
await queue.add(
"calculateUserPoints",
{ userId: user.id },
{ jobId: `calculateUserPoints:${user.id}` }
);
};
server/jobs/fetch-user-repos.ts
+1 -1
View File
@@ -62,7 +62,7 @@ export const fetchUserRepos = async (data: FetchUserRepos) => {
data,
opts: {
attempts: 5,
backoff: { type: "exponential", delay: 30000 },
backoff: { type: "exponential", delay: 3000 },
jobId: `contributors:${data.uri}`,
},
}))
server/lib/github.ts
-1
View File
@@ -32,7 +32,6 @@ const github = {
const name = $(selectors.user.name).text().trim();
const avatar = $(selectors.user.avatar).attr("src");
console.log({ avatar });
const location = $(selectors.user.location).text().trim();
const followers = intval($(selectors.user.followers).text().trim());
const following = intval($(selectors.user.following).text().trim());
server/models/repo-languages.ts
+9 -8
View File
@@ -1,23 +1,24 @@
import { InferInsertModel, InferSelectModel, relations } from "drizzle-orm";
import {
text,
sqliteTable,
varchar,
serial,
pgTable,
integer,
index,
real,
} from "drizzle-orm/sqlite-core";
doublePrecision,
} from "drizzle-orm/pg-core";
import { repositories } from "./repositories";
export const repoLanguages = sqliteTable(
export const repoLanguages = pgTable(
"repository_languages",
{
id: integer("id").primaryKey({ autoIncrement: true }),
id: serial("id").primaryKey(),
repoId: integer("repo_id")
.notNull()
.references(() => repositories.id),
name: text("name").notNull(),
percentage: real("percentage").notNull(),
name: varchar("name").notNull(),
percentage: doublePrecision("percentage").notNull(),
},
(t) => ({
nameIdx: index("repository_languages_name_idx").on(t.name),
server/models/repositories.ts
+22 -15
View File
@@ -5,37 +5,44 @@ import {
relations,
sql,
} from "drizzle-orm";
import { text, sqliteTable, integer, index } from "drizzle-orm/sqlite-core";
import {
varchar,
pgTable,
integer,
index,
serial,
jsonb,
boolean,
timestamp,
} from "drizzle-orm/pg-core";
import { users } from "./users";
import { repoLanguages } from "./repo-languages";
export const repositories = sqliteTable(
export const repositories = pgTable(
"repositories",
{
id: integer("id").primaryKey({ autoIncrement: true }),
id: serial("id").primaryKey(),
userId: integer("user_id")
.notNull()
.references(() => users.id),
name: text("name").notNull(),
uri: text("uri").notNull(),
language: text("language").notNull(),
name: varchar("name").notNull(),
uri: varchar("uri").notNull(),
language: varchar("language").notNull(),
stars: integer("stars").notNull(),
forks: integer("forks").notNull(),
lastUpdate: text("last_update").notNull(),
contributors: text("contributors", { mode: "json" }).$type<Contributor[]>(),
lastUpdate: varchar("last_update").notNull(),
contributors: jsonb("contributors").$type<Contributor[]>(),
isPending: integer("is_pending", { mode: "boolean" })
.notNull()
.default(false),
isError: integer("is_error", { mode: "boolean" }).notNull().default(false),
isPending: boolean("is_pending").notNull().default(false),
isError: boolean("is_error").notNull().default(false),
createdAt: text("created_at")
createdAt: timestamp("created_at")
.notNull()
.default(sql`CURRENT_TIMESTAMP`),
updatedAt: text("updated_at")
updatedAt: timestamp("updated_at")
.notNull()
.$onUpdate(() => sql`CURRENT_TIMESTAMP`),
.$onUpdate(() => new Date()),
},
(t) => ({
nameIdx: index("repositories_name_idx").on(t.name),
server/models/users.ts
+19 -14
View File
@@ -1,30 +1,35 @@
import { Achievement } from "@server/lib/github";
import { InferInsertModel, InferSelectModel, sql } from "drizzle-orm";
import { text, sqliteTable, integer } from "drizzle-orm/sqlite-core";
import {
varchar,
pgTable,
serial,
integer,
jsonb,
timestamp,
} from "drizzle-orm/pg-core";
export const users = sqliteTable("users", {
id: integer("id").primaryKey({ autoIncrement: true }),
username: text("username").notNull().unique(),
name: text("name").notNull(),
avatar: text("avatar"),
location: text("location"),
export const users = pgTable("users", {
id: serial("id").primaryKey(),
username: varchar("username").notNull().unique(),
name: varchar("name").notNull(),
avatar: varchar("avatar"),
location: varchar("location"),
followers: integer("followers").notNull().default(0),
following: integer("following").notNull().default(0),
achievements: text("achievements", { mode: "json" })
.$type<Achievement[]>()
.default([]),
achievements: jsonb("achievements").$type<Achievement[]>().default([]),
points: integer("points").notNull().default(0),
commits: integer("commits").notNull().default(0),
lineOfCodes: integer("line_of_codes").notNull().default(0),
githubId: integer("github_id").unique(),
accessToken: text("access_token"),
accessToken: varchar("access_token"),
createdAt: text("created_at")
createdAt: timestamp("created_at")
.notNull()
.default(sql`CURRENT_TIMESTAMP`),
updatedAt: text("updated_at")
updatedAt: timestamp("updated_at")
.notNull()
.$onUpdate(() => sql`CURRENT_TIMESTAMP`),
.$onUpdate(() => new Date()),
});
export type User = InferSelectModel<typeof users>;
server/repository/leaderboard.ts
+4 -2
View File
@@ -59,7 +59,8 @@ export class LeaderboardRepository {
.from(repositories)
.innerJoin(repoLanguages, eq(repoLanguages.repoId, repositories.id))
.groupBy(repoLanguages.name)
.orderBy(desc(sql`count(*) * avg(${repoLanguages.percentage})`));
.orderBy(desc(sql`count(*) * avg(${repoLanguages.percentage})`))
.limit(20);
const columns = [
{
@@ -116,7 +117,8 @@ export class LeaderboardRepository {
.innerJoin(repoLanguages, eq(repoLanguages.repoId, repositories.id))
.where(inArray(sql`lower(${repoLanguages.name})`, languages))
.groupBy(users.id)
.orderBy(desc(count()));
.orderBy(desc(count()))
.limit(100);
const columns = [
{