From 0142224f60c11a0addea5155993dcde2f830294a Mon Sep 17 00:00:00 2001 From: Khairul Hidayat Date: Fri, 9 Aug 2024 21:45:14 +0700 Subject: [PATCH] feat: update language leaderboard calculation --- ..._emma_frost.sql => 0000_tough_jubilee.sql} | 10 ++- server/db/migrations/meta/0000_snapshot.json | 68 ++++++++++++++++--- server/db/migrations/meta/_journal.json | 4 +- server/import-user.ts | 34 ++++++++++ server/jobs/calculate-user-points.ts | 10 +-- server/jobs/fetch-repo-data.ts | 59 +++++++++++++--- server/jobs/fetch-user-profile.ts | 1 + server/lib/github.ts | 4 ++ server/lib/utils.ts | 3 + server/models/index.ts | 3 +- server/models/repo-languages.ts | 35 ++++++++++ server/models/repositories.ts | 15 +++- server/queue-worker.ts | 2 +- server/repository/leaderboard.ts | 51 ++++++++++---- src/components/ui/image.tsx | 19 ++++++ src/pages/home/components/rank-list-item.tsx | 8 ++- src/pages/home/components/user-rank.tsx | 2 +- src/pages/home/components/view-sheet.tsx | 48 ++++++++++--- src/pages/home/data.ts | 13 ++++ 19 files changed, 330 insertions(+), 59 deletions(-) rename server/db/migrations/{0000_young_emma_frost.sql => 0000_tough_jubilee.sql} (79%) create mode 100644 server/import-user.ts create mode 100644 server/models/repo-languages.ts create mode 100644 src/components/ui/image.tsx diff --git a/server/db/migrations/0000_young_emma_frost.sql b/server/db/migrations/0000_tough_jubilee.sql similarity index 79% rename from server/db/migrations/0000_young_emma_frost.sql rename to server/db/migrations/0000_tough_jubilee.sql index b36d075..b08dfdc 100644 --- a/server/db/migrations/0000_young_emma_frost.sql +++ b/server/db/migrations/0000_tough_jubilee.sql @@ -1,3 +1,11 @@ +CREATE TABLE `repository_languages` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `repo_id` integer NOT NULL, + `name` text NOT NULL, + `percentage` real NOT NULL, + FOREIGN KEY (`repo_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint CREATE TABLE `repositories` ( `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, `user_id` integer NOT NULL, @@ -7,7 +15,6 @@ CREATE TABLE `repositories` ( `stars` integer NOT NULL, `forks` integer NOT NULL, `last_update` text NOT NULL, - `languages` text, `contributors` text, `is_pending` integer DEFAULT false NOT NULL, `is_error` integer DEFAULT false NOT NULL, @@ -34,6 +41,7 @@ CREATE TABLE `users` ( `updated_at` text NOT NULL ); --> statement-breakpoint +CREATE INDEX `repository_languages_name_idx` ON `repository_languages` (`name`);--> statement-breakpoint CREATE INDEX `repositories_name_idx` ON `repositories` (`name`);--> statement-breakpoint CREATE INDEX `repositories_uri_idx` ON `repositories` (`uri`);--> statement-breakpoint CREATE INDEX `repositories_language_idx` ON `repositories` (`language`);--> statement-breakpoint diff --git a/server/db/migrations/meta/0000_snapshot.json b/server/db/migrations/meta/0000_snapshot.json index d744dbf..d8c9689 100644 --- a/server/db/migrations/meta/0000_snapshot.json +++ b/server/db/migrations/meta/0000_snapshot.json @@ -1,9 +1,68 @@ { "version": "6", "dialect": "sqlite", - "id": "13d859a6-4c32-46a6-90e9-19740b2bb13a", + "id": "a268ec23-239a-4d86-8830-2f3c1c2110df", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { + "repository_languages": { + "name": "repository_languages", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "repo_id": { + "name": "repo_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "percentage": { + "name": "percentage", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "repository_languages_name_idx": { + "name": "repository_languages_name_idx", + "columns": [ + "name" + ], + "isUnique": false + } + }, + "foreignKeys": { + "repository_languages_repo_id_repositories_id_fk": { + "name": "repository_languages_repo_id_repositories_id_fk", + "tableFrom": "repository_languages", + "tableTo": "repositories", + "columnsFrom": [ + "repo_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, "repositories": { "name": "repositories", "columns": { @@ -63,13 +122,6 @@ "notNull": true, "autoincrement": false }, - "languages": { - "name": "languages", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, "contributors": { "name": "contributors", "type": "text", diff --git a/server/db/migrations/meta/_journal.json b/server/db/migrations/meta/_journal.json index 1dfa564..3cf21f1 100644 --- a/server/db/migrations/meta/_journal.json +++ b/server/db/migrations/meta/_journal.json @@ -5,8 +5,8 @@ { "idx": 0, "version": "6", - "when": 1723210265266, - "tag": "0000_young_emma_frost", + "when": 1723211354217, + "tag": "0000_tough_jubilee", "breakpoints": true } ] diff --git a/server/import-user.ts b/server/import-user.ts new file mode 100644 index 0000000..41efc83 --- /dev/null +++ b/server/import-user.ts @@ -0,0 +1,34 @@ +import db from "./db"; +import queue from "./lib/queue"; +import { users } from "./models"; + +const main = async () => { + const username = process.argv[2]; + if (!username) { + throw new Error("Missing username"); + } + + const [user] = await db + .insert(users) + .values({ username, name: username }) + .onConflictDoUpdate({ + target: users.username, + set: { username }, + }) + .returning(); + + await queue.add( + "fetchUserProfile", + { userId: user.id }, + { jobId: `fetchUserProfile:${user.id}` } + ); + await queue.add( + "fetchUserRepos", + { userId: user.id }, + { jobId: `fetchUserRepos:${user.id}` } + ); + + process.exit(); +}; + +main(); diff --git a/server/jobs/calculate-user-points.ts b/server/jobs/calculate-user-points.ts index 53b4493..598c6cc 100644 --- a/server/jobs/calculate-user-points.ts +++ b/server/jobs/calculate-user-points.ts @@ -37,15 +37,15 @@ export const calculateUserPoints = async (data: CalculateUserPointsType) => { : 0; // User repositories - const repos = await db - .select() - .from(repositories) - .where(eq(repositories.userId, user.id)); + const repos = await db.query.repositories.findMany({ + where: eq(repositories.userId, user.id), + with: { languages: true }, + }); points += repos.length * weights.repositories; // Languages known const languages = new Set( - repos.flatMap((i) => i.languages?.map((j) => j.lang)) + repos.flatMap((i) => i.languages?.map((j) => j.name)) ); points += languages.size * weights.languagesKnown; diff --git a/server/jobs/fetch-repo-data.ts b/server/jobs/fetch-repo-data.ts index 0a1dc20..a5e7c4c 100644 --- a/server/jobs/fetch-repo-data.ts +++ b/server/jobs/fetch-repo-data.ts @@ -2,7 +2,8 @@ import db from "@server/db"; import github from "@server/lib/github"; import queue from "@server/lib/queue"; import { repositories } from "@server/models"; -import { eq } from "drizzle-orm"; +import { repoLanguages } from "@server/models/repo-languages"; +import { and, eq, notInArray } from "drizzle-orm"; export type FetchRepoDataJobType = { id: number; @@ -10,17 +11,55 @@ export type FetchRepoDataJobType = { }; export const fetchRepoData = async (data: FetchRepoDataJobType) => { - const details = await github.getRepoDetails(data.uri); + const [repository] = await db + .select() + .from(repositories) + .where(eq(repositories.id, data.id)); - const [result] = await db - .update(repositories) - .set({ languages: details.languages }) - .where(eq(repositories.id, data.id)) - .returning(); - - if (!result) { + if (!repository) { throw new Error("Repository not found!"); } - await queue.add("calculateUserPoints", { userId: result.userId }); + const details = await github.getRepoDetails(data.uri); + const { languages } = details; + + await db.transaction(async (tx) => { + // Remove languages that don't exist anymore + const purgeLangFilter = and( + eq(repoLanguages.repoId, repository.id), + notInArray( + repoLanguages.name, + languages.map((i) => i.lang) + ) + ); + await tx.delete(repoLanguages).where(purgeLangFilter); + + // Add or update languages + for (const lang of languages) { + const [existing] = await tx + .select({ id: repoLanguages.id }) + .from(repoLanguages) + .where( + and( + eq(repoLanguages.repoId, repository.id), + eq(repoLanguages.name, lang.lang) + ) + ); + + if (existing) { + await tx + .update(repoLanguages) + .set({ percentage: lang.amount }) + .where(eq(repoLanguages.id, existing.id)); + } else { + await tx.insert(repoLanguages).values({ + repoId: repository.id, + name: lang.lang, + percentage: lang.amount, + }); + } + } + }); + + await queue.add("calculateUserPoints", { userId: repository.userId }); }; diff --git a/server/jobs/fetch-user-profile.ts b/server/jobs/fetch-user-profile.ts index 01467bc..c6d598a 100644 --- a/server/jobs/fetch-user-profile.ts +++ b/server/jobs/fetch-user-profile.ts @@ -19,6 +19,7 @@ export const fetchUserProfile = async (data: FetchUserProfileType) => { .update(users) .set({ name: details.name, + avatar: details.avatar, followers: details.followers, following: details.following, location: details.location, diff --git a/server/lib/github.ts b/server/lib/github.ts index a44e99a..757655a 100644 --- a/server/lib/github.ts +++ b/server/lib/github.ts @@ -8,6 +8,7 @@ const GITHUB_API_URL = "https://api.github.com"; const selectors = { user: { name: "h1.vcard-names > span.vcard-fullname", + avatar: ".js-profile-editable-replace img.avatar-user", location: "li[itemprop='homeLocation'] span", followers: ".js-profile-editable-area a[href$='?tab=followers'] > span", following: ".js-profile-editable-area a[href$='?tab=following'] > span", @@ -30,6 +31,8 @@ const github = { const $ = cheerio.load(response); const name = $(selectors.user.name).text().trim(); + const avatar = $(selectors.user.avatar).attr("src"); + console.log({ avatar }); const location = $(selectors.user.location).text().trim(); const followers = intval($(selectors.user.followers).text().trim()); const following = intval($(selectors.user.following).text().trim()); @@ -43,6 +46,7 @@ const github = { return { name: name || username, + avatar, username, location, followers, diff --git a/server/lib/utils.ts b/server/lib/utils.ts index ae32db2..af7c84a 100644 --- a/server/lib/utils.ts +++ b/server/lib/utils.ts @@ -12,6 +12,9 @@ export const intval = (value: any) => { export const getLanguageLogo = (language: string) => { const alias: Record = { gdscript: "godot", + gap: "godot", + html: "html5", + css: "css3", }; let lang = language.toLowerCase().replace(/[^a-zA-Z]/g, ""); diff --git a/server/models/index.ts b/server/models/index.ts index c25303f..fa6a035 100644 --- a/server/models/index.ts +++ b/server/models/index.ts @@ -1,2 +1,3 @@ export { users } from "./users"; -export { repositories } from "./repositories"; +export { repositories, repositoriesRelations } from "./repositories"; +export { repoLanguages, repoLanguagesRelations } from "./repo-languages"; diff --git a/server/models/repo-languages.ts b/server/models/repo-languages.ts new file mode 100644 index 0000000..44ac3e8 --- /dev/null +++ b/server/models/repo-languages.ts @@ -0,0 +1,35 @@ +import { InferInsertModel, InferSelectModel, relations } from "drizzle-orm"; +import { + text, + sqliteTable, + integer, + index, + real, +} from "drizzle-orm/sqlite-core"; +import { repositories } from "./repositories"; + +export const repoLanguages = sqliteTable( + "repository_languages", + { + id: integer("id").primaryKey({ autoIncrement: true }), + repoId: integer("repo_id") + .notNull() + .references(() => repositories.id), + + name: text("name").notNull(), + percentage: real("percentage").notNull(), + }, + (t) => ({ + nameIdx: index("repository_languages_name_idx").on(t.name), + }) +); + +export const repoLanguagesRelations = relations(repoLanguages, ({ one }) => ({ + repository: one(repositories, { + fields: [repoLanguages.repoId], + references: [repositories.id], + }), +})); + +export type RepositoryLanguage = InferSelectModel; +export type CreateRepositoryLanguage = InferInsertModel; diff --git a/server/models/repositories.ts b/server/models/repositories.ts index 474005d..0c7e20f 100644 --- a/server/models/repositories.ts +++ b/server/models/repositories.ts @@ -1,7 +1,13 @@ -import { Contributor, Language } from "@server/lib/github"; -import { InferInsertModel, InferSelectModel, sql } from "drizzle-orm"; +import { Contributor } from "@server/lib/github"; +import { + InferInsertModel, + InferSelectModel, + relations, + sql, +} from "drizzle-orm"; import { text, sqliteTable, integer, index } from "drizzle-orm/sqlite-core"; import { users } from "./users"; +import { repoLanguages } from "./repo-languages"; export const repositories = sqliteTable( "repositories", @@ -17,7 +23,6 @@ export const repositories = sqliteTable( stars: integer("stars").notNull(), forks: integer("forks").notNull(), lastUpdate: text("last_update").notNull(), - languages: text("languages", { mode: "json" }).$type(), contributors: text("contributors", { mode: "json" }).$type(), isPending: integer("is_pending", { mode: "boolean" }) @@ -39,5 +44,9 @@ export const repositories = sqliteTable( }) ); +export const repositoriesRelations = relations(repositories, ({ many }) => ({ + languages: many(repoLanguages), +})); + export type Repository = InferSelectModel; export type CreateRepository = InferInsertModel; diff --git a/server/queue-worker.ts b/server/queue-worker.ts index da99f28..131c723 100644 --- a/server/queue-worker.ts +++ b/server/queue-worker.ts @@ -22,7 +22,7 @@ const onJobRetriesExhausted = async (job: Job) => { const worker = new Worker(BULLMQ_JOB_NAME, handler, { connection: BULLMQ_CONNECTION, - concurrency: Number(import.meta.env.QUEUE_CONCURRENCY) || 1, + concurrency: Number(import.meta.env.QUEUE_CONCURRENCY) || 10, removeOnComplete: { count: 0 }, removeOnFail: { count: 0 }, }); diff --git a/server/repository/leaderboard.ts b/server/repository/leaderboard.ts index 63e3859..8816ec3 100644 --- a/server/repository/leaderboard.ts +++ b/server/repository/leaderboard.ts @@ -1,11 +1,11 @@ import db from "@server/db"; import { getLanguageLogo } from "@server/lib/utils"; -import { repositories, users } from "@server/models"; +import { repoLanguages, repositories, users } from "@server/models"; import { count, desc, eq, getTableColumns, inArray, sql } from "drizzle-orm"; import { HTTPException } from "hono/http-exception"; export type GetLeaderboardRes = { - columns: { title: string; selector: string }[]; + columns: { title: string; selector: string; hideOnSmall?: boolean }[]; rows: { id?: string | null; rank: number; @@ -48,12 +48,18 @@ export class LeaderboardRepository { async getTopLanguages() { const data = await db .select({ - language: repositories.language, + language: repoLanguages.name, count: count(), + total: sql`sum(${repoLanguages.percentage})`.as("total"), + avg: sql`avg(${repoLanguages.percentage})`.as("avg"), + coverage: sql`count(*) * avg(${repoLanguages.percentage})`.as( + "coverage" + ), }) .from(repositories) - .groupBy(repositories.language) - .orderBy(desc(count())); + .innerJoin(repoLanguages, eq(repoLanguages.repoId, repositories.id)) + .groupBy(repoLanguages.name) + .orderBy(desc(sql`count(*) * avg(${repoLanguages.percentage})`)); const columns = [ { @@ -62,8 +68,21 @@ export class LeaderboardRepository { }, { title: "Total Repo", + selector: "repo", + }, + { + title: "Rerata", + selector: "avg", + hideOnSmall: true, + }, + { + title: "Cakupan", selector: "sub", }, + // { + // title: "Total Persentase", + // selector: "total", + // }, ]; const rows = data @@ -71,8 +90,11 @@ export class LeaderboardRepository { .map((data, idx) => ({ rank: idx + 1, name: data.language, - sub: `${data.count} repo`, + sub: data.coverage.toFixed(0), image: getLanguageLogo(data.language), + repo: `${data.count} repo`, + total: data.total, + avg: data.avg.toFixed(1), })); return { columns, rows } satisfies GetLeaderboardRes; @@ -91,7 +113,8 @@ export class LeaderboardRepository { }) .from(repositories) .innerJoin(users, eq(users.id, repositories.userId)) - .where(inArray(sql`lower(${repositories.language})`, languages)) + .innerJoin(repoLanguages, eq(repoLanguages.repoId, repositories.id)) + .where(inArray(sql`lower(${repoLanguages.name})`, languages)) .groupBy(users.id) .orderBy(desc(count())); @@ -141,18 +164,18 @@ export class LeaderboardRepository { throw new HTTPException(404, { message: "User not found!" }); } - const repos = await db - .select() - .from(repositories) - .where(eq(repositories.userId, user.id)) - .orderBy(desc(repositories.stars), desc(repositories.forks)); + const repos = await db.query.repositories.findMany({ + where: eq(repositories.userId, user.id), + orderBy: [desc(repositories.stars), desc(repositories.forks)], + with: { languages: true }, + }); const languageMap: Record = {}; repos .flatMap((i) => i.languages || []) .forEach((i) => { - if (!languageMap[i.lang]) languageMap[i.lang] = 0; - languageMap[i.lang] += i.amount; + if (!languageMap[i.name]) languageMap[i.name] = 0; + languageMap[i.name] += i.percentage; }); const totalLangWeight = Object.values(languageMap).reduce( diff --git a/src/components/ui/image.tsx b/src/components/ui/image.tsx new file mode 100644 index 0000000..451abfb --- /dev/null +++ b/src/components/ui/image.tsx @@ -0,0 +1,19 @@ +import React, { useState } from "react"; + +type Props = React.ComponentPropsWithoutRef<"img"> & { + fallback?: string; +}; + +const Image = ({ src, fallback, ...props }: Props) => { + const [isError, setError] = useState(false); + + return ( + setError(true)} + /> + ); +}; + +export default Image; diff --git a/src/pages/home/components/rank-list-item.tsx b/src/pages/home/components/rank-list-item.tsx index 56c5445..c1993d4 100644 --- a/src/pages/home/components/rank-list-item.tsx +++ b/src/pages/home/components/rank-list-item.tsx @@ -49,7 +49,8 @@ const RankListItem = ({ "flex-1 truncate", idx === 0 && data.image ? "flex flex-row items-center gap-x-2" - : "" + : "", + col.hideOnSmall && "hidden md:flex" )} > {idx === 0 && data.image ? ( @@ -78,7 +79,10 @@ export const RankListHeader = ({ columns }: RankListHeaderProps) => {
{columns.map((col, idx) => ( -

+

{col.title}

))} diff --git a/src/pages/home/components/user-rank.tsx b/src/pages/home/components/user-rank.tsx index 9d4ba8a..1a3d40c 100644 --- a/src/pages/home/components/user-rank.tsx +++ b/src/pages/home/components/user-rank.tsx @@ -22,7 +22,7 @@ const UserRank = () => { if (!data) { return ( -
+

Pengen nama kamu masuk list ini juga?

Hayuk

-
-
- -

{data?.user.rank}

-
-

{data?.user.points + " pts"}

-
+ + + + + + + + Perhitungan Point + + + 
+                  {JSON.stringify(pointWeights, null, 2)}
+
+ +

+ Cek lebih lengkap{" "} + + disini + + . +

+ + +
{data && pendingRepos > 0 && ( @@ -202,7 +228,7 @@ const ViewSheet = () => {

{item.name}

- {item.languages?.map((i) => i.lang).join(", ") || + {item.languages?.map((i) => i.name).join(", ") || item.language}

diff --git a/src/pages/home/data.ts b/src/pages/home/data.ts index 564b715..72f4294 100644 --- a/src/pages/home/data.ts +++ b/src/pages/home/data.ts @@ -28,3 +28,16 @@ export type LeaderboardType = { value: LeaderboardTypes; query?: any; }; + +export const pointWeights = { + followers: 20, + following: 10, + achievements: 100, + repositories: 1, + contributorsAmount: 25, + stars: 10, + forks: 10, + languagesKnown: 50, + commits: 1, + lineOfCodes: 0.01, +};