feat: update language leaderboard calculation

server/db/migrations/0000_tough_jubilee.sql

@@ -1,3 +1,11 @@
+CREATE TABLE `repository_languages` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`repo_id` integer NOT NULL,
+	`name` text NOT NULL,
+	`percentage` real NOT NULL,
+	FOREIGN KEY (`repo_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
 CREATE TABLE `repositories` (
 	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
 	`user_id` integer NOT NULL,
@@ -7,7 +15,6 @@ CREATE TABLE `repositories` (
 	`stars` integer NOT NULL,
 	`forks` integer NOT NULL,
 	`last_update` text NOT NULL,
-	`languages` text,
 	`contributors` text,
 	`is_pending` integer DEFAULT false NOT NULL,
 	`is_error` integer DEFAULT false NOT NULL,
@@ -34,6 +41,7 @@ CREATE TABLE `users` (
 	`updated_at` text NOT NULL
 );
 --> statement-breakpoint
+CREATE INDEX `repository_languages_name_idx` ON `repository_languages` (`name`);--> statement-breakpoint
 CREATE INDEX `repositories_name_idx` ON `repositories` (`name`);--> statement-breakpoint
 CREATE INDEX `repositories_uri_idx` ON `repositories` (`uri`);--> statement-breakpoint
 CREATE INDEX `repositories_language_idx` ON `repositories` (`language`);--> statement-breakpoint

server/db/migrations/meta/0000_snapshot.json

@@ -1,9 +1,68 @@
 {
   "version": "6",
   "dialect": "sqlite",
-  "id": "13d859a6-4c32-46a6-90e9-19740b2bb13a",
+  "id": "a268ec23-239a-4d86-8830-2f3c1c2110df",
   "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
+    "repository_languages": {
+      "name": "repository_languages",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": true
+        },
+        "repo_id": {
+          "name": "repo_id",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "percentage": {
+          "name": "percentage",
+          "type": "real",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "repository_languages_name_idx": {
+          "name": "repository_languages_name_idx",
+          "columns": [
+            "name"
+          ],
+          "isUnique": false
+        }
+      },
+      "foreignKeys": {
+        "repository_languages_repo_id_repositories_id_fk": {
+          "name": "repository_languages_repo_id_repositories_id_fk",
+          "tableFrom": "repository_languages",
+          "tableTo": "repositories",
+          "columnsFrom": [
+            "repo_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
     "repositories": {
       "name": "repositories",
       "columns": {
@@ -63,13 +122,6 @@
           "notNull": true,
           "autoincrement": false
         },
-        "languages": {
-          "name": "languages",
-          "type": "text",
-          "primaryKey": false,
-          "notNull": false,
-          "autoincrement": false
-        },
         "contributors": {
           "name": "contributors",
           "type": "text",

server/db/migrations/meta/_journal.json

@@ -5,8 +5,8 @@
     {
       "idx": 0,
       "version": "6",
-      "when": 1723210265266,
-      "tag": "0000_young_emma_frost",
+      "when": 1723211354217,
+      "tag": "0000_tough_jubilee",
       "breakpoints": true
     }
   ]

server/import-user.ts

@@ -0,0 +1,34 @@
+import db from "./db";
+import queue from "./lib/queue";
+import { users } from "./models";
+
+const main = async () => {
+  const username = process.argv[2];
+  if (!username) {
+    throw new Error("Missing username");
+  }
+
+  const [user] = await db
+    .insert(users)
+    .values({ username, name: username })
+    .onConflictDoUpdate({
+      target: users.username,
+      set: { username },
+    })
+    .returning();
+
+  await queue.add(
+    "fetchUserProfile",
+    { userId: user.id },
+    { jobId: `fetchUserProfile:${user.id}` }
+  );
+  await queue.add(
+    "fetchUserRepos",
+    { userId: user.id },
+    { jobId: `fetchUserRepos:${user.id}` }
+  );
+
+  process.exit();
+};
+
+main();

server/jobs/calculate-user-points.ts

@@ -37,15 +37,15 @@ export const calculateUserPoints = async (data: CalculateUserPointsType) => {
     : 0;
 
   // User repositories
-  const repos = await db
-    .select()
-    .from(repositories)
-    .where(eq(repositories.userId, user.id));
+  const repos = await db.query.repositories.findMany({
+    where: eq(repositories.userId, user.id),
+    with: { languages: true },
+  });
   points += repos.length * weights.repositories;
 
   // Languages known
   const languages = new Set(
-    repos.flatMap((i) => i.languages?.map((j) => j.lang))
+    repos.flatMap((i) => i.languages?.map((j) => j.name))
   );
   points += languages.size * weights.languagesKnown;
 

server/jobs/fetch-repo-data.ts

@@ -2,7 +2,8 @@ import db from "@server/db";
 import github from "@server/lib/github";
 import queue from "@server/lib/queue";
 import { repositories } from "@server/models";
-import { eq } from "drizzle-orm";
+import { repoLanguages } from "@server/models/repo-languages";
+import { and, eq, notInArray } from "drizzle-orm";
 
 export type FetchRepoDataJobType = {
   id: number;
@@ -10,17 +11,55 @@ export type FetchRepoDataJobType = {
 };
 
 export const fetchRepoData = async (data: FetchRepoDataJobType) => {
-  const details = await github.getRepoDetails(data.uri);
+  const [repository] = await db
+    .select()
+    .from(repositories)
+    .where(eq(repositories.id, data.id));
 
-  const [result] = await db
-    .update(repositories)
-    .set({ languages: details.languages })
-    .where(eq(repositories.id, data.id))
-    .returning();
-
-  if (!result) {
+  if (!repository) {
     throw new Error("Repository not found!");
   }
 
-  await queue.add("calculateUserPoints", { userId: result.userId });
+  const details = await github.getRepoDetails(data.uri);
+  const { languages } = details;
+
+  await db.transaction(async (tx) => {
+    // Remove languages that don't exist anymore
+    const purgeLangFilter = and(
+      eq(repoLanguages.repoId, repository.id),
+      notInArray(
+        repoLanguages.name,
+        languages.map((i) => i.lang)
+      )
+    );
+    await tx.delete(repoLanguages).where(purgeLangFilter);
+
+    // Add or update languages
+    for (const lang of languages) {
+      const [existing] = await tx
+        .select({ id: repoLanguages.id })
+        .from(repoLanguages)
+        .where(
+          and(
+            eq(repoLanguages.repoId, repository.id),
+            eq(repoLanguages.name, lang.lang)
+          )
+        );
+
+      if (existing) {
+        await tx
+          .update(repoLanguages)
+          .set({ percentage: lang.amount })
+          .where(eq(repoLanguages.id, existing.id));
+      } else {
+        await tx.insert(repoLanguages).values({
+          repoId: repository.id,
+          name: lang.lang,
+          percentage: lang.amount,
+        });
+      }
+    }
+  });
+
+  await queue.add("calculateUserPoints", { userId: repository.userId });
 };

server/jobs/fetch-user-profile.ts

@@ -19,6 +19,7 @@ export const fetchUserProfile = async (data: FetchUserProfileType) => {
     .update(users)
     .set({
       name: details.name,
+      avatar: details.avatar,
       followers: details.followers,
       following: details.following,
       location: details.location,

server/lib/github.ts

@@ -8,6 +8,7 @@ const GITHUB_API_URL = "https://api.github.com";
 const selectors = {
   user: {
     name: "h1.vcard-names > span.vcard-fullname",
+    avatar: ".js-profile-editable-replace img.avatar-user",
     location: "li[itemprop='homeLocation'] span",
     followers: ".js-profile-editable-area a[href$='?tab=followers'] > span",
     following: ".js-profile-editable-area a[href$='?tab=following'] > span",
@@ -30,6 +31,8 @@ const github = {
     const $ = cheerio.load(response);
 
     const name = $(selectors.user.name).text().trim();
+    const avatar = $(selectors.user.avatar).attr("src");
+    console.log({ avatar });
     const location = $(selectors.user.location).text().trim();
     const followers = intval($(selectors.user.followers).text().trim());
     const following = intval($(selectors.user.following).text().trim());
@@ -43,6 +46,7 @@ const github = {
 
     return {
       name: name || username,
+      avatar,
       username,
       location,
       followers,

server/lib/utils.ts

@@ -12,6 +12,9 @@ export const intval = (value: any) => {
 export const getLanguageLogo = (language: string) => {
   const alias: Record<string, string> = {
     gdscript: "godot",
+    gap: "godot",
+    html: "html5",
+    css: "css3",
   };
 
   let lang = language.toLowerCase().replace(/[^a-zA-Z]/g, "");

server/models/index.ts

@@ -1,2 +1,3 @@
 export { users } from "./users";
-export { repositories } from "./repositories";
+export { repositories, repositoriesRelations } from "./repositories";
+export { repoLanguages, repoLanguagesRelations } from "./repo-languages";

server/models/repo-languages.ts

@@ -0,0 +1,35 @@
+import { InferInsertModel, InferSelectModel, relations } from "drizzle-orm";
+import {
+  text,
+  sqliteTable,
+  integer,
+  index,
+  real,
+} from "drizzle-orm/sqlite-core";
+import { repositories } from "./repositories";
+
+export const repoLanguages = sqliteTable(
+  "repository_languages",
+  {
+    id: integer("id").primaryKey({ autoIncrement: true }),
+    repoId: integer("repo_id")
+      .notNull()
+      .references(() => repositories.id),
+
+    name: text("name").notNull(),
+    percentage: real("percentage").notNull(),
+  },
+  (t) => ({
+    nameIdx: index("repository_languages_name_idx").on(t.name),
+  })
+);
+
+export const repoLanguagesRelations = relations(repoLanguages, ({ one }) => ({
+  repository: one(repositories, {
+    fields: [repoLanguages.repoId],
+    references: [repositories.id],
+  }),
+}));
+
+export type RepositoryLanguage = InferSelectModel<typeof repoLanguages>;
+export type CreateRepositoryLanguage = InferInsertModel<typeof repoLanguages>;

server/models/repositories.ts

@@ -1,7 +1,13 @@
-import { Contributor, Language } from "@server/lib/github";
-import { InferInsertModel, InferSelectModel, sql } from "drizzle-orm";
+import { Contributor } from "@server/lib/github";
+import {
+  InferInsertModel,
+  InferSelectModel,
+  relations,
+  sql,
+} from "drizzle-orm";
 import { text, sqliteTable, integer, index } from "drizzle-orm/sqlite-core";
 import { users } from "./users";
+import { repoLanguages } from "./repo-languages";
 
 export const repositories = sqliteTable(
   "repositories",
@@ -17,7 +23,6 @@ export const repositories = sqliteTable(
     stars: integer("stars").notNull(),
     forks: integer("forks").notNull(),
     lastUpdate: text("last_update").notNull(),
-    languages: text("languages", { mode: "json" }).$type<Language[]>(),
     contributors: text("contributors", { mode: "json" }).$type<Contributor[]>(),
 
     isPending: integer("is_pending", { mode: "boolean" })
@@ -39,5 +44,9 @@ export const repositories = sqliteTable(
   })
 );
 
+export const repositoriesRelations = relations(repositories, ({ many }) => ({
+  languages: many(repoLanguages),
+}));
+
 export type Repository = InferSelectModel<typeof repositories>;
 export type CreateRepository = InferInsertModel<typeof repositories>;

server/queue-worker.ts

@@ -22,7 +22,7 @@ const onJobRetriesExhausted = async (job: Job) => {
 
 const worker = new Worker(BULLMQ_JOB_NAME, handler, {
   connection: BULLMQ_CONNECTION,
-  concurrency: Number(import.meta.env.QUEUE_CONCURRENCY) || 1,
+  concurrency: Number(import.meta.env.QUEUE_CONCURRENCY) || 10,
   removeOnComplete: { count: 0 },
   removeOnFail: { count: 0 },
 });

server/repository/leaderboard.ts

@@ -1,11 +1,11 @@
 import db from "@server/db";
 import { getLanguageLogo } from "@server/lib/utils";
-import { repositories, users } from "@server/models";
+import { repoLanguages, repositories, users } from "@server/models";
 import { count, desc, eq, getTableColumns, inArray, sql } from "drizzle-orm";
 import { HTTPException } from "hono/http-exception";
 
 export type GetLeaderboardRes = {
-  columns: { title: string; selector: string }[];
+  columns: { title: string; selector: string; hideOnSmall?: boolean }[];
   rows: {
     id?: string | null;
     rank: number;
@@ -48,12 +48,18 @@ export class LeaderboardRepository {
   async getTopLanguages() {
     const data = await db
       .select({
-        language: repositories.language,
+        language: repoLanguages.name,
         count: count(),
+        total: sql<number>`sum(${repoLanguages.percentage})`.as("total"),
+        avg: sql<number>`avg(${repoLanguages.percentage})`.as("avg"),
+        coverage: sql<number>`count(*) * avg(${repoLanguages.percentage})`.as(
+          "coverage"
+        ),
       })
       .from(repositories)
-      .groupBy(repositories.language)
-      .orderBy(desc(count()));
+      .innerJoin(repoLanguages, eq(repoLanguages.repoId, repositories.id))
+      .groupBy(repoLanguages.name)
+      .orderBy(desc(sql`count(*) * avg(${repoLanguages.percentage})`));
 
     const columns = [
       {
@@ -62,8 +68,21 @@ export class LeaderboardRepository {
       },
       {
         title: "Total Repo",
+        selector: "repo",
+      },
+      {
+        title: "Rerata",
+        selector: "avg",
+        hideOnSmall: true,
+      },
+      {
+        title: "Cakupan",
         selector: "sub",
       },
+      // {
+      //   title: "Total Persentase",
+      //   selector: "total",
+      // },
     ];
 
     const rows = data
@@ -71,8 +90,11 @@ export class LeaderboardRepository {
       .map((data, idx) => ({
         rank: idx + 1,
         name: data.language,
-        sub: `${data.count} repo`,
+        sub: data.coverage.toFixed(0),
         image: getLanguageLogo(data.language),
+        repo: `${data.count} repo`,
+        total: data.total,
+        avg: data.avg.toFixed(1),
       }));
 
     return { columns, rows } satisfies GetLeaderboardRes;
@@ -91,7 +113,8 @@ export class LeaderboardRepository {
       })
       .from(repositories)
       .innerJoin(users, eq(users.id, repositories.userId))
-      .where(inArray(sql`lower(${repositories.language})`, languages))
+      .innerJoin(repoLanguages, eq(repoLanguages.repoId, repositories.id))
+      .where(inArray(sql`lower(${repoLanguages.name})`, languages))
       .groupBy(users.id)
       .orderBy(desc(count()));
 
@@ -141,18 +164,18 @@ export class LeaderboardRepository {
       throw new HTTPException(404, { message: "User not found!" });
     }
 
-    const repos = await db
-      .select()
-      .from(repositories)
-      .where(eq(repositories.userId, user.id))
-      .orderBy(desc(repositories.stars), desc(repositories.forks));
+    const repos = await db.query.repositories.findMany({
+      where: eq(repositories.userId, user.id),
+      orderBy: [desc(repositories.stars), desc(repositories.forks)],
+      with: { languages: true },
+    });
 
     const languageMap: Record<string, number> = {};
     repos
       .flatMap((i) => i.languages || [])
       .forEach((i) => {
-        if (!languageMap[i.lang]) languageMap[i.lang] = 0;
-        languageMap[i.lang] += i.amount;
+        if (!languageMap[i.name]) languageMap[i.name] = 0;
+        languageMap[i.name] += i.percentage;
       });
 
     const totalLangWeight = Object.values(languageMap).reduce(