├── bun.lockb ├── assets ├── img.png └── style.css ├── .env.example ├── tailwind.config.js ├── fly.toml ├── Dockerfile ├── package.json ├── jsconfig.json ├── LICENSE ├── README.md ├── .gitignore └── src ├── main.test.js ├── server.jsx ├── main.js └── old.jsx /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixqc/ziglist/HEAD/bun.lockb -------------------------------------------------------------------------------- /assets/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixqc/ziglist/HEAD/assets/img.png -------------------------------------------------------------------------------- /assets/style.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | GITHUB_API_KEY= 2 | GROQ_API_KEY= 3 | R2_ENDPOINT= 4 | R2_TOKEN_VALUE= 5 | R2_ACCESS_KEY_ID= 6 | R2_SECRET_ACCESS_KEY= 7 | CODEBERG_API_KEY= 8 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | darkMode: "media", 3 | content: ["./src/**/*.jsx"], 4 | theme: { 5 | extend: {}, 6 | }, 7 | plugins: [], 8 | }; 9 | -------------------------------------------------------------------------------- /fly.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for ziglist on 2024-08-01T13:25:36Z 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = 'ziglist' 7 | primary_region = 'iad' 8 | 9 | [build] 10 | 11 | [http_service] 12 | internal_port = 8080 13 | force_https = true 14 | auto_stop_machines = 'stop' 15 | auto_start_machines = true 16 | min_machines_running = 1 17 | processes = ['app'] 18 | 19 | [[vm]] 20 | size = 'shared-cpu-1x' 21 | memory = '512' 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG DENO_VERSION=1.45.3 2 | ARG BIN_IMAGE=denoland/deno:bin-${DENO_VERSION} 3 | FROM ${BIN_IMAGE} AS bin 4 | FROM frolvlad/alpine-glibc:alpine-3.13 5 | 6 | RUN apk --no-cache add ca-certificates 7 | RUN addgroup --gid 1000 deno \ 8 | && adduser --uid 1000 --disabled-password deno --ingroup deno \ 9 | && mkdir /deno-dir/ \ 10 | && chown deno:deno /deno-dir/ 11 | 12 | ENV DENO_DIR /deno-dir/ 13 | ENV DENO_INSTALL_ROOT /usr/local 14 | ARG DENO_VERSION 15 | ENV DENO_VERSION=${DENO_VERSION} 16 | 17 | COPY --from=bin /deno /bin/deno 18 | 19 | WORKDIR /deno-dir 20 | 21 | COPY src/ ./src/ 22 | COPY assets/ ./assets/ 23 | COPY deno.json deno.lock .env ./ 24 | 25 | ENTRYPOINT ["/bin/deno", "task"] 26 | CMD ["start"] 27 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ziglist", 3 | "module": "src/main.jsx", 4 | "devDependencies": { 5 | "@types/bun": "latest" 6 | }, 7 | "peerDependencies": { 8 | "typescript": "^5.0.0" 9 | }, 10 | "scripts": { 11 | "start": "IS_PROD=1 bun run src/server.jsx", 12 | "dev:main": "bun run --watch src/server.jsx", 13 | "dev:tailwind": "bunx tailwindcss -i ./assets/style.css -o ./assets/tailwind.css --minify", 14 | "dev:clean": "rm db.sqlite db.sqlite-shm db.sqlite-wal", 15 | "dev:init": "bun run dev:tailwind && bun run dev:restore", 16 | "dev": "bun run dev:tailwind --watch & bun run dev:main" 17 | }, 18 | "type": "module", 19 | "dependencies": { 20 | "hono": "^4.5.8" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | // Enable latest features 4 | "lib": ["ESNext", "DOM"], 5 | "target": "ESNext", 6 | "module": "ESNext", 7 | "moduleDetection": "force", 8 | "jsx": "react-jsx", 9 | "jsxImportSource": "hono/jsx", 10 | "allowJs": true, 11 | "checkJs": true, 12 | "noImplicitAny": false, 13 | 14 | // Bundler mode 15 | "moduleResolution": "bundler", 16 | "allowImportingTsExtensions": true, 17 | "verbatimModuleSyntax": true, 18 | "noEmit": true, 19 | 20 | // Best practices 21 | "strict": true, 22 | "skipLibCheck": true, 23 | "noFallthroughCasesInSwitch": true, 24 | 25 | // Some stricter flags (disabled by default) 26 | "noUnusedLocals": false, 27 | "noUnusedParameters": false, 28 | "noPropertyAccessFromIndexSignature": false 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2024 pixqc 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Ziglist](./assets/img.png)](https://ziglist.org) 2 | 3 | Ziglist is a web-based tool to discover Zig projects and packages. Visit [ziglist.org](https://ziglist.org). 4 | 5 | How it works: Ziglist periodically indexes GitHub for Zig-related repositories, saves it in a SQLite database, and serves it. Ziglist lives in a single JavaScript [file](./src/main.jsx). It runs on the Deno runtime. 6 | 7 | To run Ziglist locally: 8 | 9 | - Install Deno, refer to the [documentation](https://docs.deno.com/runtime/manual/getting_started/installation/) 10 | - `git clone https://github.com/pixqc/ziglist.git` 11 | - `mv .env.example .env` and fill it out 12 | - `deno task dev` 13 | 14 | Help wanted! If you found: 15 | 16 | - c/cpp repo built with Zig, that's not on Ziglist 17 | - repo on Ziglist that's not Zig-related 18 | - missing dependencies in one of the repos 19 | 20 | Please open an issue or a PR, ctrl+f for `HELP:` in the [file](./src/main.jsx). 21 | 22 | Ziglist's visual design is inspired by [https://github.com/piotrkulpinski/openalternative](https://github.com/piotrkulpinski/openalternative) 23 | 24 | Check out the [blogpost](https://pixqc.com/ziglist)! 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore 2 | 3 | # Logs 4 | 5 | logs 6 | _.log 7 | npm-debug.log_ 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | .pnpm-debug.log* 12 | 13 | # Caches 14 | 15 | .cache 16 | 17 | # Diagnostic reports (https://nodejs.org/api/report.html) 18 | 19 | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json 20 | 21 | # Runtime data 22 | 23 | pids 24 | _.pid 25 | _.seed 26 | *.pid.lock 27 | 28 | # Directory for instrumented libs generated by jscoverage/JSCover 29 | 30 | lib-cov 31 | 32 | # Coverage directory used by tools like istanbul 33 | 34 | coverage 35 | *.lcov 36 | 37 | # nyc test coverage 38 | 39 | .nyc_output 40 | 41 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 42 | 43 | .grunt 44 | 45 | # Bower dependency directory (https://bower.io/) 46 | 47 | bower_components 48 | 49 | # node-waf configuration 50 | 51 | .lock-wscript 52 | 53 | # Compiled binary addons (https://nodejs.org/api/addons.html) 54 | 55 | build/Release 56 | 57 | # Dependency directories 58 | 59 | node_modules/ 60 | jspm_packages/ 61 | 62 | # Snowpack dependency directory (https://snowpack.dev/) 63 | 64 | web_modules/ 65 | 66 | # TypeScript cache 67 | 68 | *.tsbuildinfo 69 | 70 | # Optional npm cache directory 71 | 72 | .npm 73 | 74 | # Optional eslint cache 75 | 76 | .eslintcache 77 | 78 | # Optional stylelint cache 79 | 80 | .stylelintcache 81 | 82 | # Microbundle cache 83 | 84 | .rpt2_cache/ 85 | .rts2_cache_cjs/ 86 | .rts2_cache_es/ 87 | .rts2_cache_umd/ 88 | 89 | # Optional REPL history 90 | 91 | .node_repl_history 92 | 93 | # Output of 'npm pack' 94 | 95 | *.tgz 96 | 97 | # Yarn Integrity file 98 | 99 | .yarn-integrity 100 | 101 | # dotenv environment variable files 102 | 103 | .env 104 | .env.development.local 105 | .env.test.local 106 | .env.production.local 107 | .env.local 108 | 109 | # parcel-bundler cache (https://parceljs.org/) 110 | 111 | .parcel-cache 112 | 113 | # Next.js build output 114 | 115 | .next 116 | out 117 | 118 | # Nuxt.js build / generate output 119 | 120 | .nuxt 121 | dist 122 | 123 | # Gatsby files 124 | 125 | # Comment in the public line in if your project uses Gatsby and not Next.js 126 | 127 | # https://nextjs.org/blog/next-9-1#public-directory-support 128 | 129 | # public 130 | 131 | # vuepress build output 132 | 133 | .vuepress/dist 134 | 135 | # vuepress v2.x temp and cache directory 136 | 137 | .temp 138 | 139 | # Docusaurus cache and generated files 140 | 141 | .docusaurus 142 | 143 | # Serverless directories 144 | 145 | .serverless/ 146 | 147 | # FuseBox cache 148 | 149 | .fusebox/ 150 | 151 | # DynamoDB Local files 152 | 153 | .dynamodb/ 154 | 155 | # TernJS port file 156 | 157 | .tern-port 158 | 159 | # Stores VSCode versions used for testing VSCode extensions 160 | 161 | .vscode-test 162 | 163 | # yarn v2 164 | 165 | .yarn/cache 166 | .yarn/unplugged 167 | .yarn/build-state.yml 168 | .yarn/install-state.gz 169 | .pnp.* 170 | 171 | # IntelliJ based IDEs 172 | .idea 173 | 174 | # Finder (MacOS) folder config 175 | .DS_Store 176 | 177 | tailwind.css 178 | log.txt 179 | .http-cache -------------------------------------------------------------------------------- /src/main.test.js: -------------------------------------------------------------------------------- 1 | import { expect, describe, beforeAll, afterAll, test } from "bun:test"; 2 | import { Glob } from "bun"; 3 | import { Database } from "bun:sqlite"; 4 | import { 5 | headers, 6 | extractZon, 7 | getTopRepoURL, 8 | insertUrlDependencies, 9 | upsertDependencies, 10 | initDB, 11 | upsertZigRepos, 12 | logger, 13 | getZigZonURL, 14 | fetchMetadata, 15 | zon2json, 16 | getZigBuildURL, 17 | getAllRepoURL, 18 | upsertMetadata, 19 | getNextURL, 20 | dateGenerator, 21 | repoExtractors, 22 | } from "./main.js"; 23 | 24 | const CACHE_DIR = "./.http-cache"; 25 | const DB_NAME = ":memory:"; 26 | 27 | // TODO: 28 | // - fts 29 | // - fetch -> insert -> read (server's query) should be in good state 30 | 31 | /** @typedef {{full_name: string, default_branch: string, platform: 'github' | 'codeberg'}} RepoName */ 32 | 33 | // biome-ignore format: off 34 | /** @type {Array} */ 35 | const repos = [ 36 | { full_name: "ziglang/zig", default_branch: "master", platform: "github" }, 37 | { full_name: "ggerganov/ggml", default_branch: "master", platform: "github" }, 38 | { full_name: "fairyglade/ly", default_branch: "master", platform: "github" }, 39 | { full_name: "Hejsil/zig-clap", default_branch: "master", platform: "github" }, 40 | { full_name: "dude_the_builder/zigstr", default_branch: "main", platform: "codeberg" }, 41 | { full_name: "grayhatter/player", default_branch: "main", platform: "codeberg" }, 42 | { full_name: "ziglings/exercises", default_branch: "main", platform: "codeberg" }, 43 | ]; 44 | 45 | /** 46 | * @param {'repo' | 'metadata-zig' | 'metadata-zon'} type 47 | * @param {RepoName} repo 48 | * @returns {string} 49 | */ 50 | const getCacheFilename = (type, repo) => { 51 | return `${CACHE_DIR}/${type}-${repo.platform}-${repo.full_name.replace("/", "-")}.json`; 52 | }; 53 | 54 | /** 55 | * @param {RepoName} repo 56 | * @returns {string} 57 | */ 58 | const getURL = (repo) => { 59 | if (repo.platform === "github") { 60 | return `https://api.github.com/repos/${repo.full_name}`; 61 | } else if (repo.platform === "codeberg") { 62 | return `https://codeberg.org/api/v1/repos/${repo.full_name}`; 63 | } 64 | return ""; // unreachable 65 | }; 66 | 67 | /** 68 | * @param {RepoName} repo 69 | * @returns {Promise} */ 70 | const cacheRepo = async (repo) => { 71 | const filename = getCacheFilename("repo", repo); 72 | const file = Bun.file(filename); 73 | if (file.size > 0) return; 74 | const url = getURL(repo); 75 | const res = await fetch(url, { headers: headers[repo.platform] }); 76 | const data = await res.json(); 77 | await Bun.write(file, JSON.stringify(data)); 78 | }; 79 | 80 | /** 81 | * @param {RepoName} repo 82 | * @returns {Promise} */ 83 | const cacheMetadata = async (repo) => { 84 | const zigFilename = getCacheFilename("metadata-zig", repo); 85 | const zonFilename = getCacheFilename("metadata-zon", repo); 86 | const zigFile = Bun.file(zigFilename); 87 | const zonFile = Bun.file(zonFilename); 88 | if (zigFile.size === 0) { 89 | const zigUrl = getZigBuildURL(repo); 90 | const zigResponse = await fetchMetadata(zigUrl); 91 | await Bun.write(zigFile, JSON.stringify(zigResponse)); 92 | } 93 | if (zonFile.size === 0) { 94 | const zonUrl = getZigZonURL(repo); 95 | const zonResponse = await fetchMetadata(zonUrl); 96 | await Bun.write(zonFile, JSON.stringify(zonResponse)); 97 | } 98 | }; 99 | 100 | describe("db inserts and reads", () => { 101 | let db; 102 | beforeAll(async () => { 103 | const promises = repos.flatMap((repo) => [ 104 | cacheRepo(repo), 105 | cacheMetadata(repo), 106 | ]); 107 | await Promise.all(promises); 108 | 109 | db = new Database(DB_NAME); 110 | initDB(db); 111 | }); 112 | 113 | test("multiple repo inserts should not duplicate", async () => { 114 | for (const repo of repos) { 115 | const file = Bun.file(getCacheFilename("repo", repo)); 116 | const data = await file.json(); 117 | const extractor = repoExtractors[repo.platform]; 118 | const parsed = extractor(data); 119 | 120 | upsertZigRepos(db, [parsed, parsed, parsed, parsed, parsed]); 121 | const stmt = db.prepare( 122 | `SELECT * 123 | FROM repos 124 | WHERE full_name = ? AND platform = ?`, 125 | ); 126 | 127 | const result = stmt.all(repo.full_name, repo.platform); 128 | expect(result).toHaveLength(1); 129 | expect(result[0].full_name).toBe(parsed.full_name); 130 | expect(result[0].platform).toBe(parsed.platform); 131 | expect(result[0].name).toBe(parsed.name); 132 | expect(result[0].default_branch).toBe(parsed.default_branch); 133 | expect(result[0].owner).toBe(parsed.owner); 134 | expect(result[0].created_at).toBe(parsed.created_at); 135 | expect(result[0].updated_at).toBe(parsed.updated_at); 136 | expect(result[0].pushed_at).toBe(parsed.pushed_at); 137 | expect(result[0].description).toBe(parsed.description); 138 | expect(result[0].homepage).toBe(parsed.homepage); 139 | expect(result[0].license).toBe(parsed.license); 140 | expect(result[0].language).toBe(parsed.language); 141 | expect(result[0].stars).toBe(parsed.stars); 142 | expect(result[0].forks).toBe(parsed.forks); 143 | expect(Boolean(result[0].is_fork)).toBe(parsed.is_fork); 144 | expect(Boolean(result[0].is_archived)).toBe(parsed.is_archived); 145 | } 146 | }); 147 | 148 | test("repo upsert should update properly", async () => { 149 | for (const repo of repos) { 150 | const file = Bun.file(getCacheFilename("repo", repo)); 151 | const data = await file.json(); 152 | const extractor = repoExtractors[repo.platform]; 153 | let parsed = extractor(data); 154 | parsed.description = parsed.description + "!"; 155 | upsertZigRepos(db, [parsed]); 156 | const stmt = db.prepare( 157 | `SELECT * 158 | FROM repos 159 | WHERE full_name = ? AND platform = ?`, 160 | ); 161 | const result = stmt.all(repo.full_name, repo.platform); 162 | expect(result).toHaveLength(1); 163 | expect(result[0].full_name).toBe(parsed.full_name); 164 | expect(result[0].platform).toBe(parsed.platform); 165 | expect(result[0].name).toBe(parsed.name); 166 | expect(result[0].default_branch).toBe(parsed.default_branch); 167 | expect(result[0].owner).toBe(parsed.owner); 168 | expect(result[0].created_at).toBe(parsed.created_at); 169 | expect(result[0].updated_at).toBe(parsed.updated_at); 170 | expect(result[0].pushed_at).toBe(parsed.pushed_at); 171 | expect(result[0].description).toBe(parsed.description); 172 | expect(result[0].description).toEndWith("!"); 173 | expect(result[0].homepage).toBe(parsed.homepage); 174 | expect(result[0].license).toBe(parsed.license); 175 | expect(result[0].language).toBe(parsed.language); 176 | expect(result[0].stars).toBe(parsed.stars); 177 | expect(result[0].forks).toBe(parsed.forks); 178 | expect(Boolean(result[0].is_fork)).toBe(parsed.is_fork); 179 | expect(Boolean(result[0].is_archived)).toBe(parsed.is_archived); 180 | } 181 | }); 182 | 183 | test("multiple metadata inserts should not duplicate", async () => { 184 | for (const repo of repos) { 185 | const zigFile = Bun.file(getCacheFilename("metadata-zig", repo)); 186 | const zonFile = Bun.file(getCacheFilename("metadata-zon", repo)); 187 | const buildData = await zigFile.json(); 188 | const zonData = await zonFile.json(); 189 | const zonExists = zonData.status === 200; 190 | const buildExists = buildData.status === 200; 191 | if (!zonExists) continue; 192 | const parsed = extractZon(JSON.parse(zon2json(zonData.content))); 193 | 194 | const repoStmt = db.prepare( 195 | `SELECT id 196 | FROM repos 197 | WHERE full_name = ? AND platform = ?`, 198 | ); 199 | const repoResult = repoStmt.get(repo.full_name, repo.platform); 200 | expect(repoResult).toBeDefined(); 201 | const repoId = repoResult.id; 202 | 203 | const metadata = { 204 | repo_id: repoId, 205 | min_zig_version: parsed.minimum_zig_version, 206 | build_zig_exists: buildExists, 207 | build_zig_zon_exists: zonExists, 208 | fetched_at: zonData.fetched_at, 209 | }; 210 | upsertMetadata(db, [metadata, metadata, metadata]); 211 | 212 | const stmt = db.prepare( 213 | `SELECT * 214 | FROM repo_metadata 215 | WHERE repo_id = ?`, 216 | ); 217 | const result = stmt.all(repoId); 218 | expect(result).toHaveLength(1); 219 | expect(result[0].repo_id).toBe(repoId); 220 | expect(result[0].min_zig_version).toBe(metadata.min_zig_version); 221 | expect(Boolean(result[0].build_zig_exists)).toBe(buildExists); 222 | expect(Boolean(result[0].build_zig_zon_exists)).toBe(zonExists); 223 | expect(result[0].fetched_at).toBeGreaterThan(0); 224 | } 225 | }); 226 | 227 | test("metadata upsert should update properly", async () => { 228 | for (const repo of repos) { 229 | const zigFile = Bun.file(getCacheFilename("metadata-zig", repo)); 230 | const zonFile = Bun.file(getCacheFilename("metadata-zon", repo)); 231 | const buildData = await zigFile.json(); 232 | const zonData = await zonFile.json(); 233 | const zonExists = zonData.status === 200; 234 | const buildExists = buildData.status === 200; 235 | 236 | const repoStmt = db.prepare( 237 | `SELECT id 238 | FROM repos 239 | WHERE full_name = ? AND platform = ?`, 240 | ); 241 | const repoResult = repoStmt.get(repo.full_name, repo.platform); 242 | expect(repoResult).toBeDefined(); 243 | const repoId = repoResult.id; 244 | 245 | const metadata = { 246 | repo_id: repoId, 247 | min_zig_version: "0.11.0", 248 | build_zig_exists: buildExists, 249 | build_zig_zon_exists: zonExists, 250 | fetched_at: zonData.fetched_at + 1, 251 | }; 252 | upsertMetadata(db, [metadata]); 253 | 254 | const stmt = db.prepare( 255 | `SELECT * 256 | FROM repo_metadata 257 | WHERE repo_id = ?`, 258 | ); 259 | const result = stmt.all(repoId); 260 | expect(result).toHaveLength(1); 261 | expect(result[0].repo_id).toBe(repoId); 262 | expect(result[0].min_zig_version).toBe("0.11.0"); 263 | expect(result[0].fetched_at).toBe(zonData.fetched_at + 1); 264 | } 265 | }); 266 | 267 | test("extracted zon should match database entries", async () => { 268 | for (const repo of repos) { 269 | const zonFile = Bun.file(getCacheFilename("metadata-zon", repo)); 270 | const zonData = await zonFile.json(); 271 | const zonExists = zonData.status === 200; 272 | if (!zonExists) continue; 273 | const parsed = extractZon(JSON.parse(zon2json(zonData.content))); 274 | const repoStmt = db.prepare( 275 | `SELECT id FROM repos WHERE full_name = ? AND platform = ?`, 276 | ); 277 | const repoResult = repoStmt.get(repo.full_name, repo.platform); 278 | expect(repoResult).toBeDefined(); 279 | const repoId = repoResult.id; 280 | 281 | // intentionally duplicated 282 | for (let i = 0; i < 5; i++) { 283 | if (parsed.urlDeps.length > 0) 284 | insertUrlDependencies(db, parsed.urlDeps); 285 | if (parsed.deps.length > 0) upsertDependencies(db, parsed.deps, repoId); 286 | } 287 | const urlDepStmt = db.prepare( 288 | `SELECT * 289 | FROM url_dependencies 290 | WHERE hash IN ( 291 | SELECT url_dependency_hash 292 | FROM repo_dependencies 293 | WHERE repo_id = ? 294 | )`, 295 | ); 296 | const urlDepResults = urlDepStmt.all(repoId); 297 | expect(urlDepResults).toHaveLength(parsed.urlDeps.length); 298 | for (const expectedUrlDep of parsed.urlDeps) { 299 | const actualUrlDep = urlDepResults.find( 300 | (d) => d.hash === expectedUrlDep.hash, 301 | ); 302 | expect(actualUrlDep).toBeDefined(); 303 | expect(actualUrlDep).toEqual( 304 | expect.objectContaining({ 305 | hash: expectedUrlDep.hash, 306 | name: expectedUrlDep.name, 307 | url: expectedUrlDep.url, 308 | }), 309 | ); 310 | } 311 | const depStmt = db.prepare( 312 | `SELECT * 313 | FROM repo_dependencies 314 | WHERE repo_id = ?`, 315 | ); 316 | const depResults = depStmt.all(repoId); 317 | expect(depResults).toHaveLength(parsed.deps.length); 318 | for (const expectedDep of parsed.deps) { 319 | const actualDep = depResults.find((d) => d.name === expectedDep.name); 320 | expect(actualDep).toBeDefined(); 321 | expect(actualDep).toEqual( 322 | expect.objectContaining({ 323 | repo_id: repoId, 324 | name: expectedDep.name, 325 | dependency_type: expectedDep.dependency_type, 326 | path: expectedDep.path, 327 | url_dependency_hash: expectedDep.url_dependency_hash, 328 | }), 329 | ); 330 | } 331 | } 332 | }); 333 | 334 | test("deps parsed data should match database entries", async () => { 335 | for (const repo of repos) { 336 | const zonFile = Bun.file(getCacheFilename("metadata-zon", repo)); 337 | const zonData = await zonFile.json(); 338 | if (zonData.status !== 200) continue; 339 | const parsed = extractZon(JSON.parse(zon2json(zonData.content))); 340 | const repoStmt = db.prepare( 341 | `SELECT id 342 | FROM repos 343 | WHERE full_name = ? AND platform = ?`, 344 | ); 345 | const repoResult = repoStmt.get(repo.full_name, repo.platform); 346 | expect(repoResult).toBeDefined(); 347 | const repoId = repoResult.id; 348 | 349 | const joinStmt = db.prepare( 350 | `SELECT 351 | r.*, 352 | GROUP_CONCAT(d.name) AS dependencies 353 | FROM repos r 354 | LEFT JOIN repo_dependencies d ON r.id = d.repo_id 355 | WHERE r.id = ? 356 | GROUP BY r.id 357 | `, 358 | ); 359 | const joinResult = joinStmt.get(repoId); 360 | const dbSet = 361 | joinResult.dependencies == null 362 | ? new Set() 363 | : new Set(joinResult.dependencies.split(",")); 364 | const parsedSet = new Set(parsed.deps.map((d) => d.name)); 365 | expect(dbSet).toEqual(parsedSet); 366 | } 367 | }); 368 | 369 | afterAll(() => { 370 | db.close(); 371 | logger.flush(); 372 | }); 373 | }); 374 | 375 | /** 376 | * @param {'github' | 'codeberg'} platform 377 | * @param {number} pages 378 | * @returns {Promise} 379 | */ 380 | const cacheTopRepos = async (platform, pages) => { 381 | let url = getTopRepoURL(platform); 382 | for (let i = 1; i <= pages; i++) { 383 | const filename = `./.http-cache/${platform}-top-${i}.json`; 384 | const file = Bun.file(filename); 385 | if (file.size > 0) continue; 386 | const response = await fetch(url, { headers: headers[platform] }); 387 | const data = await response.json(); 388 | await Bun.write(file, JSON.stringify(data)); 389 | // @ts-ignore - wont be undefined 390 | url = getNextURL(response); 391 | } 392 | }; 393 | 394 | /** 395 | * only the first page, go through all date ranges, github only 396 | * FIXME: refetches on --rerun-each 5 397 | * 398 | * @param {'github' | 'codeberg'} platform 399 | * @returns {Promise} 400 | */ 401 | const cacheAllRepos = async (platform) => { 402 | let idx = 1; 403 | while (true) { 404 | const filename = `./.http-cache/${platform}-all-${idx}.json`; 405 | const file = Bun.file(filename); 406 | if (file.size > 0) { 407 | idx++; 408 | continue; 409 | } 410 | const url = getAllRepoURL(platform); 411 | if (idx !== 1 && url.includes("2015-07-04")) break; 412 | const response = await fetch(url, { headers: headers[platform] }); 413 | const data = await response.json(); 414 | await Bun.write(Bun.file(filename), JSON.stringify(data)); 415 | } 416 | }; 417 | 418 | describe("fetches", () => { 419 | let db; 420 | beforeAll(async () => { 421 | await Promise.all([ 422 | cacheTopRepos("github", 2), 423 | cacheTopRepos("codeberg", 2), 424 | cacheAllRepos("github"), 425 | ]); 426 | 427 | db = new Database(DB_NAME); 428 | initDB(db); 429 | }); 430 | 431 | test("top github repos should parse", async () => { 432 | ["1", "2"].forEach(async (page) => { 433 | const filename = `./.http-cache/github-top-${page}.json`; 434 | const file = Bun.file(filename); 435 | const data = await file.json(); 436 | expect(data.items).toHaveLength(100); 437 | const extractor = repoExtractors["github"]; 438 | for (const item of data.items) expect(extractor(item)).toBeDefined(); 439 | }); 440 | }); 441 | 442 | test("top codeberg repos should parse", async () => { 443 | ["1", "2"].forEach(async (page) => { 444 | const filename = `./.http-cache/codeberg-top-${page}.json`; 445 | const file = Bun.file(filename); 446 | const data = await file.json(); 447 | expect(data.data).toHaveLength(50); 448 | const extractor = repoExtractors["codeberg"]; 449 | for (const item of data.data) expect(extractor(item)).toBeDefined(); 450 | }); 451 | }); 452 | 453 | test("fetch all should have items below 1k", async () => { 454 | const glob = new Glob("./.http-cache/github-all-*.json"); 455 | for await (const filename of glob.scan({ dot: true })) { 456 | const file = Bun.file(filename); 457 | const data = await file.json(); 458 | expect(data.total_count).toBeLessThan(1000); 459 | } 460 | }); 461 | 462 | test("generator should loop date range", async () => { 463 | const starts = []; 464 | for (let i = 0; i < 100; i++) { 465 | const { start } = dateGenerator().next().value; 466 | starts.push(start.toISOString().slice(0, 10)); 467 | } 468 | const countOccurrences = (date) => starts.filter((d) => d === date).length; 469 | expect(countOccurrences("2015-07-04")).toBeGreaterThanOrEqual(2); 470 | expect(countOccurrences("2024-01-20")).toBeGreaterThanOrEqual(2); 471 | }); 472 | 473 | afterAll(() => { 474 | db.close(); 475 | logger.flush(); 476 | }); 477 | }); 478 | -------------------------------------------------------------------------------- /src/server.jsx: -------------------------------------------------------------------------------- 1 | import { Database } from "bun:sqlite"; 2 | import { Hono } from "hono"; 3 | import { 4 | logger, 5 | initDB, 6 | fetchRepo, 7 | serverHomeQuery, 8 | serverNewQuery, 9 | serverTopQuery, 10 | serverSearchQuery, 11 | serverDependencyQuery, 12 | fetchBuildZig, 13 | processBuildZig, 14 | rebuildFts, 15 | } from "./main.js"; 16 | 17 | const SECONDLY = 1000; 18 | const MINUTELY = 60 * SECONDLY; 19 | const HOURLY = 60 * MINUTELY; 20 | const DAILY = 24 * HOURLY; 21 | 22 | /** 23 | * @param {number} unixSecond 24 | * @returns {string} 25 | */ 26 | const timeAgo = (unixSecond) => { 27 | const moment = new Date().getTime() / 1000; 28 | const diff = moment - unixSecond; 29 | const intervals = [ 30 | { label: "yr", seconds: 31536000 }, 31 | { label: "wk", seconds: 604800 }, 32 | { label: "d", seconds: 86400 }, 33 | { label: "hr", seconds: 3600 }, 34 | { label: "min", seconds: 60 }, 35 | { label: "sec", seconds: 1 }, 36 | ]; 37 | for (let i = 0; i < intervals.length; i++) { 38 | const count = Math.floor(diff / intervals[i].seconds); 39 | if (count > 0) { 40 | return `${count}${intervals[i].label} ago`; 41 | } 42 | } 43 | return "just now"; 44 | }; 45 | 46 | /** 47 | * 81930 -> 81.9k 48 | * 1000 -> 1.0k 49 | * 999 -> 999 50 | * 51 | * @param {number} num - The number to format. 52 | * @returns {string} - Formatted number as a string. 53 | */ 54 | const formatNumberK = (num) => { 55 | if (num < 1000) return num.toString(); 56 | const thousands = num / 1000; 57 | return (Math.floor(thousands * 10) / 10).toFixed(1) + "k"; 58 | }; 59 | 60 | const LucideChevronLeft = () => ( 61 | 73 | 74 | 75 | ); 76 | 77 | const LucideChevronRight = () => ( 78 | 90 | 91 | 92 | ); 93 | 94 | const LucideGithub = () => ( 95 | 107 | 108 | 109 | 110 | ); 111 | 112 | const LucideSearch = () => ( 113 | 125 | 126 | 127 | 128 | ); 129 | 130 | const LucideCircleOff = () => ( 131 | 143 | 144 | 145 | 146 | 147 | ); 148 | 149 | const SearchBar = ({ query }) => ( 150 |
151 |
152 | 159 | 165 |
166 |
167 | ); 168 | 169 | const RepoDetail = ({ kind, value }) => ( 170 |
171 | {kind} 172 |
173 |
174 |
175 |
176 | {value} 177 |
178 | ); 179 | 180 | const Badge = ({ value }) => ( 181 | 182 | {value} 183 | 184 | ); 185 | 186 | const SpecialCard = () => { 187 | return ( 188 |
189 |

190 | More features coming soon! 191 |

192 |

193 | GitLab support, zigmod+gyro support, dependency graph, etc. Feature 194 | requests? Missing dependencies in one of the pkgs/projects? Let me know! 195 |

196 | 206 | ); 207 | }; 208 | 209 | const RepoCard = ({ repo }) => { 210 | const shownDeps = 5; 211 | const deps = repo.dependencies ? repo.dependencies.split(",") : []; 212 | const repoUrl = 213 | repo.platform === "github" 214 | ? `https://github.com/${repo.full_name}` 215 | : `https://codeberg.org/${repo.full_name}`; 216 | 217 | return ( 218 | 224 |

225 | {repo.full_name} 226 |

227 | {repo.description && ( 228 |

229 | {repo.description.length > 120 230 | ? repo.description.slice(0, 120) + "..." 231 | : repo.description} 232 |

233 | )} 234 |
235 |
236 | {repo.build_zig_exists === 1 && } 237 | {repo.build_zig_zon_exists === 1 && } 238 | {repo.is_fork === 1 && } 239 | {repo.build_zig_exists === 1 && 240 | repo.language !== "Zig" && 241 | repo.language !== null && } 242 | {repo.platform === "codeberg" && } 243 |
244 | {deps.length > 0 && ( 245 |
246 | 247 | Deps: 248 | 249 | {deps.slice(0, shownDeps).map((dep) => ( 250 | 251 | ))} 252 | {deps.length > shownDeps && ( 253 | 254 |
255 |
256 |
257 |
258 | +{deps.length - shownDeps} more deps 259 | 260 | )} 261 |
262 | )} 263 | {repo.min_zig_version && ( 264 | 265 | )} 266 | 267 | 268 | 269 | ); 270 | }; 271 | 272 | const RepoGrid = ({ repos, page, currentPath }) => { 273 | const repoElements = repos.map((repo) => ); 274 | if (currentPath === "/" && page === 1) { 275 | repoElements.splice(2, 0, ); 276 | } 277 | return ( 278 |
279 | {repoElements} 280 |
281 | ); 282 | }; 283 | 284 | const DependencyList = ({ repos }) => { 285 | return ( 286 |
287 |
288 | 289 | Popular dependencies: 290 | 291 |
292 |

293 | · · · 294 |

295 | {repos.map((repo, index) => ( 296 |
297 |

298 | 304 | {repo.full_name} 305 | 306 |

307 | 308 | dependencies 309 | 310 |
    311 | {repo.dependencies.map((dep, depIndex) => ( 312 |
  • 316 | {dep.name} 317 |
    318 |
    319 |
    320 |
    321 | {dep.dependency_type === "url" && ( 322 | 323 | {dep.url} 324 | 325 | )} 326 | {dep.dependency_type === "path" && ( 327 | 328 | [path] {dep.path} 329 | 330 | )} 331 |
    • 332 | ))} 333 |
334 |
335 | ))} 336 |
337 | ); 338 | }; 339 | 340 | const tailwindcss = await Bun.file("./assets/tailwind.css").text(); 341 | const BaseLayout = ({ children }) => ( 342 | 343 | 344 | 345 | 346 | ziglist.org 347 |