├── .gitignore ├── cache-request-lifecycle.png ├── jest.setup.ts ├── tsconfig.json ├── jest.config.js ├── index.ts ├── cloudflare.ts ├── fetchTypes.ts ├── controlFlow.ts ├── package.json ├── tests ├── mswServers.ts └── index.spec.ts ├── fetchContentCloudflare.ts ├── fetchContent.ts ├── README.md └── getGithubContent.ts /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | node_modules 3 | dist 4 | types -------------------------------------------------------------------------------- /cache-request-lifecycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mkartchner994/github-contents-cache/HEAD/cache-request-lifecycle.png -------------------------------------------------------------------------------- /jest.setup.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | 3 | // Polyfill fetch for cloudflare tests 4 | beforeAll(() => { 5 | if (!globalThis.fetch) { 6 | globalThis.fetch = fetch; 7 | } 8 | }); 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "emitDeclarationOnly": true, 4 | "declaration": true, 5 | "declarationDir": "./dist" 6 | }, 7 | "files": ["index.ts", "cloudflare.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */ 2 | module.exports = { 3 | preset: "ts-jest", 4 | testEnvironment: "node", 5 | setupFilesAfterEnv: ['./jest.setup.ts'], 6 | }; 7 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | import getGithubContentFactory from "./getGithubContent"; 2 | import fetchContent from "./fetchContent"; 3 | 4 | export type { GetGithubContentCache } from "./getGithubContent"; 5 | export default getGithubContentFactory(fetchContent); 6 | -------------------------------------------------------------------------------- /cloudflare.ts: -------------------------------------------------------------------------------- 1 | import getGithubContentFactory from "./getGithubContent"; 2 | import fetchContent from "./fetchContentCloudflare"; 3 | 4 | export type { GetGithubContentCache } from "./getGithubContent"; 5 | export default getGithubContentFactory(fetchContent); 6 | -------------------------------------------------------------------------------- /fetchTypes.ts: -------------------------------------------------------------------------------- 1 | export type FetchContentArgs = { 2 | token: string; 3 | owner: string; 4 | repo: string; 5 | path: string; 6 | userAgent: string; 7 | etag?: string; 8 | }; 9 | 10 | export type FetchContentReturn = 11 | | { 12 | statusCode: 403; 13 | limit: number; 14 | remaining: number; 15 | timestampTillNextResetInSeconds: number; 16 | } 17 | | { 18 | statusCode: 200 | 304 | 404; 19 | content?: string; 20 | etag?: string; 21 | }; 22 | 23 | export type FetchContentFn = ( 24 | args: FetchContentArgs 25 | ) => Promise; 26 | -------------------------------------------------------------------------------- /controlFlow.ts: -------------------------------------------------------------------------------- 1 | type ControlFlowStepsEntry = { 2 | nextEvent?: string; 3 | [key: string]: any; 4 | }; 5 | 6 | type ControlFlowSteps = { 7 | entry?: (arg: T) => Promise; 8 | final?: boolean; 9 | [key: string]: any; 10 | }; 11 | 12 | type ControlFlowArgs = { 13 | initialStep: string; 14 | steps: { 15 | [key: string]: ControlFlowSteps; 16 | }; 17 | stepContext: T; 18 | }; 19 | 20 | type ControlFlowReturn = { 21 | step: string; 22 | data?: any; 23 | event?: string; 24 | }; 25 | 26 | async function* createControlFlow({ 27 | initialStep, 28 | steps, 29 | stepContext, 30 | }: ControlFlowArgs): AsyncGenerator { 31 | let currentStep = initialStep; 32 | let currentConfig = steps[currentStep]; 33 | while (true) { 34 | if (currentConfig.final) { 35 | return; 36 | } 37 | let data = await currentConfig.entry(stepContext); 38 | let nextEvent = data.nextEvent; 39 | delete data.nextEvent; 40 | let next = { step: currentConfig[nextEvent], data, event: nextEvent }; 41 | currentStep = next.step; 42 | currentConfig = steps[currentStep]; 43 | yield next; 44 | } 45 | } 46 | 47 | export default async function controlFlow({ 48 | initialStep, 49 | steps, 50 | stepContext, 51 | }: ControlFlowArgs): Promise { 52 | const controlFlowInstance = createControlFlow({ 53 | initialStep, 54 | steps, 55 | stepContext, 56 | }); 57 | let result; 58 | for await (const next of controlFlowInstance) { 59 | result = next; 60 | } 61 | return result; 62 | } 63 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "github-contents-cache", 3 | "version": "1.0.0", 4 | "description": "A helpful utility for retrieving and caching file contents from GitHub's contents api", 5 | "main": "dist/index.js", 6 | "types": "dist/index.d.ts", 7 | "files": [ 8 | "dist" 9 | ], 10 | "scripts": { 11 | "build": "npm test && npm run build:cleanDirs && npm run build:esbuild:node && npm run build:esbuild:cloudflare && npm run build:replaceExtDepRequire && npm run build:tsc", 12 | "build:cleanDirs": "rm -rf ./dist/", 13 | "build:esbuild:node": "esbuild index.ts --bundle --outfile=dist/index.js --platform=node --target=node14 --external:./node_modules/*", 14 | "build:esbuild:cloudflare": "esbuild cloudflare.ts --bundle --outfile=dist/cloudflare.js --platform=node --target=node14", 15 | "build:replaceExtDepRequire": "sed -i 's/\\.\\.\\/node_modules\\/follow-redirects\\/https.js/follow-redirects\\/https/g' ./dist/index.js", 16 | "build:tsc": "tsc --project tsconfig.json", 17 | "test": "jest" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "git+https://github.com/mkartchner994/github-contents-cache.git" 22 | }, 23 | "author": "Morgan Kartchner", 24 | "license": "MIT", 25 | "bugs": { 26 | "url": "https://github.com/mkartchner994/github-contents-cache/issues" 27 | }, 28 | "homepage": "https://github.com/mkartchner994/github-contents-cache#readme", 29 | "devDependencies": { 30 | "@types/jest": "^27.4.0", 31 | "@types/node": "^17.0.15", 32 | "esbuild": "^0.14.19", 33 | "esbuild-register": "^3.3.2", 34 | "jest": "^27.5.1", 35 | "msw": "^0.36.8", 36 | "node-fetch": "^2.6.7", 37 | "ts-jest": "^27.1.3", 38 | "typescript": "^4.5.5" 39 | }, 40 | "dependencies": { 41 | "follow-redirects": "^1.14.8" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tests/mswServers.ts: -------------------------------------------------------------------------------- 1 | /* istanbul ignore file */ 2 | 3 | import { setupServer } from "msw/node"; 4 | import { rest } from "msw"; 5 | 6 | const GITHUB_API_URL = 7 | "https://api.github.com/repos/mkartchner994/github-contents-cache/contents/test-file.mdx"; 8 | 9 | export const ETAG = '"abcdefghijklmnop"'; 10 | export const CONTENT = "This is a Test"; 11 | export const CONTENT_UPDATED = "This is Updated Content"; 12 | export const SECONDS_UNTIL_NEXT_RESET = (Date.now() + 5000) / 1000; 13 | 14 | export const badRequest = setupServer( 15 | rest.get(GITHUB_API_URL, (req, res) => { 16 | return res.networkError("Failed to connect"); 17 | }) 18 | ); 19 | 20 | export const foundFileOnGitHub = setupServer( 21 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 22 | const body = { 23 | content: Buffer.from(CONTENT, "utf-8").toString("base64"), 24 | }; 25 | return res(ctx.status(200), ctx.set("etag", ETAG), ctx.json(body)); 26 | }) 27 | ); 28 | 29 | export const foundFileOnGitHubBadJsonBody = setupServer( 30 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 31 | return res( 32 | ctx.status(200), 33 | ctx.set("etag", ETAG), 34 | ctx.body("{this is a bad json response}") 35 | ); 36 | }) 37 | ); 38 | 39 | export const foundFileOnGitHubUpdatedContent = setupServer( 40 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 41 | const body = { 42 | content: Buffer.from(CONTENT_UPDATED, "utf-8").toString("base64"), 43 | }; 44 | return res(ctx.status(200), ctx.set("etag", ETAG), ctx.json(body)); 45 | }) 46 | ); 47 | 48 | export const foundInCacheDidNotChange = setupServer( 49 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 50 | return res(ctx.status(304)); 51 | }) 52 | ); 53 | 54 | export const notFounOnGitHub = setupServer( 55 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 56 | return res(ctx.status(404)); 57 | }) 58 | ); 59 | 60 | export const rateLimitExceededOnGitHub = setupServer( 61 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 62 | return res( 63 | ctx.status(403), 64 | ctx.set("x-ratelimit-remaining", "0"), 65 | ctx.set("x-ratelimit-limit", "5000"), 66 | ctx.set("x-ratelimit-reset", `${SECONDS_UNTIL_NEXT_RESET}`) 67 | ); 68 | }) 69 | ); 70 | 71 | export const badCreds = setupServer( 72 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 73 | return res(ctx.status(401)); 74 | }) 75 | ); 76 | 77 | export const internalServerErrorOnGitHub = setupServer( 78 | rest.get(GITHUB_API_URL, (req, res, ctx) => { 79 | return res(ctx.status(500)); 80 | }) 81 | ); 82 | -------------------------------------------------------------------------------- /fetchContentCloudflare.ts: -------------------------------------------------------------------------------- 1 | import type { FetchContentArgs, FetchContentReturn } from "./fetchTypes"; 2 | 3 | export default async function fetchContent({ 4 | owner, 5 | repo, 6 | path, 7 | token, 8 | userAgent, 9 | etag, 10 | }: FetchContentArgs): Promise { 11 | return await new Promise((resolve, reject) => { 12 | const fileExtension = path.split(".").pop(); 13 | if (fileExtension.includes("/") || !path.includes(".")) { 14 | return reject( 15 | new Error( 16 | `The path ${path} is not a file with an extension, which is currenlty not supported in the github-contents-cache library` 17 | ) 18 | ); 19 | } 20 | fetch(`https://api.github.com/repos/${owner}/${repo}/contents/${path}`, { 21 | headers: { 22 | accept: "application/vnd.github.v3+json", 23 | // https://docs.github.com/en/rest/overview/resources-in-the-rest-api#user-agent-required 24 | "user-agent": userAgent, 25 | authorization: `token ${token}`, 26 | ...(etag ? { "If-None-Match": etag } : {}), 27 | }, 28 | }) 29 | .then(async (res) => { 30 | if (res.status === 200) { 31 | try { 32 | const json = await res.json(); 33 | const content = atob(json.content); 34 | resolve({ 35 | statusCode: 200, 36 | content: content, 37 | etag: res.headers.get("etag"), 38 | }); 39 | } catch (error) { 40 | reject( 41 | new Error( 42 | "Received a 200 response from GitHub but could not parse the response body" 43 | ) 44 | ); 45 | } 46 | } else if (res.status === 304) { 47 | resolve({ statusCode: 304 }); 48 | } else if (res.status === 404) { 49 | resolve({ statusCode: 404 }); 50 | } else if ( 51 | res.status === 403 && 52 | res.headers.get("x-ratelimit-remaining") && 53 | res.headers.get("x-ratelimit-remaining").trim() === "0" 54 | ) { 55 | // https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers 56 | const remaining = Number(res.headers.get("x-ratelimit-remaining")); 57 | const limit = Number(res.headers.get("x-ratelimit-limit")); 58 | const timestampTillNextResetInSeconds = Number( 59 | res.headers.get("x-ratelimit-reset") 60 | ); 61 | resolve({ 62 | statusCode: 403, 63 | limit, 64 | remaining, 65 | timestampTillNextResetInSeconds, 66 | }); 67 | } else if (res.status === 401 || res.status === 403) { 68 | reject( 69 | new Error( 70 | `Received HTTP response status code ${res.status} from GitHub. This means bad credentials were provided or you do not have access to the resource` 71 | ) 72 | ); 73 | } else { 74 | reject( 75 | new Error( 76 | `Received HTTP response status code ${res.status} from GitHub which is not an actionable code for the github-contents-cache library` 77 | ) 78 | ); 79 | } 80 | }) 81 | .catch((e) => { 82 | reject(new Error("Could not complete request to the GitHub api")); 83 | }); 84 | }); 85 | } 86 | -------------------------------------------------------------------------------- /fetchContent.ts: -------------------------------------------------------------------------------- 1 | import type { FetchContentArgs, FetchContentReturn } from "./fetchTypes"; 2 | import { request } from "follow-redirects/https"; 3 | import { extname } from "path"; 4 | 5 | export default async function fetchContent({ 6 | owner, 7 | repo, 8 | path, 9 | token, 10 | userAgent, 11 | etag, 12 | }: FetchContentArgs): Promise { 13 | return await new Promise((resolve, reject) => { 14 | const isFile = extname(path); 15 | if (!isFile) { 16 | return reject( 17 | new Error( 18 | `The path ${path} is not a file with an extension, which is currenlty not supported in the github-contents-cache library` 19 | ) 20 | ); 21 | } 22 | request( 23 | { 24 | hostname: "api.github.com", 25 | port: 443, 26 | path: `/repos/${owner}/${repo}/contents/${path}`, 27 | method: "GET", 28 | headers: { 29 | accept: "application/vnd.github.v3+json", 30 | // https://docs.github.com/en/rest/overview/resources-in-the-rest-api#user-agent-required 31 | "user-agent": userAgent, 32 | authorization: `token ${token}`, 33 | ...(etag ? { "If-None-Match": etag } : {}), 34 | }, 35 | }, 36 | (res) => { 37 | if (res.statusCode === 200) { 38 | const chunks = []; 39 | res 40 | .on("data", (chunk) => { 41 | chunks.push(chunk); 42 | }) 43 | .on("end", () => { 44 | try { 45 | const bodyString = Buffer.concat(chunks).toString(); 46 | const json = JSON.parse(bodyString); 47 | let content = Buffer.from(json.content, "base64").toString( 48 | "utf-8" 49 | ); 50 | resolve({ 51 | statusCode: 200, 52 | content: content, 53 | etag: res.headers.etag, 54 | }); 55 | } catch (error) { 56 | reject( 57 | new Error( 58 | "Received a 200 response from GitHub but could not parse the response body" 59 | ) 60 | ); 61 | } 62 | }); 63 | } else if (res.statusCode === 304) { 64 | // https://docs.github.com/en/rest/overview/resources-in-the-rest-api#conditional-requests 65 | resolve({ statusCode: 304 }); 66 | } else if (res.statusCode === 404) { 67 | resolve({ statusCode: 404 }); 68 | } else if ( 69 | res.statusCode === 403 && 70 | res.headers["x-ratelimit-remaining"] && 71 | res.headers["x-ratelimit-remaining"].trim() === "0" 72 | ) { 73 | // https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limit-http-headers 74 | const remaining = Number(res.headers["x-ratelimit-remaining"]); 75 | const limit = Number(res.headers["x-ratelimit-limit"]); 76 | const timestampTillNextResetInSeconds = Number( 77 | res.headers["x-ratelimit-reset"] 78 | ); 79 | resolve({ 80 | statusCode: 403, 81 | limit, 82 | remaining, 83 | timestampTillNextResetInSeconds, 84 | }); 85 | } else if (res.statusCode === 401 || res.statusCode === 403) { 86 | reject( 87 | new Error( 88 | `Received HTTP response status code ${res.statusCode} from GitHub. This means bad credentials were provided or you do not have access to the resource` 89 | ) 90 | ); 91 | } else { 92 | reject( 93 | new Error( 94 | `Received HTTP response status code ${res.statusCode} from GitHub which is not an actionable code for the github-contents-cache library` 95 | ) 96 | ); 97 | } 98 | } 99 | ) 100 | .on("error", (error) => { 101 | reject(new Error("Could not complete request to the GitHub api")); 102 | }) 103 | .end(); 104 | }); 105 | } 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # github-contents-cache 2 | 3 | A helpful utility for retrieving and caching file contents from GitHub's contents API. 4 | 5 | ## The problem 6 | 7 | You want to store files on GitHub and retrieve the contents from those files using [GitHub's contents API](https://docs.github.com/en/rest/reference/repos#contents) - using GitHub as a CMS for your .mdx blog posts, hosting .csv monthly reports, html template files, etc - but you also want to make sure to be a good API citzen and stay within the rate limit by caching file contents and only updating the cache when the file contents change using [conditional requests](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#conditional-requests). But as the [saying goes](https://martinfowler.com/bliki/TwoHardThings.html), cache invalidation is hard. Especially when you then also have to think about how to handle all of the other edge cases like - "Should I cache 404 responses? And for how long?", "What if I receive an error from GitHub? 500 response or a malformed return body?", "What if I want to explicitly ignore the cache for certain requests?" - you get the idea. 8 | 9 | ## This solution 10 | 11 | You are responsible to bring your own cache instance - redis, sqlite, disk, etc. This library will manage the lifecycle for making requests to GitHub and setting, updating, and removing contents from the cache. The diagram belows shows just an idea of things covered in the lifecycle - check the tests to see all the covered cases. 12 | 13 | ![Diagram of the cache request lifecycle](./cache-request-lifecycle.png) 14 | 15 | ## Installation 16 | 17 | This module is distributed via [npm][npm] which is bundled with [node][node] and 18 | should be installed as one of your project's `dependencies`: 19 | 20 | ``` 21 | npm install --save github-contents-cache 22 | ``` 23 | 24 | Or, you can install this module through the [yarn][yarn] package manager. 25 | 26 | ``` 27 | yarn add github-contents-cache 28 | ``` 29 | 30 | ## Supported Node Versions 31 | 32 | Current version supported is >=14 33 | 34 | ## Usage 35 | 36 | ```js 37 | import getContentsFromGithub from "github-contents-cache"; 38 | 39 | try { 40 | // Currently, this will only pull content from the default branch of the repo but we may look 41 | // at adding the ability to define a different branch in the future. 42 | const results = await getContentsFromGithub({ 43 | // REQUIRED: Personal access token with repo scope - https://github.com/settings/tokens 44 | // As always with access tokens, be careful to not commit your token! 45 | token: GITHUB_AUTH_TOKEN, 46 | // REQUIRED: Owner of the repo 47 | owner: OWNER, 48 | // REQUIRED: Repo name 49 | repo: REPO, 50 | // REQUIRED: Path from the root of the repo 51 | path: `content/${slug}.mdx`, 52 | // REQUIRED: Be a good API citizen, pass a useful user agent 53 | // https://docs.github.com/en/rest/overview/resources-in-the-rest-api#user-agent-required 54 | userAgent: "GitHub user ", 55 | // REQUIRED: Methods you provided to be able to get, set, and remove content from your cache instance 56 | // The path arg provided to the get, set, and remove methods below will be the same as the 57 | // path option given above 58 | cache: { 59 | // Should set the entry in your cache instance. If you want to make updates to the file 60 | // contents before they are stored - use the serialize method above 61 | set: async (path, entry) => { 62 | await blogCache.set(path, JSON.stringify(entry)); 63 | }, 64 | // Should return the entry exactly as it was provided as the second arg of the set method above 65 | // If a falsey value is returned, it will be assumed the entry was not found in the cache 66 | get: async (path) => { 67 | const cacheResults = await blogCache.get(path); 68 | if (!cacheResults) { 69 | return null; 70 | } 71 | return JSON.parse(cacheResults.value); 72 | }, 73 | // Should remove the cache entry 74 | remove: async (path) => { 75 | await blogCache.remove(path); 76 | }, 77 | }, 78 | // OPTIONAL: Whether or not to use the cache for this request 79 | // default: false 80 | ignoreCache: false, 81 | // OPTIONAL: How long to wait before allowing a cached 200 to look in GitHub for changes 82 | // default: 0 - always check in GitHub for changes 83 | // 304 from GitHub does not count against the api limit 84 | maxAgeInMilliseconds: 10000, 85 | // OPTIONAL: How long to wait before allowing a cached 404 to look in GitHub again. 86 | // default: Infinity - cache indefinitely (pass ignoreCache true to break the cache for the entry) 87 | // 404 from GitHub counts against the api limit 88 | max404AgeInMilliseconds: 10000, 89 | // OPTIONAL: Allows a stale (maxAgeInMilliseconds has expired) cache entry to be used while the cache entry gets refreshed in the background 90 | // default: false - does not revalidate cache entries in the background 91 | // Since the cached response will be returned before the cache entry gets revalidated, this will probably not work in a serverless environment 92 | // like AWS Lambda as execution of the script will be stop once a response is returned. In those environments, it might work best to set a 93 | // longer `maxAgeInMilliseconds` time in conjunction with an outside process (cron?) which invalidates cache entries using `ignoreCache: true` 94 | staleWhileRevalidate: false, 95 | // OPTIONAL: If you want to transform your file contents in some way before they are cached 96 | serialize: async (fileContentsString) => { 97 | const { code, frontmatter } = await bundleMDX({ 98 | source: fileContentsString, 99 | }); 100 | return { code, frontmatter }; 101 | }, 102 | }); 103 | 104 | if (results.status === "found") { 105 | // results.content will either be a string with the file contents, or 106 | // whatever was returned from the serialize method if provided 107 | return { 108 | code: results.content.code, 109 | frontmatter: results.content.frontmatter, 110 | cacheHit: results.cacheHit, 111 | }; 112 | } 113 | 114 | if (results.status === "notFound") { 115 | // We didn't find this file in GitHub 116 | throw new Response(null, { status: 404 }); 117 | } 118 | 119 | if (results.status === "error") { 120 | // These errors could range from errors thrown in the serialize or cache instance methods provided 121 | // or an unexpected error from the GitHub api 122 | console.warn(results.message, results.error); 123 | throw new Response(null, { status: 500 }); 124 | } 125 | 126 | if (results.status === "rateLimitExceeded") { 127 | // You could use the timestampTillNextResetInSeconds to implement logic to only return cached 128 | // entries until the reset happens. GitHubs api limits are pretty generous though so you hopefully 129 | // would never run into this if your content on GitHub changes infrequently and their aren't 130 | // errors coming from the provided cache instance 131 | const { content, limit, remaining, timestampTillNextResetInSeconds } = 132 | results; 133 | console.warn("We've reached our github api limit", { 134 | limit, 135 | remaining, 136 | timestampTillNextResetInSeconds, 137 | }); 138 | // Hit the limit but we found something in the cache for this request 139 | if (results.cacheHit) { 140 | return { 141 | code: results.content.code, 142 | frontmatter: results.content.frontmatter, 143 | cacheHit: results.cacheHit, 144 | }; 145 | } 146 | // Hit the limit and didn't find anything in the cache 147 | throw new Response(null, { status: 500 }); 148 | } 149 | 150 | throw new Error("Unexpected status"); 151 | } catch (error) { 152 | if (error instanceof Response) { 153 | throw error; 154 | } 155 | 156 | console.warn(error); 157 | 158 | throw new Response(null, { status: 500 }); 159 | } 160 | ``` 161 | 162 | ## LICENSE 163 | 164 | MIT 165 | 166 | [npm]: https://www.npmjs.com/ 167 | [node]: https://nodejs.org 168 | [yarn]: https://yarnpkg.com 169 | -------------------------------------------------------------------------------- /getGithubContent.ts: -------------------------------------------------------------------------------- 1 | import type { FetchContentFn } from "./fetchTypes"; 2 | import controlFlow from "./controlFlow"; 3 | 4 | type GetGithubContentCacheEntry = 5 | | { 6 | type: "found"; 7 | time: number; 8 | content: any; 9 | etag: string; 10 | } 11 | | { type: "notFound"; time: number }; 12 | 13 | type GetGithubContentCacheGetReturn = 14 | | GetGithubContentCacheEntry 15 | | null 16 | | undefined; 17 | 18 | export type GetGithubContentCache = { 19 | get: (path: string) => Promise; 20 | set: (path: string, entry: GetGithubContentCacheEntry) => Promise; 21 | remove: (path: string) => Promise; 22 | }; 23 | 24 | export interface GetGithubContentArgs { 25 | token: string; 26 | owner: string; 27 | repo: string; 28 | path: string; 29 | userAgent: string; 30 | cache: GetGithubContentCache; 31 | ignoreCache?: boolean; 32 | staleWhileRevalidate?: boolean; 33 | maxAgeInMilliseconds?: number; 34 | max404AgeInMilliseconds?: number; 35 | serialize?: (content: string) => Promise; 36 | } 37 | 38 | interface GetGithubContentArgsWithfetchContent extends GetGithubContentArgs { 39 | fetchContent: FetchContentFn; 40 | } 41 | 42 | type GetGithubContentReturn = 43 | | { status: "found"; content: any; etag: string; cacheHit: boolean } 44 | | { status: "notFound"; content: ""; cacheHit: boolean } 45 | | { 46 | status: "rateLimitExceeded"; 47 | limit: number; 48 | remaining: number; 49 | timestampTillNextResetInSeconds: number; 50 | content?: any; // If we have hit our rate limit but we still have a cached value 51 | etag?: string; 52 | cacheHit?: boolean; 53 | } 54 | | { 55 | status: "error"; 56 | message: string; 57 | error: Error; 58 | }; 59 | 60 | type GetGithubContentStepContext = { 61 | cache: GetGithubContentCache; 62 | token: GetGithubContentArgs["token"]; 63 | owner: GetGithubContentArgs["owner"]; 64 | repo: GetGithubContentArgs["repo"]; 65 | path: GetGithubContentArgs["path"]; 66 | userAgent: GetGithubContentArgs["userAgent"]; 67 | maxAgeInMilliseconds: GetGithubContentArgs["maxAgeInMilliseconds"]; 68 | max404AgeInMilliseconds: GetGithubContentArgs["max404AgeInMilliseconds"]; 69 | staleWhileRevalidate: GetGithubContentArgs["staleWhileRevalidate"]; 70 | serialize: GetGithubContentArgs["serialize"]; 71 | fetchContent: GetGithubContentArgsWithfetchContent["fetchContent"]; 72 | maxAgeInMillisecondsExpired?: boolean; 73 | cachedResults?: { 74 | time: number; 75 | content: any; 76 | etag: string; 77 | }; 78 | }; 79 | 80 | async function getGithubContent({ 81 | token, 82 | owner, 83 | repo, 84 | path, 85 | userAgent, 86 | cache, 87 | maxAgeInMilliseconds, 88 | ignoreCache = false, 89 | staleWhileRevalidate = false, 90 | max404AgeInMilliseconds = Infinity, 91 | serialize = async (content: string) => content, 92 | fetchContent, 93 | }: GetGithubContentArgsWithfetchContent): Promise { 94 | if (!token || !owner || !repo || !path || !userAgent || !cache) { 95 | throw new Error( 96 | "Please provide all of the required arguments - { token, owner, repo, path, userAgent, cache }" 97 | ); 98 | } 99 | 100 | let result = await controlFlow({ 101 | initialStep: ignoreCache ? "clearCacheEntry" : "lookInCache", 102 | stepContext: { 103 | cache, 104 | token, 105 | owner, 106 | repo, 107 | path, 108 | userAgent, 109 | serialize, 110 | fetchContent, 111 | staleWhileRevalidate, 112 | maxAgeInMilliseconds, 113 | max404AgeInMilliseconds, 114 | }, 115 | steps: { 116 | clearCacheEntry: { 117 | entry: clearCacheEntry, 118 | onCachedCleared: "lookInGithub", // Was able to clear the cache, lets get the latest from github 119 | onError: "error", // Something went wrong clearing the cache - either from a corrupt cache or a manual call to clear 120 | }, 121 | lookInCache: { 122 | entry: lookInCache, 123 | onFound: "found", // Found in the cache and the maxAgeInMilliseconds had not yet expired 124 | onFoundInCache: "lookInGithub", // Ask github if what we have in cache is stale (Does Not count against our api limit) 125 | onNotInCache: "lookInGithub", // Ask for it from github (Does count against our api limit) 126 | on404CacheExpired: "clearCacheEntry", // We found a cache 404 but it has expired 127 | on404InCache: "notFound", // We asked github earlier and they said they didn't have it 128 | onError: "error", // Unknown error we couldn't recover from 129 | }, 130 | lookInGithub: { 131 | entry: lookInGithub, 132 | onFound: "found", // Either came from the cache or from github and then we cached it 133 | on404FromGithub: "notFound", // Github said they didn't have it 134 | onRateLimitExceeded: "rateLimitExceeded", // Github said we are hitting their api to much 135 | onError: "error", // Having trouble getting data from the github api? 136 | }, 137 | found: { final: true }, // Got it! 138 | notFound: { final: true }, // Don't have it 139 | rateLimitExceeded: { final: true }, // Oops 140 | error: { final: true }, // Hopefully this never happens, but we are taking care of it if it does :thumbsup: 141 | }, 142 | }); 143 | 144 | if (result.step === "found") { 145 | return { 146 | status: "found", 147 | content: result.data.content, 148 | etag: result.data.etag, 149 | cacheHit: result.data.cacheHit, 150 | }; 151 | } 152 | if (result.step == "notFound") { 153 | return { status: "notFound", content: "", cacheHit: result.data.cacheHit }; 154 | } 155 | if (result.step == "rateLimitExceeded") { 156 | return { 157 | status: "rateLimitExceeded", 158 | limit: result.data.limit, 159 | remaining: result.data.remaining, 160 | timestampTillNextResetInSeconds: 161 | result.data.timestampTillNextResetInSeconds, 162 | content: result.data.content, 163 | etag: result.data.etag, 164 | cacheHit: result.data.cacheHit, 165 | }; 166 | } 167 | if (result.step == "error") { 168 | return { 169 | status: "error", 170 | message: result.data.message, 171 | error: result.data.error, 172 | }; 173 | } 174 | } 175 | 176 | ////// Entry Functions 177 | ////// 178 | 179 | const clearCacheEntry = async (stepContext: GetGithubContentStepContext) => { 180 | try { 181 | await stepContext.cache.remove(stepContext.path); 182 | return { nextEvent: "onCachedCleared" }; 183 | } catch (error) { 184 | return { 185 | nextEvent: "onError", 186 | message: `Error when trying to remove entry from the cache at path ${stepContext.path}`, 187 | error, 188 | }; 189 | } 190 | }; 191 | 192 | const lookInCache = async (stepContext: GetGithubContentStepContext) => { 193 | { 194 | try { 195 | const cachedResults = await stepContext.cache.get(stepContext.path); 196 | if (!cachedResults) { 197 | return { 198 | nextEvent: "onNotInCache", 199 | }; 200 | } 201 | if (cachedResults.type === "notFound") { 202 | if ( 203 | Date.now() - cachedResults.time > 204 | stepContext.max404AgeInMilliseconds 205 | ) { 206 | return { nextEvent: "on404CacheExpired" }; 207 | } 208 | return { nextEvent: "on404InCache", cacheHit: true }; 209 | } 210 | 211 | stepContext.cachedResults = cachedResults; 212 | 213 | const foundEvent = { 214 | nextEvent: "onFound", 215 | content: cachedResults.content, 216 | etag: cachedResults.etag, 217 | cacheHit: true, 218 | }; 219 | 220 | if (stepContext.maxAgeInMilliseconds) { 221 | if ( 222 | Date.now() - cachedResults.time <= 223 | stepContext.maxAgeInMilliseconds 224 | ) { 225 | return foundEvent; 226 | } 227 | stepContext.maxAgeInMillisecondsExpired = true; 228 | } 229 | if (stepContext.staleWhileRevalidate) { 230 | try { 231 | // Do not await this, we want this to happen in the background (this probably will not work right in a serverless environment) 232 | lookInGithub(stepContext); 233 | } catch (error) { 234 | // Ignore errors here - return our cached value fast but try and update the cache in the background 235 | } 236 | return foundEvent; 237 | } 238 | return { nextEvent: "onFoundInCache" }; 239 | } catch (error) { 240 | return { 241 | nextEvent: "onError", 242 | message: `Error when trying to get entry from the cache at path ${stepContext.path}`, 243 | error, 244 | }; 245 | } 246 | } 247 | }; 248 | 249 | const lookInGithub = async (stepContext: GetGithubContentStepContext) => { 250 | try { 251 | let resp = await stepContext.fetchContent({ 252 | token: stepContext.token, 253 | owner: stepContext.owner, 254 | repo: stepContext.repo, 255 | path: stepContext.path, 256 | userAgent: stepContext.userAgent, 257 | etag: stepContext.cachedResults && stepContext.cachedResults.etag, 258 | }); 259 | // If the content isn't modified return what is in our cache 260 | if (resp.statusCode === 304) { 261 | // If we got here because the maxAgeInMilliseconds provided had expired 262 | // We need to make sure to reset the `time` in the cache or else the cached time 263 | // would always be passed the maxAgeInMilliseconds from that point on 264 | if (stepContext.maxAgeInMillisecondsExpired === true) { 265 | try { 266 | await stepContext.cache.set(stepContext.path, { 267 | type: "found", 268 | time: Date.now(), 269 | content: stepContext.cachedResults.content, 270 | etag: stepContext.cachedResults.etag, 271 | }); 272 | } catch (error) { 273 | // Ignore errors we get if trying to set content to the cache 274 | // These should be handled in the cache.set method by the caller 275 | } 276 | } 277 | return { 278 | nextEvent: "onFound", 279 | content: stepContext.cachedResults.content, 280 | etag: stepContext.cachedResults.etag, 281 | cacheHit: true, 282 | }; 283 | } 284 | // This file wasn't found in github, cache the 404 response so we don't hit our api limit 285 | // Using the time field with the max404AgeInMilliseconds option to expire this cache entry 286 | if (resp.statusCode === 404) { 287 | try { 288 | await stepContext.cache.set(stepContext.path, { 289 | time: Date.now(), 290 | type: "notFound", 291 | }); 292 | } catch (error) { 293 | // Ignore errors we get if trying to set content to the cache 294 | // These should be handled in the cache.set method by the caller 295 | } 296 | return { nextEvent: "on404FromGithub", cacheHit: false }; 297 | } 298 | // There has probably been a mistake with the cache logic we've been provided? 299 | // Or this is actual demand which is a good problem to have, but we need to figure it out 300 | if (resp.statusCode === 403) { 301 | return { 302 | nextEvent: "onRateLimitExceeded", 303 | limit: resp.limit, 304 | remaining: resp.remaining, 305 | timestampTillNextResetInSeconds: resp.timestampTillNextResetInSeconds, 306 | content: 307 | (stepContext.cachedResults && stepContext.cachedResults.content) ?? 308 | "", 309 | etag: 310 | (stepContext.cachedResults && stepContext.cachedResults.etag) ?? "", 311 | cacheHit: 312 | stepContext.cachedResults && stepContext.cachedResults.content 313 | ? true 314 | : false, 315 | }; 316 | } 317 | 318 | try { 319 | resp.content = await stepContext.serialize(resp.content); 320 | } catch (error) { 321 | return { 322 | nextEvent: "onError", 323 | message: "Error occured when serializing the content", 324 | error, 325 | }; 326 | } 327 | 328 | try { 329 | await stepContext.cache.set(stepContext.path, { 330 | type: "found", 331 | time: Date.now(), 332 | content: resp.content, 333 | etag: resp.etag, 334 | }); 335 | } catch (error) { 336 | // Ignore errors we get if trying to set content to the cache 337 | // These should be handled in the cache.set method by the caller 338 | } 339 | 340 | return { 341 | nextEvent: "onFound", 342 | content: resp.content, 343 | etag: resp.etag, 344 | cacheHit: false, 345 | }; 346 | } catch (error: any) { 347 | // We didn't get back what we expected from the github api, but we have it cached 348 | // so lets return that 349 | if (stepContext.cachedResults && stepContext.cachedResults.content) { 350 | console.warn( 351 | "Received an unexpected error, but returning the value from the cache", 352 | error 353 | ); 354 | return { 355 | nextEvent: "onFound", 356 | content: stepContext.cachedResults.content, 357 | etag: stepContext.cachedResults.etag, 358 | cacheHit: true, 359 | }; 360 | } 361 | // Treat anything else as an internal server error 362 | return { 363 | nextEvent: "onError", 364 | message: `Unexpected error when looking for content on GitHub at path ${stepContext.path}`, 365 | error, 366 | }; 367 | } 368 | }; 369 | 370 | function getGithubContentFactory(fetchContent: FetchContentFn) { 371 | return (options: GetGithubContentArgs): Promise => 372 | getGithubContent({ ...options, fetchContent }); 373 | } 374 | 375 | export default getGithubContentFactory; 376 | -------------------------------------------------------------------------------- /tests/index.spec.ts: -------------------------------------------------------------------------------- 1 | import getGithubContent, { GetGithubContentCache } from "../index"; 2 | import getGithubContentCloudflare from "../cloudflare"; 3 | import { 4 | CONTENT, 5 | CONTENT_UPDATED, 6 | ETAG, 7 | SECONDS_UNTIL_NEXT_RESET, 8 | foundFileOnGitHub, 9 | foundFileOnGitHubBadJsonBody, 10 | foundFileOnGitHubUpdatedContent, 11 | foundInCacheDidNotChange, 12 | notFounOnGitHub, 13 | rateLimitExceededOnGitHub, 14 | badCreds, 15 | badRequest, 16 | internalServerErrorOnGitHub, 17 | } from "./mswServers"; 18 | 19 | // Wait until a mock is called N number of times before continuing 20 | // This is helpful for the staleWhileRevalidate tests 21 | const createWaitableMock = () => { 22 | let resolve; 23 | let times; 24 | let calledCount = 0; 25 | const mock = jest.fn(); 26 | mock.mockImplementation(() => { 27 | calledCount += 1; 28 | if (resolve && calledCount >= times) { 29 | resolve(); 30 | } 31 | }); 32 | 33 | // @ts-ignore 34 | mock.waitToHaveBeenCalled = (t) => { 35 | times = t; 36 | return new Promise((r) => { 37 | resolve = r; 38 | }); 39 | }; 40 | 41 | return mock; 42 | }; 43 | 44 | function createTestSuite(platform = "node") { 45 | const getGithubContentByPlatform = 46 | platform === "node" ? getGithubContent : getGithubContentCloudflare; 47 | 48 | function Cache({ 49 | foundInCache = false, 50 | type404 = false, 51 | typeMaxAge = false, 52 | setMockFn = (...args) => {}, 53 | }): GetGithubContentCache { 54 | return { 55 | get: async () => { 56 | if (foundInCache === false) { 57 | return null; 58 | } 59 | if (foundInCache && type404) { 60 | // time - assume this was cached 5 seconds in the past 61 | return { type: "notFound", time: Date.now() - 5000 }; 62 | } 63 | return { 64 | type: "found", 65 | // time - assume this was cached 5 seconds in the past if we are testing maxAge 66 | time: typeMaxAge ? Date.now() - 5000 : Date.now(), 67 | content: CONTENT, 68 | etag: ETAG, 69 | }; 70 | }, 71 | set: async (...args) => { 72 | setMockFn(...args); 73 | }, 74 | remove: async () => {}, 75 | }; 76 | } 77 | 78 | async function serialize(content) { 79 | return content.toLowerCase(); 80 | } 81 | 82 | function getContentFromMkartchner994(args) { 83 | return getGithubContentByPlatform({ 84 | token: "123", 85 | owner: "mkartchner994", 86 | repo: "github-contents-cache", 87 | path: "test-file.mdx", 88 | userAgent: "Github user mkartchner994 personal blog", 89 | ...args, 90 | }); 91 | } 92 | 93 | function getContentFromMkartchner994Dir(args) { 94 | return getGithubContentByPlatform({ 95 | token: "123", 96 | owner: "mkartchner994", 97 | repo: "github-contents-cache", 98 | path: "contentDir", 99 | userAgent: "Github user mkartchner994 personal blog", 100 | ...args, 101 | }); 102 | } 103 | 104 | describe(`Tests for ${platform}`, () => { 105 | test(`An error is thrown if not all of the required arguments are provided`, async () => { 106 | badCreds.listen(); 107 | await expect( 108 | getContentFromMkartchner994({}) // not providing cache 109 | ).rejects.toThrowError( 110 | "Please provide all of the required arguments - { token, owner, repo, path, userAgent, cache }" 111 | ); 112 | badCreds.close(); 113 | }); 114 | 115 | test(`Throw an error if a bad token was provided`, async () => { 116 | badCreds.listen(); 117 | const cache = Cache({ foundInCache: false }); 118 | const response = await getContentFromMkartchner994({ 119 | serialize: serialize, 120 | ignoreCache: false, 121 | cache: cache, 122 | }); 123 | // Because we don't have the actual instance of the error, checking response.error.message matches our expected error string 124 | expect(response.status).toEqual("error"); 125 | // @ts-ignore 126 | expect(response.message).toEqual( 127 | "Unexpected error when looking for content on GitHub at path test-file.mdx" 128 | ); 129 | // @ts-ignore 130 | expect(response.error.message).toEqual( 131 | "Received HTTP response status code 401 from GitHub. This means bad credentials were provided or you do not have access to the resource" 132 | ); 133 | badCreds.close(); 134 | }); 135 | 136 | test(`Throw an error if the path is not a file with an extension`, async () => { 137 | foundFileOnGitHub.listen(); 138 | const cache = Cache({ foundInCache: false }); 139 | const response = await getContentFromMkartchner994Dir({ 140 | serialize: serialize, 141 | ignoreCache: false, 142 | cache: cache, 143 | }); 144 | // Because we don't have the actual instance of the error, checking response.error.message matches our expected error string 145 | expect(response.status).toEqual("error"); 146 | // @ts-ignore 147 | expect(response.message).toEqual( 148 | "Unexpected error when looking for content on GitHub at path contentDir" 149 | ); 150 | // @ts-ignore 151 | expect(response.error.message).toEqual( 152 | "The path contentDir is not a file with an extension, which is currenlty not supported in the github-contents-cache library" 153 | ); 154 | foundFileOnGitHub.close(); 155 | }); 156 | 157 | test(`Throw an error if the request cannot be made`, async () => { 158 | badRequest.listen(); 159 | const cache = Cache({ foundInCache: false }); 160 | const response = await getContentFromMkartchner994({ 161 | serialize: serialize, 162 | ignoreCache: false, 163 | cache: cache, 164 | }); 165 | // Because we don't have the actual instance of the error, checking response.error.message matches our expected error string 166 | expect(response.status).toEqual("error"); 167 | // @ts-ignore 168 | expect(response.message).toEqual( 169 | "Unexpected error when looking for content on GitHub at path test-file.mdx" 170 | ); 171 | // @ts-ignore 172 | expect(response.error.message).toEqual( 173 | "Could not complete request to the GitHub api" 174 | ); 175 | badRequest.close(); 176 | }); 177 | 178 | test(`Throw an error if malformed json is received from GitHub`, async () => { 179 | foundFileOnGitHubBadJsonBody.listen(); 180 | const cache = Cache({ foundInCache: false }); 181 | const response = await getContentFromMkartchner994({ 182 | serialize: serialize, 183 | ignoreCache: false, 184 | cache: cache, 185 | }); 186 | // Because we don't have the actual instance of the error, checking response.error.message matches our expected error string 187 | expect(response.status).toEqual("error"); 188 | // @ts-ignore 189 | expect(response.message).toEqual( 190 | "Unexpected error when looking for content on GitHub at path test-file.mdx" 191 | ); 192 | // @ts-ignore 193 | expect(response.error.message).toEqual( 194 | "Received a 200 response from GitHub but could not parse the response body" 195 | ); 196 | foundFileOnGitHubBadJsonBody.close(); 197 | }); 198 | 199 | test(`NOT FOUND in cache, FOUND in GitHub, YES serialize method provided - return { status: "found", cacheHit: false, content: }`, async () => { 200 | foundFileOnGitHub.listen(); 201 | const cache = Cache({ foundInCache: false }); 202 | const response = await getContentFromMkartchner994({ 203 | serialize: serialize, 204 | ignoreCache: false, 205 | cache: cache, 206 | }); 207 | const expectedResponse = { 208 | status: "found", 209 | content: await serialize(CONTENT), 210 | etag: ETAG, 211 | cacheHit: false, 212 | }; 213 | expect(response).toEqual(expectedResponse); 214 | foundFileOnGitHub.close(); 215 | }); 216 | 217 | test(`NOT FOUND in cache, FOUND in GitHub, NO serialize method provided - return { status: "found", cacheHit: false, content: }`, async () => { 218 | foundFileOnGitHub.listen(); 219 | const cache = Cache({ foundInCache: false }); 220 | const response = await getContentFromMkartchner994({ 221 | ignoreCache: false, 222 | cache: cache, 223 | }); 224 | const expectedResponse = { 225 | status: "found", 226 | content: CONTENT, 227 | etag: ETAG, 228 | cacheHit: false, 229 | }; 230 | expect(response).toEqual(expectedResponse); 231 | foundFileOnGitHub.close(); 232 | }); 233 | 234 | test(`FOUND in cache, UPDATE NOT FOUND in GitHub - return { status: "found", cacheHit: true, content: }`, async () => { 235 | foundInCacheDidNotChange.listen(); 236 | const cache = Cache({ foundInCache: true }); 237 | const response = await getContentFromMkartchner994({ 238 | ignoreCache: false, 239 | serialize: serialize, 240 | cache: cache, 241 | }); 242 | const cachedResults = await cache.get("test-file.mdx"); 243 | const expectedResponse = { 244 | status: "found", 245 | // @ts-ignore 246 | content: cachedResults.content, 247 | // @ts-ignore 248 | etag: cachedResults.etag, 249 | cacheHit: true, 250 | }; 251 | expect(response).toEqual(expectedResponse); 252 | foundInCacheDidNotChange.close(); 253 | }); 254 | 255 | test(`FOUND in cache, UPDATE FOUND in GitHub - return { status: "found", cacheHit: false, content: }`, async () => { 256 | foundFileOnGitHubUpdatedContent.listen(); 257 | const cache = Cache({ foundInCache: true }); 258 | const response = await getContentFromMkartchner994({ 259 | serialize: serialize, 260 | ignoreCache: false, 261 | cache: cache, 262 | }); 263 | const cachedResults = await cache.get("test-file.mdx"); 264 | // @ts-ignore 265 | expect(cachedResults?.type).toEqual("found"); 266 | // @ts-ignore 267 | expect(cachedResults?.content).toEqual(CONTENT); 268 | const expectedResponse = { 269 | status: "found", 270 | content: await serialize(CONTENT_UPDATED), 271 | etag: ETAG, 272 | cacheHit: false, 273 | }; 274 | expect(response).toEqual(expectedResponse); 275 | foundFileOnGitHubUpdatedContent.close(); 276 | }); 277 | 278 | test(`NOT FOUND in cache, NOT FOUND in GitHub - return { status: "notFound", cacheHit: false, content: "" }`, async () => { 279 | notFounOnGitHub.listen(); 280 | const cache = Cache({ foundInCache: false }); 281 | const response = await getContentFromMkartchner994({ 282 | serialize: serialize, 283 | ignoreCache: false, 284 | cache: cache, 285 | }); 286 | const expectedResponse = { 287 | status: "notFound", 288 | content: "", 289 | cacheHit: false, 290 | }; 291 | expect(response).toEqual(expectedResponse); 292 | notFounOnGitHub.close(); 293 | }); 294 | 295 | test(`FOUND in cache, NOT FOUND in GitHub - return { status: "notFound", cacheHit: false, content: "" }`, async () => { 296 | notFounOnGitHub.listen(); 297 | const cache = Cache({ foundInCache: true }); 298 | const response = await getContentFromMkartchner994({ 299 | serialize: serialize, 300 | ignoreCache: false, 301 | cache: cache, 302 | }); 303 | const expectedResponse = { 304 | status: "notFound", 305 | content: "", 306 | cacheHit: false, 307 | }; 308 | expect(response).toEqual(expectedResponse); 309 | notFounOnGitHub.close(); 310 | }); 311 | 312 | test(`NOT FOUND in cache, GitHub rate limit exceeded - return { status: "rateLimitExceeded", cacheHit: false, content: "", ... }`, async () => { 313 | rateLimitExceededOnGitHub.listen(); 314 | const cache = Cache({ foundInCache: false }); 315 | const response = await getContentFromMkartchner994({ 316 | serialize: serialize, 317 | ignoreCache: false, 318 | cache: cache, 319 | }); 320 | const expectedResponse = { 321 | status: "rateLimitExceeded", 322 | limit: 5000, 323 | remaining: 0, 324 | timestampTillNextResetInSeconds: SECONDS_UNTIL_NEXT_RESET, 325 | content: "", 326 | etag: "", 327 | cacheHit: false, 328 | }; 329 | expect(response).toEqual(expectedResponse); 330 | rateLimitExceededOnGitHub.close(); 331 | }); 332 | 333 | test(`FOUND in cache, GitHub rate limit exceeded - return { status: "rateLimitExceeded", cacheHit: true, content: , ... }`, async () => { 334 | rateLimitExceededOnGitHub.listen(); 335 | const cache = Cache({ foundInCache: true }); 336 | const response = await getContentFromMkartchner994({ 337 | serialize: serialize, 338 | ignoreCache: false, 339 | cache: cache, 340 | }); 341 | const cachedResults = await cache.get("test-file.mdx"); 342 | const expectedResponse = { 343 | status: "rateLimitExceeded", 344 | limit: 5000, 345 | remaining: 0, 346 | timestampTillNextResetInSeconds: SECONDS_UNTIL_NEXT_RESET, 347 | // @ts-ignore 348 | content: cachedResults.content, 349 | // @ts-ignore 350 | etag: cachedResults.etag, 351 | cacheHit: true, 352 | }; 353 | expect(response).toEqual(expectedResponse); 354 | rateLimitExceededOnGitHub.close(); 355 | }); 356 | 357 | test(`FOUND in cache, ignoreCache TRUE, FOUND in GitHub - return { status: "found", cacheHit: false, content: }`, async () => { 358 | foundFileOnGitHub.listen(); 359 | const cache = Cache({ foundInCache: true }); 360 | const response = await getContentFromMkartchner994({ 361 | serialize: serialize, 362 | ignoreCache: true, 363 | cache: cache, 364 | }); 365 | const expectedResponse = { 366 | status: "found", 367 | content: await serialize(CONTENT), 368 | etag: ETAG, 369 | cacheHit: false, 370 | }; 371 | expect(response).toEqual(expectedResponse); 372 | foundFileOnGitHub.close(); 373 | }); 374 | 375 | test(`FOUND in cache, ignoreCache TRUE, NOT FOUND in GitHub - return { status: "notFound", cacheHit: false, content: "" }`, async () => { 376 | notFounOnGitHub.listen(); 377 | const cache = Cache({ foundInCache: true }); 378 | const response = await getContentFromMkartchner994({ 379 | serialize: serialize, 380 | ignoreCache: true, 381 | cache: cache, 382 | }); 383 | const expectedResponse = { 384 | status: "notFound", 385 | content: "", 386 | cacheHit: false, 387 | }; 388 | expect(response).toEqual(expectedResponse); 389 | notFounOnGitHub.close(); 390 | }); 391 | 392 | test(`FOUND in cache, INTERNAL SERVER ERROR from GitHub - return { status: "found", cacheHit: true, content: }`, async () => { 393 | internalServerErrorOnGitHub.listen(); 394 | const cache = Cache({ foundInCache: true }); 395 | const response = await getContentFromMkartchner994({ 396 | serialize: serialize, 397 | ignoreCache: false, 398 | cache: cache, 399 | }); 400 | const cachedResults = await cache.get("test-file.mdx"); 401 | const expectedResponse = { 402 | status: "found", 403 | // @ts-ignore 404 | content: cachedResults.content, 405 | // @ts-ignore 406 | etag: cachedResults.etag, 407 | cacheHit: true, 408 | }; 409 | expect(response).toEqual(expectedResponse); 410 | internalServerErrorOnGitHub.close(); 411 | }); 412 | 413 | test(`NOT FOUND in cache, INTERNAL SERVER ERROR from GitHub - return { status: "error", error: , message: "..." }`, async () => { 414 | internalServerErrorOnGitHub.listen(); 415 | const cache = Cache({ foundInCache: false }); 416 | const response = await getContentFromMkartchner994({ 417 | serialize: serialize, 418 | ignoreCache: false, 419 | cache: cache, 420 | }); 421 | // Because we don't have the actual instance of the error, checking response.error.message matches our expected error string 422 | expect(response.status).toEqual("error"); 423 | // @ts-ignore 424 | expect(response.message).toEqual( 425 | "Unexpected error when looking for content on GitHub at path test-file.mdx" 426 | ); 427 | // @ts-ignore 428 | expect(response.error.message).toEqual( 429 | "Received HTTP response status code 500 from GitHub which is not an actionable code for the github-contents-cache library" 430 | ); 431 | internalServerErrorOnGitHub.close(); 432 | }); 433 | 434 | test(`FOUND 404 in cache, max404AgeInMilliseconds provided HAS elapsed, SHOULD retry for new content - return { status: "found", cacheHit: false, content: }`, async () => { 435 | foundFileOnGitHub.listen(); 436 | const cache = Cache({ foundInCache: true, type404: true }); 437 | const response = await getContentFromMkartchner994({ 438 | serialize: serialize, 439 | ignoreCache: false, 440 | cache: cache, 441 | max404AgeInMilliseconds: 1, 442 | }); 443 | const expectedResponse = { 444 | status: "found", 445 | content: await serialize(CONTENT), 446 | etag: ETAG, 447 | cacheHit: false, 448 | }; 449 | expect(response).toEqual(expectedResponse); 450 | foundFileOnGitHub.close(); 451 | }); 452 | 453 | test(`FOUND 404 in cache, max404AgeInMilliseconds provided HAS NOT elapsed, SHOULD NOT retry for new content - return { status: "notFound", cacheHit: true, content: "" }`, async () => { 454 | foundFileOnGitHub.listen(); 455 | const cache = Cache({ foundInCache: true, type404: true }); 456 | const response = await getContentFromMkartchner994({ 457 | serialize: serialize, 458 | ignoreCache: false, 459 | cache: cache, 460 | max404AgeInMilliseconds: 10000, 461 | }); 462 | const expectedResponse = { 463 | status: "notFound", 464 | content: "", 465 | cacheHit: true, 466 | }; 467 | expect(response).toEqual(expectedResponse); 468 | foundFileOnGitHub.close(); 469 | }); 470 | 471 | test(`FOUND in cache, maxAgeInMilliseconds provided HAS elapsed, SHOULD retry for new content - return { status: "found", cacheHit: false, content: }`, async () => { 472 | foundFileOnGitHub.listen(); 473 | const setMockFn = jest.fn(); 474 | const cache = Cache({ foundInCache: true, typeMaxAge: true, setMockFn }); 475 | const response = await getContentFromMkartchner994({ 476 | serialize: serialize, 477 | ignoreCache: false, 478 | cache: cache, 479 | maxAgeInMilliseconds: 1, 480 | }); 481 | const expectedResponse = { 482 | status: "found", 483 | content: await serialize(CONTENT), 484 | etag: ETAG, 485 | cacheHit: false, 486 | }; 487 | expect(response).toEqual(expectedResponse); 488 | expect(setMockFn).toHaveBeenCalled(); 489 | foundFileOnGitHub.close(); 490 | }); 491 | 492 | test(`FOUND in cache, maxAgeInMilliseconds provided HAS elapsed, UPDATE NOT FOUND in GitHub - return { status: "found", cacheHit: true, content: }`, async () => { 493 | foundInCacheDidNotChange.listen(); 494 | const setMockFn = jest.fn(); 495 | const cache = Cache({ foundInCache: true, typeMaxAge: true, setMockFn }); 496 | const response = await getContentFromMkartchner994({ 497 | ignoreCache: false, 498 | cache: cache, 499 | maxAgeInMilliseconds: 1, 500 | }); 501 | const expectedResponse = { 502 | status: "found", 503 | content: CONTENT, 504 | etag: ETAG, 505 | cacheHit: true, 506 | }; 507 | expect(response).toEqual(expectedResponse); 508 | expect(setMockFn).toHaveBeenCalled(); 509 | foundInCacheDidNotChange.close(); 510 | }); 511 | 512 | test(`FOUND in cache, maxAgeInMilliseconds provided HAS NOT elapsed, SHOULD NOT retry for new content - return { status: "found", cacheHit: true, content: }`, async () => { 513 | foundFileOnGitHub.listen(); 514 | const setMockFn = jest.fn(); 515 | const cache = Cache({ foundInCache: true, typeMaxAge: true, setMockFn }); 516 | const response = await getContentFromMkartchner994({ 517 | ignoreCache: false, 518 | cache: cache, 519 | maxAgeInMilliseconds: 10000, 520 | }); 521 | const expectedResponse = { 522 | status: "found", 523 | content: CONTENT, 524 | etag: ETAG, 525 | cacheHit: true, 526 | }; 527 | expect(response).toEqual(expectedResponse); 528 | expect(setMockFn).not.toHaveBeenCalled(); 529 | foundFileOnGitHub.close(); 530 | }); 531 | 532 | test(`Error when 'getting' from cache - return { status: "error", error: , message: "..." }`, async () => { 533 | foundFileOnGitHub.listen(); 534 | const cache = Cache({ foundInCache: false }); 535 | const error = new Error("Error getting from cache"); 536 | cache.get = async () => { 537 | throw error; 538 | }; 539 | const response = await getContentFromMkartchner994({ 540 | serialize: serialize, 541 | ignoreCache: false, 542 | cache: cache, 543 | }); 544 | const expectedResponse = { 545 | status: "error", 546 | message: 547 | "Error when trying to get entry from the cache at path test-file.mdx", 548 | error: error, 549 | }; 550 | expect(response).toEqual(expectedResponse); 551 | foundFileOnGitHub.close(); 552 | }); 553 | 554 | test(`Error when 'setting' to cache, FOUND in GitHub - return { status: "found", cacheHit: false, content: }`, async () => { 555 | foundFileOnGitHub.listen(); 556 | const cache = Cache({ foundInCache: false }); 557 | cache.set = async () => { 558 | throw new Error("Error setting to cache"); 559 | }; 560 | const response = await getContentFromMkartchner994({ 561 | serialize: serialize, 562 | ignoreCache: false, 563 | cache: cache, 564 | }); 565 | const expectedResponse = { 566 | status: "found", 567 | content: await serialize(CONTENT), 568 | etag: ETAG, 569 | cacheHit: false, 570 | }; 571 | expect(response).toEqual(expectedResponse); 572 | foundFileOnGitHub.close(); 573 | }); 574 | 575 | test(`Error when 'setting' to cache, NOT FOUND in GitHub - return { status: "notFound", cacheHit: false, content: "" }`, async () => { 576 | notFounOnGitHub.listen(); 577 | const cache = Cache({ foundInCache: false }); 578 | cache.set = async () => { 579 | throw new Error("Error setting to cache"); 580 | }; 581 | const response = await getContentFromMkartchner994({ 582 | serialize: serialize, 583 | ignoreCache: false, 584 | cache: cache, 585 | }); 586 | const expectedResponse = { 587 | status: "notFound", 588 | content: "", 589 | cacheHit: false, 590 | }; 591 | expect(response).toEqual(expectedResponse); 592 | notFounOnGitHub.close(); 593 | }); 594 | 595 | test(`Error when 'removing' from cache - return { status: "error", error: , message: "..." }`, async () => { 596 | foundFileOnGitHub.listen(); 597 | const cache = Cache({ foundInCache: true }); 598 | const error = new Error("could not remove from cache"); 599 | cache.remove = async () => { 600 | throw error; 601 | }; 602 | const response = await getContentFromMkartchner994({ 603 | serialize: serialize, 604 | ignoreCache: true, 605 | cache: cache, 606 | }); 607 | const expectedResponse = { 608 | status: "error", 609 | message: 610 | "Error when trying to remove entry from the cache at path test-file.mdx", 611 | error: error, 612 | }; 613 | expect(response).toEqual(expectedResponse); 614 | foundFileOnGitHub.close(); 615 | }); 616 | 617 | test(`Error when serializing content - return { status: "error", error: , message: "..." }`, async () => { 618 | foundFileOnGitHub.listen(); 619 | const cache = Cache({ foundInCache: false }); 620 | const error = new Error("Could not serialize content"); 621 | const serialize = async () => { 622 | throw error; 623 | }; 624 | const response = await getContentFromMkartchner994({ 625 | serialize: serialize, 626 | ignoreCache: false, 627 | cache: cache, 628 | }); 629 | const expectedResponse = { 630 | status: "error", 631 | message: "Error occured when serializing the content", 632 | error: error, 633 | }; 634 | expect(response).toEqual(expectedResponse); 635 | foundFileOnGitHub.close(); 636 | }); 637 | }); 638 | 639 | test(`FOUND in cache, UPDATE FOUND in GitHub, staleWhileRevalidate true - return cache { status: "found", cacheHit: true, content: } - set update in background for next request`, async () => { 640 | const now = Date.now(); 641 | foundFileOnGitHub.listen(); 642 | const setMockFn = createWaitableMock(); 643 | const cache = Cache({ foundInCache: true, typeMaxAge: false, setMockFn }); 644 | const response = await getContentFromMkartchner994({ 645 | ignoreCache: false, 646 | staleWhileRevalidate: true, 647 | serialize: serialize, 648 | cache: cache, 649 | }); 650 | const cachedResults = await cache.get("test-file.mdx"); 651 | const expectedResponse = { 652 | status: "found", 653 | // @ts-ignore 654 | content: cachedResults.content, 655 | // @ts-ignore 656 | etag: cachedResults.etag, 657 | cacheHit: true, 658 | }; 659 | expect(response).toEqual(expectedResponse); 660 | // @ts-ignore 661 | await setMockFn.waitToHaveBeenCalled(1); 662 | const args = setMockFn.mock.calls[0]; 663 | expect(args[0]).toEqual("test-file.mdx"); 664 | expect(args[1].type).toEqual("found"); 665 | expect(args[1].content).toEqual(await serialize(CONTENT)); 666 | expect(args[1].etag).toEqual(ETAG); 667 | expect(args[1].time).toBeGreaterThanOrEqual(now); 668 | foundFileOnGitHub.close(); 669 | }); 670 | 671 | test(`FOUND in cache, UPDATE NOT FOUND in GitHub, maxAgeInMilliseconds EXPIRED, staleWhileRevalidate true - return cache { status: "found", cacheHit: true, content: } - set update in background for next request`, async () => { 672 | const now = Date.now(); 673 | foundInCacheDidNotChange.listen(); 674 | const setMockFn = createWaitableMock(); 675 | const cache = Cache({ foundInCache: true, typeMaxAge: true, setMockFn }); 676 | const response = await getContentFromMkartchner994({ 677 | ignoreCache: false, 678 | staleWhileRevalidate: true, 679 | cache: cache, 680 | maxAgeInMilliseconds: 1, 681 | }); 682 | const cachedResults = await cache.get("test-file.mdx"); 683 | const expectedResponse = { 684 | status: "found", 685 | // @ts-ignore 686 | content: cachedResults.content, 687 | // @ts-ignore 688 | etag: cachedResults.etag, 689 | cacheHit: true, 690 | }; 691 | expect(response).toEqual(expectedResponse); 692 | // @ts-ignore 693 | await setMockFn.waitToHaveBeenCalled(1); 694 | const args = setMockFn.mock.calls[0]; 695 | expect(args[0]).toEqual("test-file.mdx"); 696 | expect(args[1].type).toEqual("found"); 697 | // @ts-ignore 698 | expect(args[1].content).toEqual(cachedResults.content); 699 | // @ts-ignore 700 | expect(args[1].etag).toEqual(cachedResults.etag); 701 | expect(args[1].time).toBeGreaterThanOrEqual(now); 702 | foundInCacheDidNotChange.close(); 703 | }); 704 | } 705 | 706 | createTestSuite("node"); 707 | createTestSuite("cloudflare"); 708 | --------------------------------------------------------------------------------