├── screeenshot.png ├── .gitignore ├── public ├── favicon.ico ├── favicon-16x16.png ├── favicon-32x32.png ├── mstile-144x144.png ├── mstile-150x150.png ├── mstile-310x150.png ├── mstile-310x310.png ├── mstile-70x70.png ├── apple-touch-icon.png ├── android-chrome-192x192.png ├── android-chrome-512x512.png ├── staticwebapp.config.json ├── browserconfig.xml ├── site.webmanifest └── safari-pinned-tab.svg ├── postcss.config.js ├── vite.config.ts ├── src ├── main.tsx ├── components │ ├── home.tsx │ ├── loading-state.tsx │ ├── empty-state.tsx │ ├── spinner.tsx │ ├── error-state.tsx │ ├── display-commit.tsx │ ├── logo.tsx │ ├── org-listing.tsx │ ├── repo-form.tsx │ ├── picker.tsx │ ├── file-picker.tsx │ ├── json-detail-container.tsx │ └── repo-detail.tsx ├── types.ts ├── glass.svg ├── App.tsx ├── code.svg ├── favicon.svg ├── bug.svg ├── index.css ├── flat.svg ├── hooks │ └── index.ts ├── lib │ └── index.ts └── api │ └── index.ts ├── tsconfig.json ├── tailwind.config.js ├── .github └── workflows │ ├── dispatch.yml │ └── azure-static-web-apps-kind-pond-00161ce0f.yml ├── index.html ├── LICENSE ├── package.json └── README.md /screeenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/screeenshot.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | dist 4 | dist-ssr 5 | *.local 6 | yarn-error.log 7 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /public/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/favicon-16x16.png -------------------------------------------------------------------------------- /public/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/favicon-32x32.png -------------------------------------------------------------------------------- /public/mstile-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/mstile-144x144.png -------------------------------------------------------------------------------- /public/mstile-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/mstile-150x150.png -------------------------------------------------------------------------------- /public/mstile-310x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/mstile-310x150.png -------------------------------------------------------------------------------- /public/mstile-310x310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/mstile-310x310.png -------------------------------------------------------------------------------- /public/mstile-70x70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/mstile-70x70.png -------------------------------------------------------------------------------- /public/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/apple-touch-icon.png -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /public/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/android-chrome-192x192.png -------------------------------------------------------------------------------- /public/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/githubocto/flat-viewer/HEAD/public/android-chrome-512x512.png -------------------------------------------------------------------------------- /public/staticwebapp.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "routes": [ 3 | { 4 | "route": "/*.html", 5 | "rewrite": "index.html" 6 | } 7 | ], 8 | "responseOverrides": { 9 | "404": { 10 | "rewrite": "/index.html", 11 | "statusCode": 200 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import reactRefresh from "@vitejs/plugin-react-refresh"; 3 | import pluginRewriteAll from "vite-plugin-rewrite-all"; 4 | 5 | // https://vitejs.dev/config/ 6 | export default defineConfig({ 7 | plugins: [reactRefresh(), pluginRewriteAll()], 8 | }); 9 | -------------------------------------------------------------------------------- /public/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | #da532c 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import { QueryClient, QueryClientProvider } from "react-query"; 4 | 5 | import "./index.css"; 6 | import App from "./App"; 7 | 8 | const queryClient = new QueryClient(); 9 | 10 | ReactDOM.render( 11 | 12 | 13 | 14 | 15 | , 16 | document.getElementById("root") 17 | ); 18 | -------------------------------------------------------------------------------- /public/site.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "", 3 | "short_name": "", 4 | "icons": [ 5 | { 6 | "src": "/android-chrome-192x192.png", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | }, 10 | { 11 | "src": "/android-chrome-512x512.png", 12 | "sizes": "512x512", 13 | "type": "image/png" 14 | } 15 | ], 16 | "theme_color": "#ffffff", 17 | "background_color": "#ffffff", 18 | "display": "standalone" 19 | } 20 | -------------------------------------------------------------------------------- /src/components/home.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import FlatIcon from "../flat.svg"; 3 | import { RepoForm } from "./repo-form"; 4 | 5 | export function Home() { 6 | return ( 7 |
8 |
9 | Flat Logo 14 | 15 |
16 |
17 | ); 18 | } 19 | -------------------------------------------------------------------------------- /src/components/loading-state.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Spinner } from "./spinner"; 3 | 4 | interface LoadingStateProps { 5 | text?: string; 6 | } 7 | 8 | export function LoadingState(props: LoadingStateProps) { 9 | const { text = "Loading..." } = props; 10 | 11 | return ( 12 |
13 |
14 | 15 | {text} 16 |
17 |
18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 5 | "types": ["vite/client"], 6 | "allowJs": false, 7 | "skipLibCheck": false, 8 | "esModuleInterop": false, 9 | "allowSyntheticDefaultImports": true, 10 | "strict": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "module": "ESNext", 13 | "moduleResolution": "Node", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "noEmit": true, 17 | "jsx": "react" 18 | }, 19 | "include": ["./src"] 20 | } 21 | -------------------------------------------------------------------------------- /src/components/empty-state.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import Code from "../code.svg"; 3 | 4 | interface EmptyStateProps { 5 | children: React.ReactNode; 6 | alt: string; 7 | } 8 | 9 | export function EmptyState(props: EmptyStateProps) { 10 | return ( 11 |
12 |
13 | {props.alt} 14 |
15 | {props.children} 16 |
17 |
18 |
19 | ); 20 | } 21 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | const defaultTheme = require("tailwindcss/defaultTheme"); 2 | 3 | module.exports = { 4 | purge: { 5 | content: ["./index.html", "./src/**/*.{vue,js,ts,jsx,tsx}"], 6 | options: { 7 | safelist: [ 8 | "h-10", 9 | "overflow-ellipsis", 10 | "block", 11 | "whitespace-nowrap", 12 | "overflow-hidden", 13 | ], 14 | }, 15 | }, 16 | darkMode: false, // or 'media' or 'class' 17 | theme: { 18 | extend: { 19 | fontFamily: { 20 | sans: ["Inter var", ...defaultTheme.fontFamily.sans], 21 | }, 22 | }, 23 | }, 24 | variants: { 25 | extend: {}, 26 | }, 27 | plugins: [require("@tailwindcss/forms")], 28 | }; 29 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import { Endpoints } from "@octokit/types"; 2 | 3 | export type Commit = 4 | Endpoints["GET /repos/{owner}/{repo}/commits"]["response"]["data"][0]; 5 | 6 | export type Repo = { 7 | owner: string; 8 | name: string; 9 | }; 10 | 11 | export interface FlatDataTab { 12 | key?: string; 13 | value?: object[]; 14 | invalidValue?: string; 15 | } 16 | 17 | interface RepositoryLicense { 18 | key: string; 19 | name: string; 20 | url: string; 21 | } 22 | 23 | export interface Repository { 24 | name: string; 25 | description: string; 26 | id: string; 27 | topics?: string[]; 28 | stargazers_count: number; 29 | language: string; 30 | updated_at: string; 31 | license: RepositoryLicense; 32 | } 33 | -------------------------------------------------------------------------------- /src/components/spinner.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | export function Spinner() { 4 | return ( 5 | 11 | 19 | 24 | 25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /.github/workflows/dispatch.yml: -------------------------------------------------------------------------------- 1 | name: Repo Events Repository Dispatch 2 | 3 | on: 4 | - issues 5 | - issue_comment 6 | - pull_request 7 | 8 | jobs: 9 | preflight-job: 10 | name: Dispatch 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Print Outputs 14 | env: 15 | outputs: ${{ toJSON(github) }} 16 | run: | 17 | echo outputs: $outputs 18 | - name: Repository Dispatch 19 | uses: peter-evans/repository-dispatch@v1 20 | with: 21 | token: ${{ secrets.PAT }} 22 | repository: githubocto/next-devex-workflows # repo to send event to 23 | event-type: repoevents # name of the custom event 24 | client-payload: '{"event": ${{ toJSON(github) }}}' 25 | -------------------------------------------------------------------------------- /src/components/error-state.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | interface ErrorStateProps { 4 | img: string; 5 | alt: string; 6 | children: React.ReactNode; 7 | } 8 | 9 | export function ErrorState(props: ErrorStateProps) { 10 | return ( 11 |
12 |
13 | {props.alt} 18 |
19 | {props.children} 20 |
21 |
22 |
23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /src/glass.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/App.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useIsFetching } from "react-query"; 3 | import { HeadProvider, Title } from "react-head"; 4 | import { QueryParamProvider } from "use-query-params"; 5 | 6 | import { BrowserRouter as Router, Switch, Route } from "react-router-dom"; 7 | 8 | import { RepoDetail } from "./components/repo-detail"; 9 | import { OrgListing } from "./components/org-listing"; 10 | import { Home } from "./components/home"; 11 | import { useProgressBar } from "./hooks"; 12 | 13 | function App() { 14 | const isFetching = useIsFetching(); 15 | useProgressBar(isFetching); 16 | 17 | return ( 18 | 19 | Flat 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | ); 31 | } 32 | 33 | export default App; 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 GitHub OCTO 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/code.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/bug.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/components/display-commit.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import cc from "classcat"; 3 | import { parseFlatCommitMessage } from "../lib"; 4 | 5 | interface DisplayCommitProps { 6 | author?: string; 7 | message?: string; 8 | filename?: string | null; 9 | } 10 | 11 | export function DisplayCommit(props: DisplayCommitProps) { 12 | const { author, message, filename } = props; 13 | 14 | if (!message) return null; 15 | 16 | const isFlatCommit = author === "flat-data@users.noreply.github.com"; 17 | if (!isFlatCommit) 18 | return ( 19 |
20 |
21 | {message} 22 |
23 |
24 | ); 25 | 26 | const parsed = parseFlatCommitMessage(message, filename || ""); 27 | 28 | if (!parsed) 29 | return ( 30 |
31 | {message} 32 |
33 | ); 34 | 35 | const negativeDelta = parsed.file?.deltaBytes < 0; 36 | 37 | const byteClass = cc([ 38 | "text-xs font-mono", 39 | { 40 | "text-red-700 bg-red-50 p-1 rounded": negativeDelta, 41 | "text-green-700 bg-green-100 p-1 rounded": !negativeDelta, 42 | }, 43 | ]); 44 | 45 | return ( 46 |
47 |
48 | {parsed.message} 49 |
50 | 51 |
52 | 53 | {negativeDelta ? "-" : "+"} 54 | {Math.abs(parsed.file?.deltaBytes)}b 55 | 56 |
57 |
58 | ); 59 | } 60 | -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | @import "nprogress/nprogress.css"; 2 | 3 | @tailwind base; 4 | @tailwind components; 5 | @tailwind utilities; 6 | 7 | body, 8 | html { 9 | height: 100%; 10 | } 11 | 12 | html::selection, 13 | ::selection { 14 | background: theme("colors.indigo.900") !important; 15 | color: theme("colors.white"); 16 | } 17 | 18 | body, 19 | #root { 20 | @apply flex flex-col h-full; 21 | } 22 | 23 | .grayscale { 24 | filter: grayscale(1); 25 | } 26 | 27 | #nprogress .bar { 28 | background: theme("colors.indigo.400") !important; 29 | } 30 | 31 | #nprogress .peg { 32 | box-shadow: 0 0 10px theme("colors.indigo.400"), 33 | 0 0 5px theme("colors.indigo.400") !important; 34 | } 35 | 36 | #nprogress .spinner-icon { 37 | border-top-color: theme("colors.indigo.400") !important; 38 | border-left-color: theme("colors.indigo.400") !important; 39 | } 40 | 41 | .skeleton { 42 | animation: skeleton-glow 1s linear infinite alternate; 43 | background: rgba(206, 217, 224, 0.2); 44 | background-clip: padding-box !important; 45 | border-color: rgba(206, 217, 224, 0.2) !important; 46 | border-radius: 2px; 47 | box-shadow: none !important; 48 | color: transparent !important; 49 | cursor: default; 50 | pointer-events: none; 51 | user-select: none; 52 | } 53 | 54 | .skeleton *, 55 | .skeleton:after, 56 | .skeleton:before { 57 | visibility: hidden !important; 58 | } 59 | 60 | @keyframes skeleton-glow { 61 | 0% { 62 | background: rgba(206, 217, 224, 0.2); 63 | border-color: rgba(206, 217, 224, 0.2); 64 | } 65 | 66 | to { 67 | background: rgba(92, 112, 128, 0.2); 68 | border-color: rgba(92, 112, 128, 0.2); 69 | } 70 | } 71 | 72 | .max-w-prose { 73 | max-width: 80ch; 74 | } 75 | 76 | .controls { 77 | display: none; 78 | } 79 | 80 | @screen lg { 81 | .controls { 82 | display: block; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flat-viewer", 3 | "version": "0.0.0", 4 | "scripts": { 5 | "dev": "vite", 6 | "build": "vite build", 7 | "serve": "vite preview" 8 | }, 9 | "dependencies": { 10 | "@githubocto/flat-ui": "^0.13.5", 11 | "@octokit/rest": "^18.3.5", 12 | "@popperjs/core": "^2.9.1", 13 | "@primer/octicons-react": "^12.1.0", 14 | "@types/d3-dsv": "^2.0.1", 15 | "@types/lodash": "^4.14.168", 16 | "@types/lodash.debounce": "^4.0.6", 17 | "@types/lodash.truncate": "^4.0.6", 18 | "classcat": "^5.0.3", 19 | "d3-dsv": "^2.0.0", 20 | "date-fns": "^2.19.0", 21 | "downshift": "^6.1.0", 22 | "formik": "^2.2.6", 23 | "lodash": "^4.17.21", 24 | "lodash.debounce": "^4.0.8", 25 | "lodash.truncate": "^4.0.8", 26 | "nprogress": "^0.2.0", 27 | "query-string": "^6.14.1", 28 | "react": "^17.0.0", 29 | "react-dom": "^17.0.0", 30 | "react-head": "^3.4.0", 31 | "react-hot-toast": "^1.0.2", 32 | "react-icons": "^4.2.0", 33 | "react-popper": "^2.2.4", 34 | "react-portal": "^4.2.1", 35 | "react-query": "^3.12.1", 36 | "react-router-dom": "^5.2.0", 37 | "reakit": "^1.3.8", 38 | "store2": "^2.12.0", 39 | "use-query-params": "^1.2.2", 40 | "vite-plugin-rewrite-all": "^0.1.2", 41 | "wretch": "^1.7.4", 42 | "yaml": "^1.10.0", 43 | "yup": "^0.32.9" 44 | }, 45 | "devDependencies": { 46 | "@octokit/types": "^6.12.2", 47 | "@tailwindcss/forms": "^0.2.1", 48 | "@types/nprogress": "^0.2.0", 49 | "@types/react": "^17.0.0", 50 | "@types/react-dom": "^17.0.0", 51 | "@types/react-portal": "^4.0.2", 52 | "@types/react-router-dom": "^5.1.7", 53 | "@vitejs/plugin-react-refresh": "^1.3.1", 54 | "autoprefixer": "^10.2.5", 55 | "postcss": "^8.2.8", 56 | "tailwindcss": "^2.1.2", 57 | "typescript": "^4.1.2", 58 | "vite": "^2.0.5" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flat Viewer 2 | 3 | 👉🏽 👉🏽 👉🏽 **Full writeup**: [Flat Data Project](https://octo.github.com/projects/flat-data) 👈🏽 👈🏽 👈🏽 4 | 5 | Flat Viewer is a tool to view un-nested data (CSV & JSON files) in an interactive table. The table has various affordances for exploring the data, such as: 6 | 7 | - filtering 8 | - sorting 9 | - sticky header and column 10 | - diffs for specific commits that change the data 11 | 12 | ![Flat Viewer](./screeenshot.png) 13 | 14 | ## What is Flat Data? 15 | 16 | Flat Viewer is part of a larger project to make it easy to fetch and commit data into GitHub repositories. The action is intended to be run on a schedule, retrieving data from any supported target and creating a commit if there is any change to the fetched data. Flat Data builds on the [“git scraping” approach pioneered by Simon Willison](https://simonwillison.net/2020/Oct/9/git-scraping/) to offer a simple pattern for bringing working datasets into your repositories and versioning them, because developing against local datasets is faster and easier than working with data over the wire. 17 | 18 | ## Usage 19 | 20 | To use Flat Viewer, prepend `flat` to the URL of your GitHub repo: 21 | 22 | from: [`github.com/githubocto/flat-demo-covid-dashboard`](http://github.com/githubocto/flat-demo-covid-dashboard) 23 | to: [`flatgithub.com/githubocto/flat-demo-covid-dashboard`](http://flatgithub.com/githubocto/flat-demo-covid-dashboard) 24 | 25 | ## Development 26 | 27 | To run locally: 28 | 29 | ```bash 30 | yarn # to install dependencies 31 | yarn dev 32 | ``` 33 | 34 | ## Deployment 35 | 36 | flatgithub.com will automatically re-build and deploy when changes are pushed to the `main` branch. 37 | 38 | ## Issues 39 | 40 | If you run into any trouble or have questions, feel free to [open an issue](https://github.com/githubocto/flat-editor/issues). 41 | 42 | ❤️ GitHub OCTO 43 | 44 | ## License 45 | 46 | [MIT](LICENSE) 47 | -------------------------------------------------------------------------------- /.github/workflows/azure-static-web-apps-kind-pond-00161ce0f.yml: -------------------------------------------------------------------------------- 1 | name: Azure Static Web Apps CI/CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request_target: 8 | types: [opened, synchronize, reopened, closed] 9 | branches: 10 | - main 11 | 12 | jobs: 13 | build_and_deploy_job: 14 | if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.action != 'closed') 15 | runs-on: ubuntu-latest 16 | name: Build and Deploy Job 17 | steps: 18 | - uses: actions/checkout@v2 19 | with: 20 | submodules: true 21 | - name: Build And Deploy 22 | id: builddeploy 23 | uses: Azure/static-web-apps-deploy@v0.0.1-preview 24 | with: 25 | azure_static_web_apps_api_token: ${{ secrets.AZURE_STATIC_WEB_APPS_API_TOKEN_KIND_POND_00161CE0F }} 26 | repo_token: ${{ secrets.GITHUB_TOKEN }} # Used for Github integrations (i.e. PR comments) 27 | action: "upload" 28 | ###### Repository/Build Configurations - These values can be configured to match you app requirements. ###### 29 | # For more information regarding Static Web App workflow configurations, please visit: https://aka.ms/swaworkflowconfig 30 | app_location: "/" # App source code path 31 | api_location: "api" # Api source code path - optional 32 | output_location: "dist" # Built app content directory - optional 33 | ###### End of Repository/Build Configurations ###### 34 | 35 | close_pull_request_job: 36 | if: github.event_name == 'pull_request' && github.event.action == 'closed' 37 | runs-on: ubuntu-latest 38 | name: Close Pull Request Job 39 | steps: 40 | - name: Close Pull Request 41 | id: closepullrequest 42 | uses: Azure/static-web-apps-deploy@v0.0.1-preview 43 | with: 44 | azure_static_web_apps_api_token: ${{ secrets.AZURE_STATIC_WEB_APPS_API_TOKEN_KIND_POND_00161CE0F }} 45 | action: "close" 46 | -------------------------------------------------------------------------------- /src/flat.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /src/hooks/index.ts: -------------------------------------------------------------------------------- 1 | import { useQuery, UseQueryOptions, UseQueryResult } from "react-query"; 2 | import nprogress from "nprogress"; 3 | 4 | import { 5 | fetchCommits, 6 | fetchFlatYaml, 7 | fetchDataFile, 8 | FileParams, 9 | FileParamsWithSHA, 10 | listCommitsResponse, 11 | fetchFilesFromRepo, 12 | fetchOrgRepos, 13 | } from "../api"; 14 | import { Repo, FlatDataTab, Repository } from "../types"; 15 | import React from "react"; 16 | 17 | // Hooks 18 | export function useFlatYaml(repo: Repo) { 19 | return useQuery(["flat-yaml", repo], () => fetchFlatYaml(repo), { 20 | retry: false, 21 | refetchOnWindowFocus: false, 22 | enabled: Boolean(repo.owner) && Boolean(repo.name), 23 | }); 24 | } 25 | 26 | export function useCommits( 27 | params: FileParams, 28 | config: UseQueryOptions 29 | ) { 30 | return useQuery(["commits", params], () => fetchCommits(params), { 31 | retry: false, 32 | refetchOnWindowFocus: false, 33 | ...config, 34 | }); 35 | } 36 | 37 | export function useDataFile( 38 | params: FileParamsWithSHA, 39 | config?: UseQueryOptions 40 | ) { 41 | return useQuery(["data", params], async () => await fetchDataFile(params), { 42 | retry: false, 43 | refetchOnWindowFocus: false, 44 | ...config, 45 | }); 46 | } 47 | 48 | nprogress.configure({ showSpinner: false }); 49 | 50 | export function useProgressBar(numFetching: number) { 51 | React.useEffect(() => { 52 | if (numFetching > 0) { 53 | nprogress.start(); 54 | } else { 55 | nprogress.done(); 56 | } 57 | }, [numFetching]); 58 | } 59 | 60 | export function useGetFiles( 61 | { owner, name }: Repo, 62 | config?: UseQueryOptions 63 | ) { 64 | return useQuery( 65 | ["files", owner, name], 66 | () => fetchFilesFromRepo({ owner, name }), 67 | { 68 | retry: false, 69 | refetchOnWindowFocus: false, 70 | ...config, 71 | } 72 | ); 73 | } 74 | 75 | export function useOrgFlatRepos( 76 | orgName: string, 77 | config?: UseQueryOptions 78 | ) { 79 | return useQuery(["org", orgName], () => fetchOrgRepos(orgName), { 80 | retry: false, 81 | refetchOnWindowFocus: false, 82 | ...config, 83 | }); 84 | } 85 | -------------------------------------------------------------------------------- /src/lib/index.ts: -------------------------------------------------------------------------------- 1 | const COMMIT_META_REGEXP = /\n(.*)/s; 2 | const COMMIT_MESSAGE_REGEXP = /^[^\(]+/; 3 | 4 | interface FlatFileMeta { 5 | name: string; 6 | deltaBytes: number; 7 | date: Date; 8 | source?: string; 9 | } 10 | 11 | interface FlatFileCollection { 12 | files: FlatFileMeta[]; 13 | } 14 | 15 | export function parseFlatCommitMessage(message: string, filename: string) { 16 | if (!message) return; 17 | 18 | const messageMatch = message.match(COMMIT_MESSAGE_REGEXP); 19 | const metaMatch = message.match(COMMIT_META_REGEXP); 20 | 21 | if (!messageMatch) return; 22 | const extractedMessage = messageMatch[0].trim(); 23 | 24 | if (!metaMatch) return; 25 | try { 26 | const parsed = JSON.parse(metaMatch[0]) as FlatFileCollection; 27 | 28 | const fileIndex = parsed.files.findIndex((d) => d.name === filename); 29 | 30 | return { 31 | message: extractedMessage, 32 | file: parsed.files[fileIndex], 33 | }; 34 | } catch (e) { 35 | return {}; 36 | } 37 | } 38 | 39 | export interface GridState { 40 | filters: FilterMap; 41 | sort: string[]; 42 | stickyColumnName?: string; 43 | } 44 | 45 | export type FilterValue = string | number | [number, number]; 46 | export type FilterMap = Record; 47 | 48 | export function encodeFilterString(filters: Record) { 49 | return encodeURI( 50 | Object.keys(filters) 51 | .map((columnName) => { 52 | const value = filters[columnName]; 53 | return [ 54 | columnName, 55 | typeof value === "string" 56 | ? value 57 | : Array.isArray(value) 58 | ? value.join(",") 59 | : "", 60 | ].join("="); 61 | }) 62 | .join("&") 63 | ); 64 | } 65 | 66 | export function decodeFilterString(filterString?: string | null) { 67 | if (!filterString) return undefined; 68 | const splitFilters = decodeURI(filterString).split("&") || []; 69 | let filters = {}; 70 | splitFilters.forEach((filter) => { 71 | const [key, value] = filter.split("="); 72 | if (!key || !value) return; 73 | const isArray = value?.split(",").length === 2; 74 | // @ts-ignore 75 | filters[key] = isArray ? value.split(",").map((d) => +d) : value; 76 | }); 77 | 78 | return filters; 79 | } 80 | -------------------------------------------------------------------------------- /src/components/logo.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | export function Logo() { 4 | return ( 5 | 12 | 16 | 20 | 21 | ); 22 | } 23 | -------------------------------------------------------------------------------- /public/safari-pinned-tab.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 7 | 8 | Created by potrace 1.11, written by Peter Selinger 2001-2013 9 | 10 | 12 | 31 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/components/org-listing.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { RouteComponentProps } from "react-router"; 3 | import { Link } from "react-router-dom"; 4 | import formatDistance from "date-fns/formatDistance"; 5 | import { GoStar } from "react-icons/go"; 6 | 7 | import { useOrgFlatRepos } from "../hooks"; 8 | import { ErrorState } from "./error-state"; 9 | import { Spinner } from "./spinner"; 10 | import Bug from "../bug.svg"; 11 | import { Repository } from "../types"; 12 | 13 | interface OrgListingProps extends RouteComponentProps<{ org: string }> {} 14 | 15 | interface RepoListingProps { 16 | repos: Repository[]; 17 | org: string; 18 | } 19 | 20 | function RepoListing(props: RepoListingProps) { 21 | return ( 22 |
    23 | {props.repos.map((repo) => { 24 | const lastUpdated = formatDistance( 25 | Date.parse(repo.updated_at), 26 | new Date(), 27 | { 28 | addSuffix: true, 29 | } 30 | ); 31 | 32 | return ( 33 |
  • 34 | 35 | 36 | {props.org}/{repo.name} 37 | 38 |
    39 |

    {repo.description}

    40 |
    41 |
      42 |
    • 43 | 44 | {repo.stargazers_count} 45 |
      • 46 |
    • {repo.language}
      • 47 | {Boolean(repo.license) &&
    • {repo.license.name}
      • } 48 |
    • Updated {lastUpdated}
      • 49 |
    50 | 51 |
    • 52 | ); 53 | })} 54 |
55 | ); 56 | } 57 | 58 | export function OrgListing(props: OrgListingProps) { 59 | const { match } = props; 60 | const { org } = match.params; 61 | const { data = [], status } = useOrgFlatRepos(org); 62 | 63 | return ( 64 |
65 |
66 |
67 |
68 | Flat Viewer a simple tool for 69 | exploring flat data files in GitHub repositories. 70 |
71 |
72 |
73 | {status === "loading" && ( 74 |
75 |
76 | 77 |

Loading organization...

78 |
79 |
80 | )} 81 | {status === "success" && 82 | (data.length > 0 ? ( 83 |
84 |
85 | Repositories tagged{" "} 86 | 87 | flat-data 88 | {" "} 89 | in the {org} organization. 90 |
91 | 92 |
93 | ) : ( 94 | 95 |
96 | Hmm, we couldn't find any repos with the topic{" "} 97 | 98 | flat-data 99 | {" "} 100 | in this organization 101 |
102 | 103 | ))} 104 | {status === "error" && ( 105 |
106 | 107 | Hmm, we could not load the organization. 108 | 109 |
110 | )} 111 |
112 | ); 113 | } 114 | -------------------------------------------------------------------------------- /src/components/repo-form.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Formik, FormikProps, Form, Field } from "formik"; 3 | import { object, string } from "yup"; 4 | import { useHistory, Link } from "react-router-dom"; 5 | import cc from "classcat"; 6 | 7 | import { Repo } from "../types"; 8 | 9 | const initialValues: Repo = { 10 | owner: "", 11 | name: "", 12 | }; 13 | 14 | const validationSchema = object().shape({ 15 | owner: string().required("Please enter a repository owner"), 16 | name: string().optional(), 17 | }); 18 | 19 | function RepoFormComponent(props: FormikProps) { 20 | const makeFieldClass = (name: keyof Repo, index: number) => 21 | cc([ 22 | `appearance-none rounded-none relative block w-full px-3 py-2 border border-gray-300 placeholder-gray-500 text-gray-900 focus:outline-none focus:z-10 text-sm`, 23 | { 24 | "border-red-200 bg-red-50 focus:ring-red-500 focus:border-red-500": 25 | Boolean(props.errors[name]), 26 | "focus:ring-gray-500 focus:border-gray-500": !Boolean( 27 | props.errors[name] 28 | ), 29 | "rounded-t-md": index === 0, 30 | "rounded-b-md": index === 1, 31 | }, 32 | ]); 33 | 34 | return ( 35 |
36 |
37 |
38 |
39 | 42 | 49 |
50 |
51 | 54 | 61 |
62 |
63 |
64 | 70 |
71 |
72 |
73 | 76 |
77 | 78 | or, alternatively 79 | 80 |
81 |
82 |
83 |
84 |

Start with an example:

85 |
86 | 90 | githubocto/flat-demo-bitcoin-price 91 | 92 |
93 |
94 |
95 |
96 | or read the writeup 97 |
98 |
99 | ); 100 | } 101 | 102 | export function RepoForm() { 103 | const history = useHistory(); 104 | return ( 105 | { 112 | history.push(`/${values.owner}/` + (values.name || '')) 113 | }} 114 | /> 115 | ); 116 | } 117 | -------------------------------------------------------------------------------- /src/components/picker.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useSelect, UseSelectStateChange } from "downshift"; 3 | import { usePopper } from "react-popper"; 4 | import { ChevronDownIcon } from "@primer/octicons-react"; 5 | import cc from "classcat"; 6 | 7 | interface PickerProps { 8 | label?: string; 9 | placeholder: string; 10 | items: Item[]; 11 | value?: Item; 12 | onChange: (selected: Item) => void; 13 | itemRenderer: (item: Item) => React.ReactNode; 14 | selectedItemRenderer: (item: Item) => React.ReactNode; 15 | disclosureClass?: string; 16 | } 17 | 18 | export function Picker(props: PickerProps) { 19 | const { 20 | items, 21 | value, 22 | onChange, 23 | itemRenderer, 24 | selectedItemRenderer, 25 | placeholder, 26 | label, 27 | disclosureClass, 28 | } = props; 29 | 30 | const handleSelectedItemChange = (changes: UseSelectStateChange) => { 31 | if (changes.selectedItem) { 32 | onChange(changes.selectedItem); 33 | } 34 | }; 35 | 36 | const { 37 | isOpen, 38 | getToggleButtonProps, 39 | getLabelProps, 40 | getMenuProps, 41 | highlightedIndex, 42 | getItemProps, 43 | } = useSelect({ 44 | items, 45 | selectedItem: value, 46 | onSelectedItemChange: handleSelectedItemChange, 47 | }); 48 | 49 | const [ 50 | referenceElement, 51 | setReferenceElement, 52 | ] = React.useState(null); 53 | const [ 54 | popperElement, 55 | setPopperElement, 56 | ] = React.useState(null); 57 | 58 | const { styles, attributes, forceUpdate } = usePopper( 59 | referenceElement, 60 | popperElement, 61 | { 62 | placement: "bottom-start", 63 | modifiers: [ 64 | { 65 | name: "offset", 66 | options: { 67 | offset: [0, 4], 68 | }, 69 | }, 70 | ], 71 | } 72 | ); 73 | 74 | // Popper has the wrong position on mount, this hack seems to fix it... 75 | React.useEffect(() => { 76 | if (isOpen && forceUpdate) { 77 | forceUpdate(); 78 | } 79 | }, [isOpen, forceUpdate]); 80 | 81 | return ( 82 |
83 | {label && ( 84 | 90 | )} 91 |
92 | 103 |
114 |
115 |
    119 | {items.map((item, index) => { 120 | const isHighlighted = highlightedIndex === index; 121 | 122 | return ( 123 |
  • 133 |
    134 | {itemRenderer(item)} 135 |
    136 |
    • 137 | ); 138 | })} 139 |
140 |
141 |
142 |
143 |
144 | ); 145 | } 146 | -------------------------------------------------------------------------------- /src/components/file-picker.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useCombobox } from "downshift"; 3 | import { FileIcon } from "@primer/octicons-react"; 4 | import cc from "classcat"; 5 | 6 | interface FilePickerProps { 7 | label?: string; 8 | placeholder: string; 9 | items: string[]; 10 | value?: string; 11 | onChange: (newValue: string) => void; 12 | itemRenderer: (item: string) => React.ReactNode; 13 | disclosureClass?: string; 14 | isClearable?: boolean; 15 | } 16 | 17 | export function FilePicker(props: FilePickerProps) { 18 | const { 19 | items, 20 | value, 21 | onChange, 22 | itemRenderer, 23 | placeholder, 24 | label, 25 | disclosureClass, 26 | isClearable = false, 27 | } = props; 28 | 29 | const [inputValue, setInputValue] = React.useState(""); 30 | const inputElement = React.useRef(null); 31 | 32 | const filteredItems = (items || []).filter((file: string) => { 33 | const hasFilterString = value === inputValue || file.includes(inputValue); 34 | return hasFilterString; 35 | }); 36 | 37 | const { 38 | isOpen, 39 | getToggleButtonProps, 40 | getLabelProps, 41 | getMenuProps, 42 | getInputProps, 43 | getComboboxProps, 44 | highlightedIndex, 45 | getItemProps, 46 | openMenu, 47 | closeMenu, 48 | } = useCombobox({ 49 | selectedItem: value || "", 50 | items: filteredItems, 51 | onInputValueChange: ({ inputValue }) => { 52 | setInputValue(inputValue || ""); 53 | if (filteredItems.includes(inputValue || "")) onChange(inputValue || ""); 54 | if (!inputValue && isClearable) onChange(""); 55 | }, 56 | onSelectedItemChange: ({ selectedItem }) => { 57 | if (!selectedItem) return; 58 | onChange(selectedItem); 59 | if (inputElement.current) inputElement.current.blur(); 60 | }, 61 | }); 62 | 63 | return ( 64 |
65 | {label && ( 66 | 72 | )} 73 | 74 |
75 | 76 | 77 |
81 | 82 | 83 | { 91 | if (isOpen) return; 92 | openMenu(); 93 | e.target.select(); 94 | }, 95 | ref: inputElement, 96 | })} 97 | /> 98 | 99 | 109 | 110 | {isClearable && !!value && ( 111 | 120 | )} 121 |
122 |
    137 | {isOpen && ( 138 | <> 139 | {filteredItems.map((item, index) => ( 140 |
  • 150 | {itemRenderer(item)} 151 |
    • 152 | ))} 153 | {!items && ( 154 |
    155 | Loading... 156 |
    157 | )} 158 | {!filteredItems.length && ( 159 |
    160 | No files found 161 | {inputValue && ` that include "${inputValue}"`} 162 |
    163 | )} 164 | 165 | )} 166 |
167 |
168 |
169 | ); 170 | } 171 | -------------------------------------------------------------------------------- /src/components/json-detail-container.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Grid } from "@githubocto/flat-ui"; 3 | import { Toaster } from "react-hot-toast"; 4 | import cc from "classcat"; 5 | import truncate from "lodash.truncate"; 6 | 7 | import { decodeFilterString, encodeFilterString, GridState } from "../lib"; 8 | import { useDataFile } from "../hooks"; 9 | import { LoadingState } from "./loading-state"; 10 | import { ErrorState } from "./error-state"; 11 | import { EmptyState } from "./empty-state"; 12 | import Bug from "../bug.svg"; 13 | import { StringParam, useQueryParams } from "use-query-params"; 14 | 15 | interface JSONDetailProps { 16 | sha: string; 17 | previousSha?: string; 18 | filename: string; 19 | owner: string; 20 | name: string; 21 | } 22 | 23 | export function JSONDetail(props: JSONDetailProps) { 24 | const [query, setQuery] = useQueryParams({ 25 | tab: StringParam, 26 | stickyColumnName: StringParam, 27 | sort: StringParam, 28 | filters: StringParam, 29 | }); 30 | 31 | const { sha, previousSha, filename, owner, name } = props; 32 | const queryResult = useDataFile( 33 | { 34 | sha, 35 | filename, 36 | owner, 37 | name, 38 | }, 39 | { 40 | onSuccess: (data) => { 41 | const tab = 42 | query.tab && data.find((d) => d.key === query.tab) 43 | ? query.tab 44 | : (data.find((d) => d.key) || {}).key; 45 | setQuery({ tab }, "replaceIn"); 46 | }, 47 | } 48 | ); 49 | 50 | const pastQueryResult = useDataFile( 51 | { 52 | // @ts-ignore 53 | sha: previousSha, 54 | filename, 55 | owner, 56 | name, 57 | }, 58 | { 59 | enabled: Boolean(previousSha), 60 | } 61 | ); 62 | 63 | const { data = [], isError } = queryResult; 64 | const { data: diffData = [] } = pastQueryResult; 65 | 66 | const showKeyPicker = data.length > 1; 67 | 68 | const tabIndex = data.findIndex((d) => d?.key === query.tab) || 0; 69 | const tabData = data[tabIndex] || {}; 70 | const tabDiffData = diffData[tabIndex] || {}; 71 | 72 | const decodedFilterString = decodeFilterString(query.filters); 73 | const [hasMounted, setHasMounted] = React.useState(false); 74 | 75 | const onTabChange = (tab: string) => 76 | setQuery( 77 | { 78 | tab, 79 | sort: undefined, 80 | stickyColumnName: undefined, 81 | filters: undefined, 82 | }, 83 | "replaceIn" 84 | ); 85 | 86 | const onGridChange = (newState: GridState) => { 87 | if (!hasMounted) { 88 | setHasMounted(true); 89 | return; 90 | } 91 | 92 | setQuery( 93 | { 94 | sort: newState.sort.join(","), 95 | stickyColumnName: newState.stickyColumnName, 96 | filters: encodeFilterString(newState.filters), 97 | }, 98 | "replaceIn" 99 | ); 100 | }; 101 | 102 | React.useEffect(() => { 103 | if (!hasMounted) return; 104 | setHasMounted(false); 105 | 106 | setQuery( 107 | { 108 | sort: undefined, 109 | stickyColumnName: undefined, 110 | filters: undefined, 111 | }, 112 | "replaceIn" 113 | ); 114 | }, [filename]); 115 | 116 | const date = new Date().toLocaleDateString(); 117 | const downloadFilename = `${owner}_${name}__${filename}__${date}`.replace( 118 | /\./g, 119 | "-" 120 | ); 121 | 122 | if (queryResult.status === "loading") { 123 | return ; 124 | } else if (queryResult.status === "error") { 125 | return ( 126 | 127 | Oh no, we couldn't load{" "} 128 | {filename} for some 129 | reason. 130 | 131 | ); 132 | } 133 | 134 | return ( 135 | <> 136 | 137 |
138 | {showKeyPicker && ( 139 |
140 |
141 | {data.map(({ key, value }) => { 142 | const tabClass = cc([ 143 | "h-8 px-3 flex-shrink-0 appearance-none focus:outline-none focus:ring-2 focus:ring-offset-1 focus:ring-indigo-600 border-b relative rounded-tl rounded-tr", 144 | { 145 | "text-indigo-600 font-medium bg-white": key === query.tab, 146 | "bg-transparent border-transparent hover:bg-indigo-700 hover:border-indigo-200 focus:bg-indigo-700 focus:border-indigo-200 text-white": 147 | key !== query.tab, 148 | }, 149 | ]); 150 | return ( 151 | 162 | ); 163 | })} 164 |
165 |
166 | )} 167 | {showKeyPicker && !query.tab && ( 168 | 169 |
170 |
171 | Hmm, it looks like your data file has multiple keys with array 172 | data. 173 |
174 | Select the tab of the key you'd like to visualize. 175 |
176 |
177 | 178 | )} 179 | {!!tabData.value && ( 180 |
181 | 192 |
193 | )} 194 | {isError && ( 195 | 196 | Oh no, we couldn't load{" "} 197 | {filename} for some 198 | reason. 199 | 200 | )} 201 | {!tabData.value && queryResult.status === "success" && ( 202 | 203 | Oh no, we can't load that type of data from{" "} 204 | {filename}. 205 |
206 | 
207 |               {truncate(tabData.invalidValue, { length: 3000 })}
208 |
209 | 210 | )} 211 |
212 | 213 | ); 214 | } 215 | -------------------------------------------------------------------------------- /src/api/index.ts: -------------------------------------------------------------------------------- 1 | import wretch from "wretch"; 2 | import { Endpoints } from "@octokit/types"; 3 | import store from "store2"; 4 | import YAML from "yaml"; 5 | 6 | import { Repo, Repository } from "../types"; 7 | import { csvParse, tsvParse } from "d3-dsv"; 8 | 9 | export type listCommitsResponse = 10 | Endpoints["GET /repos/{owner}/{repo}/commits"]["response"]; 11 | 12 | const githubApiURL = `https://api.github.com`; 13 | const cachedPat = store.get("flat-viewer-pat"); 14 | 15 | let githubWretch = cachedPat 16 | ? wretch(githubApiURL).auth(`token ${cachedPat}`) 17 | : wretch(githubApiURL); 18 | 19 | export async function fetchFlatYaml(repo: Repo) { 20 | let res; 21 | try { 22 | res = await fetchFile( 23 | `https://raw.githubusercontent.com/${repo.owner}/${repo.name}/main/.github/workflows/flat.yaml` 24 | ); 25 | } catch (e) { 26 | try { 27 | res = await fetchFile( 28 | `https://raw.githubusercontent.com/${repo.owner}/${repo.name}/main/.github/workflows/flat.yml` 29 | ); 30 | } catch (e) { 31 | throw new Error("Flat YAML not found"); 32 | } 33 | } 34 | return res && res.length > 0; 35 | } 36 | export function fetchFile(url: string) { 37 | return wretch() 38 | .url(url) 39 | .get() 40 | .notFound(() => { 41 | throw new Error("File not found"); 42 | }) 43 | .text((res) => { 44 | return res; 45 | }); 46 | } 47 | 48 | const ignoredFiles = ["package.json", "tsconfig.json"]; 49 | const ignoredFolders = [".vscode", ".github"]; 50 | const getFilesFromRes = (res: any) => { 51 | return res.tree 52 | .map((file: any) => file.path) 53 | .filter((path: string) => { 54 | const extension = path.split(".").pop() || ""; 55 | const validExtensions = [ 56 | "csv", 57 | "tsv", 58 | "json", 59 | "geojson", 60 | "topojson", 61 | "yml", 62 | "yaml", 63 | ]; 64 | return ( 65 | validExtensions.includes(extension) && 66 | !ignoredFiles.includes(path.split("/").slice(-1)[0]) && 67 | !ignoredFolders.includes(path.split("/")[0]) 68 | ); 69 | }); 70 | }; 71 | 72 | function tryBranch(owner: string, name: string, branch: string) { 73 | return githubWretch 74 | .url(`/repos/${owner}/${name}/git/trees/${branch}?recursive=1`) 75 | .get() 76 | .notFound((e) => { 77 | throw new Error("File not found"); 78 | }) 79 | .error(401, () => { 80 | // clear PAT 81 | store.remove("flat-viewer-pat"); 82 | console.log("PAT expired"); 83 | githubWretch = wretch(githubApiURL); 84 | }) 85 | .error(403, (e: any) => { 86 | const message = JSON.parse(e.message).message; 87 | if (message.includes("API rate limit exceeded")) { 88 | throw new Error("Rate limit exceeded"); 89 | } 90 | throw new Error(e); 91 | }) 92 | .json((res) => { 93 | return getFilesFromRes(res); 94 | }); 95 | } 96 | 97 | export async function fetchFilesFromRepo({ owner, name }: Repo) { 98 | try { 99 | const files = await tryBranch(owner, name, "main"); 100 | if (typeof files !== "string") return files; 101 | } catch (e) { 102 | try { 103 | const files = await tryBranch(owner, name, "master"); 104 | if (typeof files !== "string") return files; 105 | } catch (e) { 106 | if (e.message == "Rate limit exceeded") { 107 | throw new Error("Rate limit exceeded"); 108 | } 109 | throw new Error(e); 110 | } 111 | } 112 | } 113 | 114 | export interface FileParams { 115 | filename?: string | null; 116 | owner: string; 117 | name: string; 118 | } 119 | 120 | export interface FileParamsWithSHA extends FileParams { 121 | sha: string; 122 | } 123 | 124 | export function fetchCommits(params: FileParams) { 125 | const { name, owner, filename } = params; 126 | 127 | return githubWretch 128 | .url(`/repos/${owner}/${name}/commits`) 129 | .query({ 130 | path: filename, 131 | }) 132 | .get() 133 | .json((res: any) => { 134 | if (res.length === 0) { 135 | throw new Error("No commits..."); 136 | } 137 | 138 | return res; 139 | }); 140 | } 141 | 142 | export async function fetchDataFile(params: FileParamsWithSHA) { 143 | const { filename, name, owner, sha } = params; 144 | if (!filename) return []; 145 | const fileType = filename.split(".").pop() || ""; 146 | const validTypes = [ 147 | "csv", 148 | "tsv", 149 | "json", 150 | "geojson", 151 | "topojson", 152 | "yml", 153 | "yaml", 154 | ]; 155 | if (!validTypes.includes(fileType)) return []; 156 | // const githubWretch = cachedPat 157 | // ? wretch( 158 | // `https://raw.githubusercontent.com/${owner}/${name}/${sha}/${filename}` 159 | // ).auth(`token ${cachedPat}`) 160 | // : 161 | 162 | let res; 163 | const text = await wretch( 164 | `https://raw.githubusercontent.com/${owner}/${name}/${sha}/${filename}` 165 | ) 166 | .get() 167 | .notFound(async () => { 168 | if (cachedPat) { 169 | const data = await githubWretch 170 | .url(`/repos/${owner}/${name}/contents/${filename}`) 171 | .get() 172 | .json(); 173 | const content = atob(data.content); 174 | return content; 175 | } else { 176 | throw new Error("Data file not found"); 177 | } 178 | }) 179 | .text(); 180 | 181 | let data: any; 182 | try { 183 | if (fileType === "csv") { 184 | data = csvParse(text); 185 | } else if ( 186 | ["geojson", "topojson"].includes(fileType) || 187 | filename.endsWith(".geo.json") 188 | ) { 189 | data = JSON.parse(text); 190 | if (data.features) { 191 | const features = data.features.map((feature: any) => { 192 | let geometry = {} as Record; 193 | Object.keys(feature?.geometry).forEach((key) => { 194 | geometry[`geometry.${key}`] = feature.geometry[key]; 195 | }); 196 | let properties = {} as Record; 197 | Object.keys(feature?.properties).forEach((key) => { 198 | properties[`properties.${key}`] = feature.properties[key]; 199 | }); 200 | const { geometry: g, properties: p, ...restOfKeys } = feature; 201 | return { ...restOfKeys, ...geometry, ...properties }; 202 | }); 203 | // make features the first key of the object 204 | const { features: f, ...restOfData } = data; 205 | data = { features, ...restOfData }; 206 | } 207 | } else if (fileType === "json") { 208 | data = JSON.parse(text); 209 | } else if (fileType === "tsv") { 210 | data = tsvParse(text); 211 | } else if (fileType === "yml" || fileType === "yaml") { 212 | data = YAML.parse(text); 213 | } else { 214 | return [ 215 | { 216 | invalidValue: stringifyValue(text), 217 | }, 218 | ]; 219 | } 220 | } catch (e) { 221 | console.log(e); 222 | return [ 223 | { 224 | invalidValue: stringifyValue(text), 225 | }, 226 | ]; 227 | } 228 | 229 | if (typeof data !== "object") { 230 | return [ 231 | { 232 | invalidValue: stringifyValue(data), 233 | }, 234 | ]; 235 | } 236 | 237 | const isArray = Array.isArray(data); 238 | if (isArray) { 239 | return [ 240 | { 241 | value: data, 242 | }, 243 | ]; 244 | } 245 | 246 | const keys = Object.keys(data); 247 | 248 | const isObjectOfObjects = 249 | keys.length && 250 | !Object.values(data).find((d) => typeof d !== "object" || Array.isArray(d)); 251 | 252 | if (!isObjectOfObjects) 253 | return keys.map((key) => { 254 | const value = data[key]; 255 | if (!Array.isArray(value)) { 256 | return { 257 | key, 258 | invalidValue: stringifyValue(value), 259 | }; 260 | } 261 | 262 | if (typeof value[0] === "string") { 263 | return { 264 | key, 265 | value: value.map((d) => ({ value: d })), 266 | }; 267 | } 268 | 269 | return { 270 | key, 271 | value, 272 | }; 273 | }); 274 | 275 | let parsedData = []; 276 | keys.forEach((key) => { 277 | parsedData = [...parsedData, { ...data[key], id: key }]; 278 | }); 279 | return [ 280 | { 281 | value: parsedData, 282 | }, 283 | ]; 284 | } 285 | 286 | export async function fetchOrgRepos(orgName: string) { 287 | const res = await githubWretch 288 | .url(`/search/repositories`) 289 | .query({ q: `topic:flat-data org:${orgName}`, per_page: 100 }) 290 | .get() 291 | .json(); 292 | 293 | return res.items; 294 | } 295 | 296 | const stringifyValue = (data: any) => { 297 | if (typeof data === "object") return JSON.stringify(data, undefined, 2); 298 | return data.toString(); 299 | }; 300 | -------------------------------------------------------------------------------- /src/components/repo-detail.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { RouteComponentProps } from "react-router-dom"; 3 | import { GoThreeBars, GoX } from "react-icons/go"; 4 | import { BsArrowRightShort } from "react-icons/bs"; 5 | import { 6 | useDisclosureState, 7 | Disclosure, 8 | DisclosureContent, 9 | } from "reakit/Disclosure"; 10 | import formatDistance from "date-fns/formatDistance"; 11 | import { useQueryParam, StringParam } from "use-query-params"; 12 | import toast, { Toaster } from "react-hot-toast"; 13 | import { ErrorState } from "./error-state"; 14 | import Bug from "../bug.svg"; 15 | 16 | import { 17 | BookmarkIcon, 18 | CommitIcon, 19 | LinkExternalIcon, 20 | RepoIcon, 21 | } from "@primer/octicons-react"; 22 | import { Title } from "react-head"; 23 | 24 | import { useCommits, useGetFiles } from "../hooks"; 25 | import { Repo } from "../types"; 26 | 27 | import { JSONDetail } from "./json-detail-container"; 28 | import { parseFlatCommitMessage } from "../lib"; 29 | import { Picker } from "./picker"; 30 | import { FilePicker } from "./file-picker"; 31 | import { DisplayCommit } from "./display-commit"; 32 | import truncate from "lodash/truncate"; 33 | 34 | interface RepoDetailProps extends RouteComponentProps {} 35 | 36 | export function RepoDetail(props: RepoDetailProps) { 37 | const { match } = props; 38 | const { owner, name } = match.params; 39 | const [filename, setFilename] = useQueryParam("filename", StringParam); 40 | const [selectedSha, setSelectedSha] = useQueryParam("sha", StringParam); 41 | const disclosure = useDisclosureState(); 42 | 43 | const { 44 | data: files, 45 | status: filesStatus, 46 | error: filesError, 47 | } = useGetFiles( 48 | { owner, name }, 49 | { 50 | onSuccess: (data) => { 51 | if (!data.length) return; 52 | setFilename(filename || data[0], "replaceIn"); 53 | }, 54 | } 55 | ); 56 | 57 | // Hook for fetching commits, once we've determined this is a Flat repo. 58 | const { data: commits = [] } = useCommits( 59 | { owner, name, filename }, 60 | { 61 | enabled: Boolean(filename), 62 | onSuccess: (commits) => { 63 | const mostRecentCommitSha = commits[0].sha; 64 | 65 | if (commits.length > 0) { 66 | if (selectedSha) { 67 | if (commits.some((commit) => commit.sha === selectedSha)) { 68 | // noop 69 | } else { 70 | toast.error( 71 | "Hmm, we couldn't find a commit by that SHA. Reverting to the most recent commit.", 72 | { 73 | duration: 4000, 74 | } 75 | ); 76 | setSelectedSha(mostRecentCommitSha, "replaceIn"); 77 | } 78 | } else { 79 | setSelectedSha(mostRecentCommitSha, "replaceIn"); 80 | } 81 | } 82 | }, 83 | } 84 | ); 85 | 86 | const repoUrl = `https://github.com/${owner}/${name}`; 87 | 88 | const parsedCommit = selectedSha 89 | ? parseFlatCommitMessage( 90 | commits?.find((commit) => commit.sha === selectedSha)?.commit.message || 91 | "", 92 | filename || "" 93 | ) 94 | : null; 95 | const dataSource = parsedCommit?.file?.source; 96 | 97 | const selectedShaIndex = commits.findIndex((d) => d.sha === selectedSha); 98 | const selectedShaPrevious = 99 | selectedShaIndex !== -1 100 | ? (commits[selectedShaIndex + 1] || {}).sha 101 | : undefined; 102 | 103 | const controls = ( 104 |
105 | 131 | {!!(files || []).length && ( 132 |
133 |

Data File

134 | { 138 | setFilename(newFilename); 139 | }} 140 | items={files || []} 141 | itemRenderer={(item) => ( 142 | {item} 143 | )} 144 | /> 145 |
146 | )} 147 | 148 | {Boolean(filename) && ( 149 |
150 |

Commit

151 | {commits && ( 152 | 153 | label="Choose a commit" 154 | placeholder="Select a SHA" 155 | onChange={setSelectedSha} 156 | value={selectedSha || ""} 157 | items={commits.map((commit) => commit.sha)} 158 | disclosureClass="appearance-none bg-indigo-700 hover:bg-indigo-800 focus:bg-indigo-800 h-9 px-2 rounded text-white text-xs focus:outline-none focus:ring-2 focus:ring-indigo-400 w-full lg:max-w-md" 159 | itemRenderer={(sha) => { 160 | const commit = commits.find((commit) => commit.sha === sha); 161 | return ( 162 |
163 | 168 |
169 |
170 |

171 | {formatDistance( 172 | new Date(commit?.commit.author?.date || ""), 173 | new Date(), 174 | { addSuffix: true } 175 | )} 176 |

177 |
178 |
179 |
180 | ); 181 | }} 182 | selectedItemRenderer={(sha) => ( 183 |
184 | 185 |
186 | commit.sha === sha)?.commit 189 | .message 190 | } 191 | author={ 192 | commits.find((commit) => commit.sha === sha)?.commit 193 | .author?.email 194 | } 195 | filename={filename} 196 | /> 197 |
198 |
199 | )} 200 | /> 201 | )} 202 |
203 | )} 204 | 205 | {!!dataSource && ( 206 | 227 | )} 228 |
229 | ); 230 | 231 | return ( 232 | 233 | 234 | {owner}/{name} – Flat 235 | 236 | 237 | 238 |
239 |
240 |
241 | Flat Viewer a simple tool for 242 | exploring flat data files in GitHub repositories. 243 |
244 | 248 | {disclosure.visible ? : } 249 | 250 |
251 | 255 | {controls} 256 | 257 | 261 | {!disclosure.visible && ( 262 |
263 | {truncate(`${owner}/${name}`)} 264 | {Boolean(filename) && ( 265 | <> 266 | 267 | 268 | {" "} 269 | {truncate(filename || "")} 270 | 271 | )} 272 |
273 | )} 274 | 275 |
{controls}
276 |
277 | 278 | 279 | {selectedSha && Boolean(filename) && filesStatus !== "error" && ( 280 | 288 | )} 289 | 290 | {match && 291 | !(files || []).length && 292 | filesStatus !== "loading" && 293 | !selectedSha && ( 294 | 295 | {files 296 | ? "Hmm, we couldn't find any files in that repo" 297 | : // @ts-ignore 298 | filesError && filesError?.message === "Error: Rate limit exceeded" 299 | ? // @ts-ignore 300 | filesError?.message 301 | : "Hmm, are you sure that's a public GitHub repo?"} 302 | 303 | )} 304 | 305 | ); 306 | } 307 | --------------------------------------------------------------------------------