├── .github └── workflows │ └── flat.yaml ├── LICENSE ├── README.md ├── btc-price-postprocessed.json ├── btc-price.json └── postprocess.js /.github/workflows/flat.yaml: -------------------------------------------------------------------------------- 1 | name: Flat 2 | 3 | on: 4 | push: 5 | paths: 6 | - .github/workflows/flat.yml # Only run a new workflow every time this file (flat.yaml) file changes 7 | workflow_dispatch: # Required even though this is currently empty 8 | schedule: 9 | - cron: '0 0 1 * *' # Run once a month 10 | 11 | jobs: 12 | scheduled: 13 | runs-on: ubuntu-latest 14 | steps: # This workflow has 3 steps 15 | # The first step is to check out the repository so it can read the files inside of it and do other operations 16 | - name: Check out repo 17 | uses: actions/checkout@v3 18 | # This step installs Deno, which is a new Javascript runtime that improves on Node. We'll use it for postprocessing later 19 | - name: Setup deno 20 | uses: denoland/setup-deno@main 21 | with: 22 | deno-version: v1.x 23 | # The third step is a Flat Action step. We fetch the data in the http_url and save it as downloaded_filename 24 | - name: Fetch data 25 | uses: githubocto/flat@mr/node16 26 | with: 27 | http_url: https://api.coindesk.com/v2/bpi/currentprice.json # The data to fetch every 5 minutes 28 | downloaded_filename: btc-price.json # The http_url gets saved and renamed in our repository as btc-price.json 29 | postprocess: postprocess.js # A postprocessing javascript or typescript file written in Deno 30 | # mask: true # optional param to mask the source http_url from the commit. Look at README for more options 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 GitHub OCTO 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flat Data Demo - Bitcoin Price 2 | 3 | This demo is part of a larger Flat Data project created by [GitHub OCTO](https://octo.github.com/). Read more about the project [here](https://octo.github.com/projects/flat-data). 4 | 5 | ## What this demo does 6 | 7 | This repository uses a [Flat Data Action](https://github.com/githubocto/flat) to fetch the current price of Bitcoin [from this link](https://api.coindesk.com/v2/bpi/currentprice.json) and downloads that data to `btc-price.json` and a filtered version of the data to `btc-price-postprocessed.json`. Both files are updated every 5 minutes if there are changes. 8 | 9 | diagram 10 | 11 | ## Using Python 12 | 13 | While this example uses a javascript Deno file to run the postprocessing tasks, you can also use Python as specified in this example: [https://github.com/pierrotsmnrd/flat_data_py_example](https://github.com/pierrotsmnrd/flat_data_py_example). Thank you [@pierrotsmnrd](https://github.com/pierrotsmnrd)! 14 | 15 | ## Tutorial using the Github GUI 16 | 17 | This is a super simple example of how to use Flat Data using the Github GUI. 18 | 19 | ### Part I: Getting data into our GitHub repository 20 | 21 | #### Setting up a Flat Data project 22 | 23 | 1. **Create a new GitHub repository**: create a new GitHub repository. 24 | 25 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/1.newrepo.png) 26 | 27 | 2. **Create a flat.yaml file inside of a .github/workflows/ folder**: You can use the GitHub GUI to create a new “flat.yaml” file inside of a very specific folder: .github/workflows/. You can create the folders by just typing out the path as you create the flat.yaml file. 28 | 29 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/2.yaml.png) 30 | 31 | 3. **Add the following code to the flat.yaml file:** 32 | 33 | ```yaml 34 | name: Flat 35 | 36 | on: 37 | push: 38 | paths: 39 | - .github/workflows/flat.yml # Only run a new workflow every time this file (flat.yaml) file changes 40 | workflow_dispatch: # Required even though this is currently empty 41 | schedule: 42 | - cron: '*/5 * * * *' # Run this workflow every 5 minutes 43 | 44 | jobs: 45 | scheduled: 46 | runs-on: ubuntu-latest 47 | steps: # This workflow has 3 steps 48 | # The first step is to check out the repository so it can read the files inside of it and do other operations 49 | - name: Check out repo 50 | uses: actions/checkout@v2 51 | # This step installs Deno, which is a new Javascript runtime that improves on Node. We'll use it for postprocessing later 52 | - name: Setup deno 53 | uses: denoland/setup-deno@main 54 | with: 55 | deno-version: v1.x 56 | # The third step is a Flat Action step. We fetch the data in the http_url and save it as downloaded_filename 57 | - name: Fetch data 58 | uses: githubocto/flat@v3 59 | with: 60 | http_url: https://api.coindesk.com/v2/bpi/currentprice.json # The data to fetch every 5 minutes 61 | downloaded_filename: btc-price.json # The http_url gets saved and renamed in our repository as btc-price.json 62 | ``` 63 | 64 | Let’s discuss what’s happening in a few of these lines of code: 65 | 66 | a. This section tells the workflow to manually start any time that there is a change made to the flat.yaml file 67 | 68 | ```yaml 69 | push: 70 | paths: 71 | - .github/workflows/flat.yml 72 | ``` 73 | 74 | b. This section determines how often the Action will run. It uses a utility called [cron](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule) to schedule time-based jobs. Cron jobs have a very particular syntax, and in this example we’re telling the workflow to run every 5 minutes (Actions schedules run at most every 5 minutes). You can use a [handy page like this one](https://crontab.guru/) to figure out any schedule (every hour, once a month, every Saturday, etc). 75 | 76 | ```yaml 77 | schedule: 78 | - cron: '*/5 * * * *' 79 | ``` 80 | 81 | c. Finally, we have a section for the Flat Action itself. The action takes at minimum two parameters. The `http_url` parameter specifies what endpoint or data we want the Action to download. The `downloaded_filename` specifies what to rename the downloaded data once we commit it to our repo. 82 | 83 | ```yaml 84 | with: 85 | http_url: https://api.coindesk.com/v2/bpi/currentprice.json 86 | downloaded_filename: btc-price.json 87 | ``` 88 | 89 | 90 | 4. **Commit the flat.yaml file:** Once you commit the flat.yaml file to your folder, you will have officially completed writing your first Github Action! GitHub Actions is a service that lets us automate all kinds of workflows in a repository by writing instructions and scripts in a YAML file. In this example our action uses another action itself, the Flat Action. 91 | 92 | 93 | #### Running the Flat Action 94 | 95 | 1. **Launch a GitHub Action workflow:** Without doing anything, our Action will run every 5 minutes and fetch any new Bitcoin price data. For the sake of this tutorial, we’ll also trigger it manually so you can see the result instantly. 96 | 97 | Navigate to the “Actions” tab, click on the “Flat” workflow on the left hand column, click on “Run Workflow” dropdown and then button. 98 | 99 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/3.run.png) 100 | 101 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/4.action.png) 102 | 103 | 2. **Check for the btc-price.json file:** After a few seconds you should see the btc-price.json file show up in your repository. It’s the same data as was in the original endpoint! 104 | 105 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/5.jsonfile.png) 106 | 107 | 3. **Check the commit history:** If you check the commit history for the repository, you’ll see that Flat Action logs when a particular file changed and was committed to your repository. 108 | 109 | At this point you’ve used Flat Data to get data on a schedule (in this case every 5 minutes) into your repo. Congrats! But Flat Data can do a bit more. Read on to see how you can add an extra step to process your data in more advanced ways. 110 | 111 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/6.commit.png) 112 | 113 | ### Part II: Additional processing for our downloaded data 114 | 115 | We hope you’re already starting to see how the Flat Action can be useful. It allows you to download any kind of data (JSON, CSV, images, text files, zip folders, etc.) into your repositories at a repeatable schedule. As long as what you’re fetching has a URL, it can be downloaded into a GitHub repo. 116 | 117 | But what if you want to process or change the data in some way before it gets added to your repository? We’ve built in a way for you to run these post-processing tasks by specifying a `postprocess` parameter. Let’s take a look at how to do that. 118 | 119 | 1. **Create a postprocess script:** Create a new file in the repository and call it postprocess.js. 120 | 121 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/7.postprocess.png) 122 | 123 | 2. **Add code to the postprocess script:** The following code written in Javascript and using Deno (more on this in a bit) fetches a particular key inside of the JSON data that represents the Bitcoin price in USD. It saves just this data point into a new JSON file. 124 | 125 | ```javascript 126 | // This can be a typescript file as well 127 | 128 | // Helper library written for useful postprocessing tasks with Flat Data 129 | // Has helper functions for manipulating csv, json, excel, zip, and image files 130 | import { readJSON, writeJSON } from 'https://deno.land/x/flat@0.0.11/mod.ts' 131 | 132 | // Step 1: Read the downloaded_filename JSON 133 | const filename = Deno.args[0] // Same name as downloaded_filename `const filename = 'btc-price.json';` 134 | const json = await readJSON(filename) 135 | console.log(json) 136 | 137 | // Step 2: Filter specific data we want to keep and write to a new JSON file 138 | const currencyRates = Object.values(json.bpi); // convert property values into an array 139 | const filteredCurrencyRates = currencyRates.map(rate => ({ 140 | currency: rate.description, 141 | bitcoinRate: rate.rate 142 | })); 143 | 144 | // Step 3. Write a new JSON file with our filtered data 145 | const newFilename = `btc-price-postprocessed.json` // name of a new file to be saved 146 | await writeJSON(newFilename, filteredCurrencyRates) // create a new JSON file with just the Bitcoin price 147 | console.log("Wrote a post process file") 148 | ``` 149 | 150 | 3. **Add a postprocess parameter to flat.yaml:** Go back to the flat.yaml file and click on the pencil icon to edit it. Add the `postprocess` line below downloaded_filename. This tells the Flat Action that we want to pass the downloaded file through a script that will process it and do additional work. 151 | 152 | ```yaml 153 | - name: Fetch data 154 | uses: githubocto/flat@v3 155 | with: 156 | http_url: https://api.coindesk.com/v2/bpi/currentprice.json # The data to fetch every 5 minutes 157 | downloaded_filename: btc-price.json # The http_url gets saved and renamed in our repository as btc-price.json 158 | postprocess: postprocess.js # A postprocessing javascript or typescript file 159 | ``` 160 | 161 | 162 | 4. **Check the new postprocessed-btc-price.json file:** In the repository you should now see a new file with just the Bitcoin price in USD. 163 | 164 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/8.postprocessfile.png) 165 | 166 | 5. **Remove the original downloaded data (optional):** If you optionally only wanted to keep the postprocessed-btc-price.json file and not the original data, you can add the following lines to the postprocess script to simply delete it before it gets committed to the repository. 167 | 168 | ```javascript 169 | import { readJSON, writeJSON, removeFile } from 'https://deno.land/x/flat@0.0.11/mod.ts' 170 | 171 | const filename = Deno.args[0] 172 | 173 | … 174 | 175 | await removeFile(filename) // or await removeFile(‘btc-price.json’) 176 | ``` 177 | 178 | 179 | 180 | 💡What is Deno?💡 181 | 182 | You might be wondering what Deno is and why the postprocess script was written in this language. Deno was created by the same engineers who wrote Node.js, but it was designed to improve on the capabilities of Node.js. Think of it as the successor to Node.js. Deno is more secure by default and does not need centralized package manager, to name just a few of it’s very handy features. 183 | 184 | If you know how to program with Node, Deno programming will be a breeze and extremely familiar. To get you started quickly, we’ve written a handful of examples and helper functions to get you started. Go to this repo to learn more. 185 | 186 | 187 | ### Part III: Visualizing our data for easy sharing 188 | 189 | For tabular data, we built a Flat Viewer app you can use to visualize your data files with a GUI. 190 | 191 | Check out the Flat Viewer for this repo: [https://flatgithub.com/githubocto/flat-demo-bitcoin-price](https://flatgithub.com/githubocto/flat-demo-bitcoin-price) 192 | 193 | How to use Flat Viewer: 194 | 195 | 1. **Make sure your repository is public:** Although you can use the Flat Action with any repository — whether private or public — the Flat Viewer only works with public repositories for now. 196 | 197 | 2. **Go to the Flat Viewer URL for your repository:** all you need to do to go to the Flat Viewer is change the domain of your repository from “github.com” to “flatgithub.com”. In this example instead of using the URL: [https://github.com/githubocto/flat-demo-bitcoin-price](https://github.com/githubocto/flat-demo-bitcoin-price) we want to navigate to [https://flatgithub.com/githubocto/flat-demo-bitcoin-price](https://flatgithub.com/githubocto/flat-demo-bitcoin-price) 198 | 199 | 3. **Select ‘btc-price-postprocessed.json’ from the dropdown:** you should be able to see the json file data in a neat tabular format. This is a super simple file, but we provide more useful filtering and formatting features for more complex files. 200 | 201 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/9.flatviewer.png) 202 | 203 | 4. **Look at the commit history:** in the commit dropdown you can select previous versions of the file and see how the bitcoin price has changed over time. 204 | 205 | ![](https://raw.githubusercontent.com/githubocto/flat-demo-bitcoin-price/readme-assets/assets/10.commit.png) 206 | 207 | ## Using the Flat Editor VSCode extension 208 | 209 | You can also create Flat Data Action files easily with Flat Editor, a VSCode extension: [https://github.com/githubocto/flat-editor](https://github.com/githubocto/flat-editor). 210 | 211 | 212 | ## License 213 | 214 | [MIT](LICENSE) 215 | -------------------------------------------------------------------------------- /btc-price-postprocessed.json: -------------------------------------------------------------------------------- 1 | [{"currency":"United States Dollar","bitcoinRate":"103,136.514"},{"currency":"British Pound Sterling","bitcoinRate":"82,827.594"},{"currency":"Euro","bitcoinRate":"98,606.449"},{"currency":"Chinese Yuan","bitcoinRate":"749,462.108"}] -------------------------------------------------------------------------------- /btc-price.json: -------------------------------------------------------------------------------- 1 | {"time":{"updated":"Feb 1, 2025 00:21:32 UTC","updatedISO":"2025-02-01T00:21:32+00:00","updateduk":"Feb 1, 2025 at 00:21 GMT"},"disclaimer":"This data was produced from the CoinDesk Bitcoin Price Index (USD). Non-USD currency data converted using hourly conversion rate from openexchangerates.org","chartName":"Bitcoin","bpi":{"USD":{"code":"USD","symbol":"$","rate":"103,136.514","description":"United States Dollar","rate_float":103136.5143},"GBP":{"code":"GBP","symbol":"£","rate":"82,827.594","description":"British Pound Sterling","rate_float":82827.5938},"EUR":{"code":"EUR","symbol":"€","rate":"98,606.449","description":"Euro","rate_float":98606.4491},"CNY":{"code":"CNY","symbol":"¥","rate":"749,462.108","description":"Chinese Yuan","rate_float":749462.1081}}} -------------------------------------------------------------------------------- /postprocess.js: -------------------------------------------------------------------------------- 1 | // This can be a typescript file as well 2 | 3 | // Helper library written for useful postprocessing tasks with Flat Data 4 | // Has helper functions for manipulating csv, txt, json, excel, zip, and image files 5 | import { readJSON, writeJSON, removeFile } from 'https://deno.land/x/flat@0.0.14/mod.ts' 6 | 7 | // Step 1: Read the downloaded_filename JSON 8 | const filename = Deno.args[0] // Same name as downloaded_filename `const filename = 'btc-price.json';` 9 | const json = await readJSON(filename) 10 | console.log(json) 11 | 12 | // Step 2: Filter specific data we want to keep and write to a new JSON file 13 | const currencyRates = Object.values(json.bpi); // convert property values into an array 14 | const filteredCurrencyRates = currencyRates.map(rate => ({ 15 | currency: rate.description, 16 | bitcoinRate: rate.rate 17 | })); 18 | 19 | // Step 3. Write a new JSON file with our filtered data 20 | const newFilename = `btc-price-postprocessed.json` // name of a new file to be saved 21 | await writeJSON(newFilename, filteredCurrencyRates) // create a new JSON file with just the Bitcoin price 22 | console.log("Wrote a post process file") 23 | 24 | // Optionally delete the original file 25 | // await removeFile('./btc-price.json') // equivalent to removeFile('btc-price.json') 26 | --------------------------------------------------------------------------------