├── .env.example ├── .gitignore ├── .rspec ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin ├── console ├── release └── setup ├── jekyll-airtable.gemspec ├── lib ├── jekyll-airtable.rb ├── jekyll-airtable │ ├── api.rb │ ├── client.rb │ ├── configuration.rb │ └── version.rb └── jekyll │ └── airtable_fetcher.rb └── spec ├── fixtures ├── 404.md ├── _config.yml ├── _layouts │ └── some_default.html ├── _my_collection │ ├── custom_permalink.md │ ├── custom_permalink_2.md │ ├── test.html │ └── this-has-non-standard-chars.md ├── _other_things │ └── test2.html ├── _posts │ ├── 2013-12-12-dec-the-second.md │ ├── 2014-03-02-march-the-second.md │ ├── 2014-03-04-march-the-fourth.md │ ├── 2014-05-11-exclude-this-post.md │ ├── 2015-01-18-jekyll-last-modified-at.md │ ├── 2016-04-01-错误.html │ ├── 2016-04-02-错误.html │ └── 2016-04-03-错误.html ├── feeds │ └── atom.xml ├── images │ └── hubot.png ├── index.html ├── jekyll-last-modified-at │ └── page.html ├── some-subfolder │ ├── exclude-this-page.html │ ├── htm.htm │ ├── index.html │ ├── test_index.html │ ├── this-is-a-subfile.html │ ├── this-is-a-subpage.html │ └── xhtml.xhtml └── static_files │ ├── 404.html │ ├── excluded.pdf │ ├── html_file.html │ └── test.pdf ├── jekyll-airtable_spec.rb ├── jekyll └── airtable_spec.rb └── spec_helper.rb /.env.example: -------------------------------------------------------------------------------- 1 | SYNC_WITH_AIRTABLE='false' 2 | AIRTABLE_API_KEY='key' 3 | AIRTABLE_BASE_UID='appb' 4 | AIRTABLE_TABLE_NAMES='Table 1, Table 2, Table 3' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | # rspec failure tracking 11 | .rspec_status 12 | .env 13 | .byebug_history -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: ruby 3 | rvm: 4 | - 2.4.2 5 | before_install: gem install bundler -v 1.16.1 6 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at galih0muhammad@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | git_source(:github) do |repo_name| 4 | repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/") 5 | "https://github.com/#{repo_name}.git" 6 | end 7 | 8 | # Specify your gem's dependencies in jekyll-airtable.gemspec 9 | gemspec -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | jekyll-airtable (0.4.1) 5 | faraday (~> 0.11.0) 6 | faraday_middleware (~> 0.10.1) 7 | hashie 8 | jekyll (~> 3.3) 9 | 10 | GEM 11 | remote: https://rubygems.org/ 12 | specs: 13 | addressable (2.5.2) 14 | public_suffix (>= 2.0.2, < 4.0) 15 | byebug (10.0.2) 16 | colorator (1.1.0) 17 | concurrent-ruby (1.0.5) 18 | diff-lcs (1.3) 19 | dotenv (2.5.0) 20 | em-websocket (0.5.1) 21 | eventmachine (>= 0.12.9) 22 | http_parser.rb (~> 0.6.0) 23 | eventmachine (1.2.7) 24 | faraday (0.11.0) 25 | multipart-post (>= 1.2, < 3) 26 | faraday_middleware (0.10.1) 27 | faraday (>= 0.7.4, < 1.0) 28 | ffi (1.9.25) 29 | forwardable-extended (2.6.0) 30 | hashie (3.6.0) 31 | http_parser.rb (0.6.0) 32 | i18n (0.9.5) 33 | concurrent-ruby (~> 1.0) 34 | jekyll (3.8.4) 35 | addressable (~> 2.4) 36 | colorator (~> 1.0) 37 | em-websocket (~> 0.5) 38 | i18n (~> 0.7) 39 | jekyll-sass-converter (~> 1.0) 40 | jekyll-watch (~> 2.0) 41 | kramdown (~> 1.14) 42 | liquid (~> 4.0) 43 | mercenary (~> 0.3.3) 44 | pathutil (~> 0.9) 45 | rouge (>= 1.7, < 4) 46 | safe_yaml (~> 1.0) 47 | jekyll-sass-converter (1.5.2) 48 | sass (~> 3.4) 49 | jekyll-watch (2.0.0) 50 | listen (~> 3.0) 51 | kramdown (1.17.0) 52 | liquid (4.0.0) 53 | listen (3.1.5) 54 | rb-fsevent (~> 0.9, >= 0.9.4) 55 | rb-inotify (~> 0.9, >= 0.9.7) 56 | ruby_dep (~> 1.2) 57 | mercenary (0.3.6) 58 | multipart-post (2.0.0) 59 | pathutil (0.16.1) 60 | forwardable-extended (~> 2.6) 61 | public_suffix (3.0.3) 62 | rake (10.5.0) 63 | rb-fsevent (0.10.3) 64 | rb-inotify (0.9.10) 65 | ffi (>= 0.5.0, < 2) 66 | rouge (3.2.1) 67 | rspec (3.8.0) 68 | rspec-core (~> 3.8.0) 69 | rspec-expectations (~> 3.8.0) 70 | rspec-mocks (~> 3.8.0) 71 | rspec-core (3.8.0) 72 | rspec-support (~> 3.8.0) 73 | rspec-expectations (3.8.1) 74 | diff-lcs (>= 1.2.0, < 2.0) 75 | rspec-support (~> 3.8.0) 76 | rspec-mocks (3.8.0) 77 | diff-lcs (>= 1.2.0, < 2.0) 78 | rspec-support (~> 3.8.0) 79 | rspec-support (3.8.0) 80 | ruby_dep (1.5.0) 81 | safe_yaml (1.0.4) 82 | sass (3.6.0) 83 | sass-listen (~> 4.0.0) 84 | sass-listen (4.0.0) 85 | rb-fsevent (~> 0.9, >= 0.9.4) 86 | rb-inotify (~> 0.9, >= 0.9.7) 87 | 88 | PLATFORMS 89 | ruby 90 | 91 | DEPENDENCIES 92 | bundler (~> 1.16) 93 | byebug 94 | dotenv 95 | jekyll-airtable! 96 | rake (~> 10.0) 97 | rspec (~> 3.0) 98 | 99 | BUNDLED WITH 100 | 1.16.1 101 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Galih Muhammad 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Jekyll::Airtable 2 | 3 | This gem enables you to easily integrate Airtable with Jekyll site and use it as a database. Everytime the Jekyll build is triggered, the gem would automatically send API request to the Airtable base and tables you specify from the environment variable and then store the records as collections, grouped according to the table names. 4 | 5 | ## Installation 6 | 7 | Add this line to your Jekyll site Gemfile: 8 | 9 | ```ruby 10 | gem 'jekyll-airtable' 11 | ``` 12 | 13 | And then execute: 14 | 15 | $ bundle install 16 | 17 | ## Usage 18 | 19 | 1. Because you will have to mention/use your API key, it is VERY RECOMMENDED to use dotenv so you do not have your API key lying around in plain sight. 20 | ```ruby 21 | gem 'dotenv' 22 | ``` 23 | run bundle install again 24 | 25 | 2. Add this to your repo .gitignore (create one if does not exist): 26 | ``` 27 | .env 28 | ``` 29 | 30 | 3. Copy the .env.example in this repo to the root of your project, rename it to .env. 31 | 4. Set your Airtable API Key in the .env 32 | 33 | ``` 34 | AIRTABLE_API_KEY='your_airtable_api_key' 35 | ``` 36 | 37 | 5. You need to add a custom plugin to get the dotenv to work, you do this by creating a folder ```_plugins``` (if does not exist already) inside your Jekyll repo 38 | 6. Inside the ```/_plugins ```, create a file called "environment_variables_generator.rb", with this as the content: 39 | 40 | ```ruby 41 | # Plugin to add environment variables to the `site` object in Liquid templates 42 | require 'dotenv' 43 | 44 | module Jekyll 45 | class EnvironmentVariablesGenerator < Generator 46 | priority :highest 47 | 48 | def generate(site) 49 | Dotenv.overload 50 | site.config['env'] = Dotenv.overload 51 | 52 | site.config['AIRTABLE_API_KEY'] = ENV['AIRTABLE_API_KEY'] 53 | end 54 | end 55 | end 56 | ``` 57 | 58 | Now the secret keys can be accessed by Jekyll without being visible to the public. 59 | 60 | 7. Now, you need to declare and hook the plugins in the config.yml 61 | ```yml 62 | plugins: 63 | - jekyll-airtable 64 | - environment_variables_generator 65 | 66 | airtable: 67 | enable_sync: true 68 | base_uid: 'appp7C1MblPCnMePn' 69 | tables: 70 | - name: 'Use Cases' 71 | - name: 'Whitepapers' 72 | - name: 'Quotes' 73 | long_text_columns: 74 | - 'column name' 75 | ``` 76 | 77 | 8. Finally, you can execute the plugin using ```sh bundle exec jekyll serve ``` or ```sh bundle exec jekyll build ``` 78 | 79 | For production, you also have to set those keys and values. 80 | 81 | 9. You then need to set the collections on the ```_config.yml ``` so that Jekyll recognizes them. The snippets below here are taken from https://github.com/galliani/airbase/blob/master/_config.yml. The collections on that repo are "use_cases", "whitepapers", and "tutorials", taken from the Airtable of "Use Cases", "Whitepapers", and "Tutorials" respectively. 82 | 83 | ```yml 84 | collections_dir: collections 85 | collections: 86 | use_cases: 87 | output: true 88 | whitepapers: 89 | output: true 90 | 91 | defaults: 92 | - scope: 93 | type: "whitepapers" 94 | values: 95 | layout: "page" # any jekyll layout file you already have in the _layouts that you want to use for this collection type. 96 | ``` 97 | 98 | 99 | ## Conventions 100 | 101 | There are a number of "enforced" conventions you have to know and can take advantage of. 102 | 103 | 1. By default, the slug of the record (which is the file name) come from this sources, in order of priority: 1) The column labelled 'Slug' / 'slug' in your Airtable table, 2) The primary key of each row (the first column on each table), 3) or the Airtable UID of the record. So if you want to set each record's slug manually, you can do so by creating slug column in your Airtable table.**GOTCHA:** Should slugs that are derived from priority 1 and 2 are both longer than 100 char, then it will automatically use the Airtable UID. 104 | 105 | 2. Attachments are not copied into your Jekyll repo but their informations, such as url and so on, will be stored in "_data/airtable/attachments.yml_" folder, if none exist before, this plugin will automatically create it for you. 106 | 107 | 108 | 3. Long text with paragraphs are now supported. 109 | 110 | ## Development 111 | 112 | After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 113 | 114 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 115 | 116 | ## Contributing 117 | 118 | Bug reports and pull requests are welcome on GitHub at https://github.com/galliani/jekyll-airtable. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. 119 | 120 | ## License 121 | 122 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 123 | 124 | ## Code of Conduct 125 | 126 | Everyone interacting in the Jekyll::Airtable project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/galliani/jekyll-airtable/blob/master/CODE_OF_CONDUCT.md). 127 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | def relative_to_root(path) 3 | File.expand_path(path, File.dirname(__dir__)) 4 | end 5 | 6 | require "bundler/setup" 7 | require "jekyll" 8 | require_relative("../lib/jekyll-airtable.rb") 9 | require 'byebug' 10 | 11 | # You can add fixtures and/or initialization code here to make experimenting 12 | # with your gem easier. You can also use a different console, if you like. 13 | 14 | # (If you use this, don't forget to add pry to your Gemfile!) 15 | # require "pry" 16 | # Pry.start 17 | 18 | SOURCE_DIR = relative_to_root("spec/fixtures") 19 | DEST_DIR = relative_to_root("spec/dest") 20 | 21 | def source_dir(*files) 22 | File.join(SOURCE_DIR, *files) 23 | end 24 | 25 | def dest_dir(*files) 26 | File.join(DEST_DIR, *files) 27 | end 28 | 29 | def config(overrides = {}) 30 | Jekyll.configuration( 31 | "source" => source_dir, 32 | "destination" => dest_dir, 33 | "url" => "http://example.org" 34 | ).merge(overrides) 35 | end 36 | 37 | def site(configuration = config) 38 | Jekyll::Site.new(configuration) 39 | end 40 | 41 | require "irb" 42 | IRB.start(__FILE__) 43 | -------------------------------------------------------------------------------- /bin/release: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Tag and push a release. 3 | 4 | set -e 5 | 6 | bundle exec rake release -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /jekyll-airtable.gemspec: -------------------------------------------------------------------------------- 1 | 2 | lib = File.expand_path("../lib", __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require "jekyll-airtable/version" 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "jekyll-airtable" 8 | spec.version = Jekyll::Airtable::VERSION 9 | spec.authors = ["Galih Muhammad"] 10 | spec.email = ["galih0muhammad@gmail.com"] 11 | 12 | spec.summary = "Jekyll plugin to integrate Airtable" 13 | spec.description = "Airtable support in Jekyll site to easily fetch and store data" 14 | spec.homepage = "https://github.com/galliani/jekyll-airtable" 15 | spec.license = "MIT" 16 | 17 | spec.files = `git ls-files -z`.split("\x0").reject do |f| 18 | f.match(%r{^(test|spec|features)/}) 19 | end 20 | spec.bindir = "exe" 21 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 22 | spec.require_paths = ["lib"] 23 | 24 | spec.required_ruby_version = ">= 2.3.0" 25 | 26 | spec.add_development_dependency "bundler", "~> 1.16" 27 | spec.add_development_dependency "rake", "~> 10.0" 28 | spec.add_development_dependency "rspec", "~> 3.0" 29 | spec.add_development_dependency "byebug" 30 | spec.add_development_dependency "dotenv" 31 | spec.add_dependency 'faraday', '~> 0.11.0' 32 | spec.add_dependency 'faraday_middleware', '~> 0.10.1' 33 | spec.add_dependency "jekyll", "~> 3.3" 34 | spec.add_dependency "hashie" 35 | end 36 | -------------------------------------------------------------------------------- /lib/jekyll-airtable.rb: -------------------------------------------------------------------------------- 1 | require "jekyll-airtable/version" 2 | require "jekyll-airtable/configuration" 3 | require "jekyll-airtable/api" 4 | require "jekyll-airtable/client" 5 | 6 | module Jekyll 7 | module Airtable 8 | extend Configuration 9 | 10 | def self.client(options = {}) 11 | Jekyll::Airtable::Client.new(options) 12 | end 13 | 14 | # Delegate to Airtable::Client 15 | def self.method_missing(method, *args, &block) 16 | return super unless client.respond_to?(method) 17 | client.send(method, *args, &block) 18 | end 19 | 20 | # Delegate to Airtable::Client 21 | def self.respond_to?(method, include_all=false) 22 | return client.respond_to?(method, include_all) || super 23 | end 24 | end 25 | end 26 | 27 | require "jekyll/airtable_fetcher" -------------------------------------------------------------------------------- /lib/jekyll-airtable/api.rb: -------------------------------------------------------------------------------- 1 | require_relative "./configuration" 2 | 3 | module Jekyll 4 | module Airtable 5 | class API 6 | attr_accessor *Configuration::VALID_OPTIONS_KEYS 7 | 8 | # Creates a new API 9 | def initialize(options={}) 10 | options = Airtable.options.merge(options) 11 | 12 | Configuration::VALID_OPTIONS_KEYS.each do |key| 13 | send("#{key}=", options[key]) 14 | end 15 | end 16 | 17 | def config 18 | conf = {} 19 | 20 | Configuration::VALID_OPTIONS_KEYS.each do |key| 21 | conf[key] = send key 22 | end 23 | 24 | conf 25 | end 26 | 27 | def versioned_base_endpoint_url 28 | config[:endpoint] + config[:api_version] + '/' + config[:base_uid] + '/' 29 | end 30 | 31 | def processing_query_params(request_obj, params, params_key) 32 | keys = [ 33 | :fields, 34 | :filter_by_formula, 35 | :max_records, 36 | :page_size, 37 | :sort, 38 | :view, 39 | :cell_format, 40 | :time_zone, 41 | :user_locale, 42 | :offset 43 | ] 44 | 45 | return nil unless keys.include?(params_key) || 46 | params[params_key].nil? || 47 | params[params_key] == '' 48 | 49 | key_string = params_key.to_s.split('_').each_with_index.map do |a, i| 50 | if i == 0 then a else a.capitalize end 51 | end.join 52 | 53 | request_obj.params[key_string] = params[params_key] 54 | end 55 | end 56 | end 57 | end -------------------------------------------------------------------------------- /lib/jekyll-airtable/client.rb: -------------------------------------------------------------------------------- 1 | require 'faraday' 2 | require 'uri' 3 | require 'json' 4 | 5 | module Jekyll 6 | module Airtable 7 | class Client < API 8 | 9 | def list_records(table_name:, params: {}) 10 | @connection = Faraday.new(url: versioned_base_endpoint_url) do |faraday| 11 | faraday.response :logger # log requests to STDOUT 12 | faraday.request :url_encoded 13 | faraday.adapter Faraday.default_adapter # make requests with Net::HTTP 14 | end 15 | 16 | @connection.authorization(:Bearer, Airtable.api_key) 17 | 18 | records = [] 19 | offset = nil 20 | counter = 1 21 | 22 | begin 23 | looper = "request no. #{counter}" 24 | puts 'Sending ' + looper 25 | 26 | params[:offset] = offset if !offset.nil? 27 | 28 | data = send_get_request(table_name, params) 29 | puts "Response received for the " + looper 30 | 31 | records << data['records'] 32 | offset = data['offset'] 33 | 34 | # Pause for 1 second, just to be safe 35 | sleep 1 36 | counter += 1 37 | end while !offset.nil? 38 | 39 | records.flatten 40 | end 41 | 42 | private 43 | 44 | def send_get_request(table_name, params) 45 | response = @connection.get do |req| 46 | req.url URI.escape(table_name) 47 | 48 | default_params = params 49 | default_params.keys.each do |key| 50 | processing_query_params(req, default_params, key) 51 | end 52 | end 53 | 54 | JSON.parse(response.body) 55 | end 56 | 57 | end 58 | end 59 | end -------------------------------------------------------------------------------- /lib/jekyll-airtable/configuration.rb: -------------------------------------------------------------------------------- 1 | module Jekyll 2 | module Airtable 3 | module Configuration 4 | API_URL = 'https://api.airtable.com/' 5 | API_VERSION = 'v0' 6 | 7 | # An array of valid keys in the options hash 8 | VALID_OPTIONS_KEYS = [ 9 | :endpoint, 10 | :api_version, 11 | :api_key, 12 | :base_uid 13 | ].freeze 14 | 15 | attr_accessor *VALID_OPTIONS_KEYS 16 | 17 | # When this module is extended, set all configuration options to their default values 18 | def self.extended(base) 19 | base.reset 20 | end 21 | 22 | # Create a hash of options and their values 23 | def options 24 | VALID_OPTIONS_KEYS.inject({}) do |option, key| 25 | option.merge!(key => send(key)) 26 | end 27 | end 28 | 29 | # Convenience method to allow configuration options to be set in a block 30 | # To be called from the base class 31 | def configure 32 | yield self 33 | end 34 | 35 | # Reset all configuration options to defaults 36 | def reset 37 | self.endpoint = API_URL 38 | self.api_version = API_VERSION 39 | self.api_key = nil 40 | self.base_uid = '' 41 | end 42 | end 43 | end 44 | end -------------------------------------------------------------------------------- /lib/jekyll-airtable/version.rb: -------------------------------------------------------------------------------- 1 | module Jekyll 2 | module Airtable 3 | VERSION = "0.4.1" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /lib/jekyll/airtable_fetcher.rb: -------------------------------------------------------------------------------- 1 | require "json" 2 | require 'fileutils' 3 | require 'yaml' 4 | require 'hashie' 5 | require 'uri' 6 | 7 | module Jekyll 8 | class AirtableFetcher < Jekyll::Generator 9 | safe true 10 | priority :lowest 11 | 12 | def generate(site) 13 | jekyll_config = Hashie::Mash.new(site.config) 14 | return false if should_generate_be_prevented?(jekyll_config) 15 | # For storing hashes of attachments that will be saved to the data file 16 | @attachments_hash = {} 17 | setup_directories 18 | 19 | client = setting_up_airtable_client 20 | 21 | @table_names.each do |table_name| 22 | records = client.list_records(table_name: table_name) 23 | next if records.size == 0 24 | 25 | if records.class.name == 'Hash' 26 | error = records["error"] 27 | if error.present? 28 | puts error['message'] 29 | next 30 | end 31 | end 32 | 33 | directory_name = "collections/_" + to_snake(table_name) 34 | Dir.mkdir(directory_name) unless File.exists?(directory_name) 35 | 36 | records.each do |record| 37 | fields = record['fields'] 38 | uid = record['id'] 39 | out_file = create_page_for_the_record(directory_name, uid, fields) 40 | 41 | fields.each do |key, value| 42 | snake_key = to_snake(key) 43 | 44 | if is_a_long_text?(jekyll_config, table_name, key) 45 | write_long_text_to_file(snake_key, value, out_file) 46 | next 47 | end 48 | 49 | if value.class.name == 'Array' 50 | out_file.puts("#{snake_key}:") 51 | write_array_values(out_file, value) 52 | else 53 | value = stringify_value_if_necessary(value) 54 | 55 | out_file.puts("#{snake_key}: #{value}") 56 | end 57 | end 58 | 59 | out_file.puts(front_matter_mark) 60 | 61 | out_file.close 62 | end 63 | end 64 | 65 | write_attachments_data_file if @attachments_hash.keys.size > 0 66 | end 67 | 68 | private 69 | 70 | def should_generate_be_prevented?(config) 71 | is_enabled = config.airtable.enable_sync 72 | return true if !is_enabled 73 | 74 | @api_key = config.to_hash['AIRTABLE_API_KEY'] 75 | @base_uid = config.airtable.base_uid 76 | @table_names = config.airtable.tables.map{ |t| t.name } 77 | 78 | return true if @api_key.nil? || @api_key == '' || @base_uid.nil? || @base_uid == '' 79 | false 80 | end 81 | 82 | def setting_up_airtable_client 83 | Airtable.configure { |config| config.api_key = @api_key } 84 | 85 | Airtable.client(base_uid: @base_uid) 86 | end 87 | 88 | def front_matter_mark 89 | '---' 90 | end 91 | 92 | def to_snake(string) 93 | string.split(' ').map(&:downcase).join('_') 94 | end 95 | 96 | def to_dash(string) 97 | string.split(' ').map(&:downcase).join('-') 98 | end 99 | 100 | def write_array_values(file, array) 101 | array.each do |element| 102 | if is_this_is_an_attachment?(element) 103 | # Store only the uid of the attachment in the front matter 104 | value = element['id'] 105 | 106 | # Store the hash into the hash of hashes 107 | @attachments_hash[value] = element 108 | else 109 | # - { name: 'Low', color: '#306d8c' } 110 | value = stringify_value_if_necessary(element) 111 | end 112 | 113 | file.puts(" - #{value}") 114 | end 115 | end 116 | 117 | def stringify_value_if_necessary(value) 118 | begin 119 | return "'#{value}'" if value.include?(':') 120 | 121 | value 122 | rescue NoMethodError 123 | value 124 | end 125 | end 126 | 127 | def is_this_is_an_attachment?(value) 128 | is_hash = value.class.name == 'Hash' 129 | return false unless is_hash 130 | 131 | value.keys.map(&:to_s).include?('filename') 132 | end 133 | 134 | def setup_directories 135 | Dir.mkdir('_data') unless File.exists?('_data') 136 | Dir.mkdir('_data/airtable') unless File.exists?('_data/airtable') 137 | 138 | Dir.mkdir('collections') unless File.exists?('collections') 139 | end 140 | 141 | def write_attachments_data_file 142 | File.open("_data/airtable/attachments.yml", "w") do |f| 143 | f.write(@attachments_hash.to_yaml) 144 | end 145 | end 146 | 147 | def create_page_for_the_record(directory_name, uid, fields) 148 | # We use the first field as the primary key 149 | # Then find the value of the primary key to be stored as the slug, which 150 | # will be used as file name and the path to the record in the url. 151 | # However, if the record has field called 'slug', it will be used instead 152 | pkey = fields.keys.first 153 | slug = fields['slug'].nil? ? fields[pkey] : fields['slug'] 154 | slug = slug.length > 100 ? uid : slug 155 | 156 | if is_an_uri?(slug) 157 | filename = to_dash(slug) + '.md' 158 | else 159 | filename = to_dash(uid) + '.md' 160 | end 161 | 162 | out_file = File.new("#{directory_name}/#{filename}", "w") 163 | out_file.puts(front_matter_mark) 164 | out_file.puts("uid: #{uid}") 165 | 166 | out_file 167 | end 168 | 169 | def is_a_long_text?(config, table_name, key) 170 | table = config.airtable.tables.select{ |a| a.name == table_name }.first 171 | return false if table.nil? 172 | 173 | list = table.long_text_columns || [] 174 | list.include?(key) 175 | end 176 | 177 | def write_long_text_to_file(snake_key, text, out_file) 178 | out_file.puts("#{snake_key}: |") 179 | 180 | lines = text.split("\n") 181 | 182 | lines.each do |line| 183 | out_file.puts(" " + line) 184 | end 185 | end 186 | 187 | def is_an_uri?(string) 188 | return true if string.include?('http') || 189 | string.include?('://') || 190 | string.include?('www.') 191 | 192 | uri = URI.parse(string) 193 | %w( http https ).include?(uri.scheme) 194 | rescue URI::BadURIError 195 | false 196 | rescue URI::InvalidURIError 197 | false 198 | end 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /spec/fixtures/404.md: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | 404. That's an error. 5 | -------------------------------------------------------------------------------- /spec/fixtures/_config.yml: -------------------------------------------------------------------------------- 1 | timezone: UTC 2 | 3 | defaults: 4 | - 5 | scope: 6 | path: "" 7 | type: page 8 | values: 9 | layout: some_default 10 | - 11 | scope: 12 | path: "static_files/excluded.pdf" 13 | values: 14 | sitemap: false 15 | - 16 | scope: 17 | path: "static_files/html_file.html" 18 | values: 19 | sitemap: false 20 | -------------------------------------------------------------------------------- /spec/fixtures/_layouts/some_default.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | THIS IS MY LAYOUT 4 | {{ content }} 5 | -------------------------------------------------------------------------------- /spec/fixtures/_my_collection/custom_permalink.md: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: /permalink/ 3 | --- 4 | 5 | # Custom permalink 6 | -------------------------------------------------------------------------------- /spec/fixtures/_my_collection/custom_permalink_2.md: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: /permalink/unique_name.html 3 | --- 4 | 5 | # Unique html name 6 | -------------------------------------------------------------------------------- /spec/fixtures/_my_collection/test.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | This is just a test. 5 | -------------------------------------------------------------------------------- /spec/fixtures/_my_collection/this-has-non-standard-chars.md: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: this url has an ümlaut 3 | --- 4 | 5 | # URL contains characters that need to be URI encoded 6 | -------------------------------------------------------------------------------- /spec/fixtures/_other_things/test2.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | This file shouldn't show up in the sitemap. 5 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2013-12-12-dec-the-second.md: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | December the twelfth, actually. 5 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2014-03-02-march-the-second.md: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | March the second! 5 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2014-03-04-march-the-fourth.md: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | March the fourth! 5 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2014-05-11-exclude-this-post.md: -------------------------------------------------------------------------------- 1 | --- 2 | sitemap: false 3 | --- 4 | 5 | This post should not appear in the sitemap. 6 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2015-01-18-jekyll-last-modified-at.md: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | Please don't modify this file. It's modified time is important. 5 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2016-04-01-错误.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2016-04-02-错误.html: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: "/2016/04/02/错误.html" 3 | --- 4 | -------------------------------------------------------------------------------- /spec/fixtures/_posts/2016-04-03-错误.html: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: "/2016/04/03/%E9%94%99%E8%AF%AF.html" 3 | --- 4 | -------------------------------------------------------------------------------- /spec/fixtures/feeds/atom.xml: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /spec/fixtures/images/hubot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galliani/jekyll-airtable/8bcea310f10fa8365f8f9b7d0d15ad1f7e4c43f6/spec/fixtures/images/hubot.png -------------------------------------------------------------------------------- /spec/fixtures/index.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | HERE IS MY SITE I AM SO EXCITED TO BE USING GITHUB PAGES 5 | -------------------------------------------------------------------------------- /spec/fixtures/jekyll-last-modified-at/page.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | This is a page with a modified time. 5 | -------------------------------------------------------------------------------- /spec/fixtures/some-subfolder/exclude-this-page.html: -------------------------------------------------------------------------------- 1 | --- 2 | sitemap: false 3 | --- 4 | 5 | Exclude this page 6 | -------------------------------------------------------------------------------- /spec/fixtures/some-subfolder/htm.htm: -------------------------------------------------------------------------------- 1 | This file has an .htm extension, and should be included in the sitemap 2 | -------------------------------------------------------------------------------- /spec/fixtures/some-subfolder/index.html: -------------------------------------------------------------------------------- 1 | static subfolder index.html file that should be indexed as permalink 2 | -------------------------------------------------------------------------------- /spec/fixtures/some-subfolder/test_index.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | The permalink of this page does not end with a '/', but with a filename 5 | -------------------------------------------------------------------------------- /spec/fixtures/some-subfolder/this-is-a-subfile.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galliani/jekyll-airtable/8bcea310f10fa8365f8f9b7d0d15ad1f7e4c43f6/spec/fixtures/some-subfolder/this-is-a-subfile.html -------------------------------------------------------------------------------- /spec/fixtures/some-subfolder/this-is-a-subpage.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | This is a subpage! 5 | -------------------------------------------------------------------------------- /spec/fixtures/some-subfolder/xhtml.xhtml: -------------------------------------------------------------------------------- 1 | This file has an .xhtml extension, and should be included in the sitemap 2 | -------------------------------------------------------------------------------- /spec/fixtures/static_files/404.html: -------------------------------------------------------------------------------- 1 | 404. That's an error. 2 | -------------------------------------------------------------------------------- /spec/fixtures/static_files/excluded.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galliani/jekyll-airtable/8bcea310f10fa8365f8f9b7d0d15ad1f7e4c43f6/spec/fixtures/static_files/excluded.pdf -------------------------------------------------------------------------------- /spec/fixtures/static_files/html_file.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galliani/jekyll-airtable/8bcea310f10fa8365f8f9b7d0d15ad1f7e4c43f6/spec/fixtures/static_files/html_file.html -------------------------------------------------------------------------------- /spec/fixtures/static_files/test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galliani/jekyll-airtable/8bcea310f10fa8365f8f9b7d0d15ad1f7e4c43f6/spec/fixtures/static_files/test.pdf -------------------------------------------------------------------------------- /spec/jekyll-airtable_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/galliani/jekyll-airtable/8bcea310f10fa8365f8f9b7d0d15ad1f7e4c43f6/spec/jekyll-airtable_spec.rb -------------------------------------------------------------------------------- /spec/jekyll/airtable_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | RSpec.describe Jekyll::Airtable do 4 | it "has a version number" do 5 | expect(Jekyll::Airtable::VERSION).not_to be nil 6 | end 7 | 8 | let(:site) { make_site } 9 | before { site.process } 10 | 11 | context "full page rendering" do 12 | let(:content) { File.read(dest_dir("page.html")) } 13 | 14 | # Specify what to display on the page.html 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # spec_helper.rb 2 | $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) 3 | require "jekyll" 4 | require "jekyll-airtable" 5 | 6 | Jekyll.logger.log_level = :error 7 | 8 | RSpec.configure do |config| 9 | config.run_all_when_everything_filtered = true 10 | config.filter_run :focus 11 | config.order = "random" 12 | 13 | SOURCE_DIR = File.expand_path("../fixtures", __FILE__) 14 | DEST_DIR = File.expand_path("../dest", __FILE__) 15 | 16 | def source_dir(*files) 17 | File.join(SOURCE_DIR, *files) 18 | end 19 | 20 | def dest_dir(*files) 21 | File.join(DEST_DIR, *files) 22 | end 23 | 24 | CONFIG_DEFAULTS = { 25 | "source" => source_dir, 26 | "destination" => dest_dir, 27 | "gems" => ["jekyll-airtable"] 28 | }.freeze 29 | 30 | def make_page(options = {}) 31 | page = Jekyll::Page.new(site, CONFIG_DEFAULTS["source"], "", "page.md") 32 | page.data = options 33 | page 34 | end 35 | 36 | def make_site(options = {}) 37 | site_config = Jekyll.configuration(CONFIG_DEFAULTS.merge(options)) 38 | Jekyll::Site.new(site_config) 39 | end 40 | 41 | def make_context(registers = {}, environments = {}) 42 | Liquid::Context.new(environments, {}, 43 | { :site => site, :page => page }.merge(registers)) 44 | end 45 | end --------------------------------------------------------------------------------