├── .editorconfig ├── .github └── workflows │ └── test.yml ├── .gitignore ├── .nvmrc ├── LICENSE ├── README.md ├── lerna.json ├── netlify.toml ├── package.json ├── packages ├── .gitkeep ├── algolia-fragmenter │ ├── .eslintrc.js │ ├── LICENSE │ ├── README.md │ ├── index.js │ ├── lib │ │ └── transformer.js │ ├── package.json │ └── test │ │ ├── .eslintrc.js │ │ ├── fixtures │ │ ├── install-source.html │ │ ├── massive-example.html │ │ └── minimal-example.html │ │ ├── fragmenter.test.js │ │ └── utils │ │ ├── assertions.js │ │ ├── index.js │ │ └── overrides.js ├── algolia-indexer │ ├── .eslintrc.js │ ├── LICENSE │ ├── README.md │ ├── index.js │ ├── lib │ │ └── IndexFactory.js │ ├── package.json │ └── test │ │ ├── .eslintrc.js │ │ ├── IndexFactory.test.js │ │ └── utils │ │ ├── assertions.js │ │ ├── index.js │ │ └── overrides.js ├── algolia-netlify │ ├── .env.example │ ├── .nvmrc │ ├── LICENSE │ ├── README.md │ ├── functions │ │ ├── post-published.js │ │ └── post-unpublished.js │ ├── netlify.toml │ ├── package.json │ └── test │ │ ├── .eslintrc.js │ │ ├── hello.test.js │ │ └── utils │ │ ├── assertions.js │ │ ├── index.js │ │ └── overrides.js └── algolia │ ├── .eslintrc.js │ ├── LICENSE │ ├── README.md │ ├── bin │ └── cli.js │ ├── example.config.json │ ├── index.js │ ├── lib │ └── utils.js │ ├── package.json │ └── test │ ├── .eslintrc.js │ ├── hello.test.js │ └── utils │ ├── assertions.js │ ├── index.js │ └── overrides.js ├── renovate.json ├── test ├── hello.test.js └── utils │ ├── assertions.js │ ├── index.js │ └── overrides.js └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 4 9 | end_of_line = lf 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | 13 | [*.hbs] 14 | insert_final_newline = false 15 | 16 | [*.json] 17 | indent_size = 2 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [*.{yml,yaml}] 23 | indent_size = 2 24 | 25 | [Makefile] 26 | indent_style = tab 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - main 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | node: [ 16, 18 ] 13 | env: 14 | FORCE_COLOR: 1 15 | name: Node ${{ matrix.node }} 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-node@v3 19 | with: 20 | node-version: ${{ matrix.node }} 21 | - run: yarn global add lerna 22 | 23 | - run: yarn 24 | - run: yarn test 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Node template 2 | 3 | # Logs 4 | logs 5 | *.log 6 | npm-debug.log* 7 | yarn-debug.log* 8 | yarn-error.log* 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | *.pid.lock 15 | 16 | # Directory for instrumented libs generated by jscoverage/JSCover 17 | lib-cov 18 | 19 | # Coverage directory used by tools like istanbul 20 | coverage 21 | 22 | # nyc test coverage 23 | .nyc_output 24 | 25 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 26 | .grunt 27 | 28 | # Bower dependency directory (https://bower.io/) 29 | bower_components 30 | 31 | # node-waf configuration 32 | .lock-wscript 33 | 34 | # Compiled binary addons (https://nodejs.org/api/addons.html) 35 | build/Release 36 | 37 | # Dependency directories 38 | node_modules/ 39 | jspm_packages/ 40 | 41 | # Typescript v1 declaration files 42 | typings/ 43 | 44 | # Optional npm cache directory 45 | .npm 46 | 47 | # Optional eslint cache 48 | .eslintcache 49 | 50 | # Optional REPL history 51 | .node_repl_history 52 | 53 | # Output of 'npm pack' 54 | *.tgz 55 | 56 | # Yarn Integrity file 57 | .yarn-integrity 58 | 59 | # dotenv environment variables file 60 | .env 61 | 62 | # IDE 63 | .idea/* 64 | *.iml 65 | *.sublime-* 66 | .vscode/* 67 | 68 | # OSX 69 | .DS_Store 70 | 71 | # Algolia Custom 72 | packages/*/build/* 73 | 74 | packages/algolia/config*.json 75 | 76 | # Local Netlify folder 77 | .netlify 78 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 18 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-2025 Ghost Foundation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ghost Algolia tools 2 | 3 | Ghost Algolia tools offers tools to index and fragment Ghost posts to an Algolia index. It consists of two user facing tools: 4 | 5 | - `algolia`, which is a CLI tool to batch index the full content of a Ghost install to a defined Algolia index 6 | - `algolia-netlify`, which uses Netlify Functions to listen to Ghost webhooks and add, update, and remove posts to an Algolia index 7 | 8 | 9 | ## Usage 10 | 11 | ### Algolia Netlify package 12 | 13 | You can start using the Algolia Netlify package by clicking on this deplooy button. You can find the detailed install and user instructions over [here](https://github.com/TryGhost/algolia/tree/master/packages/algolia-netlify). 14 | 15 | [![Deploy to Netlify](https://www.netlify.com/img/deploy/button.svg)](https://app.netlify.com/start/deploy?repository=https://github.com/TryGhost/algolia) 16 | 17 | ### Ghost Algolia CLI 18 | 19 | While the Algolia Netlify tool is useful to maintain your search index, the Ghost Algolia CLI is good for the initial indexing of the full post content of a site. See full install and user instructions in the package description [here](https://github.com/TryGhost/algolia/tree/master/packages/algolia). 20 | 21 | ## Develop 22 | 23 | This is a mono repository, managed with [lerna](https://lernajs.io/). 24 | 25 | 1. `git clone` this repo & `cd` into it as usual 26 | 2. `yarn setup` is mapped to `lerna bootstrap` 27 | - installs all external dependencies 28 | - links all internal dependencies 29 | 30 | To add a new package to the repo: 31 | - install [slimer](https://github.com/TryGhost/slimer) 32 | - run `slimer new ` 33 | 34 | 35 | ## Run 36 | 37 | - `yarn dev` 38 | 39 | 40 | ## Test 41 | 42 | - `yarn lint` run just eslint 43 | - `yarn test` run lint and tests 44 | 45 | 46 | ## Publish 47 | 48 | - `yarn ship` is an alias for `lerna publish` 49 | - Publishes all packages which have changed 50 | - Also updates any packages which depend on changed packages 51 | 52 | 53 | # Copyright & License 54 | 55 | Copyright (c) 2013-2025 Ghost Foundation - Released under the [MIT license](LICENSE). 56 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "independent", 3 | "npmClient": "yarn", 4 | "packages": ["packages/*"], 5 | "command": { 6 | "publish": { 7 | "allowBranch": "main", 8 | "message": "Published new versions" 9 | } 10 | }, 11 | "local": { 12 | "public": "true", 13 | "repo": "https://github.com/TryGhost/algolia", 14 | "scope": "@tryghost" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | base = "packages/algolia-netlify" 3 | command = "NODE_ENV=production yarn build" 4 | functions = "build/functions" 5 | publish = "" 6 | 7 | [build.environment] 8 | NODE_VERSION = "16" 9 | AWS_LAMBDA_JS_RUNTIME = "nodejs16.x" 10 | 11 | [template.environment] 12 | ALGOLIA_ACTIVE = "TRUE or FALSE. Set to TRUE to trigger indexing." 13 | ALGOLIA_APP_ID = "Algolia Application ID" 14 | ALGOLIA_API_KEY = "An Algolia Admin API Key or a generated one" 15 | ALGOLIA_INDEX = "Name of the Algolia index" 16 | NETLIFY_KEY = "User-defined key to authorize post requests to the Netlify function" 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "repository": "https://github.com/TryGhost/algolia", 4 | "author": "Ghost Foundation", 5 | "license": "MIT", 6 | "workspaces": [ 7 | "packages/*" 8 | ], 9 | "eslintIgnore": [ 10 | "**/node_modules/**" 11 | ], 12 | "scripts": { 13 | "dev": "echo \"Implement me!\"", 14 | "setup": "yarn", 15 | "test:parent": "NODE_ENV=testing mocha './test/**/*.test.js'", 16 | "test": "yarn test:parent && lerna run test", 17 | "lint": "lerna run lint", 18 | "preship": "yarn test", 19 | "ship": "lerna publish" 20 | }, 21 | "devDependencies": { 22 | "eslint": "8.54.0", 23 | "eslint-plugin-ghost": "3.4.0", 24 | "mocha": "10.2.0", 25 | "should": "13.2.3", 26 | "sinon": "17.0.1" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /packages/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TryGhost/algolia/7b875425981d7c8b3f4f0d6afd05c0c995f9d4fc/packages/.gitkeep -------------------------------------------------------------------------------- /packages/algolia-fragmenter/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['ghost'], 3 | extends: [ 4 | 'plugin:ghost/node' 5 | ] 6 | }; 7 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-2025 Ghost Foundation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/README.md: -------------------------------------------------------------------------------- 1 | # Algolia Fragmenter 2 | 3 | Fragment transformer converts a Ghost post into an Algolia Object and breaks down large HTML strings into sensible fragments based on headings. 4 | 5 | ## Install 6 | 7 | `npm install @tryghost/algolia-fragmenter --save` 8 | 9 | or 10 | 11 | `yarn add @tryghost/algolia-fragmenter` 12 | 13 | 14 | ## Usage 15 | 16 | 17 | ## Develop 18 | 19 | This is a mono repository, managed with [lerna](https://lernajs.io/). 20 | 21 | Follow the instructions for the top-level repo. 22 | 1. `git clone` this repo & `cd` into it as usual 23 | 2. Run `yarn` to install top-level dependencies. 24 | 25 | 26 | ## Run 27 | 28 | - `yarn dev` 29 | 30 | 31 | ## Test 32 | 33 | - `yarn lint` run just eslint 34 | - `yarn test` run lint and tests 35 | 36 | 37 | 38 | 39 | # Copyright & License 40 | 41 | Copyright (c) 2013-2025 Ghost Foundation - Released under the [MIT license](LICENSE). 42 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/transformer'); 2 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/lib/transformer.js: -------------------------------------------------------------------------------- 1 | const HtmlExtractor = require(`algolia-html-extractor`); 2 | const Extractor = new HtmlExtractor(); 3 | 4 | /** 5 | * Utility function, takes the output of HTML Extractor, and reduces it back down 6 | * So that there is a group of HTML/content per heading 7 | * 8 | * @param {Array} accumulator 9 | * @param {Object} fragment 10 | */ 11 | const reduceFragmentsUnderHeadings = (accumulator, fragment) => { 12 | const existingFragment = accumulator.find(existing => existing.anchor === fragment.anchor); 13 | 14 | if (existingFragment) { 15 | // Merge our fragments together 16 | if (fragment.node && fragment.node.tagName === `PRE`) { 17 | // For pre-tags, we don't keep all the markup 18 | existingFragment.html += ` ${fragment.content}`; // keep a space 19 | existingFragment.content += ` ${fragment.content}`; // keep a space 20 | } else { 21 | existingFragment.html += fragment.html; 22 | existingFragment.content += ` ${fragment.content}`; // keep a space 23 | } 24 | } else { 25 | // If we don't already have a matching fragment with this anchor, add it 26 | accumulator.push(fragment); 27 | } 28 | 29 | return accumulator; 30 | }; 31 | 32 | /** 33 | * Fragment Transformer 34 | * breaks down large HTML strings into sensible fragments based on headings 35 | */ 36 | module.exports.fragmentTransformer = (recordAccumulator, node) => { 37 | let htmlFragments = Extractor 38 | // These are the top-level HTML elements that we keep - this results in a lot of fragments 39 | .run(node.html, {cssSelector: `p,pre,td,li`}) 40 | // Use the utility function to merge fragments so that there is one-per-heading 41 | .reduce(reduceFragmentsUnderHeadings, []); 42 | 43 | // convert our fragments for this node into valid objects, and merge int the 44 | const records = htmlFragments.reduce((fragmentAccumulator, fragment, index) => { 45 | // Don't need a reference to the html node type 46 | delete fragment.node; 47 | // For now at least, we're not going to index the content string 48 | // The HTML string is already very long, and there are size limits 49 | delete fragment.content; 50 | // If we have an anchor, change the URL to be a deep link 51 | if (fragment.anchor) { 52 | fragment.url = `${node.url}#${fragment.anchor}`; 53 | } 54 | 55 | let objectID = `${node.objectID}_${index}`; 56 | 57 | // TODO: switch this on in verbose mode only 58 | // // If fragments are too long, we need this to see which fragment it was 59 | // console.log(`Created fragment: `, objectID, fragment.url || node.url, fragment.html.length); // eslint-disable-line no-console 60 | 61 | return [ 62 | ...fragmentAccumulator, 63 | {...node, ...fragment, objectID: objectID} 64 | ]; 65 | }, []); 66 | 67 | return [...recordAccumulator, ...records]; 68 | }; 69 | 70 | module.exports._testReduceFragmentsUnderHeadings = reduceFragmentsUnderHeadings; 71 | 72 | /** 73 | * Algolia Object Transformer 74 | * takes a Ghost post and selects the properties needed to send to Algolia 75 | * 76 | * @param {Array} posts 77 | */ 78 | module.exports.transformToAlgoliaObject = (posts, ignoreSlugs) => { 79 | const algoliaObjects = []; 80 | 81 | posts.map((post) => { 82 | // Define the properties we need for Algolia 83 | const algoliaPost = { 84 | objectID: post.id, 85 | slug: post.slug, 86 | url: post.url, 87 | html: post.html, 88 | image: post.feature_image, 89 | title: post.title, 90 | tags: [], 91 | authors: [] 92 | }; 93 | 94 | // If we have an array of slugs to ignore, and the current 95 | // post slug is in that list, skip this loop iteration 96 | if (ignoreSlugs) { 97 | if (ignoreSlugs.includes(post.slug)) { 98 | return false; 99 | } 100 | } 101 | 102 | if (post.tags && post.tags.length) { 103 | post.tags.forEach((tag) => { 104 | algoliaPost.tags.push({name: tag.name, slug: tag.slug}); 105 | }); 106 | } 107 | 108 | if (post.authors && post.authors.length) { 109 | post.authors.forEach((author) => { 110 | algoliaPost.authors.push({name: author.name, slug: author.slug}); 111 | }); 112 | } 113 | 114 | algoliaObjects.push(algoliaPost); 115 | 116 | return algoliaPost; 117 | }); 118 | 119 | return algoliaObjects; 120 | }; 121 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@tryghost/algolia-fragmenter", 3 | "version": "0.2.7", 4 | "repository": "https://github.com/TryGhost/algolia/tree/master/packages/algolia-fragmenter", 5 | "author": "Ghost Foundation", 6 | "license": "MIT", 7 | "main": "index.js", 8 | "scripts": { 9 | "dev": "echo \"Implement me!\"", 10 | "test": "NODE_ENV=testing mocha './test/**/*.test.js'", 11 | "lint": "eslint . --ext .js --cache", 12 | "posttest": "yarn lint" 13 | }, 14 | "files": [ 15 | "index.js", 16 | "lib" 17 | ], 18 | "publishConfig": { 19 | "access": "public" 20 | }, 21 | "devDependencies": { 22 | "mocha": "10.2.0", 23 | "should": "13.2.3", 24 | "sinon": "17.0.1" 25 | }, 26 | "dependencies": { 27 | "algolia-html-extractor": "0.0.1" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['ghost'], 3 | extends: [ 4 | 'plugin:ghost/test' 5 | ] 6 | }; 7 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/fixtures/install-source.html: -------------------------------------------------------------------------------- 1 |

This guide is for installing a local development copy of Ghost from source, primarily for the purpose of modifying Ghost core

2 |

Pre-requisites

3 |

Before getting started, you'll need these global packages to be installed:

4 | 8 |

The install these global packages

9 | 
yarn global add knex-migrator grunt-cli ember-cli bower
10 |
11 |

Create GitHub forks

12 |

First you'll need to make forks of both the Ghost and Ghost-Admin respositories. Click on the fork button right at the top, wait for a copy to be created over on your personal GitHub account, and you should be all set!

13 |

Fork

14 |
15 |

Configure repositories

16 |

The next step is to configure the Git repositories for local development

17 |

Ghost Core

18 |

The main Ghost repository contains the full Ghost package, including the Admin client and default theme which will also be automatically set up

19 | 
# First clone Ghost and make it your working dir
 20 | git clone git@github.com:TryGhost/Ghost.git && cd Ghost
21 |

Properly rename your references

22 | 
# Rename origin to upstream
 23 | git remote rename origin upstream
 24 | 
 25 | # Ensure it has the correct path
 26 | git remote set-url upstream git@github.com:TryGhost/Ghost.git
 27 | 
 28 | # Add your fork as an origin, editing in <YourUsername>!
 29 | git remote add origin git@github.com:<YourUsername>/Ghost.git
30 |

Ghost Admin

31 |

Because Ghost-Admin is a submodule repository of the main Ghost repository, the same steps need to be repeated to configure Git here, too.

32 | 
# Switch to Ghost-Admin dir
 33 | cd core/client
34 |

Properly rename your references again

35 | 
# Rename origin to upstream
 36 | git remote rename origin upstream
 37 | 
 38 | # Ensure admin also has the correct path
 39 | git remote set-url upstream git@github.com:TryGhost/Ghost-Admin.git
 40 | 
 41 | # Add your fork as an origin, editing in <YourUsername>!
 42 | git remote add origin git@github.com:<YourUsername>/Ghost-Admin.git
43 |

Bring Ghost-Admin up to date

44 | 
# Quick check that everything is on latest
 45 | git checkout master && git pull upstream master
 46 | 
 47 | # Then return to Ghost root directory
 48 | cd ../../
49 |
50 |

Run setup & installation

51 | 
# Only ever run this once
 52 | yarn setup
53 |

The setup task will install dependencies, initialise the database, set up git hooks & initialise submodules and run a first build of the admin. The very first build generally takes a while, so now's a good time to re-open that Reddit tab.

54 |
55 |

Start Ghost

56 | 
# Run Ghost in development mode
 57 | grunt dev
58 |

Ghost is now running at http://localhost:2368/

59 |
60 |

Stay up to date

61 |

When your working copies become out of date due to upstream changes, this is the command always brings you back up to latest master

62 | 
# Update EVERYTHING
 63 | grunt master
64 |

That's it, you're done with the install! The rest of this guide is about working with your new development copy of Ghost.

65 |
66 |

Dev Commands

67 |

When running locally there are a number development utility commands which come in handy for running tests, building packages, and other helpful tasks.

68 |

Running Ghost

69 |

The most commonly used commands for running the core codebase locally

70 | 
grunt dev
 71 | # Default way of running Ghost in development mode
 72 | # Builds admin files on start & then watches for changes
 73 | 
 74 | grunt dev --server
 75 | # Ignores admin changes
 76 | 
 77 | grunt dev --no-server-watch
 78 | # Ignores server changes
 79 | 
 80 | grunt build
 81 | # Build admin client manually
 82 | 
 83 | grunt prod
 84 | # Build full Ghost package for production
85 |

Database tools

86 |

Ghost uses it's own tool called knex-migrator to manage database migrations

87 | 
knex-migrator reset
 88 | # Wipe the database
 89 | 
 90 | knex-migrator init
 91 | # Populate a fresh database
92 |

Server Tests

93 |

Tests run with SQlite. To use MySQL, prepend commands with NODE_ENV=testing-mysql

94 | 
grunt test-all
 95 | # Run all tests
 96 | 
 97 | grunt test-unit
 98 | # Run unit tests
 99 | 
100 | grunt test-integration
101 | # Run integration tests
102 | 
103 | grunt test-functional
104 | # Run functional tests
105 | 
106 | grunt test:path/to/test.js
107 | # Run a single test
108 | 
109 | grunt lint
110 | # Make sure your code doesn't suck
111 |

Client Tests

112 |

Client tests should always be run inside the core/client directory. Any time you have grunt dev running the client tests will be available at http://localhost:4200/tests

113 | 
ember test
114 | # Run all tests in Chrome + Firefox
115 | 
116 | ember test --server
117 | # Run all tests, leave results open, and watch for changes
118 | 
119 | ember test -f 'gh-my-component'
120 | # Run tests where `describe()` or `it()` matches supplied argument
121 | # Note: Case sensitive
122 | 
123 | ember test --launch=chrome
124 | # Run all tests in Chrome only
125 | 
126 | ember test -s -f 'Acceptance: Settings - General' --launch=chrome
127 | # Most useful test comment for continuous local development
128 | # Targets specific test of area being worked on
129 | # Only using Chrome to keep resource usage minimal
130 |
131 |

Troubleshooting

132 |

Some common Ghost development problems and their solutions

133 |

ERROR: (EADDRINUSE) Cannot start Ghost
134 | This error means that Ghost is already running, and you need to stop it

135 |

ERROR: ENOENT
136 | This error means that the mentioned file doesn't exist

137 |

ERROR Error: Cannot find module
138 | Install did not complete. Remove your node_modules and re-run yarn

139 |

Error: Cannot find module './build/default/DTraceProviderBindings'
140 | You switched node versions. Remove your node_modules and re-run yarn

141 |

ENOENT: no such file or directory, stat 'path/to/favicon.ico' at Error (native)
142 | Your admin client has not been built. Run grunt prod for production or grunt dev

143 |

TypeError: Cannot read property 'tagName' of undefined
144 | You can't run ember test at the same time as grunt dev. Wait for tests to finish before continuing and wait for the "Build successful" message before loading admin.

145 |

yarn.lock conflicts
146 | When rebasing a feature branch it's possible you'll get conflicts on yarn.lock because there were dependency changes in both master and <feature-branch>.

147 |
    148 |
  1. Note what dependencies have changed in package.json
    149 | (Eg. dev-1 was added and dev dep dev-2 was removed)
    2. 150 |
  2. git reset HEAD package.json yarn.lock - unstages the files
    3. 151 |
  3. git checkout -- package.json yarn.lock - removes local changes
    4. 152 |
  4. yarn add dev-1 -D - re-adds the dependency and updates yarn.lock
    5. 153 |
  5. yarn remove dev-2 - removes the dependency and updates yarn.lock
    6. 154 |
  6. git add package.json yarn.lock - re-stage the changes
    7. 155 |
  7. git rebase --continue - continue with the rebase
    8. 156 |
157 |

It's always more reliable to let yarn auto-generate the lockfile rather than trying to manually merge potentially incompatible changes.

158 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/fixtures/massive-example.html: -------------------------------------------------------------------------------- 1 |

The open-source Ghost editor is robust and extensible.

2 |

Overview

3 |

More than just a formatting toolbar, the rich editing experience within Ghost allows authors to pull in dynamic blocks of content like photos, videos, tweets, embeds, code and markdown.

4 |

For these author-specified options to work, themes need to support the HTML markup and CSS classes that are output by the {{content}} helper. The following guide explains how these options interact with the theme layer and how you can ensure your theme is compatible with the latest version of the Ghost editor.

5 |

<figure> and <figcaption>

6 |

Images and embeds will be output using the semantic <figure> and <figcaption> elements. For example:

7 |
Rendered Output
8 | 
<figure class="kg-image-card">
  9 |     <img class="kg-image" src="https://casper.ghost.org/v1.25.0/images/koenig-demo-1.jpg">
 10 |     <figcaption>An example image</figcaption>
 11 | </figure>
12 |

The following CSS classes are used:

13 | 18 |

This is only relevant when authors use the built-in image and embed cards, and themes must also support images and embeds that are not wrapped in <figure> elements to maintain compatibility with the Markdown and HTML cards.

19 |

Image size options

20 |

The editor allows three size options for images: normal, wide and full width. These size options are achieved by adding kg-width-wide and kg-width-full classes to the <figure> elements in the HTML output. Here's an example for wide images:

21 |
Rendered Output
22 | 
<figure class="kg-image-card kg-width-wide">
 23 |     <img class="kg-image" src="https://casper.ghost.org/v1.25.0/images/koenig-demo-1.jpg">
 24 | </figure>
25 |

Normal width image cards do not have any extra CSS classes.

26 |

Image size implementations

27 |

The specific implementation required for making images wider than their container width will depend on your theme's existing styles. The default Ghost theme Casper uses flexbox to implement layout using the following HTML and CSS:

28 |
Rendered Output
29 | 
<div class="content">
 30 |   <article>
 31 |     <h1>Image size implementation</h1>
 32 | 
 33 |     <p>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce at interdum ipsum.</p>
 34 | 
 35 | 
 36 |     <figure class="kg-image-card kg-width-full">
 37 |       <img class="kg-image" src="https://casper.ghost.org/v1.25.0/images/koenig-demo-2.jpg">
 38 |       <figcaption>A full-width image</figcaption>
 39 |     </figure>
 40 | 
 41 |     <p>Fusce interdum velit tristique, scelerisque libero et, venenatis nisi. Maecenas euismod luctus neque nec finibus.</p>
 42 | 
 43 |     <figure class="kg-image-card kg-width-wide">
 44 |       <img class="kg-image" src="https://casper.ghost.org/v1.25.0/images/koenig-demo-1.jpg">
 45 |       <figcaption>A wide image</figcaption>
 46 |     </figure>
 47 | 
 48 |     <p>Suspendisse sed lacus efficitur, euismod nisi a, sollicitudin orci.</p>
 49 |   </article>
 50 | </div>
 51 | 
 52 | <footer>An example post</footer>
53 |

And the CSS:

54 |
style.css
55 | 
.content {
 56 |   width: 70%;
 57 |   margin: 0 auto;
 58 |  }
 59 | 
 60 | article {
 61 |   display: flex;
 62 |   flex-direction: column;
 63 |   align-items: center;
 64 | }
 65 | 
 66 | article img {
 67 |   display: block;
 68 |   max-width: 100%;
 69 | }
 70 | 
 71 | .kg-width-wide img {
 72 |   max-width: 85vw;
 73 | }
 74 | 
 75 | .kg-width-full img {
 76 |   max-width: 100vw;
 77 | }
 78 | 
 79 | article figure {
 80 |   margin: 0;
 81 | }
 82 | 
 83 | article figcaption {
 84 |   text-align: center;
 85 | }
 86 | 
 87 | body {
 88 |   margin: 0;
 89 | }
 90 | 
 91 | header, footer {
 92 |   padding: 15px 25px;
 93 |   background-color: #000;
 94 |   color: #fff;
 95 | }
 96 | 
 97 | h1 {
 98 |   width: 100%;
 99 | }
100 |

Negative margin and transforms example

101 |

Traditional CSS layout doesn't support many elegant methods for breaking elements out of their container. The following example uses negative margins and transforms to acheive breakout. Themes that are based on Casper use similar techniques.

102 |
style.css
103 | 
.content {
104 |   width: 70%;
105 |   margin: 0 auto;
106 |  }
107 | 
108 | article img {
109 |   display: block;
110 |   max-width: 100%;
111 | }
112 | 
113 | .kg-width-wide {
114 |   position: relative;
115 |   width: 85vw;
116 |   min-width: 100%;
117 |   margin: auto calc(50% - 50vw);
118 |   transform: translateX(calc(50vw - 50%));
119 | }
120 | 
121 | .kg-width-full {
122 |   position: relative;
123 |   width: 100vw;
124 |   left: 50%;
125 |   right: 50%;
126 |   margin-left: -50vw;
127 |   margin-right: -50vw;
128 | }
129 | 
130 | article figure {
131 |   margin: 0;
132 | }
133 | 
134 | article figcaption {
135 |   text-align: center;
136 | }
137 | 
138 | body {
139 |   margin: 0;
140 | }
141 | 
142 | header, footer {
143 |   padding: 15px 25px;
144 |   background-color: #000;
145 |   color: #fff;
146 | }
147 | 148 |

The image gallery card requires some CSS and JS in your theme to function correctly. Themes will be validated to ensure they have styles for the gallery markup:

149 | 154 |

Example gallery HTML:

155 |
Rendered Output
156 | 
<figure class="kg-card kg-gallery-card kg-width-wide">
157 |     <div class="kg-gallery-container">
158 |         <div class="kg-gallery-row">
159 |             <div class="kg-gallery-image">
160 |                 <img src="/content/images/1.jpg" width="6720" height="4480">
161 |             </div>
162 |             <div class="kg-gallery-image">
163 |                 <img src="/content/images/2.jpg" width="4946" height="3220">
164 |             </div>
165 |             <div class="kg-gallery-image">
166 |                 <img src="/content/images/3.jpg" width="5560" height="3492">
167 |             </div>
168 |         </div>
169 |         <div class="kg-gallery-row">
170 |             <div class="kg-gallery-image">
171 |                 <img src="/content/images/4.jpg" width="3654" height="5473">
172 |             </div>
173 |             <div class="kg-gallery-image">
174 |                 <img src="/content/images/5.jpg" width="4160" height="6240">
175 |             </div>
176 |             <div class="kg-gallery-image">
177 |                 <img src="/content/images/6.jpg" width="2645" height="3967">
178 |             </div>
179 |         </div>
180 |         <div class="kg-gallery-row">
181 |             <div class="kg-gallery-image">
182 |                 <img src="/content/images/7.jpg" width="3840" height="5760">
183 |             </div>
184 |             <div class="kg-gallery-image">
185 |                 <img src="/content/images/8.jpg" width="3456" height="5184">
186 |             </div>
187 |         </div>
188 |     </div>
189 | </figure>
190 |

For a better view of how to support the gallery card in your theme, use the Casper implementation, which is a generic solution that works for most themes.

191 |

Summary

192 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/fixtures/minimal-example.html: -------------------------------------------------------------------------------- 1 |

This guide is for installing a local development copy of Ghost from source, primarily for the purpose of modifying Ghost core

2 |

Pre-requisites

3 |

Before getting started, you'll need these global packages to be installed:

4 | 11 |

The install these global packages

12 | 
yarn global add knex-migrator grunt-cli ember-cli bower
13 |
14 |

Create GitHub forks

15 |

First you'll need to make forks of both the Ghost and Ghost-Admin respositories. Click on the fork button right at the top, wait for a copy to be created over on your personal GitHub account, and you should be all set!

16 |

Fork

17 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/fragmenter.test.js: -------------------------------------------------------------------------------- 1 | // Switch these lines once there are useful utils 2 | // const testUtils = require('./utils'); 3 | require('./utils'); 4 | 5 | const transforms = require('../'); 6 | const fs = require('fs'); 7 | const path = require('path'); 8 | 9 | const readFixture = (fileName) => { 10 | return fs.readFileSync(path.join(__dirname, `fixtures`, `${fileName}.html`), {encoding: `utf8`}); 11 | }; 12 | 13 | describe('Algolia Transforms', function () { 14 | it('Can reduce fragments under headings correctly', function () { 15 | let fragments = [ 16 | { 17 | html: '

Before getting started, you\'ll need these global packages to be installed:

', 18 | content: 'Before getting started, you\'ll need these global packages to be installed:', 19 | headings: ['Pre-requisites'], 20 | anchor: 'pre-requisites', 21 | customRanking: {position: 1, heading: 80}, 22 | objectID: '931c35eda23999b8124728b4bf4979eb' 23 | }, 24 | { 25 | html: '
  • A supported version of Node.js - Ideally installed via nvm
    • ', 26 | content: 'A supported version of Node.js - Ideally installed via nvm', 27 | headings: ['Pre-requisites'], 28 | anchor: 'pre-requisites', 29 | 30 | customRanking: {position: 2, heading: 80}, 31 | objectID: 'dcbe237f62a97a33e1a9d8880c577933' 32 | }, 33 | { 34 | html: '
  • Yarn - to manage all the packages
    • ', 35 | content: 'Yarn - to manage all the packages', 36 | headings: ['Pre-requisites'], 37 | anchor: 'pre-requisites', 38 | 39 | customRanking: {position: 3, heading: 80}, 40 | objectID: '49e23962e997062e388a060735442633' 41 | }, 42 | { 43 | html: '
    yarn global add knex-migrator grunt-cli ember-cli bower
    ', 44 | content: 'yarn global add knex-migrator grunt-cli ember-cli bower', 45 | headings: ['Pre-requisites', 'The install these global packages'], 46 | anchor: 'the-install-these-global-packages', 47 | 48 | customRanking: {position: 4, heading: 60}, 49 | objectID: 'b1a9a06228097949e1b9f0cfcb7fe352' 50 | } 51 | ]; 52 | 53 | let reducedFragments = fragments.reduce(transforms._testReduceFragmentsUnderHeadings, []); 54 | 55 | // We start with 4 elements, and end up with 2 56 | reducedFragments.should.have.lengthOf(2); 57 | // The content gets merged to contain all 3 strings 58 | reducedFragments[0].content.should.match(/getting started/); 59 | reducedFragments[0].content.should.match(/supported version/); 60 | reducedFragments[0].content.should.match(/manage all the packages/); 61 | }); 62 | 63 | it('Processes minimal example correctly', function () { 64 | const fakeNode = { 65 | objectID: `abc`, 66 | title: `Install from Source`, 67 | url: `/install/source/`, 68 | html: readFixture(`minimal-example`) 69 | }; 70 | let reducedFragments = transforms.fragmentTransformer([], fakeNode); 71 | 72 | reducedFragments.should.have.a.lengthOf(4); 73 | reducedFragments[1].url.should.eql('/install/source/#pre-requisites'); 74 | }); 75 | 76 | it('merges multiple nodes correctly', function () { 77 | const fakeNodes = [{ 78 | objectID: `abc`, 79 | title: `Install from Source`, 80 | url: `/install/source/`, 81 | html: readFixture(`minimal-example`) 82 | }, { 83 | objectID: `def`, 84 | title: `Install Test`, 85 | url: `/install/test/`, 86 | html: `

    I am a test

    Testing

    I am a subtest

    ` 87 | }]; 88 | 89 | let reducedFragments = fakeNodes.reduce(transforms.fragmentTransformer, []); 90 | 91 | reducedFragments.should.have.a.lengthOf(6); 92 | reducedFragments[0].url.should.eql('/install/source/'); 93 | reducedFragments[1].url.should.eql('/install/source/#pre-requisites'); 94 | reducedFragments[4].url.should.eql('/install/test/'); 95 | reducedFragments[5].url.should.eql('/install/test/#testing'); 96 | }); 97 | 98 | it('Processes massive example correctly', function () { 99 | const fakeNode = { 100 | objectID: `abc`, 101 | title: `Install from Source`, 102 | url: `/install/source/`, 103 | html: readFixture(`massive-example`) 104 | }; 105 | 106 | let reducedFragments = [fakeNode].reduce(transforms.fragmentTransformer, []); 107 | 108 | reducedFragments.forEach((fragment) => { 109 | JSON.stringify(fragment).length.should.be.below(10000); 110 | }); 111 | }); 112 | }); 113 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/utils/assertions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom Should Assertions 3 | * 4 | * Add any custom assertions to this file. 5 | */ 6 | 7 | // Example Assertion 8 | // should.Assertion.add('ExampleAssertion', function () { 9 | // this.params = {operator: 'to be a valid Example Assertion'}; 10 | // this.obj.should.be.an.Object; 11 | // }); 12 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/utils/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test Utilities 3 | * 4 | * Shared utils for writing tests 5 | */ 6 | 7 | // Require overrides - these add globals for tests 8 | require('./overrides'); 9 | 10 | // Require assertions - adds custom should assertions 11 | require('./assertions'); 12 | -------------------------------------------------------------------------------- /packages/algolia-fragmenter/test/utils/overrides.js: -------------------------------------------------------------------------------- 1 | // This file is required before any test is run 2 | 3 | // Taken from the should wiki, this is how to make should global 4 | // Should is a global in our eslint test config 5 | global.should = require('should').noConflict(); 6 | should.extend(); 7 | 8 | // Sinon is a simple case 9 | // Sinon is a global in our eslint test config 10 | global.sinon = require('sinon'); 11 | -------------------------------------------------------------------------------- /packages/algolia-indexer/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['ghost'], 3 | extends: [ 4 | 'plugin:ghost/node' 5 | ] 6 | }; 7 | -------------------------------------------------------------------------------- /packages/algolia-indexer/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-2025 Ghost Foundation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/algolia-indexer/README.md: -------------------------------------------------------------------------------- 1 | # Algolia Indexer 2 | 3 | IndexFactory that takes over the part of talking to the Algolia API. 4 | 5 | ## Install 6 | 7 | `npm install @tryghost/algolia-indexer --save` 8 | 9 | or 10 | 11 | `yarn add @tryghost/algolia-indexer` 12 | 13 | 14 | ## Usage 15 | 16 | 17 | ## Develop 18 | 19 | This is a mono repository, managed with [lerna](https://lernajs.io/). 20 | 21 | Follow the instructions for the top-level repo. 22 | 1. `git clone` this repo & `cd` into it as usual 23 | 2. Run `yarn` to install top-level dependencies. 24 | 25 | 26 | ## Run 27 | 28 | - `yarn dev` 29 | 30 | 31 | ## Test 32 | 33 | - `yarn lint` run just eslint 34 | - `yarn test` run lint and tests 35 | 36 | 37 | 38 | 39 | # Copyright & License 40 | 41 | Copyright (c) 2013-2025 Ghost Foundation - Released under the [MIT license](LICENSE). 42 | -------------------------------------------------------------------------------- /packages/algolia-indexer/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/IndexFactory'); 2 | -------------------------------------------------------------------------------- /packages/algolia-indexer/lib/IndexFactory.js: -------------------------------------------------------------------------------- 1 | const algoliaSearch = require('algoliasearch'); 2 | 3 | // Any defined settings will override those in the algolia UI 4 | // TODO: make this a custom setting 5 | const REQUIRED_SETTINGS = { 6 | // We chunk our pages into small algolia entries, and mark them as distinct by slug 7 | // This ensures we get one result per page, whichever is ranked highest 8 | distinct: true, 9 | attributeForDistinct: `slug`, 10 | // This ensures that chunks higher up on a page rank higher 11 | customRanking: [`desc(customRanking.heading)`, `asc(customRanking.position)`], 12 | // Defines the order algolia ranks various attributes in 13 | searchableAttributes: [`title`, `headings`, `html`, `url`, `tags.name`, `tags`, `authors.name`, `authors`], 14 | // Add slug to attributes we can filter by in order to find fragments to remove/delete 15 | attributesForFaceting: [`filterOnly(slug)`] 16 | }; 17 | 18 | /** 19 | * @param {options} options 20 | * @param {string} options.code 21 | * @param {number} options.statusCode 22 | * @param {object} options.originalError 23 | * @returns {Error} 24 | */ 25 | const AlgoliaError = ({code, statusCode, originalError}) => { 26 | let error = new Error({ message: 'Error processing Algolia' }); // eslint-disable-line 27 | 28 | error.errorType = 'AlgoliaError'; 29 | error.code = code; 30 | if (statusCode) { 31 | error.status = statusCode; 32 | } 33 | if (originalError.message) { 34 | error.message = originalError.message; 35 | } 36 | error.originalError = originalError; 37 | 38 | return error; 39 | }; 40 | 41 | /** 42 | * @typedef {object} AlgoliaSettings 43 | * @property {string} apiKey 44 | * @property {string} appId 45 | * @property {string} index 46 | * @property {object} indexSettings 47 | * @property {string} indexSettings.distinct 48 | * @property {string} indexSettings.attributeForDistinct 49 | * @property {string} indexSettings.customRanking 50 | * @property {string} indexSettings.searchableAttributes 51 | * @property {string} indexSettings.attributesForFaceting 52 | */ 53 | 54 | /** 55 | * @typedef IIndexFactory 56 | * @property {() => void} initClient 57 | * @property {() => Promise} initIndex 58 | * @property {({updateSettings?: boolean}) => Promise} setSettingsForIndex 59 | * @property {(fragments: object[]) => Promise} save 60 | * @property {(slug: string) => Promise} delete 61 | * @property {(fragments: object[]) => Promise} deleteObjects 62 | */ 63 | 64 | /** @implements IIndexFactory */ 65 | class IndexFactory { 66 | /** 67 | * @param {AlgoliaSettings} algoliaSettings 68 | */ 69 | constructor(algoliaSettings = {}) { 70 | if (!algoliaSettings.apiKey || !algoliaSettings.appId || !algoliaSettings.index || algoliaSettings.index.length < 1) { 71 | throw new Error('Algolia appId, apiKey, and index is required!'); // eslint-disable-line 72 | } 73 | this.index = []; 74 | this.options = algoliaSettings; 75 | 76 | this.options.indexSettings = algoliaSettings.indexSettings || REQUIRED_SETTINGS; 77 | } 78 | 79 | /** 80 | * @returns {void} 81 | */ 82 | initClient() { 83 | this.client = algoliaSearch(this.options.appId, this.options.apiKey); 84 | } 85 | 86 | /** 87 | * @returns {Promise} 88 | */ 89 | async initIndex() { 90 | this.initClient(); 91 | this.index = await this.client.initIndex(this.options.index); 92 | } 93 | 94 | /** 95 | * @param {object} options 96 | * @param {boolean?} options.updateSettings 97 | * @returns {Promise} 98 | */ 99 | async setSettingsForIndex(options = {}) { 100 | options.updateSettings = options?.updateSettings ?? true; 101 | 102 | try { 103 | await this.initIndex(); 104 | if (options.updateSettings) { 105 | await this.index.setSettings(this.options.indexSettings); 106 | } 107 | return await this.index.getSettings(); 108 | } catch (error) { 109 | throw AlgoliaError({code: error.code, statusCode: error.status, originalError: error}); 110 | } 111 | } 112 | 113 | /** 114 | * @param {object[]} fragments 115 | * @returns {Promise} 116 | */ 117 | async save(fragments) { 118 | console.log(`Saving ${fragments.length} fragments to Algolia index...`); // eslint-disable-line no-console 119 | try { 120 | await this.index.saveObjects(fragments); 121 | } catch (error) { 122 | throw AlgoliaError({code: error.code, statusCode: error.status, originalError: error}); 123 | } 124 | } 125 | 126 | /** 127 | * @param {string} slug 128 | * @returns {Promise} 129 | */ 130 | async delete(slug) { 131 | console.log(`Removing all fragments with post slug "${slug}"...`); // eslint-disable-line no-console 132 | try { 133 | await this.index.deleteBy({filters: `slug:${slug}`}); 134 | } catch (error) { 135 | throw AlgoliaError({code: error.code, statusCode: error.status, originalError: error}); 136 | } 137 | } 138 | 139 | /** 140 | * @param {object[]} fragments 141 | * @returns {Promise} 142 | */ 143 | async deleteObjects(fragments) { 144 | console.log(`Deleting ${fragments.length} fragments from Algolia index...`); // eslint-disable-line no-console 145 | try { 146 | await this.index.deleteObjects(fragments); 147 | } catch (error) { 148 | throw AlgoliaError({code: error.code, statusCode: error.status, originalError: error}); 149 | } 150 | } 151 | } 152 | 153 | module.exports = IndexFactory; 154 | -------------------------------------------------------------------------------- /packages/algolia-indexer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@tryghost/algolia-indexer", 3 | "version": "0.3.1", 4 | "repository": "https://github.com/TryGhost/algolia/tree/master/packages/algolia-indexer", 5 | "author": "Ghost Foundation", 6 | "license": "MIT", 7 | "main": "index.js", 8 | "scripts": { 9 | "dev": "echo \"Implement me!\"", 10 | "test": "NODE_ENV=testing mocha './test/**/*.test.js'", 11 | "lint": "eslint . --ext .js --cache", 12 | "posttest": "yarn lint" 13 | }, 14 | "files": [ 15 | "index.js", 16 | "lib" 17 | ], 18 | "publishConfig": { 19 | "access": "public" 20 | }, 21 | "devDependencies": { 22 | "mocha": "10.2.0", 23 | "should": "13.2.3", 24 | "sinon": "17.0.1" 25 | }, 26 | "dependencies": { 27 | "@tryghost/errors": "1.2.27", 28 | "algoliasearch": "4.20.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/algolia-indexer/test/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['ghost'], 3 | extends: [ 4 | 'plugin:ghost/test' 5 | ] 6 | }; 7 | -------------------------------------------------------------------------------- /packages/algolia-indexer/test/IndexFactory.test.js: -------------------------------------------------------------------------------- 1 | // Switch these lines once there are useful utils 2 | // const testUtils = require('./utils'); 3 | require('./utils'); 4 | const sinon = require('sinon'); 5 | 6 | const IndexFactory = require('../'); 7 | 8 | describe('IndexFactory', function () { 9 | let sandbox; 10 | let mockIndex; 11 | 12 | beforeEach(function () { 13 | sandbox = sinon.createSandbox(); 14 | 15 | // Mocking algoliasearch client and index 16 | mockIndex = { 17 | setSettings: sandbox.stub(), 18 | getSettings: sandbox.stub(), 19 | saveObjects: sandbox.stub(), 20 | deleteBy: sandbox.stub(), 21 | deleteObjects: sandbox.stub() 22 | }; 23 | }); 24 | 25 | afterEach(function () { 26 | sandbox.restore(); 27 | }); 28 | 29 | function createMockedAlgoliaIndex(settings) { 30 | const algoliaIndex = new IndexFactory(settings); 31 | 32 | // Immediately stub the initClient and initIndex methods after instantiation 33 | sandbox.stub(algoliaIndex, 'initClient').callsFake(function () { 34 | this.client = {}; // You can mock further if needed 35 | }); 36 | 37 | sandbox.stub(algoliaIndex, 'initIndex').callsFake(async function () { 38 | this.initClient(); 39 | this.index = mockIndex; 40 | }); 41 | 42 | return algoliaIndex; 43 | } 44 | 45 | it('throws error when settings are not passed', async function () { 46 | let algoliaIndex; 47 | try { 48 | algoliaIndex = await createMockedAlgoliaIndex(); 49 | } catch (error) { 50 | should.exist(error); 51 | should.not.exist(algoliaIndex); 52 | error.message.should.eql('Algolia appId, apiKey, and index is required!'); 53 | } 54 | }); 55 | 56 | describe('setSettingsForIndex', function () { 57 | it('updates settings by default', async function () { 58 | const algoliaIndex = await createMockedAlgoliaIndex({appId: 'test', apiKey: 'test', index: 'ALGOLIA'}); 59 | 60 | mockIndex.getSettings.resolves({some: 'settings'}); // Provide a mocked response for getSettings 61 | 62 | const settings = await algoliaIndex.setSettingsForIndex(); 63 | 64 | mockIndex.setSettings.should.have.been.called; 65 | mockIndex.getSettings.should.have.been.called; 66 | 67 | should.exist(settings); 68 | }); 69 | 70 | it('does not update Algolia settings when set to false', async function () { 71 | const algoliaIndex = await createMockedAlgoliaIndex({appId: 'test', apiKey: 'test', index: 'ALGOLIA'}); 72 | 73 | mockIndex.getSettings.resolves({some: 'settings'}); // Provide a mocked response for getSettings 74 | 75 | const settings = await algoliaIndex.setSettingsForIndex({updateSettings: false}); 76 | 77 | mockIndex.setSettings.should.have.not.been.called; 78 | mockIndex.getSettings.should.have.been.called; 79 | 80 | should.exist(settings); 81 | }); 82 | 83 | it('throws AlgoliaError when an error occurs', async function () { 84 | const algoliaIndex = await createMockedAlgoliaIndex({appId: 'test', apiKey: 'test', index: 'ALGOLIA'}); 85 | 86 | mockIndex.getSettings.rejects(new Error('Test Error')); // Simulating an error 87 | 88 | try { 89 | await algoliaIndex.setSettingsForIndex(); 90 | } catch (error) { 91 | should.exist(error); 92 | error.errorType.should.eql('AlgoliaError'); 93 | } 94 | }); 95 | }); 96 | 97 | // TODO: Add tests for the other methods like save, delete, deleteObjects, etc. 98 | }); 99 | -------------------------------------------------------------------------------- /packages/algolia-indexer/test/utils/assertions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom Should Assertions 3 | * 4 | * Add any custom assertions to this file. 5 | */ 6 | 7 | // Example Assertion 8 | // should.Assertion.add('ExampleAssertion', function () { 9 | // this.params = {operator: 'to be a valid Example Assertion'}; 10 | // this.obj.should.be.an.Object; 11 | // }); 12 | -------------------------------------------------------------------------------- /packages/algolia-indexer/test/utils/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test Utilities 3 | * 4 | * Shared utils for writing tests 5 | */ 6 | 7 | // Require overrides - these add globals for tests 8 | require('./overrides'); 9 | 10 | // Require assertions - adds custom should assertions 11 | require('./assertions'); 12 | -------------------------------------------------------------------------------- /packages/algolia-indexer/test/utils/overrides.js: -------------------------------------------------------------------------------- 1 | // This file is required before any test is run 2 | 3 | // Taken from the should wiki, this is how to make should global 4 | // Should is a global in our eslint test config 5 | global.should = require('should').noConflict(); 6 | should.extend(); 7 | 8 | // Sinon is a simple case 9 | // Sinon is a global in our eslint test config 10 | global.sinon = require('sinon'); 11 | -------------------------------------------------------------------------------- /packages/algolia-netlify/.env.example: -------------------------------------------------------------------------------- 1 | ALGOLIA_ACTIVE = "TRUE or FALSE. Set to TRUE to trigger indexing." 2 | ALGOLIA_APP_ID = "Algolia Application ID" 3 | ALGOLIA_API_KEY = "An Algolia Admin API Key or a generated one" 4 | ALGOLIA_INDEX = "Name of the Algolia index" 5 | NETLIFY_KEY = "User-defined key to authorize post requests to the Netlify function" 6 | -------------------------------------------------------------------------------- /packages/algolia-netlify/.nvmrc: -------------------------------------------------------------------------------- 1 | 18 2 | -------------------------------------------------------------------------------- /packages/algolia-netlify/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-2025 Ghost Foundation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/algolia-netlify/README.md: -------------------------------------------------------------------------------- 1 | # Algolia Netlify 2 | 3 | [Netlify Functions](https://www.netlify.com/products/functions/) to listen to [Ghost Webhooks](https://ghost.org/docs/api/webhooks/) on post changes and update defined [Algolia](https://www.algolia.com/) search index. 4 | 5 | ## Usage 6 | 7 | ### Set up Algolia 8 | 9 | First step is to grab the API keys and Application ID from Algolia. For the setup we need both, the "Search-Only API Key" as well as the "Admin API Key". 10 | 11 | The Admin API Key can either be the general one, or can be created just for this specific search index. 12 | 13 | If you decide to create a new API key, you want to make sure that the generated key has the following authorizations on your index: 14 | 15 | - Search (`search`) 16 | - Add records (`addObject`) 17 | - Delete records (`deleteObject`) 18 | - List indexes (`listIndexes`) 19 | - Delete index (`deleteIndex`) 20 | 21 | ### Set up Netlify Functions 22 | 23 | The Ghost Algolia tooling uses [Ghost Webhooks](https://ghost.org/docs/api/webhooks/) to index and update posts. The scripts that receive and process the webhooks are hosted by [Netlify Functions](https://www.netlify.com/products/functions/): 24 | 25 | 1. Deploy to Netlify by clicking on this button: 26 | [![Deploy to Netlify](https://www.netlify.com/img/deploy/button.svg)](https://app.netlify.com/start/deploy?repository=https://github.com/TryGhost/algolia) 27 | 2. Click 'Connect to Github' and give Netlify permission 28 | 3. Configure your site 29 | - Choose a repository name 30 | - Set 'TRUE' to trigger indexing 31 | - Algolia Application ID 32 | - The Algolia Admin API key or and API key with the permissions as described above 33 | - The name of the index you want to use 34 | - Set the `NETLIFY_KEY` to be used with the target URL 35 | 36 | ### Set up Ghost Webhooks 37 | 38 | Ghost webhooks will initiate posts to be indexed to Algolia. This can be a new entry, an update, or a removal. On Ghost's admin panel, create a new **Custom Integration** (Ghost Admin → Settings → Integrations → Custom Integrations) and the following **webhooks**: 39 | 40 | 1. `post.published` 41 | - Name: Post published 42 | - Event: Post published 43 | - Target URL: the endpoint of the post-published function, found on Netlify's admin panel plus the `NETLIFY_KEY` as a query parameter as defined in the configuration data above (https://YOUR-SITE-ID.netlify.com/.netlify/functions/post-published?key=NETLIFY_KEY) 44 | 45 | 2. `post.published.edited` 46 | - Name: Post updated 47 | - Event: Published post updated 48 | - Target URL: the endpoint of the post-published function, found on Netlify's admin panel plus the `NETLIFY_KEY` as a query parameter as defined in the configuration data above (https://YOUR-SITE-ID.netlify.com/.netlify/functions/post-published?key=NETLIFY_KEY) 49 | 50 | 3. `post.unpublished` 51 | - Name: Post unpublished 52 | - Event: Post unpublished 53 | - Target URL: the endpoint of the post-published function, found on Netlify's admin panel plus the `NETLIFY_KEY` as a query parameter as defined in the configuration data above (https://YOUR-SITE-ID.netlify.com/.netlify/functions/post-unpublished?key=NETLIFY_KEY) 54 | 55 | 4. `post.deleted` 56 | - Name: Post deleted 57 | - Event: Post deleted 58 | - Target URL: the endpoint of the post-published function, found on Netlify's admin panel plus the `NETLIFY_KEY` as a query parameter as defined in the configuration data above (https://YOUR-SITE-ID.netlify.com/.netlify/functions/post-unpublished?key=NETLIFY_KEY) 59 | 60 | These webhooks will trigger an index on every **future change of posts**. 61 | 62 | > To run an initial index of all the content, you can use the handy CLI from our Ghost Algolia tooling. Head over [here](https://github.com/TryGhost/algolia/tree/master/packages/algolia) and follow the instructions from there. 63 | 64 | 65 | ## Security 66 | 67 | To avoid unauthorized access to the Netlify functions endpoints, we highly recommend to setup the `NETLIFY_KEY` variable. This key is currently optional but will be enforced in the future. 68 | 69 | ## Develop 70 | 71 | This is a mono repository, managed with [lerna](https://lernajs.io/). 72 | 73 | Follow the instructions for the top-level repo. 74 | 1. `git clone` this repo & `cd` into it as usual 75 | 2. Run `yarn` to install top-level dependencies. 76 | 77 | To run this package locally, you will need to copy the existing `.env.example` file to `.env` and fill it with the correct keys. 78 | 79 | By running 80 | 81 | - `yarn serve` 82 | 83 | you will create a server on `localhost:9000` where your functions will be exposed to listen to (e. g. http://localhost:9000/.netlify/functions/post-unpublished), so you can use them in your local Ghost instance as Webhook target URL. 84 | 85 | 86 | ## Test 87 | 88 | - `yarn lint` run just eslint 89 | - `yarn test` run lint and tests 90 | 91 | 92 | # Copyright & License 93 | 94 | Copyright (c) 2013-2025 Ghost Foundation - Released under the [MIT license](LICENSE). 95 | -------------------------------------------------------------------------------- /packages/algolia-netlify/functions/post-published.js: -------------------------------------------------------------------------------- 1 | const IndexFactory = require('@tryghost/algolia-indexer'); 2 | const transforms = require('@tryghost/algolia-fragmenter'); 3 | 4 | exports.handler = async (event) => { 5 | const {key} = event.queryStringParameters; 6 | 7 | // TODO: Deprecate this in the future and make the key mandatory 8 | if (key && key !== process.env.NETLIFY_KEY) { 9 | return { 10 | statusCode: 401, 11 | body: `Unauthorized` 12 | }; 13 | } 14 | 15 | if (process.env.ALGOLIA_ACTIVE !== 'TRUE') { 16 | return { 17 | statusCode: 200, 18 | body: `Algolia is not activated` 19 | }; 20 | } 21 | 22 | if (!event.headers['user-agent'].includes('https://github.com/TryGhost/Ghost')) { 23 | return { 24 | statusCode: 401, 25 | body: `Unauthorized` 26 | }; 27 | } 28 | 29 | const algoliaSettings = { 30 | appId: process.env.ALGOLIA_APP_ID, 31 | apiKey: process.env.ALGOLIA_API_KEY, 32 | index: process.env.ALGOLIA_INDEX 33 | }; 34 | 35 | let {post} = JSON.parse(event.body); 36 | post = (post && Object.keys(post.current).length > 0 && post.current) || {}; 37 | 38 | if (!post || Object.keys(post).length < 1) { 39 | return { 40 | statusCode: 200, 41 | body: `No valid request body detected` 42 | }; 43 | } 44 | 45 | const node = []; 46 | 47 | // Transformer methods need an Array of Objects 48 | node.push(post); 49 | 50 | // Transform into Algolia object with the properties we need 51 | const algoliaObject = transforms.transformToAlgoliaObject(node); 52 | 53 | // Create fragments of the post 54 | const fragments = algoliaObject.reduce(transforms.fragmentTransformer, []); 55 | 56 | try { 57 | // Instanciate the Algolia indexer, which connects to Algolia and 58 | // sets up the settings for the index. 59 | const index = new IndexFactory(algoliaSettings); 60 | await index.setSettingsForIndex(); 61 | await index.save(fragments); 62 | console.log('Fragments successfully saved to Algolia index'); // eslint-disable-line no-console 63 | return { 64 | statusCode: 200, 65 | body: `Post "${post.title}" has been added to the index.` 66 | }; 67 | } catch (error) { 68 | console.log(error); // eslint-disable-line no-console 69 | return { 70 | statusCode: 500, 71 | body: JSON.stringify({msg: error.message}) 72 | }; 73 | } 74 | }; 75 | -------------------------------------------------------------------------------- /packages/algolia-netlify/functions/post-unpublished.js: -------------------------------------------------------------------------------- 1 | const IndexFactory = require('@tryghost/algolia-indexer'); 2 | 3 | exports.handler = async (event) => { 4 | const {key} = event.queryStringParameters; 5 | 6 | // TODO: Deprecate this in the future and make the key mandatory 7 | if (key && key !== process.env.NETLIFY_KEY) { 8 | return { 9 | statusCode: 401, 10 | body: `Unauthorized` 11 | }; 12 | } 13 | 14 | if (process.env.ALGOLIA_ACTIVE !== 'TRUE') { 15 | return { 16 | statusCode: 200, 17 | body: `Algolia is not activated` 18 | }; 19 | } 20 | 21 | if (!event.headers['user-agent'].includes('https://github.com/TryGhost/Ghost')) { 22 | return { 23 | statusCode: 401, 24 | body: `Unauthorized` 25 | }; 26 | } 27 | 28 | const algoliaSettings = { 29 | appId: process.env.ALGOLIA_APP_ID, 30 | apiKey: process.env.ALGOLIA_API_KEY, 31 | index: process.env.ALGOLIA_INDEX 32 | }; 33 | 34 | const {post} = JSON.parse(event.body); 35 | 36 | // Updated posts are in `post.current`, deleted are in `post.previous` 37 | const {slug} = (post.current && Object.keys(post.current).length && post.current) 38 | || (post.previous && Object.keys(post.previous).length && post.previous); 39 | 40 | if (!slug) { 41 | return { 42 | statusCode: 200, 43 | body: `No valid request body detected` 44 | }; 45 | } 46 | 47 | try { 48 | // Instanciate the Algolia indexer, which connects to Algolia and 49 | // sets up the settings for the index. 50 | const index = new IndexFactory(algoliaSettings); 51 | await index.initIndex(); 52 | await index.delete(slug); 53 | console.log(`Fragments for slug "${slug}" successfully removed from Algolia index`); // eslint-disable-line no-console 54 | return { 55 | statusCode: 200, 56 | body: `Post "${slug}" has been removed from the index.` 57 | }; 58 | } catch (error) { 59 | console.log(error); // eslint-disable-line no-console 60 | return { 61 | statusCode: 500, 62 | body: JSON.stringify({msg: error.message}) 63 | }; 64 | } 65 | }; 66 | -------------------------------------------------------------------------------- /packages/algolia-netlify/netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | base = "packages/algolia-netlify" 3 | command = "yarn build" 4 | functions = "build/functions" 5 | publish = "" 6 | 7 | [build.environment] 8 | NODE_VERSION = "18" 9 | AWS_LAMBDA_JS_RUNTIME = "nodejs18.x" 10 | 11 | [template.environment] 12 | ALGOLIA_ACTIVE = "TRUE or FALSE. Set to TRUE to trigger indexing." 13 | ALGOLIA_APP_ID = "Algolia Application ID" 14 | ALGOLIA_API_KEY = "An Algolia Admin API Key or a generated one" 15 | ALGOLIA_INDEX = "Name of the Algolia index" 16 | NETLIFY_KEY = "User-defined key to authorize post requests to the Netlify function" 17 | -------------------------------------------------------------------------------- /packages/algolia-netlify/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@tryghost/algolia-netlify", 3 | "version": "0.3.4", 4 | "repository": "https://github.com/TryGhost/algolia/tree/master/packages/algolia-netlify", 5 | "author": "Ghost Foundation", 6 | "license": "MIT", 7 | "main": "index.js", 8 | "scripts": { 9 | "dev": "yarn build && netlify dev", 10 | "test": "NODE_ENV=testing mocha './test/**/*.test.js'", 11 | "lint": "eslint . --ext .js --cache", 12 | "posttest": "yarn lint", 13 | "build": "NODE_ENV=production netlify functions:build --functions build/functions --src functions" 14 | }, 15 | "files": [ 16 | "index.js", 17 | "lib" 18 | ], 19 | "publishConfig": { 20 | "access": "public" 21 | }, 22 | "eslintConfig": { 23 | "extends": [ 24 | "airbnb-base", 25 | "plugin:ghost/browser" 26 | ], 27 | "plugins": [ 28 | "ghost" 29 | ] 30 | }, 31 | "eslintIgnore": [ 32 | "webpack.functions.js", 33 | "build/*" 34 | ], 35 | "devDependencies": { 36 | "eslint": "8.54.0", 37 | "eslint-config-airbnb-base": "15.0.0", 38 | "eslint-plugin-ghost": "3.4.0", 39 | "eslint-plugin-import": "2.29.0", 40 | "mocha": "10.2.0", 41 | "netlify-cli": "16.3.1", 42 | "should": "13.2.3", 43 | "sinon": "17.0.1" 44 | }, 45 | "dependencies": { 46 | "@tryghost/algolia-fragmenter": "^0.2.7", 47 | "@tryghost/algolia-indexer": "^0.3.1" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /packages/algolia-netlify/test/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['ghost'], 3 | extends: [ 4 | 'plugin:ghost/test' 5 | ] 6 | }; 7 | -------------------------------------------------------------------------------- /packages/algolia-netlify/test/hello.test.js: -------------------------------------------------------------------------------- 1 | // Switch these lines once there are useful utils 2 | // const testUtils = require('./utils'); 3 | require('./utils'); 4 | 5 | describe('Hello world', function () { 6 | it('Runs a test', function () { 7 | // TODO: Write me! 8 | 'hello'.should.eql('hello'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /packages/algolia-netlify/test/utils/assertions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom Should Assertions 3 | * 4 | * Add any custom assertions to this file. 5 | */ 6 | 7 | // Example Assertion 8 | // should.Assertion.add('ExampleAssertion', function () { 9 | // this.params = {operator: 'to be a valid Example Assertion'}; 10 | // this.obj.should.be.an.Object; 11 | // }); 12 | -------------------------------------------------------------------------------- /packages/algolia-netlify/test/utils/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test Utilities 3 | * 4 | * Shared utils for writing tests 5 | */ 6 | 7 | // Require overrides - these add globals for tests 8 | require('./overrides'); 9 | 10 | // Require assertions - adds custom should assertions 11 | require('./assertions'); 12 | -------------------------------------------------------------------------------- /packages/algolia-netlify/test/utils/overrides.js: -------------------------------------------------------------------------------- 1 | // This file is required before any test is run 2 | 3 | // Taken from the should wiki, this is how to make should global 4 | // Should is a global in our eslint test config 5 | global.should = require('should').noConflict(); 6 | should.extend(); 7 | 8 | // Sinon is a simple case 9 | // Sinon is a global in our eslint test config 10 | global.sinon = require('sinon'); 11 | -------------------------------------------------------------------------------- /packages/algolia/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['ghost'], 3 | extends: [ 4 | 'plugin:ghost/node' 5 | ] 6 | }; 7 | -------------------------------------------------------------------------------- /packages/algolia/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-2025 Ghost Foundation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/algolia/README.md: -------------------------------------------------------------------------------- 1 | # Algolia Ghost CLI 2 | 3 | CLI tool to initially index the full Ghost post content into an Algolia index. 4 | 5 | ## Install 6 | 7 | `npm install @tryghost/algolia --save` 8 | 9 | or 10 | 11 | `yarn add @tryghost/algolia` 12 | 13 | 14 | ## Usage 15 | 16 | To use the CLI, install the dependencies with `yarn install` or `npm install`. 17 | 18 | Copy the existing `example.config.json` to e. g. `config.json` and replace the relevant values for Ghost and Algolia. 19 | `indexSettings` reflects the current default settings and can either be overwritten, or removed from the config file. 20 | 21 | To run the batch index, run 22 | 23 | ```bash 24 | yarn algolia index [options] 25 | ``` 26 | 27 | ### Caveats 28 | 29 | The [Fragmenter](https://github.com/TryGhost/algolia/tree/master/packages/algolia-fragmenter) breaks down large HTML pieces into smaller chunks by its headings. Sometimes the fragment is still too big and Algolia will throw an error listing the post id that caused the large fragment. The post id can be used to get the post slug, which then can be excluded from the batch run like this: 30 | 31 | ```bash 32 | yarn algolia index -s post-slug-to-exclude,and-another-post-slug-to-exclude 33 | ``` 34 | 35 | ### Flags 36 | 37 | - `pathToConfig`, needs to be the relative (from this package) path to the config JSON file that contains the Ghost and Algolia API keys and settings (see [usage](#usage) above) 38 | 39 | - `-s, --skip`, takes a comma separated list of post slugs that need to be **excluded** from the index (see [caveats](#caveats) above) 40 | 41 | - `-V, --verbose`, switches on verbose mode, but there's not much too see here (yet) 42 | - `-l, --limit`, limit the amount of posts to receive. Default is 'all' 43 | - `-p --page`, define the page to fetch posts from. To be used in combination with `limit`. 44 | - `-sjs --skipjsonslugs`, uses a list of slugs in `config.json` to skip before they're uploaded. This method will request all data from Ghost and skip at the point it would normally upload to Algolia. If you're getting `414 Request-URI Too Large` errors using `-s`, this is the method to use. 45 | 46 | ## Develop 47 | 48 | This is a mono repository, managed with [lerna](https://lernajs.io/). 49 | 50 | Follow the instructions for the top-level repo. 51 | 1. `git clone` this repo & `cd` into it as usual 52 | 2. Run `yarn` to install top-level dependencies. 53 | 54 | 55 | ## Run 56 | 57 | - `yarn dev` 58 | 59 | 60 | ## Test 61 | 62 | - `yarn lint` run just eslint 63 | - `yarn test` run lint and tests 64 | 65 | 66 | 67 | 68 | # Copyright & License 69 | 70 | Copyright (c) 2013-2025 Ghost Foundation - Released under the [MIT license](LICENSE). 71 | -------------------------------------------------------------------------------- /packages/algolia/bin/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const prettyCLI = require('@tryghost/pretty-cli'); 3 | const ui = require('@tryghost/pretty-cli').ui; 4 | const fs = require('fs-extra'); 5 | const utils = require('../lib/utils'); 6 | const GhostContentAPI = require('@tryghost/content-api'); 7 | const transforms = require('@tryghost/algolia-fragmenter'); 8 | const IndexFactory = require('@tryghost/algolia-indexer'); 9 | 10 | prettyCLI.preface('Command line utilities to batch index content from Ghost to Algolia'); 11 | 12 | prettyCLI.command({ 13 | id: 'algolia', 14 | flags: 'index ', 15 | desc: 'Run a batch index of all Ghost posts to Algolia', 16 | paramsDesc: ['Path to a valid config JSON file'], 17 | setup: (sywac) => { 18 | sywac.boolean('-V --verbose', { 19 | defaultValue: false, 20 | desc: 'Show verbose output' 21 | }); 22 | sywac.array('-s --skip', { 23 | defaultValue: [], 24 | desc: 'Comma separated list of post slugs to exclude from indexing' 25 | }); 26 | sywac.number('-l --limit', { 27 | desc: 'Amount of posts we want to fetch from Ghost' 28 | }); 29 | sywac.number('-p --page', { 30 | desc: 'Use page to navigate through posts when setting a limit' 31 | }); 32 | sywac.array('-sjs --skipjsonslugs', { 33 | defaultValue: false, 34 | desc: 'Exclude post slugs from config JSON file' 35 | }); 36 | }, 37 | run: async (argv) => { 38 | const mainTimer = Date.now(); 39 | let context = {errors: [], posts: []}; 40 | 41 | if (argv.verbose) { 42 | ui.log.info(`Received config file ${argv.pathToConfig}`); 43 | } 44 | 45 | // 1. Read the config files and verify everything 46 | try { 47 | const config = await fs.readJSON(argv.pathToConfig); 48 | context = Object.assign(context, config); 49 | 50 | utils.verifyConfig(context); 51 | } catch (error) { 52 | context.errors.push(error); 53 | return ui.log.error('Failed loading JSON config file:', context.errors); 54 | } 55 | 56 | // 2. Fetch all posts from the Ghost instance 57 | try { 58 | const timer = Date.now(); 59 | const params = {limit: 'all', include: 'tags,authors'}; 60 | const ghost = new GhostContentAPI({ 61 | url: context.ghost.apiUrl, 62 | key: context.ghost.apiKey, 63 | version: 'canary' 64 | }); 65 | 66 | if (argv.skip && argv.skip.length > 0) { 67 | const filterSlugs = argv.skip.join(','); 68 | 69 | params.filter = `slug:-[${filterSlugs}]`; 70 | } 71 | 72 | if (argv.limit) { 73 | params.limit = argv.limit; 74 | } 75 | 76 | ui.log.info(`Fetching ${params.limit} posts from Ghost...`); 77 | 78 | if (argv.page) { 79 | ui.log.info(`...from page #${argv.page}.`); 80 | params.page = argv.page; 81 | } 82 | 83 | context.posts = await ghost.posts.browse(params); 84 | 85 | ui.log.info(`Done fetching posts in ${Date.now() - timer}ms.`); 86 | } catch (error) { 87 | context.errors.push(error); 88 | return ui.log.error('Could not fetch posts from Ghost', context.errors); 89 | } 90 | 91 | // 3. Transform into Algolia objects and create fragments 92 | try { 93 | const timer = Date.now(); 94 | 95 | ui.log.info('Transforming and fragmenting posts...'); 96 | 97 | if (argv.skipjsonslugs) { 98 | const ignoreSlugsCount = context.ignore_slugs.length; 99 | 100 | ui.log.info(`Skipping the ${ignoreSlugsCount} slugs in ${argv.pathToConfig}`); 101 | } 102 | 103 | context.posts = transforms.transformToAlgoliaObject(context.posts, context.ignore_slugs); 104 | 105 | context.fragments = context.posts.reduce(transforms.fragmentTransformer, []); 106 | 107 | // we don't need the posts anymore 108 | delete context.posts; 109 | 110 | ui.log.info(`Done transforming and fragmenting posts in ${Date.now() - timer}ms.`); 111 | } catch (error) { 112 | context.errors.push(error); 113 | return ui.log.error('Error fragmenting posts', context.errors); 114 | } 115 | 116 | // 4. Save to Algolia 117 | try { 118 | let timer = Date.now(); 119 | 120 | ui.log.info('Connecting to Algolia index and setting it up...'); 121 | 122 | // Instanciate the Algolia indexer, which connects to Algolia and 123 | const index = new IndexFactory(context.algolia); 124 | // sets up the settings for the index. 125 | await index.setSettingsForIndex(); 126 | 127 | ui.log.info(`Done setting up Alolia index in ${Date.now() - timer}ms.`); 128 | 129 | timer = Date.now(); 130 | 131 | ui.log.info('Saving fragments to Algolia...'); 132 | 133 | await index.save(context.fragments); 134 | 135 | ui.log.ok(`${context.fragments.length} Fragments successfully saved to Algolia index in ${Date.now() - timer}ms.`); 136 | } catch (error) { 137 | context.errors.push(error); 138 | return ui.log.error('Error saving fragments', context.errors); 139 | } 140 | 141 | // Report success 142 | ui.log.ok(`Successfully indexed all the things in ${Date.now() - mainTimer}ms.`); 143 | } 144 | }); 145 | 146 | prettyCLI.style({ 147 | usageCommandPlaceholder: () => '' 148 | }); 149 | 150 | prettyCLI.groupOrder([ 151 | 'Commands:', 152 | 'Arguments:', 153 | 'Required Options:', 154 | 'Options:', 155 | 'Global Options:' 156 | ]); 157 | 158 | prettyCLI.parseAndExit(); 159 | -------------------------------------------------------------------------------- /packages/algolia/example.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "ghost": { 3 | "apiKey": "", 4 | "apiUrl": "" 5 | }, 6 | "ignore_slugs": [ 7 | "the-slug-of-a-post-with-a-huge-amount-of-content-that-will-cause-errors" 8 | ], 9 | "algolia": { 10 | "index": "", 11 | "apiKey": "", 12 | "appId": "", 13 | "indexSettings": { 14 | "distinct": true, 15 | "attributeForDistinct": "slug", 16 | "customRanking": [ 17 | "desc(customRanking.heading)", 18 | "asc(customRanking.position)" 19 | ], 20 | "searchableAttributes": [ 21 | "title", 22 | "headings", 23 | "html", 24 | "url", 25 | "tags.name", 26 | "tags", 27 | "authors.name", 28 | "authors" 29 | ], 30 | "attributesForFaceting": [ 31 | "filterOnly(slug)" 32 | ] 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /packages/algolia/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TryGhost/algolia/7b875425981d7c8b3f4f0d6afd05c0c995f9d4fc/packages/algolia/index.js -------------------------------------------------------------------------------- /packages/algolia/lib/utils.js: -------------------------------------------------------------------------------- 1 | const errors = require('@tryghost/errors'); 2 | 3 | module.exports.verifyConfig = ({ghost, algolia}) => { 4 | if (!ghost || !algolia) { 5 | throw new errors.BadRequestError({message: 'Config has the wrong format. Check `example.json` for reference.'}); 6 | } 7 | 8 | // Check for all Ghost keys 9 | if (!ghost.apiKey || !ghost.apiUrl) { 10 | throw new errors.BadRequestError({message: 'Ghost apiUrl or apiKey are missing.'}); 11 | } 12 | 13 | // Check for all Ghost keys 14 | if (!algolia.apiKey || !algolia.appId || !algolia.index) { 15 | throw new errors.BadRequestError({message: 'Algolia index, appId or apiKey are missing.'}); 16 | } 17 | 18 | if (algolia.indexSettings && Object.keys(algolia.indexSettings) < 1) { 19 | throw new errors.BadRequestError({message: 'Algolia indexSettings are empty. Please remove or provide own settings.'}); 20 | } 21 | 22 | return; 23 | }; 24 | -------------------------------------------------------------------------------- /packages/algolia/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@tryghost/algolia", 3 | "version": "0.2.11", 4 | "repository": "https://github.com/TryGhost/algolia/tree/master/packages/algolia", 5 | "author": "Ghost Foundation", 6 | "license": "MIT", 7 | "bin": { 8 | "algolia": "./bin/cli.js" 9 | }, 10 | "scripts": { 11 | "dev": "algolia", 12 | "algolia": "algolia", 13 | "test": "NODE_ENV=testing mocha './test/**/*.test.js'", 14 | "lint": "eslint . --ext .js --cache", 15 | "posttest": "yarn lint" 16 | }, 17 | "files": [ 18 | "bin", 19 | "index.js", 20 | "lib" 21 | ], 22 | "publishConfig": { 23 | "access": "public" 24 | }, 25 | "devDependencies": { 26 | "mocha": "10.2.0", 27 | "should": "13.2.3", 28 | "sinon": "17.0.1" 29 | }, 30 | "dependencies": { 31 | "@tryghost/algolia-fragmenter": "^0.2.7", 32 | "@tryghost/algolia-indexer": "^0.3.1", 33 | "@tryghost/content-api": "1.11.20", 34 | "@tryghost/errors": "1.2.27", 35 | "@tryghost/pretty-cli": "1.2.39", 36 | "fs-extra": "11.1.1" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /packages/algolia/test/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['ghost'], 3 | extends: [ 4 | 'plugin:ghost/test' 5 | ] 6 | }; 7 | -------------------------------------------------------------------------------- /packages/algolia/test/hello.test.js: -------------------------------------------------------------------------------- 1 | // Switch these lines once there are useful utils 2 | // const testUtils = require('./utils'); 3 | require('./utils'); 4 | 5 | describe('Hello world', function () { 6 | it('Runs a test', function () { 7 | // TODO: Write me! 8 | 'hello'.should.eql('hello'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /packages/algolia/test/utils/assertions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom Should Assertions 3 | * 4 | * Add any custom assertions to this file. 5 | */ 6 | 7 | // Example Assertion 8 | // should.Assertion.add('ExampleAssertion', function () { 9 | // this.params = {operator: 'to be a valid Example Assertion'}; 10 | // this.obj.should.be.an.Object; 11 | // }); 12 | -------------------------------------------------------------------------------- /packages/algolia/test/utils/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test Utilities 3 | * 4 | * Shared utils for writing tests 5 | */ 6 | 7 | // Require overrides - these add globals for tests 8 | require('./overrides'); 9 | 10 | // Require assertions - adds custom should assertions 11 | require('./assertions'); 12 | -------------------------------------------------------------------------------- /packages/algolia/test/utils/overrides.js: -------------------------------------------------------------------------------- 1 | // This file is required before any test is run 2 | 3 | // Taken from the should wiki, this is how to make should global 4 | // Should is a global in our eslint test config 5 | global.should = require('should').noConflict(); 6 | should.extend(); 7 | 8 | // Sinon is a simple case 9 | // Sinon is a global in our eslint test config 10 | global.sinon = require('sinon'); 11 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "@tryghost:quietJS" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /test/hello.test.js: -------------------------------------------------------------------------------- 1 | // Switch these lines once there are useful utils 2 | // const testUtils = require('./utils'); 3 | require('./utils'); 4 | 5 | describe('Hello world', function () { 6 | it('Runs a test', function () { 7 | // TODO: Write me! 8 | 'hello'.should.eql('hello'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /test/utils/assertions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom Should Assertions 3 | * 4 | * Add any custom assertions to this file. 5 | */ 6 | 7 | // Example Assertion 8 | // should.Assertion.add('ExampleAssertion', function () { 9 | // this.params = {operator: 'to be a valid Example Assertion'}; 10 | // this.obj.should.be.an.Object; 11 | // }); 12 | -------------------------------------------------------------------------------- /test/utils/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Test Utilities 3 | * 4 | * Shared utils for writing tests 5 | */ 6 | 7 | // Require overrides - these add globals for tests 8 | require('./overrides'); 9 | 10 | // Require assertions - adds custom should assertions 11 | require('./assertions'); 12 | -------------------------------------------------------------------------------- /test/utils/overrides.js: -------------------------------------------------------------------------------- 1 | // This file is required before any test is run 2 | 3 | // Taken from the should wiki, this is how to make should global 4 | // Should is a global in our eslint test config 5 | global.should = require('should').noConflict(); 6 | should.extend(); 7 | 8 | // Sinon is a simple case 9 | // Sinon is a global in our eslint test config 10 | global.sinon = require('sinon'); 11 | --------------------------------------------------------------------------------