├── .eslintignore
├── output
└── .gitignore
├── .prettierignore
├── .prettierrc
├── assets
├── logo.png
├── query1.png
├── query2.png
├── Screenshot.png
└── yamlScreenshot.png
├── src
├── etc
│ └── .example.env
├── config
│ ├── winston.ts
│ └── config.ts
├── graphql
│ ├── resolvers
│ │ └── csv.ts
│ └── schema.ts
├── service
│ └── csv.ts
├── server.ts
└── lib
│ └── graphQLFactories.ts
├── .gitignore
├── example.hcsvconfig.yaml
├── tsconfig.json
├── .eslintrc.js
├── package.json
└── README.md
/.eslintignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | build/
3 |
--------------------------------------------------------------------------------
/output/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | */
3 | !.gitignore
4 |
--------------------------------------------------------------------------------
/.prettierignore:
--------------------------------------------------------------------------------
1 | *.sql
2 | README.md
3 | build/
4 | node_modules/
5 |
--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "singleQuote": false,
3 | "printWidth": 80
4 | }
5 |
--------------------------------------------------------------------------------
/assets/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/happy-machine/hasura-csv/HEAD/assets/logo.png
--------------------------------------------------------------------------------
/assets/query1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/happy-machine/hasura-csv/HEAD/assets/query1.png
--------------------------------------------------------------------------------
/assets/query2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/happy-machine/hasura-csv/HEAD/assets/query2.png
--------------------------------------------------------------------------------
/assets/Screenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/happy-machine/hasura-csv/HEAD/assets/Screenshot.png
--------------------------------------------------------------------------------
/assets/yamlScreenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/happy-machine/hasura-csv/HEAD/assets/yamlScreenshot.png
--------------------------------------------------------------------------------
/src/etc/.example.env:
--------------------------------------------------------------------------------
1 | # global
2 | APP_NAME=hasura-csv
3 |
4 | # ports
5 | HOST=0.0.0.0
6 | PORT=5000
7 | NODE_ENV=development
8 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | .env
3 | hcsvconfig.yaml
4 | /etc/.env
5 | /logs/
6 | .vscode/
7 | builds/
8 | build/
9 | cache/
10 | *.swp
11 | .DS_Store
12 |
--------------------------------------------------------------------------------
/src/config/winston.ts:
--------------------------------------------------------------------------------
1 | import winston from "winston";
2 |
3 | export default winston.createLogger({
4 | transports: [new winston.transports.Console()],
5 | });
6 |
--------------------------------------------------------------------------------
/src/graphql/resolvers/csv.ts:
--------------------------------------------------------------------------------
1 | import { csvWriter } from "../../service/csv";
2 |
3 | export default async (graphQLArgs) => {
4 | const { args, context, info } = graphQLArgs;
5 | const writer = await csvWriter(
6 | `${info.fieldName}_${context.timestamp}.csv`,
7 | Object.keys(args),
8 | context.timestamp
9 | );
10 | await writer.writeRecords([args]);
11 | };
12 |
--------------------------------------------------------------------------------
/example.hcsvconfig.yaml:
--------------------------------------------------------------------------------
1 | resolvers:
2 | - name: person_csv
3 | resolver_target: csv
4 | fields:
5 | - arg: age
6 | type: integer
7 | - arg: name
8 | type: string
9 | - name: giraffe_csv
10 | resolver_target: csv
11 | fields:
12 | - arg: eye_colour
13 | type: integer
14 | - arg: neck_length
15 | type: float
16 |
--------------------------------------------------------------------------------
/src/config/config.ts:
--------------------------------------------------------------------------------
1 | import dotenv from "dotenv";
2 |
3 | dotenv.config({ path: "./src/etc/.env" });
4 |
5 | export const OUTPUT_DIR = process.env.OUTPUT_DIR;
6 | export const NODE_ENV = process.env.NODE_ENV;
7 | export const HOST = process.env.HOST;
8 | export const MODE = process.env.MODE;
9 | export const PORT = Number(process.env.PORT);
10 | export const APP_NAME = process.env.APP_NAME;
11 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "module": "commonjs",
4 | "esModuleInterop": true,
5 | "target": "es2019",
6 | "moduleResolution": "node",
7 | "sourceMap": true,
8 | "outDir": "build"
9 | },
10 | "lib": ["dom", "esnext", "es2017", "esnext", "esnext.asynciterable"],
11 | "typeRoots": ["node_modules/@types"],
12 | "include": ["src/**/*.ts", "src/**/*.json", "src/**/*.js"]
13 | }
14 |
--------------------------------------------------------------------------------
/src/graphql/schema.ts:
--------------------------------------------------------------------------------
1 | import { GraphQLObjectType, GraphQLSchema } from "graphql";
2 | import { graphQLFieldFactory } from "../lib/graphQLFactories";
3 | import yaml from "js-yaml";
4 | import fs from "fs";
5 |
6 | const doc = yaml.safeLoad(fs.readFileSync("./hcsvconfig.yaml", "utf8"));
7 |
8 | const RootQuery = new GraphQLObjectType({
9 | name: "RootQueryType",
10 | fields: graphQLFieldFactory(doc),
11 | });
12 |
13 | export const schema = new GraphQLSchema({
14 | query: RootQuery,
15 | });
16 |
--------------------------------------------------------------------------------
/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | env: {
3 | browser: true,
4 | es6: true,
5 | node: true
6 | },
7 | extends: [
8 | "eslint:recommended",
9 | "plugin:@typescript-eslint/eslint-recommended",
10 | "plugin:prettier/recommended"
11 | ],
12 | globals: {
13 | Atomics: "readonly",
14 | SharedArrayBuffer: "readonly"
15 | },
16 | parser: "@typescript-eslint/parser",
17 | parserOptions: {
18 | ecmaVersion: 2018,
19 | sourceType: "module"
20 | },
21 | plugins: ["@typescript-eslint"],
22 | rules: {
23 | // https://stackoverflow.com/questions/55280555/typescript-eslint-eslint-plugin-error-route-is-defined-but-never-used-no-un
24 | "no-unused-vars": "off",
25 | "@typescript-eslint/no-unused-vars": "error"
26 | }
27 | };
28 |
--------------------------------------------------------------------------------
/src/service/csv.ts:
--------------------------------------------------------------------------------
1 | import { createObjectCsvWriter } from "csv-writer";
2 | import { OUTPUT_DIR } from "../config/config";
3 | import logger from "../config/winston";
4 | import fs from "fs";
5 |
6 | const fsPromises = fs.promises;
7 | let currentTimeStamp;
8 |
9 | export const csvWriter = async (filename, keys, timestamp) => {
10 | /**
11 | * Use the timestamp from the request to chose filename to use
12 | ***/
13 |
14 | const path = `${OUTPUT_DIR}${filename}`;
15 | try {
16 | await fsPromises.writeFile(path, "", { flag: "wx" });
17 | } catch (e) {
18 | if (!e.code.includes("EEXIST")) {
19 | logger.error(e);
20 | }
21 | }
22 | const out = createObjectCsvWriter({
23 | path,
24 | header: keys.map((key) => ({ id: key, title: key })),
25 | append: currentTimeStamp == timestamp ? true : false,
26 | });
27 | currentTimeStamp = timestamp;
28 | return out;
29 | };
30 |
--------------------------------------------------------------------------------
/src/server.ts:
--------------------------------------------------------------------------------
1 | import logger from "./config/winston";
2 | import express from "express";
3 | import { graphqlHTTP } from "express-graphql";
4 | import cors from "cors";
5 | import { schema } from "./graphql/schema";
6 | import { HOST, PORT, NODE_ENV, APP_NAME } from "./config/config";
7 | const app = express();
8 |
9 | app.use(
10 | cors({
11 | origin: "*",
12 | })
13 | );
14 |
15 | app.use(express.json());
16 | app.use(express.urlencoded({ extended: false }));
17 |
18 | app.get("/alive", (req, res) => {
19 | res.status(200).send(JSON.stringify({ data: "run" }));
20 | logger.debug(`${APP_NAME} is alive.`);
21 | });
22 |
23 | app.use(
24 | "/graphql",
25 | [],
26 | graphqlHTTP(() => ({
27 | schema,
28 | graphiql: false,
29 | context: {
30 | timestamp: Date.now(),
31 | },
32 | }))
33 | );
34 |
35 | app.use("*", (req, res) => {
36 | logger.error("No path found at ", req.path);
37 | return res.status(404).json({ message: `${APP_NAME}: Path not found` });
38 | });
39 |
40 | app.listen(PORT, HOST, () => {
41 | logger.info(`${APP_NAME} listening on port ${PORT} in ${NODE_ENV} mode.`);
42 | });
43 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hasura-csv",
3 | "version": "0.0.1",
4 | "description": "Utility to transform hasura resolver output to csv files",
5 | "main": "build/server.js",
6 | "scripts": {
7 | "start": "MODE=local nodemon --watch 'src/**/*.ts' --exec 'ts-node' src/server.ts",
8 | "build": "tsc",
9 | "test": "NODE_ENV=test PORT=5001 MODE=local mocha -r ts-node/register ./src/tests/*.ts --exit",
10 | "test-watch": "NODE_ENV=test PORT=5001 MODE=local mocha -r ts-node/register ./src/tests/*.ts --watch --watch-extensions ts --reporter min",
11 | "lint": "eslint --ext .js,.ts ./ || prettier --write */**",
12 | "lint-ts": "eslint --ext .js,.ts ./",
13 | "prettier": "prettier --write */**"
14 | },
15 | "author": "happy-machine",
16 | "license": "ISC",
17 | "dependencies": {
18 | "axios": "^0.21.0",
19 | "cors": "^2.8.5",
20 | "csv-writer": "^1.6.0",
21 | "dotenv": "^8.2.0",
22 | "express": "^4.17.1",
23 | "express-graphql": "^0.11.0",
24 | "fluent-logger": "^3.4.1",
25 | "fs": "0.0.1-security",
26 | "graphql": "^15.4.0",
27 | "js-yaml": "^3.14.0",
28 | "nodemon": "^2.0.6",
29 | "ts-node": "^9.0.0",
30 | "winston": "^3.3.3"
31 | },
32 | "devDependencies": {
33 | "@types/cors": "^2.8.8",
34 | "@types/express": "^4.17.8",
35 | "@types/graphql": "^14.5.0",
36 | "eslint": "^7.12.1",
37 | "eslint-config-prettier": "^6.15.0",
38 | "eslint-plugin-prettier": "^3.1.4",
39 | "mocha": "^8.2.1",
40 | "prettier": "^2.1.2",
41 | "typescript": "^4.0.5"
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/lib/graphQLFactories.ts:
--------------------------------------------------------------------------------
1 | import logger from "../config/winston";
2 | import {
3 | GraphQLInt,
4 | GraphQLFloat,
5 | GraphQLID,
6 | GraphQLString,
7 | GraphQLObjectType,
8 | } from "graphql";
9 |
10 | const types = {
11 | string: GraphQLString,
12 | integer: GraphQLInt,
13 | float: GraphQLFloat,
14 | id: GraphQLID,
15 | };
16 |
17 | export const graphQLTypeFactory = (resolver) => {
18 | const mappedFields = resolver.fields.map((field) => ({
19 | [field.arg]: { type: types[field.type] },
20 | }));
21 | mappedFields.push({ status: { type: GraphQLString } });
22 | return new GraphQLObjectType({
23 | name: resolver.name,
24 | fields: () => Object.assign({}, ...mappedFields),
25 | });
26 | };
27 |
28 | export const graphQLFieldFactory = (doc) => {
29 | try {
30 | const out = doc.resolvers.map((resolver) => {
31 | return {
32 | [resolver.name]: {
33 | type: graphQLTypeFactory(resolver),
34 | args: Object.assign(
35 | {},
36 | ...resolver.fields.map((field) => ({
37 | [field.arg]: {
38 | type: types[field.type],
39 | },
40 | }))
41 | ),
42 | async resolve(_, args, context, info) {
43 | try {
44 | const graphQLResolver = await import(
45 | `../graphql/resolvers/${resolver.resolver_target}.ts`
46 | );
47 | await graphQLResolver.default({
48 | args,
49 | context,
50 | info,
51 | });
52 | return { ...args, status: "done" };
53 | } catch (e) {
54 | logger.error(e);
55 | return e;
56 | }
57 | },
58 | },
59 | };
60 | });
61 | return Object.assign({}, ...out);
62 | } catch (e) {
63 | logger.error(e);
64 | }
65 | };
66 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Hasura-csv
2 | 
3 |
4 | ### What is hasura-csv?
5 |
6 | One of the many ways to use Hasura is to allow power users to interact with an API through Hasura instead of a website designed primarily for visualising data. One often requested feature that was particularly useful for my team was the ability to download CSV data which a site might normally provide a browser interface for.
7 |
8 | This service will spin up a remote GraphQL schema from a yaml file which will allow the quick creation of custom field/s that can forward the results from a Hasura resolver to the hasura-csv service, where it will be written to the output folder as a csv file.
9 |
10 | #### How do i use it?
11 |
12 | Lets say that I have a Person resolver in my Hasura schema which looks like this
13 |
14 | 
15 |
16 | But I only want their names and ages.
17 |
18 | First of all I create the following hcsvconfig.yaml (copy the example.hcsvconfig.yaml for a quick leg up).
19 |
20 | I need to create a target resolver in my yaml which Hasura will connect to.
21 |
22 | **!!!This must be a different name from the actual resolver we want to output from!!!**
23 |
24 | (i use person_csv here for the Person resolver).
25 |
26 | 
27 |
28 | Here i've also added a seperate link for the giraffe resolver 🦒 add as many fields and resolver links as you like. Each remote resolver will correspond to one Hasura resolver.
29 |
30 | **npm install** and run the service with **npm run start**.
31 |
32 | - Go to your Hasura console and select **REMOTE SCHEMAS**.
33 | - Enter the url of hasura-csv (by default this will be http://localhost:5000/graphql)
34 | - Name your remote schema (i generally use 'CSV')
35 | - Select the resolver you wish to output in the **DATA** tab and select **relationships** and **Add a remote schema relationship**
36 |
37 | Now name the field which you will want to use in the Person resolver to indicate you want to also send the output of the resolver to hasura-csvs person-csv, i use CSV (this can be the same for every table if you want)
38 |
39 | 
40 |
41 | Now to output to hasura-csv simply add the CSV field to the resolver you have linked it to in a UI query, for example:
42 |
43 | 
44 |
45 |
46 | *The subfields will only affect the display in Hasura, the linked fields only will be transmitted, status is the optional 'OK' response from the service.*
47 |
48 | The output will be written to './output/person_csv_{{timestamp}}'
49 |
50 |
51 | #### Setting up the API:
52 |
53 |
54 | >cp ./etc/.example-env ./etc/.env
55 | >cp ./example.hcsvconfig.yaml ./hcsvconfig.yaml
56 |
57 | Then set secrets and paths in .env, and setup your yaml configuration before running.
58 |
59 | #### TBD:
60 |
61 |
62 | - Typescript is not yet doing much.
63 |
64 |
65 | - I have only tested this on a couple of thousand rows, please feel free to PR :pray:
66 |
67 |
68 | #### Script commands:
69 |
70 | **npm start**
71 | **npm run build** build only
72 | **npm run prettier** prettier code formatting
73 | **npm run lint** run linter
74 | **npm run lint-ts** run linter on typescript
75 |
--------------------------------------------------------------------------------