2 |
3 |
108 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: "3.8"
2 | services:
3 | # When scaling the opal-server to multiple nodes and/or multiple workers, we use
4 | # a *broadcast* channel to sync between all the instances of opal-server.
5 | # Under the hood, this channel is implemented by encode/broadcaster (see link below).
6 | # At the moment, the broadcast channel can be either: postgresdb, redis or kafka.
7 | # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here:
8 | # https://github.com/encode/broadcaster#available-backends
9 | broadcast_channel:
10 | image: postgres:alpine
11 | environment:
12 | - POSTGRES_DB=postgres
13 | - POSTGRES_USER=postgres
14 | - POSTGRES_PASSWORD=postgres
15 | # example postgresdb with data that opal-client can pull and store in OPA cache
16 | example_db:
17 | image: postgres
18 | restart: always
19 | environment:
20 | - POSTGRES_DB=postgres
21 | - POSTGRES_USER=postgres
22 | - POSTGRES_PASSWORD=postgres
23 | logging:
24 | options:
25 | max-size: 10m
26 | max-file: "3"
27 | ports:
28 | - '5438:5432'
29 | volumes:
30 | # copy the sql script to create tables
31 | - ./sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
32 | # copy the sql script to fill tables
33 | - ./sql/fill_tables.sql:/docker-entrypoint-initdb.d/fill_tables.sql
34 | opal_server:
35 | # by default we run opal-server from latest official image
36 | image: permitio/opal-server:latest
37 | environment:
38 | # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel)
39 | - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres
40 | # number of uvicorn workers to run inside the opal-server container
41 | - UVICORN_NUM_WORKERS=4
42 | # the git repo hosting our policy
43 | # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`)
44 | # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy
45 | # - for more info, see: https://github.com/permitio/opal/blob/master/docs/HOWTO/track_a_git_repo.md
46 | - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo
47 | # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy).
48 | # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits.
49 | # for more info see: https://github.com/permitio/opal/blob/master/docs/HOWTO/track_a_git_repo.md
50 | - OPAL_POLICY_REPO_POLLING_INTERVAL=30
51 | # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc).
52 | # the data sources represents from where the opal clients should get a "complete picture" of the data they need.
53 | # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server.
54 | # Add "fetch_key":"city_id" to get an object with keys of city_id, rather than a list
55 | - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"postgresql://postgres@example_db:5432/postgres","config":{"fetcher":"PostgresFetchProvider","query":"SELECT * from city;","connection_params":{"password":"postgres"}},"topics":["policy_data"],"dst_path":"cities"}]}}
56 | ports:
57 | # exposes opal server on the host machine, you can access the server at: http://localhost:7002
58 | - "7002:7002"
59 | depends_on:
60 | - broadcast_channel
61 | opal_client:
62 | # by default we run opal-client from latest official image
63 | build:
64 | context: .
65 | environment:
66 | - OPAL_SERVER_URL=http://opal_server:7002
67 | - OPAL_FETCH_PROVIDER_MODULES=opal_common.fetcher.providers,opal_fetcher_postgres.provider
68 | - OPAL_INLINE_OPA_LOG_FORMAT=http
69 | ports:
70 | # exposes opal client on the host machine, you can access the client at: http://localhost:7000
71 | - "7766:7000"
72 | # exposes the OPA agent (being run by OPAL) on the host machine
73 | # you can access the OPA api that you know and love at: http://localhost:8181
74 | # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/
75 | - "8181:8181"
76 | depends_on:
77 | - opal_server
78 | - example_db
79 | # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments
80 | # to make sure that opal-server is already up before starting the client.
81 | command: sh -c "wait-for-it --service opal_server:7002 --service example_db:5432 --timeout 60 -- /start.sh"
82 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = ["setuptools>=61.0", "setuptools_scm[toml]>=7.1"]
3 | build-backend = "setuptools.build_meta"
4 |
5 | [project]
6 | name = 'opal-fetcher-postgres'
7 | authors = [
8 | { name="Asaf Cohen", email="asaf@permit.io" },
9 | ]
10 | description="An OPAL fetch provider to bring authorization state from Postgres DB."
11 | readme = "README.md"
12 | keywords = [ "Open Policy Agent", "OPA", "OPAL", "Open Policy Administration Layer", "Postgres", "Permit.io" ]
13 | requires-python = ">=3.7"
14 | license = {text = "Apache-2.0"}
15 | classifiers = [
16 | 'Operating System :: OS Independent',
17 | 'License :: OSI Approved :: Apache Software License',
18 | "Topic :: Software Development :: Libraries :: Python Modules",
19 | 'Programming Language :: Python',
20 | 'Programming Language :: Python :: 3',
21 | 'Programming Language :: Python :: 3.7',
22 | 'Programming Language :: Python :: 3.8',
23 | 'Programming Language :: Python :: 3.9',
24 | ]
25 | dependencies = [
26 | 'opal-common>=0.1.11',
27 | 'asyncpg',
28 | "pydantic",
29 | 'tenacity',
30 | 'click',
31 | ]
32 | dynamic = ["version"]
33 |
34 | [project.urls]
35 | "Source" = "https://github.com/permitio/opal-fetcher-postgres"
36 | "Bug Tracker" = "https://github.com/permitio/opal-fetcher-postgres/issues"
37 |
38 | [tool.setuptools_scm]
39 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 |
3 | setup(
4 | name="opal_fetcher_postgres",
5 | version="0.0.4",
6 | )
7 |
--------------------------------------------------------------------------------
/sql/create_tables.sql:
--------------------------------------------------------------------------------
1 | -- Creation of product table
2 | CREATE TABLE IF NOT EXISTS product (
3 | product_id INT NOT NULL,
4 | name varchar(250) NOT NULL,
5 | PRIMARY KEY (product_id)
6 | );
7 |
8 | -- Creation of country table
9 | CREATE TABLE IF NOT EXISTS country (
10 | country_id INT NOT NULL,
11 | country_name varchar(450) NOT NULL,
12 | PRIMARY KEY (country_id)
13 | );
14 |
15 | -- Creation of city table
16 | CREATE TABLE IF NOT EXISTS city (
17 | city_id INT NOT NULL,
18 | city_name varchar(450) NOT NULL,
19 | country_id INT NOT NULL,
20 | PRIMARY KEY (city_id),
21 | CONSTRAINT fk_country
22 | FOREIGN KEY(country_id)
23 | REFERENCES country(country_id)
24 | );
25 |
26 | -- Creation of store table
27 | CREATE TABLE IF NOT EXISTS store (
28 | store_id INT NOT NULL,
29 | name varchar(250) NOT NULL,
30 | city_id INT NOT NULL,
31 | PRIMARY KEY (store_id),
32 | CONSTRAINT fk_city
33 | FOREIGN KEY(city_id)
34 | REFERENCES city(city_id)
35 | );
36 |
37 | -- Creation of user table
38 | CREATE TABLE IF NOT EXISTS users (
39 | user_id INT NOT NULL,
40 | name varchar(250) NOT NULL,
41 | PRIMARY KEY (user_id)
42 | );
43 |
44 | -- Creation of status_name table
45 | CREATE TABLE IF NOT EXISTS status_name (
46 | status_name_id INT NOT NULL,
47 | status_name varchar(450) NOT NULL,
48 | PRIMARY KEY (status_name_id)
49 | );
50 |
51 | -- Creation of sale table
52 | CREATE TABLE IF NOT EXISTS sale (
53 | sale_id varchar(200) NOT NULL,
54 | amount DECIMAL(20,3) NOT NULL,
55 | date_sale TIMESTAMP,
56 | product_id INT NOT NULL,
57 | user_id INT NOT NULL,
58 | store_id INT NOT NULL,
59 | PRIMARY KEY (sale_id),
60 | CONSTRAINT fk_product
61 | FOREIGN KEY(product_id)
62 | REFERENCES product(product_id),
63 | CONSTRAINT fk_user
64 | FOREIGN KEY(user_id)
65 | REFERENCES users(user_id),
66 | CONSTRAINT fk_store
67 | FOREIGN KEY(store_id)
68 | REFERENCES store(store_id)
69 | );
70 |
71 | -- Creation of order_status table
72 | CREATE TABLE IF NOT EXISTS order_status (
73 | order_status_id varchar(200) NOT NULL,
74 | update_at TIMESTAMP,
75 | sale_id varchar(200) NOT NULL,
76 | status_name_id INT NOT NULL,
77 | PRIMARY KEY (order_status_id),
78 | CONSTRAINT fk_sale
79 | FOREIGN KEY(sale_id)
80 | REFERENCES sale(sale_id),
81 | CONSTRAINT fk_status_name
82 | FOREIGN KEY(status_name_id)
83 | REFERENCES status_name(status_name_id)
84 | );
85 |
--------------------------------------------------------------------------------
/sql/fill_tables.sql:
--------------------------------------------------------------------------------
1 | -- Set params
2 | set session my.number_of_sales = '2000000';
3 | set session my.number_of_users = '500000';
4 | set session my.number_of_products = '300';
5 | set session my.number_of_stores = '500';
6 | set session my.number_of_coutries = '100';
7 | set session my.number_of_cities = '30';
8 | set session my.status_names = '5';
9 | set session my.start_date = '2019-01-01 00:00:00';
10 | set session my.end_date = '2020-02-01 00:00:00';
11 |
12 | -- load the pgcrypto extension to gen_random_uuid ()
13 | CREATE EXTENSION pgcrypto;
14 |
15 | -- Filling of products
16 | INSERT INTO product
17 | select id, concat('Product ', id)
18 | FROM GENERATE_SERIES(1, current_setting('my.number_of_products')::int) as id;
19 |
20 | -- Filling of countries
21 | INSERT INTO country
22 | select id, concat('Country ', id)
23 | FROM GENERATE_SERIES(1, current_setting('my.number_of_coutries')::int) as id;
24 |
25 | -- Filling of cities
26 | INSERT INTO city
27 | select id
28 | , concat('City ', id)
29 | , floor(random() * (current_setting('my.number_of_coutries')::int) + 1)::int
30 | FROM GENERATE_SERIES(1, current_setting('my.number_of_cities')::int) as id;
31 |
32 | -- Filling of stores
33 | INSERT INTO store
34 | select id
35 | , concat('Store ', id)
36 | , floor(random() * (current_setting('my.number_of_cities')::int) + 1)::int
37 | FROM GENERATE_SERIES(1, current_setting('my.number_of_stores')::int) as id;
38 |
39 | -- Filling of users
40 | INSERT INTO users
41 | select id
42 | , concat('User ', id)
43 | FROM GENERATE_SERIES(1, current_setting('my.number_of_users')::int) as id;
44 |
45 | -- Filling of users
46 | INSERT INTO status_name
47 | select status_name_id
48 | , concat('Status Name ', status_name_id)
49 | FROM GENERATE_SERIES(1, current_setting('my.status_names')::int) as status_name_id;
50 |
51 | -- Filling of sales
52 | INSERT INTO sale
53 | select gen_random_uuid ()
54 | , round(CAST(float8 (random() * 10000) as numeric), 3)
55 | , TO_TIMESTAMP(start_date, 'YYYY-MM-DD HH24:MI:SS') +
56 | random()* (TO_TIMESTAMP(end_date, 'YYYY-MM-DD HH24:MI:SS')
57 | - TO_TIMESTAMP(start_date, 'YYYY-MM-DD HH24:MI:SS'))
58 | , floor(random() * (current_setting('my.number_of_products')::int) + 1)::int
59 | , floor(random() * (current_setting('my.number_of_users')::int) + 1)::int
60 | , floor(random() * (current_setting('my.number_of_stores')::int) + 1)::int
61 | FROM GENERATE_SERIES(1, current_setting('my.number_of_sales')::int) as id
62 | , current_setting('my.start_date') as start_date
63 | , current_setting('my.end_date') as end_date;
64 |
65 | -- Filling of order_status
66 | INSERT INTO order_status
67 | select gen_random_uuid ()
68 | , date_sale + random()* (date_sale + '5 days' - date_sale)
69 | , sale_id
70 | , floor(random() * (current_setting('my.status_names')::int) + 1)::int
71 | from sale;
72 |
--------------------------------------------------------------------------------
/src/opal_fetcher_postgres/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/permitio/opal-fetcher-postgres/f96e8b6dc2079f8f1b441041c08a608cb1f37d7e/src/opal_fetcher_postgres/__init__.py
--------------------------------------------------------------------------------
/src/opal_fetcher_postgres/provider.py:
--------------------------------------------------------------------------------
1 | """
2 | Simple fetch provider for postgres db.
3 |
4 | This fetcher also serves as an example how to build custom OPAL Fetch Providers.
5 | """
6 | from typing import Optional, List
7 |
8 | import asyncpg
9 | from asyncpg.transaction import Transaction
10 | from asyncpg.exceptions import DataError
11 | from pydantic import BaseModel, Field
12 | from tenacity import wait, stop, retry_unless_exception_type
13 |
14 | from opal_common.fetcher.fetch_provider import BaseFetchProvider
15 | from opal_common.fetcher.events import FetcherConfig, FetchEvent
16 | from opal_common.logger import logger
17 |
18 |
19 | class PostgresConnectionParams(BaseModel):
20 | """
21 | if one does not want to pass all postgres arguments in the dsn (in OPAL - the url is the dsn),
22 | one can also use this dict to pass specific arguments.
23 | """
24 |
25 | database: Optional[str] = Field(None, description="the database name")
26 | user: Optional[str] = Field(None, description="user name used to authenticate")
27 | password: Optional[str] = Field(None, description="password used to authenticate")
28 | host: Optional[str] = Field(
29 | None,
30 | description="database host address (defaults to UNIX socket if not provided)",
31 | )
32 | port: Optional[str] = Field(
33 | None, description="connection port number (defaults to 5432 if not provided)"
34 | )
35 |
36 |
37 | class PostgresFetcherConfig(FetcherConfig):
38 | """
39 | Config for PostgresFetchProvider, instance of `FetcherConfig`.
40 |
41 | When an OPAL client receives an update, it contains a list of `DataSourceEntry` objects.
42 | Each `DataSourceEntry` has a `config` key - which is usually an instance of a subclass of `FetcherConfig`.
43 |
44 | When writing a custom provider, you must:
45 | - derive your class (inherit) from FetcherConfig
46 | - override the `fetcher` key with your fetcher class name
47 | - (optional): add any fields relevant to a data entry of your fetcher.
48 | - In this example: since we pull data from PostgreSQL - we added a `query` key to hold the SQL query.
49 | """
50 |
51 | fetcher: str = "PostgresFetchProvider"
52 | connection_params: Optional[PostgresConnectionParams] = Field(
53 | None,
54 | description="these params can override or complement parts of the dsn (connection string)",
55 | )
56 | query: str = Field(
57 | ..., description="the query to run against postgres in order to fetch the data"
58 | )
59 | fetch_one: bool = Field(
60 | False,
61 | description="whether we fetch only one row from the results of the SELECT query",
62 | )
63 | fetch_key: str = Field(
64 | None,
65 | description="column name to use as key to transform the data to Object format rather than list/array",
66 | )
67 |
68 |
69 | class PostgresFetchEvent(FetchEvent):
70 | """
71 | A FetchEvent shape for the Postgres Fetch Provider.
72 |
73 | When writing a custom provider, you must create a custom FetchEvent subclass, just like this class.
74 | In your own class, you must set the value of the `fetcher` key to be your custom provider class name.
75 | """
76 |
77 | fetcher: str = "PostgresFetchProvider"
78 | config: PostgresFetcherConfig = None
79 |
80 |
81 | class PostgresFetchProvider(BaseFetchProvider):
82 | """
83 | An OPAL fetch provider for postgres.
84 |
85 | We fetch data from a postgres database by running a SELECT query,
86 | transforming the results to json and dumping the results into the policy store.
87 |
88 | When writing a custom provider, you must:
89 | - derive your provider class (inherit) from BaseFetchProvider
90 | - create a custom config class, as shown above, that derives from FetcherConfig
91 | - create a custom event class, as shown above, that derives from FetchEvent
92 |
93 | At minimum, your custom provider class must implement:
94 | - __init__() - and call super().__init__(event)
95 | - parse_event() - this method gets a `FetchEvent` object and must transform this object to *your own custom event class*.
96 | - Notice that `FetchEvent` is the base class
97 | - Notice that `PostgresFetchEvent` is the custom event class
98 | - _fetch_() - your custom fetch method, can use the data from your event
99 | and config to figure out *what and how to fetch* and actually do it.
100 | - _process_() - if your fetched data requires some processing, you should do it here.
101 | - The return type from this method must be json-able, i.e: can be serialized to json.
102 |
103 | You may need to implement:
104 | - __aenter__() - if your provider has state that needs to be cleaned up,
105 | (i.e: http session, postgres connection, etc) the state may be initialized in this method.
106 | - __aexit__() - if you initialized stateful objects (i.e: acquired resources) in your __aenter__, you must release them in __aexit__
107 | """
108 |
109 | RETRY_CONFIG = {
110 | "wait": wait.wait_random_exponential(),
111 | "stop": stop.stop_after_attempt(10),
112 | "retry": retry_unless_exception_type(
113 | DataError
114 | ), # query error (i.e: invalid table, etc)
115 | "reraise": True,
116 | }
117 |
118 | def __init__(self, event: PostgresFetchEvent) -> None:
119 | if event.config is None:
120 | event.config = PostgresFetcherConfig()
121 | super().__init__(event)
122 | self._connection: Optional[asyncpg.Connection] = None
123 | self._transaction: Optional[Transaction] = None
124 |
125 | def parse_event(self, event: FetchEvent) -> PostgresFetchEvent:
126 | return PostgresFetchEvent(**event.dict(exclude={"config"}), config=event.config)
127 |
128 | async def __aenter__(self):
129 | self._event: PostgresFetchEvent # type casting
130 |
131 | dsn: str = self._event.url
132 | connection_params: dict = (
133 | {}
134 | if self._event.config.connection_params is None
135 | else self._event.config.connection_params.dict(exclude_none=True)
136 | )
137 |
138 | # connect to the postgres database
139 | self._connection: asyncpg.Connection = await asyncpg.connect(
140 | dsn, **connection_params
141 | )
142 | # start a readonly transaction (we don't want OPAL client writing data due to security!)
143 | self._transaction: Transaction = self._connection.transaction(readonly=True)
144 | await self._transaction.__aenter__()
145 |
146 | return self
147 |
148 | async def __aexit__(self, exc_type=None, exc_val=None, tb=None):
149 | # End the transaction
150 | if self._transaction is not None:
151 | await self._transaction.__aexit__(exc_type, exc_val, tb)
152 | # Close the connection
153 | if self._connection is not None:
154 | await self._connection.close()
155 |
156 | async def _fetch_(self):
157 | self._event: PostgresFetchEvent # type casting
158 |
159 | if self._event.config is None:
160 | logger.warning(
161 | "incomplete fetcher config: postgres data entries require a query to specify what data to fetch!"
162 | )
163 | return
164 |
165 | logger.debug(f"{self.__class__.__name__} fetching from {self._url}")
166 |
167 | if self._event.config.fetch_one:
168 | row = await self._connection.fetchrow(self._event.config.query)
169 | return [row]
170 | else:
171 | return await self._connection.fetch(self._event.config.query)
172 |
173 | async def _process_(self, records: List[asyncpg.Record]):
174 | self._event: PostgresFetchEvent # type casting
175 |
176 | # when fetch_one is true, we want to return a dict (and not a list)
177 | if self._event.config.fetch_one:
178 | if records and len(records) > 0:
179 | # we transform the asyncpg record to a dict that we can be later serialized to json
180 | return dict(records[0])
181 | else:
182 | return {}
183 | else:
184 | if self._event.config.fetch_key is None:
185 | # we transform the asyncpg records to a list-of-dicts that we can be later serialized to json
186 | return [dict(record) for record in records]
187 | else:
188 | # we transform the asyncpg records to a dict-of-dicts that we can be later serialized to json
189 | res_dct = map(lambda i: (records[i][self._event.config.fetch_key], dict(records[i])), range(len(records)))
190 | return dict(res_dct)
191 |
--------------------------------------------------------------------------------