├── 101 ├── weather.csv ├── weather1-bare.py ├── weather2-bare.py ├── weather1-flow.py └── weather2-tasks.py ├── 102 ├── retry-flow.py ├── logflow.py ├── results1.py ├── caching1.py ├── caching2.py ├── retries-delay.py └── weatherflow.py ├── 103 ├── retrieve-secret-block.py ├── create-secret-block.py ├── block-json-create.py └── serve-two-flows.py ├── 104 ├── params.py ├── flows.py ├── serve-two-flows-scheduled.py ├── serve-with-params.py └── serve-with-schedule.py ├── 105 └── subflow.py ├── 201 ├── requirements.txt ├── flows2.py ├── serve-from-gh.py ├── deploy-with-schedule.py ├── deploy-2-flows.py ├── deploy-no-code-baked.py └── deploy-2-flows-from-gh.py ├── pacc-logo.png ├── README.md ├── Dockerfile-example ├── .prefectignore └── .gitignore /101/weather.csv: -------------------------------------------------------------------------------- 1 | 30.1 -------------------------------------------------------------------------------- /201/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas -------------------------------------------------------------------------------- /pacc-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/discdiver/pacc-2023/HEAD/pacc-logo.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Prefect Associate Certification Course 2 | 3 | ![PACC logo](pacc-logo.png) 4 | -------------------------------------------------------------------------------- /201/flows2.py: -------------------------------------------------------------------------------- 1 | from prefect import flow 2 | 3 | 4 | @flow 5 | def pipe2(): 6 | print("hi") 7 | return None 8 | -------------------------------------------------------------------------------- /103/retrieve-secret-block.py: -------------------------------------------------------------------------------- 1 | from prefect.blocks.system import Secret 2 | 3 | secret_block = Secret.load("secret-thing") 4 | print(secret_block.get()) 5 | -------------------------------------------------------------------------------- /104/params.py: -------------------------------------------------------------------------------- 1 | from prefect import flow 2 | 3 | 4 | @flow(log_prints=True) 5 | def person(name: str = "Jeff", height: int = 6): 6 | print(name, height) 7 | -------------------------------------------------------------------------------- /103/create-secret-block.py: -------------------------------------------------------------------------------- 1 | from prefect.blocks.system import Secret 2 | 3 | my_secret_block = Secret(value="shhh!-it's-a-secret") 4 | my_secret_block.save(name="secret-thing") 5 | -------------------------------------------------------------------------------- /Dockerfile-example: -------------------------------------------------------------------------------- 1 | FROM prefecthq/prefect:2-latest 2 | COPY requirements.txt /opt/prefect/201/requirements.txt 3 | RUN python -m pip install -r requirements.txt 4 | COPY . /opt/prefect/pacc-2023/ 5 | WORKDIR /opt/prefect/pacc-2023/ 6 | 7 | -------------------------------------------------------------------------------- /103/block-json-create.py: -------------------------------------------------------------------------------- 1 | from prefect.blocks.system import JSON 2 | 3 | autos = JSON(value=dict(cars=["tesla", "fiat", "chevy"], trucks=["rivian", "ford"])) 4 | autos.save(name="json-block-ex", overwrite=True) 5 | 6 | # careful don't use .save with capital letters or underscores or spaces! 7 | -------------------------------------------------------------------------------- /102/retry-flow.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow 3 | 4 | 5 | @flow(retries=4) 6 | def fetch(): 7 | cat_fact = httpx.get("https://httpstat.us/Random/200,500") 8 | if cat_fact.status_code >= 400: 9 | raise Exception() 10 | print(cat_fact.text) 11 | 12 | 13 | if __name__ == "__main__": 14 | fetch() 15 | -------------------------------------------------------------------------------- /102/logflow.py: -------------------------------------------------------------------------------- 1 | from prefect import flow, get_run_logger 2 | 3 | 4 | @flow(name="log-example-flow") 5 | def log_it(): 6 | logger = get_run_logger() 7 | logger.info("INFO level log message.") 8 | logger.debug("You only see this message if the logging level is set to DEBUG. 🙂") 9 | 10 | 11 | if __name__ == "__main__": 12 | log_it() 13 | -------------------------------------------------------------------------------- /102/results1.py: -------------------------------------------------------------------------------- 1 | from prefect import flow, task 2 | import pandas as pd 3 | 4 | 5 | @task(persist_result=True) 6 | def my_task(): 7 | df = pd.DataFrame(dict(a=[2, 3], b=[4, 5])) 8 | return df 9 | 10 | 11 | @flow 12 | def my_flow(): 13 | res = my_task() 14 | return "success" 15 | 16 | 17 | if __name__ == "__main__": 18 | my_flow() 19 | -------------------------------------------------------------------------------- /102/caching1.py: -------------------------------------------------------------------------------- 1 | from prefect import flow, task 2 | from prefect.tasks import task_input_hash 3 | 4 | 5 | @task(cache_key_fn=task_input_hash) 6 | def hello_task(name_input): 7 | print(f"Hello {name_input}!") 8 | 9 | 10 | @flow 11 | def hello_flow(name_input): 12 | hello_task(name_input) 13 | 14 | 15 | if __name__ == "__main__": 16 | hello_flow("Liz") 17 | -------------------------------------------------------------------------------- /104/flows.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow, task 3 | 4 | 5 | @task 6 | def fetch_cat_fact(): 7 | return httpx.get("https://catfact.ninja/fact?max_length=140").json()["fact"] 8 | 9 | 10 | @task 11 | def formatting(fact: str): 12 | return fact.title() 13 | 14 | 15 | @flow 16 | def pipe(): 17 | fact = fetch_cat_fact() 18 | print(formatting(fact)) 19 | -------------------------------------------------------------------------------- /102/caching2.py: -------------------------------------------------------------------------------- 1 | from prefect import flow, task 2 | from prefect.tasks import task_input_hash 3 | from datetime import timedelta 4 | 5 | @task(cache_key_fn=task_input_hash, cache_expiration=timedelta(minutes=1)) 6 | def hello_task(name_input): 7 | print(f"Hello {name_input}!") 8 | 9 | @flow 10 | def hello_flow(name_input): 11 | hello_task(name_input) 12 | 13 | 14 | if __name__ == "__main__": 15 | hello_flow("Marvin") 16 | -------------------------------------------------------------------------------- /102/retries-delay.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow, task 3 | 4 | 5 | @task(retries=4, retry_delay_seconds=0.1) 6 | def fetch_cat_fact(): 7 | cat_fact = httpx.get("https://httpstat.us/Random/200,500", verify=False) 8 | if cat_fact.status_code >= 400: 9 | raise Exception() 10 | print(cat_fact.text) 11 | 12 | 13 | @flow 14 | def fetch(): 15 | fetch_cat_fact() 16 | 17 | 18 | if __name__ == "__main__": 19 | fetch() 20 | -------------------------------------------------------------------------------- /103/serve-two-flows.py: -------------------------------------------------------------------------------- 1 | import time 2 | from prefect import flow, serve 3 | 4 | 5 | @flow 6 | def slow_flow(sleep: int = 60): 7 | "Sleepy flow - sleeps the provided amount of time (in seconds)." 8 | time.sleep(sleep) 9 | 10 | 11 | @flow 12 | def fast_flow(): 13 | "Fastest flow this side of the Atlantic." 14 | return 15 | 16 | 17 | if __name__ == "__main__": 18 | slow_deploy = slow_flow.to_deployment(name="sleeper") 19 | fast_deploy = fast_flow.to_deployment(name="fast") 20 | serve(slow_deploy, fast_deploy) 21 | -------------------------------------------------------------------------------- /101/weather1-bare.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | 3 | 4 | def fetch_weather(lat: float = 38.9, lon: float = -77.0): 5 | base_url = "https://api.open-meteo.com/v1/forecast/" 6 | weather = httpx.get( 7 | base_url, 8 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 9 | ) 10 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 11 | print(f"Most recent temp C: {most_recent_temp} degrees") 12 | return most_recent_temp 13 | 14 | 15 | if __name__ == "__main__": 16 | fetch_weather() 17 | -------------------------------------------------------------------------------- /.prefectignore: -------------------------------------------------------------------------------- 1 | # prefect artifacts 2 | .prefectignore 3 | 4 | # python artifacts 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | *.egg-info/ 9 | *.egg 10 | 11 | # Type checking artifacts 12 | .mypy_cache/ 13 | .dmypy.json 14 | dmypy.json 15 | .pyre/ 16 | 17 | # IPython 18 | profile_default/ 19 | ipython_config.py 20 | *.ipynb_checkpoints/* 21 | 22 | # Environments 23 | .python-version 24 | .env 25 | .venv 26 | env/ 27 | venv/ 28 | 29 | # MacOS 30 | .DS_Store 31 | 32 | # Dask 33 | dask-worker-space/ 34 | 35 | # Editors 36 | .idea/ 37 | .vscode/ 38 | 39 | # VCS 40 | .git/ 41 | .hg/ 42 | -------------------------------------------------------------------------------- /104/serve-two-flows-scheduled.py: -------------------------------------------------------------------------------- 1 | import time 2 | from prefect import flow, serve 3 | 4 | 5 | @flow 6 | def slow_flow(sleep: int = 60): 7 | "Sleepy flow - sleeps the provided amount of time (in seconds)." 8 | time.sleep(sleep) 9 | 10 | 11 | @flow 12 | def fast_flow(): 13 | "Fastest flow this side of the Atlantic." 14 | return 15 | 16 | 17 | if __name__ == "__main__": 18 | slow_deploy = slow_flow.to_deployment(name="sleeper-scheduling", interval="200") 19 | fast_deploy = fast_flow.to_deployment(name="fast-scheduling", cron="* * * * *") 20 | serve(slow_deploy, fast_deploy) 21 | -------------------------------------------------------------------------------- /104/serve-with-params.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow 3 | 4 | 5 | @flow() 6 | def fetch_weather(lat: float, lon: float): 7 | base_url = "https://api.open-meteo.com/v1/forecast/" 8 | weather = httpx.get( 9 | base_url, 10 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 11 | ) 12 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 13 | print(f"Most recent temp C: {most_recent_temp} degrees") 14 | return most_recent_temp 15 | 16 | 17 | if __name__ == "__main__": 18 | fetch_weather.serve(name="deploy-params", parameters={"lat": 11, "lon": 12}) 19 | -------------------------------------------------------------------------------- /105/subflow.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow 3 | 4 | 5 | @flow 6 | def fetch_cat_fact(): 7 | return httpx.get("https://catfact.ninja/fact?max_length=140").json()["fact"] 8 | 9 | 10 | @flow 11 | def fetch_dog_fact(): 12 | return httpx.get( 13 | "https://dogapi.dog/api/v2/facts", 14 | headers={"accept": "application/json"}, 15 | ).json()["data"][0]["attributes"]["body"] 16 | 17 | 18 | @flow(log_prints=True) 19 | def animal_facts(): 20 | cat_fact = fetch_cat_fact() 21 | dog_fact = fetch_dog_fact() 22 | print(f"🐱: {cat_fact} \n🐶: {dog_fact}") 23 | 24 | 25 | if __name__ == "__main__": 26 | animal_facts() 27 | -------------------------------------------------------------------------------- /104/serve-with-schedule.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow 3 | 4 | 5 | @flow() 6 | def fetch_weather(lat: float = 3, lon: float = -30): 7 | base_url = "https://api.open-meteo.com/v1/forecast/" 8 | weather = httpx.get( 9 | base_url, 10 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 11 | ) 12 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 13 | print(f"Most recent temp C: {most_recent_temp} degrees") 14 | return most_recent_temp 15 | 16 | 17 | if __name__ == "__main__": 18 | fetch_weather.serve(name="scheduled-deploy", interval="3600") 19 | # or 20 | # fetch_weather.serve(name="scheduled-deploy2", cron="0 0 * * *") 21 | -------------------------------------------------------------------------------- /201/serve-from-gh.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from prefect import flow 3 | from prefect 4 | 5 | 6 | @flow() 7 | def fetch_weather(lat: float = 3, lon: float = -30): 8 | base_url = "https://api.open-meteo.com/v1/forecast/" 9 | weather = requests.get( 10 | base_url, 11 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 12 | ) 13 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 14 | print(f"Most recent temp C: {most_recent_temp} degrees") 15 | return most_recent_temp 16 | 17 | 18 | if __name__ == "__main__": 19 | fetch_weather.from_source( 20 | source="https://github.com/org/repo.git", 21 | entrypoint="path/to/flow.py:my_flow").serve(name="gh-deploy", interval="300") 22 | 23 | -------------------------------------------------------------------------------- /201/deploy-with-schedule.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from prefect import flow 3 | 4 | 5 | @flow() 6 | def fetch_weather(lat: float = 3, lon: float = -30): 7 | base_url = "https://api.open-meteo.com/v1/forecast/" 8 | weather = requests.get( 9 | base_url, 10 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 11 | ) 12 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 13 | print(f"Most recent temp C: {most_recent_temp} degrees") 14 | return most_recent_temp 15 | 16 | 17 | if __name__ == "__main__": 18 | fetch_weather.deploy( 19 | name="scheduled-deploy", 20 | image="discdiver/img100:0.1", 21 | interval="3600", 22 | work_pool_name="dock1", 23 | push=False, 24 | ) 25 | -------------------------------------------------------------------------------- /101/weather2-bare.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | 3 | 4 | def fetch_weather(lat: float, lon: float): 5 | base_url = "https://api.open-meteo.com/v1/forecast/" 6 | weather = httpx.get( 7 | base_url, 8 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 9 | ) 10 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 11 | return most_recent_temp 12 | 13 | 14 | def save_weather(temp: float): 15 | with open("weather.csv", "w+") as w: 16 | w.write(str(temp)) 17 | return "Successfully wrote temp" 18 | 19 | 20 | def pipeline(lat: float = 38.9, lon: float = -77.0): 21 | temp = fetch_weather(lat, lon) 22 | result = save_weather(temp) 23 | return result 24 | 25 | 26 | if __name__ == "__main__": 27 | pipeline() 28 | -------------------------------------------------------------------------------- /101/weather1-flow.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow 3 | 4 | 5 | @flow() 6 | def fetch_weather(lat: float = 38.9, lon: float = -77.0): 7 | base_url = "https://api.open-meteo.com/v1/forecast/" 8 | weather = httpx.get( 9 | base_url, 10 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 11 | ) 12 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 13 | print(f"Most recent temp C: {most_recent_temp} degrees") 14 | return most_recent_temp 15 | 16 | 17 | if __name__ == "__main__": 18 | fetch_weather() 19 | 20 | # if __name__ == "__main__": 21 | # fetch_weather.serve(name="deploy-1") 22 | 23 | 24 | # if __name__ == "__main__": 25 | # fetch_weather.serve(name="deploy3", parameters={"lat": 11, "lon": 12}) 26 | -------------------------------------------------------------------------------- /101/weather2-tasks.py: -------------------------------------------------------------------------------- 1 | import httpx # requests capability, but can work with async 2 | from prefect import flow, task 3 | 4 | 5 | @task 6 | def fetch_weather(lat: float, lon: float): 7 | base_url = "https://api.open-meteo.com/v1/forecast/" 8 | weather = httpx.get( 9 | base_url, 10 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 11 | ) 12 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 13 | return most_recent_temp 14 | 15 | 16 | @task 17 | def save_weather(temp: float): 18 | with open("weather.csv", "w+") as w: 19 | w.write(str(temp)) 20 | return "Successfully wrote temp" 21 | 22 | 23 | @flow 24 | def pipeline(lat: float = 38.9, lon: float = -77.0): 25 | temp = fetch_weather(lat, lon) 26 | result = save_weather(temp) 27 | return result 28 | 29 | 30 | if __name__ == "__main__": 31 | pipeline() 32 | -------------------------------------------------------------------------------- /201/deploy-2-flows.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from prefect import flow, deploy, serve 3 | 4 | 5 | @flow(log_prints=True) 6 | def hi(): 7 | print("hello") 8 | 9 | 10 | @flow(log_prints=True) 11 | def fetch_weather(lat: float = 3, lon: float = -30): 12 | base_url = "https://api.open-meteo.com/v1/forecast/" 13 | weather = requests.get( 14 | base_url, 15 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 16 | ) 17 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 18 | print(f"Most recent temp C: {most_recent_temp} degrees") 19 | 20 | 21 | if __name__ == "__main__": 22 | dep1 = fetch_weather.to_deployment(name="weather-deploy") 23 | dep2 = hi.to_deployment(name="hey") 24 | deploy( 25 | dep1, 26 | dep2, 27 | work_pool_name="dock1", 28 | image="discdiver/whatevs-image:1.0", 29 | push=False, 30 | ) 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python artifacts 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.egg-info/ 6 | *.egg 7 | build/ 8 | dist/ 9 | sdist/ 10 | 11 | # Test artifacts 12 | .coverage 13 | .coverage.*.* 14 | .prefect-results 15 | .pytest_cache/ 16 | 17 | # Type checking artifacts 18 | .mypy_cache/ 19 | .dmypy.json 20 | dmypy.json 21 | .pyre/ 22 | 23 | # IPython 24 | profile_default/ 25 | ipython_config.py 26 | *.ipynb_checkpoints/* 27 | 28 | # Profiling 29 | /prof 30 | 31 | # Environments 32 | .python-version 33 | .env 34 | .venv 35 | env/ 36 | venv/ 37 | 38 | # Documentation artifacts 39 | docs/api-ref/schema.json 40 | site/ 41 | .cache/ 42 | 43 | # UI artifacts 44 | src/prefect/orion/ui/* 45 | **/node_modules 46 | 47 | # Databases 48 | *.db 49 | 50 | # MacOS 51 | .DS_Store 52 | 53 | # Dask 54 | dask-worker-space/ 55 | 56 | # Editors 57 | .idea/ 58 | .vscode/ 59 | 60 | # Docker 61 | Dockerfile 62 | 63 | # prefect.yaml 64 | prefect.yaml -------------------------------------------------------------------------------- /102/weatherflow.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | from prefect import flow, task 3 | from prefect.artifacts import create_markdown_artifact 4 | 5 | 6 | @task 7 | def mark_it_down(temp): 8 | markdown_report = f"""# Weather Report 9 | 10 | ## Recent weather 11 | 12 | | Time | Temperature | 13 | |:--------------|-------:| 14 | | Now | {temp} | 15 | | In 1 hour | {temp + 2} | 16 | """ 17 | create_markdown_artifact( 18 | key="weather-report", 19 | markdown=markdown_report, 20 | description="Very scientific weather report", 21 | ) 22 | 23 | 24 | @flow 25 | def fetch_weather(lat: float, lon: float): 26 | base_url = "https://api.open-meteo.com/v1/forecast/" 27 | weather = httpx.get( 28 | base_url, 29 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 30 | ) 31 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 32 | mark_it_down(most_recent_temp) 33 | 34 | 35 | if __name__ == "__main__": 36 | fetch_weather(38.9, -77.0) 37 | -------------------------------------------------------------------------------- /201/deploy-no-code-baked.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from prefect import flow 3 | 4 | 5 | @flow(log_prints=True) 6 | def fetch_weather(lat: float = 38.736946, lon: float = -9.142685): 7 | base_url = "https://api.open-meteo.com/v1/forecast/" 8 | weather = requests.get( 9 | base_url, 10 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 11 | ) 12 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 13 | print(f"Most recent temp C: {most_recent_temp} degrees") 14 | print("Hello from changed code pushed to GitHub") # push code, don't redeploy 15 | return most_recent_temp 16 | 17 | 18 | if __name__ == "__main__": 19 | fetch_weather.from_source( 20 | source="https://github.com/discdiver/pacc-2023.git", # gotcha: add .git to the end of the repo URL 21 | entrypoint="201/deploy-no-code-baked.py:fetch_weather", 22 | ).deploy( 23 | name="lisbon-weather", 24 | work_pool_name="dock1", 25 | image="discdiver/no-code-image:1.0", 26 | push=False, 27 | ) 28 | -------------------------------------------------------------------------------- /201/deploy-2-flows-from-gh.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from prefect import flow, deploy 3 | from prefect.runner.storage import RemoteStorage 4 | 5 | 6 | @flow(log_prints=True) 7 | def hi(): 8 | print("hello") 9 | 10 | 11 | @flow(log_prints=True) 12 | def fetch_weather(lat: float = 3, lon: float = -30): 13 | base_url = "https://api.open-meteo.com/v1/forecast/" 14 | weather = requests.get( 15 | base_url, 16 | params=dict(latitude=lat, longitude=lon, hourly="temperature_2m"), 17 | ) 18 | most_recent_temp = float(weather.json()["hourly"]["temperature_2m"][0]) 19 | print(f"Most recent temp C: {most_recent_temp} degrees") 20 | 21 | 22 | if __name__ == "__main__": 23 | dep1 = fetch_weather.from_source( 24 | source=RemoteStorage( 25 | url="az://my-container/my-folder", account_name="my-account-name" 26 | ), 27 | entrypoint="flows.py:my_flow", 28 | ).to_deployment(name="weather-deploy") 29 | 30 | dep2 = hi.to_deployment(name="hey") 31 | 32 | deploy( 33 | dep1, 34 | dep2, 35 | work_pool_name="dock1", 36 | image="discdiver/whatevs-image:1.0", 37 | push=False, 38 | ) 39 | --------------------------------------------------------------------------------