├── COVID-19_Simulation-Card.pdf ├── tutorials ├── assets │ ├── agent_and_planner_model.png │ ├── distributed_rl_architecture.png │ ├── foundation_snapshot_rendered.jpg │ └── prod_eq_welfare.svg ├── rllib │ ├── __init__.py │ ├── utils │ │ ├── __init__.py │ │ ├── remote.py │ │ └── saving.py │ ├── phase1 │ │ └── config.yaml │ ├── phase2 │ │ └── config.yaml │ └── env_wrapper.py ├── utils │ └── __init__.py └── LICENSE.txt ├── CODEOWNERS ├── Simulation_Card_Foundation_Economic_Simulation_Framework.pdf ├── ai_economist ├── foundation │ ├── scenarios │ │ ├── simple_wood_and_stone │ │ │ ├── map_txt │ │ │ │ ├── quadrant_8x8_4each_8clump.txt │ │ │ │ ├── top_wood_bottom_stone_14x14.txt │ │ │ │ ├── env-pure_and_mixed-15x15.txt │ │ │ │ ├── env-pure_and_mixed-25x25.txt │ │ │ │ ├── quadrant_25x25_20each_30clump.txt │ │ │ │ ├── uniform_25x25_25each_65clump.txt │ │ │ │ ├── closed_quadrant_25x25_20each_30clump.txt │ │ │ │ ├── quadrant_25x25_20each_30clump_no_water.txt │ │ │ │ ├── env-pure_and_mixed-40x40.txt │ │ │ │ ├── quadrant_40x40_50each.txt │ │ │ │ └── quadrant_40x40_50each_no_water.txt │ │ │ └── __init__.py │ │ ├── covid19 │ │ │ ├── __init__.py │ │ │ ├── covid19_build.cu │ │ │ └── key_to_check_activation_code_against │ │ ├── utils │ │ │ ├── __init__.py │ │ │ ├── social_metrics.py │ │ │ └── rewards.py │ │ ├── one_step_economy │ │ │ └── __init__.py │ │ └── __init__.py │ ├── base │ │ ├── __init__.py │ │ └── registrar.py │ ├── entities │ │ ├── __init__.py │ │ ├── endogenous.py │ │ ├── resources.py │ │ └── landmarks.py │ ├── agents │ │ ├── __init__.py │ │ ├── mobiles.py │ │ └── planners.py │ ├── components │ │ ├── __init__.py │ │ ├── utils.py │ │ ├── simple_labor.py │ │ ├── move.py │ │ └── build.py │ ├── __init__.py │ └── utils.py ├── datasets │ ├── covid19_datasets │ │ ├── data_and_fitted_params │ │ │ ├── real_world_data.npz │ │ │ └── model_constants.json │ │ ├── __init__.py │ │ ├── README.md │ │ ├── us_deaths.py │ │ ├── us_vaccinations.py │ │ ├── us_unemployment.py │ │ └── us_policies.py │ └── __init__.py ├── training │ ├── __init__.py │ ├── run_configs │ │ └── covid_and_economy_environment.yaml │ └── training_script.py ├── real_business_cycle │ ├── rbc │ │ ├── __init__.py │ │ ├── networks.py │ │ └── util.py │ ├── train_single_exp.py │ ├── train_multi_exps.py │ ├── train_bestresponse.py │ ├── README.md │ └── experiment_utils.py └── __init__.py ├── .gitmodules ├── tests ├── __init__.py ├── run_covid19_cpu_gpu_consistency_checks.py └── test_env.py ├── SECURITY.md ├── format_and_lint.sh ├── setup.py ├── CONTRIBUTING.md ├── LICENSE.txt ├── CHANGELOG.md ├── requirements.txt ├── .gitignore ├── .github └── workflows │ └── linters.yml ├── CODE_OF_CONDUCT.md └── README.md /COVID-19_Simulation-Card.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/ai-economist/HEAD/COVID-19_Simulation-Card.pdf -------------------------------------------------------------------------------- /tutorials/assets/agent_and_planner_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/ai-economist/HEAD/tutorials/assets/agent_and_planner_model.png -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Comment line immediately above ownership line is reserved for related gus information. Please be careful while editing. 2 | #ECCN:Open Source 3 | -------------------------------------------------------------------------------- /tutorials/assets/distributed_rl_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/ai-economist/HEAD/tutorials/assets/distributed_rl_architecture.png -------------------------------------------------------------------------------- /tutorials/assets/foundation_snapshot_rendered.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/ai-economist/HEAD/tutorials/assets/foundation_snapshot_rendered.jpg -------------------------------------------------------------------------------- /Simulation_Card_Foundation_Economic_Simulation_Framework.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/ai-economist/HEAD/Simulation_Card_Foundation_Economic_Simulation_Framework.pdf -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/quadrant_8x8_4each_8clump.txt: -------------------------------------------------------------------------------- 1 | WWWW@WW W; WW @ WWW;SW S@ S ;S @ ;@@ @@@ @@; S @ ;S S @ ;S @ ; S@ ; 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ai_economist/climate_cooperation"] 2 | path = ai_economist/climate_cooperation_competition 3 | url = https://github.com/mila-iqia/climate-cooperation-competition/ 4 | -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/data_and_fitted_params/real_world_data.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/ai-economist/HEAD/ai_economist/datasets/covid19_datasets/data_and_fitted_params/real_world_data.npz -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /tutorials/rllib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /tutorials/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/training/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /tutorials/rllib/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/foundation/base/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/rbc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/covid19/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/one_step_economy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist import foundation 8 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/top_wood_bottom_stone_14x14.txt: -------------------------------------------------------------------------------- 1 | WW W ; A WW A ;WW WA W ; A A ; ; AA A ; A ; ; ; ; A ; A SS S;SA S AS;S SA SA -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/env-pure_and_mixed-15x15.txt: -------------------------------------------------------------------------------- 1 | WWW @ ;WSS @ ;WWW @ ;WWW @ ;WSS ;SWS ; @ ;@@@@ @@@@ @@@; @ ;WWW @ S ; WW SS ;WWW SS;W W @ ;W @ ; @ S ; -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | 3 | Please report any security issue to [security@salesforce.com](mailto:security@salesforce.com) 4 | as soon as it is discovered. This library limits its runtime dependencies in 5 | order to reduce the total cost of ownership as much as can be, but all consumers 6 | should remain vigilant and have their security stakeholders review all third-party 7 | products (3PP) like this one and their dependencies. -------------------------------------------------------------------------------- /ai_economist/foundation/entities/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from .endogenous import endogenous_registry 8 | from .landmarks import landmark_registry 9 | from .resources import resource_registry 10 | -------------------------------------------------------------------------------- /ai_economist/foundation/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist.foundation.base.base_agent import agent_registry 8 | 9 | from . import mobiles, planners 10 | 11 | # Import files that add Agent class(es) to agent_registry 12 | # ------------------------------------------------------- 13 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/covid19/covid19_build.cu: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021, salesforce.com, inc. 2 | // All rights reserved. 3 | // SPDX-License-Identifier: BSD-3-Clause 4 | // For full license text, see the LICENSE file in the repo root 5 | // or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | #ifndef CUDA_INCLUDES_COVID19_CONST_H_ 8 | #define CUDA_INCLUDES_COVID19_CONST_H_ 9 | 10 | #include "../../components/covid19_components_step.cu" 11 | #include "covid19_env_step.cu" 12 | 13 | #endif // CUDA_INCLUDES_COVID19_CONST_H_ 14 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist.foundation.base.base_env import scenario_registry 8 | 9 | from .covid19 import covid19_env 10 | from .one_step_economy import one_step_economy 11 | from .simple_wood_and_stone import dynamic_layout, layout_from_file 12 | 13 | # Import files that add Scenario class(es) to scenario_registry 14 | # ------------------------------------------------------------- 15 | -------------------------------------------------------------------------------- /ai_economist/foundation/components/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist.foundation.base.base_component import component_registry 8 | 9 | from . import ( 10 | build, 11 | continuous_double_auction, 12 | covid19_components, 13 | move, 14 | redistribution, 15 | simple_labor, 16 | ) 17 | 18 | # Import files that add Component class(es) to component_registry 19 | # --------------------------------------------------------------- 20 | -------------------------------------------------------------------------------- /ai_economist/foundation/agents/mobiles.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist.foundation.base.base_agent import BaseAgent, agent_registry 8 | 9 | 10 | @agent_registry.add 11 | class BasicMobileAgent(BaseAgent): 12 | """ 13 | A basic mobile agent represents an individual actor in the economic simulation. 14 | 15 | "Mobile" refers to agents of this type being able to move around in the 2D world. 16 | """ 17 | 18 | name = "BasicMobileAgent" 19 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/env-pure_and_mixed-25x25.txt: -------------------------------------------------------------------------------- 1 | WW W @ ; SWWW @ ;SSWW @ ;WSSSW @ ;WSSWW ;WS WS ; WWS ;SWW S @ ; S W @ ; WS W @ ; @ ; @ ;@@@@ @@@@@@@@@@@@ @@@; @ ; @ ; W @ SSSSS; WW @ SSS ; @ S SS; WW W @ SSS S;W WW SSSS ;WWW S SS; WWWW S S ;WW W @ S ; W @ S ; W @ S SSS; -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/quadrant_25x25_20each_30clump.txt: -------------------------------------------------------------------------------- 1 | WWWWW W @ W ;WW W @ W W WW; W @ W W;SW S S @ ; @ W W ; SS ; S ; ; ;S S @ ; @ ; @ ;@@@@@ @@@@@@@ @@@@@; S @ ;S S @ ; S @ ;SS ; SS ;SS ; S ; @ ; @ ; @ ; @ ;S @ ; -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/uniform_25x25_25each_65clump.txt: -------------------------------------------------------------------------------- 1 | SSSSS; SS SSSS; SS SSSS; S SSSS; SS ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; W; W WW; WW ;W WW W W WWW;WWWWW WW W WWWWW; -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/closed_quadrant_25x25_20each_30clump.txt: -------------------------------------------------------------------------------- 1 | WWWWW W @ W ;WW W @ W W WW; W @ W W;SW S S @ ; @ W W ; SS @ ; S @ ; @ ; @ ;S S @ ; @ ; @ ;@@@@@@@@@@@@@@@@@@@@@@@@@; S @ ;S S @ ; S @ ;SS @ ; SS @ ;SS @ ; S @ ; @ ; @ ; @ ; @ ;S @ ; -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/quadrant_25x25_20each_30clump_no_water.txt: -------------------------------------------------------------------------------- 1 | WWWWW W W ;WW W W W WW; W W W;SW S S ; W W ; SS ; S ; ; ;S S ; ; ; ; S ;S S ; S ;SS ; SS ;SS ; S ; ; ; ; ;S ; -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/README.md: -------------------------------------------------------------------------------- 1 | ## List of COVID-19 datasources used 2 | 3 | 1. **US state government policies** (Oxford Covid-19 Government Response Tracker (OxCGRT)) 4 | 5 | https://github.com/OxCGRT/USA-covid-policy 6 | 7 | 8 | 2. **US federal government direct payments** (Committee for a Responsible Federal Budget) 9 | 10 | https://www.covidmoneytracker.org/ 11 | 12 | https://docs.google.com/spreadsheets/d/1Nr_J5wLfUT4IzqSXkYbdOXrRgEkBxhX0/edit#gid=682404301 13 | 14 | 15 | 3. **US deaths data** (COVID-19 Data Repository by the Center for Systems Science and Engineering (CSSE) at Johns Hopkins University) 16 | 17 | https://github.com/CSSEGISandData/COVID-19 18 | 19 | 20 | 4. **US unemployment** (Bureau of Labor and Statistics) 21 | 22 | https://www.bls.gov/lau/ 23 | 24 | 25 | 5. **US vaccinations** (Our World in Data) 26 | 27 | https://ourworldindata.org/covid-vaccinations -------------------------------------------------------------------------------- /ai_economist/foundation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist.foundation import utils 8 | from ai_economist.foundation.agents import agent_registry as agents 9 | from ai_economist.foundation.components import component_registry as components 10 | from ai_economist.foundation.entities import endogenous_registry as endogenous 11 | from ai_economist.foundation.entities import landmark_registry as landmarks 12 | from ai_economist.foundation.entities import resource_registry as resources 13 | from ai_economist.foundation.scenarios import scenario_registry as scenarios 14 | 15 | 16 | def make_env_instance(scenario_name, **kwargs): 17 | scenario_class = scenarios.get(scenario_name) 18 | return scenario_class(**kwargs) 19 | -------------------------------------------------------------------------------- /format_and_lint.sh: -------------------------------------------------------------------------------- 1 | ROOT_DIR="ai_economist/" 2 | 3 | echo -e "\n\nRunning ISORT to sort imports ..." 4 | isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --use-parentheses --line-width=88 $ROOT_DIR 5 | 6 | echo -e "\n\nRunning BLACK to format code ..." 7 | black --line-length 88 $ROOT_DIR 8 | 9 | echo -e "\n\nRunning FLAKE8 check to see if there are Python syntax errors ..." 10 | flake8 --count --select=E9,F63,F7,F82 --show-source --statistics $ROOT_DIR 11 | 12 | echo -e "\n\nRunning FLAKE8 formatting check ..." 13 | flake8 --ignore=E203,C901,W503,F401 --count --max-complexity=15 --max-line-length=88 --statistics $ROOT_DIR 14 | 15 | echo -e "\n\nRunning PYLINT check ..." 16 | pylint --disable \ 17 | bad-continuation,\ 18 | duplicate-code,\ 19 | import-error,\ 20 | invalid-name,\ 21 | missing-module-docstring,\ 22 | missing-function-docstring,\ 23 | no-self-use,\ 24 | too-few-public-methods,\ 25 | too-many-arguments,\ 26 | too-many-branches,\ 27 | too-many-instance-attributes,\ 28 | too-many-lines,\ 29 | too-many-locals,\ 30 | too-many-nested-blocks,\ 31 | too-many-public-methods,\ 32 | too-many-statements, \ 33 | $ROOT_DIR 34 | 35 | echo -e "\nPlease verify that FLAKE8 and PYLINT run successfully (above). If there are any errors, please fix them." 36 | 37 | echo -e "\n\nRunning PYTEST ..." 38 | pytest 39 | -------------------------------------------------------------------------------- /ai_economist/foundation/entities/endogenous.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist.foundation.base.registrar import Registry 8 | 9 | 10 | class Endogenous: 11 | """Base class for endogenous entity classes. 12 | 13 | Endogenous entities are those that, conceptually, describe the internal state 14 | of an agent. This provides a convenient way to separate physical entities (which 15 | may exist in the world, be exchanged among agents, or are otherwise in principal 16 | observable by others) from endogenous entities (such as the amount of labor 17 | effort an agent has experienced). 18 | 19 | Endogenous entities are registered in the "endogenous" portion of an agent's 20 | state and should only be observable by the agent itself. 21 | """ 22 | 23 | name = None 24 | 25 | def __init__(self): 26 | assert self.name is not None 27 | 28 | 29 | endogenous_registry = Registry(Endogenous) 30 | 31 | 32 | @endogenous_registry.add 33 | class Labor(Endogenous): 34 | """Labor accumulated through working. Included in all environments by default.""" 35 | 36 | name = "Labor" 37 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | with open('requirements.txt') as f: 7 | requirements = f.read().splitlines() 8 | 9 | setuptools.setup( 10 | name="ai-economist", 11 | version="1.7.1", 12 | author="Stephan Zheng, Alex Trott, Sunil Srinivasa", 13 | author_email="stephan.zheng@salesforce.com", 14 | description="Foundation: An Economics Simulation Framework", 15 | long_description=long_description, 16 | long_description_content_type="text/markdown", 17 | url="https://github.com/salesforce/ai-economist", 18 | packages=setuptools.find_packages(), 19 | package_data={ 20 | "ai_economist": [ 21 | "foundation/scenarios/simple_wood_and_stone/map_txt/*.txt", 22 | "foundation/scenarios/covid19/*.cu", 23 | "foundation/scenarios/covid19/key_to_check_activation_code_against", 24 | "foundation/components/*.cu", 25 | "datasets/covid19_datasets/data_and_fitted_params/*" 26 | ], 27 | }, 28 | include_package_data=True, 29 | install_requires=requirements, 30 | classifiers=[ 31 | "Programming Language :: Python :: 3", 32 | "License :: OSI Approved :: BSD License", 33 | "Operating System :: OS Independent", 34 | ], 35 | python_requires=">=3.7", 36 | ) 37 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to AI-Economist 2 | 3 | Thanks for considering contributing to AI-Economist! 4 | 5 | A typical workflow is to start a discussion about bugs or new features in a Github Issue, and work together with us to choose a strategy for a fix or enhancement. We recommend that all code changes are submitted via a Pull Request so we can go over the review process and apply the necessary changes. Please keep the PRs small so they are easy to understand and quick to review. 6 | 7 | # What to Contribute? 8 | 9 | Some of the things we are particularly interested in, include: 10 | 11 | - New Components 12 | - New Scenarios 13 | - Learning algorithms 14 | - Policy models 15 | - Documentation, examples, and tutorials 16 | 17 | However, we are always interested in hearing from you and your ideas! 18 | 19 | # Code Formatting and Checks 20 | 21 | Code is formatted using [isort](https://github.com/timothycrosley/isort) and [black](https://black.readthedocs.io/en/stable/). We also run a flake8 and pylint check on the code, with some exceptions, such as ignoring bad-continuation (C0330) errors. Before your code can be reviewed, please make sure it passes these basic checks. 22 | We have also included a [shell script](https://github.com/salesforce/ai-economist/blob/master/format_and_lint.sh) in the home directory that you can use to format as well as check the code. 23 | 24 | If you're adding new functionality that merits a quick unit test, please try to include that under the `tests` folder :). 25 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, Salesforce.com, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | # Release 1.7 (2022-01-11) 4 | - Real Business Cycle simulation (in CUDA C) involving consumers, firms and the government. 5 | - A GPU-based PyTorch training code 6 | 7 | # Release 1.6 (2021-12-17) 8 | - Tutorial for training the covid and economy environment with WarpDrive 9 | 10 | # Release 1.5 (2021-11-05) 11 | - Saez tax policy 12 | - Add the split-world scenario class 13 | - Extend fixed skill/starting location logic to more than 4 agents 14 | 15 | # Release 1.4 (2021-10-15) 16 | - One-step economy scenario and the simple labor component 17 | - 3 more layout maps for the gather-trade-build scenario 18 | - Tutorials for multi-agent training with RLlib 19 | 20 | # Release 1.3 (2021-09-24) 21 | - Training code for training the COVID-19 and the economy environment on a GPU with WarpDrive 22 | - CUDA C scenario and component codes 23 | - Environment wrapper 24 | - Environment CPU / GPU consistency checker 25 | 26 | # Release 1.2.3 (2021-09-09) 27 | - COVID-19 and the economy - related scenario and components. 28 | - COVID-19 simulation card. 29 | - Real world data, model constants and fitted parameters 30 | 31 | # Release 1.1.1 (2020-08-06) 32 | - Fixed layout maps 33 | 34 | # Release 1.1 (2020-08-06) 35 | - Links for the tutorials to run on Colab and set python version from 3.7 -> 3.6 36 | 37 | # Release 1.0 (2020-06-15) 38 | - Tutorials with examples. 39 | - Code documentation + black + isort + flake8 + pylint. 40 | - Initial commit with 41 | - Components: build/move/continuous double auction/redistribution. 42 | - Scenarios: wood and stone. -------------------------------------------------------------------------------- /ai_economist/foundation/agents/planners.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | from ai_economist.foundation.base.base_agent import BaseAgent, agent_registry 8 | 9 | 10 | @agent_registry.add 11 | class BasicPlanner(BaseAgent): 12 | """ 13 | A basic planner agent represents a social planner that sets macroeconomic policy. 14 | 15 | Unlike the "mobile" agent, the planner does not represent an embodied agent in 16 | the world environment. BasicPlanner modifies the BaseAgent class to remove 17 | location as part of the agent state. 18 | 19 | Also unlike the "mobile" agent, the planner agent is expected to be unique -- 20 | that is, there should only be 1 planner. For this reason, BasicPlanner ignores 21 | the idx argument during construction and always sets its agent index as "p". 22 | """ 23 | 24 | name = "BasicPlanner" 25 | 26 | def __init__(self, *args, **kwargs): 27 | super().__init__(*args, **kwargs) 28 | del self.state["loc"] 29 | 30 | # Overwrite any specified index so that this one is always indexed as 'p' 31 | # (make a separate class of planner if you want there to be multiple planners 32 | # in a game) 33 | self._idx = "p" 34 | 35 | @property 36 | def loc(self): 37 | """ 38 | Planner agents do not occupy any location. 39 | """ 40 | raise AttributeError("BasicPlanner agents do not occupy a location.") 41 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/env-pure_and_mixed-40x40.txt: -------------------------------------------------------------------------------- 1 | W WW ; W ; W W W ; W ; W W ; W W W ; W W ; WW ; ; ; ; ; ; ; ; ; SS W ; WW W; ; S S S; W WW S ; S S W ; W WS S S ; ; ; ; ; ; ; ; ; S ;S S ; ; SS ; ; S SS S ; SS S S ;S SS S S ;SSSS S SS ; -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/quadrant_40x40_50each.txt: -------------------------------------------------------------------------------- 1 | WWWWWWWW WW @@ WW ;WWWWWWWW WW @@ WW ;WWW W @@ WW W WWW; W @@ WW WW; W @@ WW WW;SSW S SS @@ ; @@ W W ; @@ W W ; SSS ; SSS ; S ; ; ; ;SS SS @@ ;SS SS @@ ; @@ ; @@ ; @@ ;@@@@@@@@ @@@@@@@@@@@@ @@@@@@@@;@@@@@@@@ @@@@@@@@@@@@ @@@@@@@@; S @@ ;SS SS @@ ;SS SS @@ ; S @@ ; S @@ ;SSS ; SSS ; SSS ;SSS ; SS ; SS ; @@ ; @@ ; @@ ; @@ ; @@ ; @@ ;SS @@ ;SS @@ ; -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/simple_wood_and_stone/map_txt/quadrant_40x40_50each_no_water.txt: -------------------------------------------------------------------------------- 1 | WWWWWWWW WW WW ;WWWWWWWW WW WW ;WWW W WW W WWW; W WW WW; W WW WW;SSW S SS ; W W ; W W ; SSS ; SSS ; S ; ; ; ;SS SS ;SS SS ; ; ; ; ; ; S ;SS SS ;SS SS ; S ; S ;SSS ; SSS ; SSS ;SSS ; SS ; SS ; ; ; ; ; ; ;SS ;SS ; -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/data_and_fitted_params/model_constants.json: -------------------------------------------------------------------------------- 1 | {"DATE_FORMAT": "%Y-%m-%d", "STRINGENCY_POLICY_KEY": "StringencyIndex", "NUM_STRINGENCY_LEVELS": 10, "SIR_SMOOTHING_STD": 10, "SIR_MORTALITY": 0.02, "SIR_GAMMA": 0.07142857142857142, "US_STATE_IDX_TO_STATE_NAME": {"0": "Alabama", "1": "Alaska", "2": "Arizona", "3": "Arkansas", "4": "California", "5": "Colorado", "6": "Connecticut", "7": "Delaware", "8": "District of Columbia", "9": "Florida", "10": "Georgia", "11": "Hawaii", "12": "Idaho", "13": "Illinois", "14": "Indiana", "15": "Iowa", "16": "Kansas", "17": "Kentucky", "18": "Louisiana", "19": "Maine", "20": "Maryland", "21": "Massachusetts", "22": "Michigan", "23": "Minnesota", "24": "Mississippi", "25": "Missouri", "26": "Montana", "27": "Nebraska", "28": "Nevada", "29": "New Hampshire", "30": "New Jersey", "31": "New Mexico", "32": "New York", "33": "North Carolina", "34": "North Dakota", "35": "Ohio", "36": "Oklahoma", "37": "Oregon", "38": "Pennsylvania", "39": "Rhode Island", "40": "South Carolina", "41": "South Dakota", "42": "Tennessee", "43": "Texas", "44": "Utah", "45": "Vermont", "46": "Virginia", "47": "Washington", "48": "West Virginia", "49": "Wisconsin", "50": "Wyoming"}, "US_STATE_POPULATION": [4903185, 740995, 7278717, 3017804, 39512223, 5758736, 3565287, 973764, 705749, 21477737, 10617423, 1415872, 1787065, 12671821, 6732219, 3155070, 2913314, 4467673, 4648794, 1344212, 6045680, 6892503, 9986857, 5639632, 2976149, 6626371, 1068778, 1934408, 3080156, 1359711, 8882190, 2096829, 19453561, 10488084, 762062, 11689100, 3956971, 4217737, 12801989, 1059361, 5148714, 884659, 6829174, 28995881, 3205958, 623989, 8535519, 7614893, 1792147, 5822434, 578759], "US_POPULATION": 328737916, "GDP_PER_CAPITA": 65300} -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/covid19/key_to_check_activation_code_against: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpgIBAAKCAQEAk1+Qz0/Qg4OOGrskBJnVI9KVGTEUvsldHUV4AzLeecYZSV5+ 3 | FZUQpl8lq1mstUZZ0xMlGSHz2t+AAJxyEro8mAj9gAp1qeN58pAX2k29DOt4YRnp 4 | sTF1UG+nrV2aW+jfH16aeVsjWY+Nq+GxGyE3Q5bsxOhnOg0TUaB6RY8SBE/scTHn 5 | bfNsgTc5EuiAAGqYYYdu12n5zeyvfjGW7bBf4Q9t0F0bI+YdZQY9HD35KAoNcqFQ 6 | dvd2vKbojejkn+WyO1amnZxgAhVjpT61FV4u18jPN0Qrt0LHuF5kUVzYal+73ySY 7 | BbwEo4onEn9xvUlQGFJWmv4OPwbI3d4nLqP+mQIHK9xUXfK97QKCAQABeR2EO0uu 8 | ERyRXa5Mh7xsOEq/OJ9sQq+si8B5gDyyM1SW61wQMKF4Wiqw68bMCVvGRwScZD+T 9 | XwBEBJMm9lCVx/UfOWqYSNFCk/YBefv9AI0Kg5lfCMZQuTdjMcbJdjoR5xoiCbO1 10 | ya7oOU8mfWx/SV0o/698b/zMVBKBBQDNZaN9pmtTOgm3G1QnM9ZlmrdlKYpe9Ihs 11 | 3sG4437QaPhumdZi8IoLBGMyYL2O38pG34LJjIkP8Efj1QVTndIIZX8CKghir++j 12 | nUAyofFt7/PBS2k7gQ/1gFISwHxKjmzl/Fc25o7ahlLbO+i2UnRiB9IXcmiGDXMv 13 | tY09oXhxCtTZAoGBAMEkMTzoiqKjXLwKLyFIF5QzXqQKcGqfC8NhQMsm43K0TgHg 14 | Sv1fLdnKw0FWSG30gppBorAY9p5FoI+AWwTSd+AJhz7T1y/shpJx1oBR8qKWO5kO 15 | gMru9kRRb0zb5hydakie3mujz7GUPiXrntKZjC4QYLar0USPulJnU+UTF6QjAoGB 16 | AMNWJqG1ybrk0sNkWJJDW+MnMT0T9o0E+CtbRHqMHh7K1LF9Sc/qh0gLfDo51+kr 17 | pscLaaJiF1Q8phzDhW9QDeNv+4lknNqMFBCFtzns1wVDlXL4U87oqhuBSs6IZAuO 18 | CGVefYKgefdwn64rcyRNala44BbiMJKwRoDvvgH1FvATAoGAV1YK9ZHB1RkXkZ5a 19 | uBePXvkScaujH4DxadMGf2tBuI1wIpVwhxOQ56yDwYoAuexXPUa8BAx2V69/LFo7 20 | H/yDYqzndA8WwZLy8oy7Ug+fFLtCp7VhkEwMPciBq6KjzUyShIBlgZOx5m5kTbfu 21 | Cs2JQU35YHeompcpLooRG1/cFZkCgYAyVlWABzmgSKJL9ohwlSBBZFCjQ1mjN6uc 22 | uRJxncqfCe3XQ5erFjuWMfPayWONBsWexNucJFc7Iz2LzCOXkUsftldEEET9f/2w 23 | PrbsEu8khNTLqUcow2Whz+A8C0dV6p2cqtTKR1XlSmNVqP30lmpHcmF+R3M/J1ON 24 | K7S9zJJ+zwKBgHIuCATGCGCOzAsUo80OQL46j74SxRV3H1CJASLKzatiTo54dbO6 25 | 86w+N6BfYtYeRlnX1CTGl6bHqVUMBBlKws8Ig3gV3xFS8BiSav8zQ2m99JuhlVHF 26 | Ocfowmuad3WXYvYXQ5IeP2JM/3q7BoPLg1DKP4GGZlNbatMRI+H0HimV 27 | -----END RSA PRIVATE KEY----- -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/train_single_exp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import argparse 8 | 9 | from experiment_utils import run_experiment_batch_parallel 10 | from rbc.constants import all_agents_export_experiment_template 11 | 12 | if __name__ == "__main__": 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--dry-run", action="store_true") 15 | parser.add_argument("--experiment-dir", type=str, default="experiment/experiment") 16 | parser.add_argument("--group-name", type=str, default="default_group") 17 | parser.add_argument("--job-name-base", type=str, default="rollout") 18 | parser.add_argument("--num-consumers", type=int, default=100) 19 | parser.add_argument("--num-firms", type=int, default=10) 20 | parser.add_argument("--num-governments", type=int, default=1) 21 | parser.add_argument("--run-only", action="store_true") 22 | parser.add_argument("--seed-from-timestamp", action="store_true") 23 | 24 | args = parser.parse_args() 25 | ( 26 | default_cfg_dict, 27 | consumption_choices, 28 | work_choices, 29 | price_and_wage, 30 | tax_choices, 31 | default_firm_action, 32 | default_government_action, 33 | ) = all_agents_export_experiment_template( 34 | args.num_firms, args.num_consumers, args.num_governments 35 | ) 36 | 37 | if not args.dry_run: 38 | # for dirs in experiment dir, run job 39 | experiment = args.experiment_dir 40 | run_experiment_batch_parallel( 41 | experiment, 42 | consumption_choices, 43 | work_choices, 44 | price_and_wage, 45 | tax_choices, 46 | group_name=args.group_name, 47 | consumers_only=False, 48 | no_firms=False, 49 | default_firm_action=default_firm_action, 50 | default_government_action=default_government_action, 51 | ) 52 | -------------------------------------------------------------------------------- /ai_economist/foundation/entities/resources.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | from ai_economist.foundation.base.registrar import Registry 10 | 11 | 12 | class Resource: 13 | """Base class for Resource entity classes. 14 | 15 | Resource classes describe entities that can be a part of an agent's inventory. 16 | 17 | Resources can also be a part of the world as collectible entities: for each 18 | Resource class with Resource.collectible=True, a complementary 19 | ResourceSourceBlock Landmark class will be created in landmarks.py. For each 20 | collectible resource in the environment, the world map will include a resource 21 | source block channel (representing landmarks where collectible resources are 22 | generated) and a resource channel (representing locations where collectible 23 | resources have generated). 24 | """ 25 | 26 | name = None 27 | color = None # array of RGB values [0 - 1] 28 | collectible = None # Is this something that exists in the world? 29 | # (versus something that can only be owned) 30 | 31 | def __init__(self): 32 | assert self.name is not None 33 | assert self.color is not None 34 | assert self.collectible is not None 35 | 36 | 37 | resource_registry = Registry(Resource) 38 | 39 | 40 | @resource_registry.add 41 | class Wood(Resource): 42 | """Wood resource. collectible.""" 43 | 44 | name = "Wood" 45 | color = np.array([107, 143, 113]) / 255.0 46 | collectible = True 47 | 48 | 49 | @resource_registry.add 50 | class Stone(Resource): 51 | """Stone resource. collectible.""" 52 | 53 | name = "Stone" 54 | color = np.array([241, 233, 219]) / 255.0 55 | collectible = True 56 | 57 | 58 | @resource_registry.add 59 | class Coin(Resource): 60 | """Coin resource. Included in all environments by default. Not collectible.""" 61 | 62 | name = "Coin" 63 | color = np.array([229, 211, 82]) / 255.0 64 | collectible = False 65 | -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/us_deaths.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import os 8 | from io import BytesIO 9 | 10 | import pandas as pd 11 | import requests 12 | 13 | 14 | class DatasetCovidDeathsUS: 15 | """ 16 | Class to load COVID-19 deaths data for the US. 17 | Source: https://github.com/CSSEGISandData/COVID-19 18 | Note: in this dataset, reporting deaths only started on the 22nd of January 2020, 19 | 20 | Attributes: 21 | df: Timeseries dataframe of confirmed COVID deaths for all the US states 22 | """ 23 | 24 | def __init__(self, data_dir="", download_latest_data=True): 25 | if not os.path.exists(data_dir): 26 | print( 27 | "Creating a dynamic data directory to store " 28 | "COVID-19 deaths data: {}".format(data_dir) 29 | ) 30 | os.makedirs(data_dir) 31 | 32 | filename = "daily_us_deaths.csv" 33 | if download_latest_data or filename not in os.listdir(data_dir): 34 | print( 35 | "Fetching latest U.S. COVID-19 deaths data from John Hopkins, " 36 | "and saving it in {}".format(data_dir) 37 | ) 38 | 39 | req = requests.get( 40 | "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/" 41 | "csse_covid_19_data/csse_covid_19_time_series/" 42 | "time_series_covid19_deaths_US.csv" 43 | ) 44 | self.df = pd.read_csv(BytesIO(req.content)) 45 | self.df.to_csv( 46 | os.path.join(data_dir, filename) 47 | ) # Note: performs an overwrite 48 | else: 49 | print( 50 | "Not fetching the latest U.S. COVID-19 deaths data from John Hopkins." 51 | " Using whatever was saved earlier in {}!!".format(data_dir) 52 | ) 53 | assert filename in os.listdir(data_dir) 54 | self.df = pd.read_csv(os.path.join(data_dir, filename), low_memory=False) 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.4 2 | appnope==0.1.2 3 | argon2-cffi==20.1.0 4 | astroid==2.5.6 5 | async-generator==1.10 6 | attrs==21.2.0 7 | backcall==0.2.0 8 | beautifulsoup4==4.9.3 9 | black==21.5b1 10 | bleach==3.3.0 11 | bs4==0.0.1 12 | certifi==2020.12.5 13 | cffi==1.14.5 14 | chardet==4.0.0 15 | click==8.0.1 16 | cycler==0.10.0 17 | decorator==5.0.9 18 | defusedxml==0.7.1 19 | entrypoints==0.3 20 | et-xmlfile==1.1.0 21 | flake8==3.9.2 22 | GPUtil==1.4.0 23 | idna==2.10 24 | iniconfig==1.1.1 25 | ipykernel==5.5.5 26 | ipython==7.31.1 27 | ipython-genutils==0.2.0 28 | ipywidgets==7.6.3 29 | isort==5.8.0 30 | jedi==0.18.0 31 | Jinja2==3.0.1 32 | jsonschema==3.2.0 33 | jupyter==1.0.0 34 | jupyter-client==6.1.12 35 | jupyter-console==6.4.0 36 | jupyter-core==4.7.1 37 | jupyterlab-pygments==0.1.2 38 | jupyterlab-widgets==1.0.0 39 | kiwisolver==1.3.1 40 | lazy-object-proxy==1.6.0 41 | lz4==3.1.3 42 | MarkupSafe==2.0.1 43 | matplotlib==3.2.1 44 | matplotlib-inline==0.1.2 45 | mccabe==0.6.1 46 | mistune==0.8.4 47 | mypy-extensions==0.4.3 48 | nbclient==0.5.3 49 | nbconvert==6.0.7 50 | nbformat==5.1.3 51 | nest-asyncio==1.5.1 52 | notebook==6.4.1 53 | numpy==1.21.0 54 | openpyxl==3.0.7 55 | packaging==20.9 56 | pandas==1.2.4 57 | pandocfilters==1.4.3 58 | parso==0.8.2 59 | pathspec==0.8.1 60 | pexpect==4.8.0 61 | pickleshare==0.7.5 62 | Pillow==9.0.1 63 | pluggy==0.13.1 64 | prometheus-client==0.10.1 65 | prompt-toolkit==3.0.18 66 | ptyprocess==0.7.0 67 | py==1.10.0 68 | pycodestyle==2.7.0 69 | pycparser==2.20 70 | pycryptodome==3.10.1 71 | pyflakes==2.3.1 72 | Pygments==2.9.0 73 | pylint==2.8.2 74 | pyparsing==2.4.7 75 | pyrsistent==0.17.3 76 | pytest==6.2.4 77 | python-dateutil==2.8.1 78 | pytz==2021.1 79 | pyyaml==5.4.1 80 | pyzmq==22.0.3 81 | qtconsole==5.1.0 82 | QtPy==1.9.0 83 | regex==2021.4.4 84 | requests==2.25.1 85 | scipy==1.6.3 86 | Send2Trash==1.5.0 87 | six==1.16.0 88 | soupsieve==2.2.1 89 | terminado==0.10.0 90 | testpath==0.5.0 91 | toml==0.10.2 92 | tornado==6.1 93 | tqdm==4.60.0 94 | traitlets==5.0.5 95 | typing-extensions==3.10.0.0 96 | urllib3==1.26.5 97 | wcwidth==0.2.5 98 | webencodings==0.5.1 99 | widgetsnbextension==3.5.1 100 | wrapt==1.12.1 101 | -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/us_vaccinations.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import os 8 | from io import BytesIO 9 | 10 | import pandas as pd 11 | import requests 12 | 13 | 14 | class DatasetCovidVaccinationsUS: 15 | """ 16 | Class to load COVID-19 vaccination data for the US. 17 | Source: https://ourworldindata.org/covid-vaccinations 18 | 19 | Attributes: 20 | df: Timeseries dataframe of COVID vaccinations for all the US states 21 | """ 22 | 23 | def __init__(self, data_dir="", download_latest_data=True): 24 | if not os.path.exists(data_dir): 25 | print( 26 | "Creating a dynamic data directory to store COVID-19 " 27 | "vaccination data: {}".format(data_dir) 28 | ) 29 | os.makedirs(data_dir) 30 | 31 | filename = "daily_us_vaccinations.csv" 32 | if download_latest_data or filename not in os.listdir(data_dir): 33 | print( 34 | "Fetching latest U.S. COVID-19 vaccination data from " 35 | "Our World in Data, and saving it in {}".format(data_dir) 36 | ) 37 | 38 | req = requests.get( 39 | "https://raw.githubusercontent.com/owid/covid-19-data/master/" 40 | "public/data/vaccinations/us_state_vaccinations.csv" 41 | ) 42 | self.df = pd.read_csv(BytesIO(req.content)) 43 | 44 | # Rename New York State to New York for consistency with other datasets 45 | self.df = self.df.replace("New York State", "New York") 46 | 47 | # Interpolate missing values 48 | self.df = self.df.interpolate(method="linear") 49 | 50 | self.df.to_csv( 51 | os.path.join(data_dir, filename) 52 | ) # Note: performs an overwrite 53 | else: 54 | print( 55 | "Not fetching the latest U.S. COVID-19 deaths data from " 56 | "Our World in Data. Using whatever was saved earlier in {}!!".format( 57 | data_dir 58 | ) 59 | ) 60 | assert filename in os.listdir(data_dir) 61 | self.df = pd.read_csv(os.path.join(data_dir, filename), low_memory=False) 62 | -------------------------------------------------------------------------------- /tutorials/rllib/utils/remote.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | 10 | def remote_env_fun(trainer, env_function): 11 | """ 12 | Create a dictionary with the following mapping: 13 | result[env_wrapper.env_id] = env_function(env) 14 | where each entry in the dictionary comes from one of the active envs in the trainer. 15 | env_function must be a function that takes an environment as its single argument 16 | """ 17 | 18 | nested_env_ids_and_results = trainer.workers.foreach_worker( 19 | lambda w: [(env.env_id, env_function(env)) for env in w.async_env.envs] 20 | ) 21 | nested_env_ids_and_results = nested_env_ids_and_results[ 22 | 1: 23 | ] # Ignore the local worker 24 | 25 | # Store them first this way in case they don't come out sorted 26 | # (gets sorted by env_id before being returned) 27 | result = {} 28 | 29 | for worker_stuff in nested_env_ids_and_results: 30 | for env_id, output in worker_stuff: 31 | result[env_id] = output 32 | return result 33 | 34 | 35 | def get_trainer_envs(trainer): 36 | return remote_env_fun(trainer, lambda env: env) 37 | 38 | 39 | def collect_stored_rollouts(trainer): 40 | aggregate_rollouts = {} 41 | 42 | rollout_dict = remote_env_fun(trainer, lambda e: e.rollout) 43 | n_envs = len(rollout_dict) 44 | 45 | for env_id, env_rollout in rollout_dict.items(): 46 | for k, v in env_rollout.items(): 47 | if k not in aggregate_rollouts: 48 | sz = v.shape 49 | sz = [sz[0], n_envs] + sz[1:] 50 | aggregate_rollouts[k] = np.zeros(sz) 51 | aggregate_rollouts[k][:, env_id] = v 52 | 53 | return aggregate_rollouts 54 | 55 | 56 | def accumulate_and_broadcast_saez_buffers(trainer): 57 | component_name = "PeriodicBracketTax" 58 | 59 | def extract_local_saez_buffers(env_wrapper): 60 | return env_wrapper.env.get_component(component_name).get_local_saez_buffer() 61 | 62 | replica_buffers = remote_env_fun(trainer, extract_local_saez_buffers) 63 | 64 | global_buffer = [] 65 | for local_buffer in replica_buffers.values(): 66 | global_buffer += local_buffer 67 | 68 | def set_global_buffer(env_wrapper): 69 | env_wrapper.env.get_component(component_name).set_global_saez_buffer( 70 | global_buffer 71 | ) 72 | 73 | _ = remote_env_fun(trainer, set_global_buffer) 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | *.py[cod] 3 | *$py.class 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | downloads/ 14 | eggs/ 15 | .eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | wheels/ 22 | share/python-wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .nox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | *.py,cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | cover/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | db.sqlite3-journal 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | .pybuilder/ 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | # For a library or package, you might want to ignore these files since the code is 86 | # intended to run in multiple environments; otherwise, check them in: 87 | # .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 97 | __pypackages__/ 98 | 99 | # Celery stuff 100 | celerybeat-schedule 101 | celerybeat.pid 102 | 103 | # SageMath parsed files 104 | *.sage.py 105 | 106 | # Environments 107 | .env 108 | .venv 109 | env/ 110 | venv/ 111 | ENV/ 112 | env.bak/ 113 | venv.bak/ 114 | 115 | # Spyder project settings 116 | .spyderproject 117 | .spyproject 118 | 119 | # Rope project settings 120 | .ropeproject 121 | 122 | # mkdocs documentation 123 | /site 124 | 125 | # mypy 126 | .mypy_cache/ 127 | .dmypy.json 128 | dmypy.json 129 | 130 | # Pyre type checker 131 | .pyre/ 132 | 133 | # pytype static type analyzer 134 | .pytype/ 135 | 136 | # Cython debug symbols 137 | cython_debug/ 138 | 139 | # PyCharm stuff 140 | .idea/ 141 | 142 | .DS_Store -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/utils/social_metrics.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | 10 | def get_gini(endowments): 11 | """Returns the normalized Gini index describing the distribution of endowments. 12 | 13 | https://en.wikipedia.org/wiki/Gini_coefficient 14 | 15 | Args: 16 | endowments (ndarray): The array of endowments for each of the agents in the 17 | simulated economy. 18 | 19 | Returns: 20 | Normalized Gini index for the distribution of endowments (float). A value of 1 21 | indicates everything belongs to 1 agent (perfect inequality), whereas a 22 | value of 0 indicates all agents have equal endowments (perfect equality). 23 | 24 | Note: 25 | Uses a slightly different method depending on the number of agents. For fewer 26 | agents (<30), uses an exact but slow method. Switches to using a much faster 27 | method for more agents, where both methods produce approximately equivalent 28 | results. 29 | """ 30 | n_agents = len(endowments) 31 | 32 | if n_agents < 30: # Slower. Accurate for all n. 33 | diff_ij = np.abs( 34 | endowments.reshape((n_agents, 1)) - endowments.reshape((1, n_agents)) 35 | ) 36 | diff = np.sum(diff_ij) 37 | norm = 2 * n_agents * endowments.sum(axis=0) 38 | unscaled_gini = diff / (norm + 1e-10) 39 | gini = unscaled_gini / ((n_agents - 1) / n_agents) 40 | return gini 41 | 42 | # Much faster. Slightly overestimated for low n. 43 | s_endows = np.sort(endowments) 44 | return 1 - (2 / (n_agents + 1)) * np.sum( 45 | np.cumsum(s_endows) / (np.sum(s_endows) + 1e-10) 46 | ) 47 | 48 | 49 | def get_equality(endowments): 50 | """Returns the complement of the normalized Gini index (equality = 1 - Gini). 51 | 52 | Args: 53 | endowments (ndarray): The array of endowments for each of the agents in the 54 | simulated economy. 55 | 56 | Returns: 57 | Normalized equality index for the distribution of endowments (float). A value 58 | of 0 indicates everything belongs to 1 agent (perfect inequality), 59 | whereas a value of 1 indicates all agents have equal endowments (perfect 60 | equality). 61 | """ 62 | return 1 - get_gini(endowments) 63 | 64 | 65 | def get_productivity(coin_endowments): 66 | """Returns the total coin inside the simulated economy. 67 | 68 | Args: 69 | coin_endowments (ndarray): The array of coin endowments for each of the 70 | agents in the simulated economy. 71 | 72 | Returns: 73 | Total coin endowment (float). 74 | """ 75 | return np.sum(coin_endowments) 76 | -------------------------------------------------------------------------------- /ai_economist/training/run_configs/covid_and_economy_environment.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | # YAML configuration for the tag continuous environment 8 | name: "covid_and_economy_environment" 9 | # Environment settings 10 | env: 11 | collate_agent_step_and_reset_data: True 12 | components: 13 | - ControlUSStateOpenCloseStatus: 14 | action_cooldown_period: 28 15 | - FederalGovernmentSubsidy: 16 | num_subsidy_levels: 20 17 | subsidy_interval: 90 18 | max_annual_subsidy_per_person: 20000 19 | - VaccinationCampaign: 20 | daily_vaccines_per_million_people: 3000 21 | delivery_interval: 1 22 | vaccine_delivery_start_date: "2021-01-12" 23 | economic_reward_crra_eta: 2 24 | episode_length: 540 25 | flatten_masks: True 26 | flatten_observations: False 27 | health_priority_scaling_agents: 0.3 28 | health_priority_scaling_planner: 0.45 29 | infection_too_sick_to_work_rate: 0.1 30 | multi_action_mode_agents: False 31 | multi_action_mode_planner: False 32 | n_agents: 51 33 | path_to_data_and_fitted_params: "" 34 | pop_between_age_18_65: 0.6 35 | risk_free_interest_rate: 0.03 36 | world_size: [1, 1] 37 | start_date: "2020-03-22" 38 | use_real_world_data: False 39 | use_real_world_policies: False 40 | # Trainer settings 41 | trainer: 42 | num_envs: 60 # number of environment replicas 43 | num_episodes: 1000 # number of episodes to run the training for 44 | train_batch_size: 5400 # total batch size used for training per iteration (across all the environments) 45 | # Policy network settings 46 | policy: # list all the policies below 47 | a: 48 | to_train: True # flag indicating whether the model needs to be trained 49 | algorithm: "PPO" # algorithm used to train the policy 50 | vf_loss_coeff: 1 # loss coefficient schedule for the value function loss 51 | entropy_coeff: 0.05 # loss coefficient schedule for the entropy loss 52 | gamma: 0.98 # discount factor 53 | lr: 0.0001 # learning rate 54 | model: 55 | type: "fully_connected" 56 | fc_dims: [256, 256] 57 | model_ckpt_filepath: "" 58 | p: 59 | to_train: True 60 | algorithm: "PPO" 61 | vf_loss_coeff: 1 62 | entropy_coeff: # annealing entropy over time 63 | - [0, 0.5] 64 | - [50000000, 0.05] 65 | gamma: 0.98 66 | lr: 0.0001 67 | model: 68 | type: "fully_connected" 69 | fc_dims: [256, 256] 70 | model_ckpt_filepath: "" 71 | # Checkpoint saving setting 72 | saving: 73 | metrics_log_freq: 100 # How often (in iterations) to print the metrics 74 | model_params_save_freq: 500 # How often (in iterations) to save the model parameters 75 | basedir: "/tmp" # base folder used for saving 76 | name: "covid19_and_economy" # experiment name 77 | tag: "experiments" # experiment tag 78 | -------------------------------------------------------------------------------- /ai_economist/foundation/entities/landmarks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | from ai_economist.foundation.base.registrar import Registry 10 | from ai_economist.foundation.entities.resources import resource_registry 11 | 12 | 13 | class Landmark: 14 | """Base class for Landmark entity classes. 15 | 16 | Landmark classes describe the entities that exist exclusively in the environment 17 | world. In other words, they represent entities that should not be included in an 18 | agent's inventory and are only observable through observations from the 19 | spatial world. 20 | 21 | Landmark classes describe the following properties: 22 | ownable: If each instance of the landmark belongs to an agent. For example, a 23 | "House" is ownable and belongs to the agent that constructs it whereas 24 | "Water" is not ownable. 25 | solid: If the landmark creates a physical barrier to movement (that is, 26 | if agents are prevented from occupying cells with the landmark). 27 | Importantly, if the landmark is ownable, the agent that owns a given 28 | landmark can occupy its cell even if the landmark is solid. 29 | """ 30 | 31 | name = None 32 | color = None # array of RGB values [0 - 1] 33 | ownable = None 34 | solid = True # Solid = Cannot be passed through 35 | # (unless it is owned by the agent trying to pass through) 36 | 37 | def __init__(self): 38 | assert self.name is not None 39 | assert self.color is not None 40 | assert self.ownable is not None 41 | 42 | # No agent can pass through this landmark 43 | self.blocking = self.solid and not self.ownable 44 | 45 | # Only the agent that owns this landmark can pass through it 46 | self.private = self.solid and self.ownable 47 | 48 | # This landmark does not belong to any agent and it does not inhibit movement 49 | self.public = not self.solid and not self.ownable 50 | 51 | 52 | landmark_registry = Registry(Landmark) 53 | 54 | # Registering each collectible resource's source block 55 | # allows treating source blocks in a specific way 56 | for resource_name in resource_registry.entries: 57 | resource = resource_registry.get(resource_name) 58 | if not resource.collectible: 59 | continue 60 | 61 | @landmark_registry.add 62 | class SourceBlock(Landmark): 63 | """Special Landmark for generating resources. Not ownable. Not solid.""" 64 | 65 | name = "{}SourceBlock".format(resource.name) 66 | color = np.array(resource.color) 67 | ownable = False 68 | solid = False 69 | 70 | 71 | @landmark_registry.add 72 | class House(Landmark): 73 | """House landmark. Ownable. Solid.""" 74 | 75 | name = "House" 76 | color = np.array([220, 20, 220]) / 255.0 77 | ownable = True 78 | solid = True 79 | 80 | 81 | @landmark_registry.add 82 | class Water(Landmark): 83 | """Water Landmark. Not ownable. Solid.""" 84 | 85 | name = "Water" 86 | color = np.array([50, 50, 250]) / 255.0 87 | ownable = False 88 | solid = True 89 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: linters 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | linter: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Set up Python 3.7 16 | uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.7 19 | - name: Install dependencies 20 | run: | 21 | # INSTALLS 22 | python -m pip install --upgrade pip 23 | pip install -r requirements.txt 24 | echo "PYTHONPATH=/home/runner/work/ai-economist-opensource/ai-economist-opensource/" >> $GITHUB_ENV 25 | 26 | - name: Sort imports with isort 27 | run: | 28 | # isort config compatible with black 29 | isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --use-parentheses --line-width=88 --check-only -rc ai_economist/ 30 | continue-on-error: false 31 | 32 | - name: Format code with black 33 | run: | 34 | black --line-length 88 --check ai_economist/ 35 | continue-on-error: false 36 | 37 | - name: Lint code with flake8 38 | run: | 39 | # stop the build if there are Python syntax errors or undefined names 40 | flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ai_economist ai_economist/ 41 | # exit-zero treats all errors as warnings. Also outputs error counts 42 | # Add as many codes under --ignore for the checks to ignore 43 | # E203: Whitespace before ':' (will be taken care of by Black) 44 | # C901: Function is too complex 45 | # W503: Line break occurred before a binary operator (Black is expected to take care of formatting) 46 | # F401: Module imported but unused 47 | flake8 --ignore=E203,C901,W503,F401 --count --max-complexity=15 --max-line-length=88 --statistics ai_economist/ 48 | 49 | - name: Lint code with pylint 50 | run: | 51 | # Add as many codes under --disable= for the checks to disable 52 | # Note: exit-zero disregards all errors 53 | # *** CODES CURRENTLY DISABLED *** 54 | # C0103: invalid-name 55 | # C0114: missing-module-docstring 56 | # C0116: missing-function-docstring 57 | # C0302: too-many-lines 58 | # C0330: bad-continuation 59 | # E0401: import-error 60 | # R0201: no-self-use 61 | # R0801: duplicate-code 62 | # R0902: too-many-instance-attributes 63 | # R0903: too-few-public-methods 64 | # R0904: too-many-public-methods 65 | # R0912: too-many-branches 66 | # R0913: too-many-arguments 67 | # R0914: too-many-locals 68 | # R0915: too-many-statements 69 | # R1702: too-many-nested-blocks 70 | 71 | pylint --disable \ 72 | bad-continuation,\ 73 | duplicate-code,\ 74 | import-error,\ 75 | invalid-name,\ 76 | missing-module-docstring,\ 77 | missing-function-docstring,\ 78 | no-self-use,\ 79 | too-few-public-methods,\ 80 | too-many-arguments,\ 81 | too-many-branches,\ 82 | too-many-instance-attributes,\ 83 | too-many-lines,\ 84 | too-many-locals,\ 85 | too-many-nested-blocks,\ 86 | too-many-public-methods,\ 87 | too-many-statements \ 88 | ai_economist/ 89 | 90 | - name: Test with pytest 91 | run: | 92 | pytest 93 | -------------------------------------------------------------------------------- /ai_economist/foundation/base/registrar.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | 8 | class Registry: 9 | """Utility for registering sets of similar classes and looking them up by name. 10 | 11 | Registries provide a simple API for getting classes used to build environment 12 | instances. Their main purpose is to organize such "building block" classes (i.e. 13 | Components, Scenarios, Agents) for easy reference as well as to ensure that all 14 | classes within a particular registry inherit from the same Base Class. 15 | 16 | Args: 17 | base_class (class): The class that all entries in the registry must be a 18 | subclass of. 19 | 20 | Example: 21 | class BaseClass: 22 | pass 23 | 24 | registry = Registry(BaseClass) 25 | 26 | @registry.add 27 | class ExampleSubclassA(BaseClass): 28 | name = "ExampleA" 29 | pass 30 | 31 | @registry.add 32 | class ExampleSubclassB(BaseClass): 33 | name = "ExampleB" 34 | pass 35 | 36 | print(registry.entries) 37 | # ["ExampleA", "ExampleB"] 38 | 39 | assert registry.has("ExampleA") 40 | assert registry.get("ExampleB") is ExampleSubclassB 41 | """ 42 | 43 | def __init__(self, base_class=None): 44 | self.base_class = base_class 45 | self._entries = [] 46 | self._lookup = dict() 47 | 48 | def add(self, cls): 49 | """Add cls to this registry. 50 | 51 | Args: 52 | cls: The class to add to this registry. Must be a subclass of 53 | self.base_class. 54 | 55 | Returns: 56 | cls (to allow decoration with @registry.add) 57 | 58 | See Registry class docstring for example. 59 | """ 60 | assert "." not in cls.name 61 | if self.base_class: 62 | assert issubclass(cls, self.base_class) 63 | self._lookup[cls.name.lower()] = cls 64 | if cls.name not in self._entries: 65 | self._entries.append(cls.name) 66 | return cls 67 | 68 | def get(self, cls_name): 69 | """Return registered class with name cls_name. 70 | 71 | Args: 72 | cls_name (str): Name of the registered class to get. 73 | 74 | Returns: 75 | Registered class cls, where cls.name matches cls_name (ignoring casing). 76 | 77 | See Registry class docstring for example. 78 | """ 79 | if cls_name.lower() not in self._lookup: 80 | raise KeyError('"{}" is not a name of a registered class'.format(cls_name)) 81 | return self._lookup[cls_name.lower()] 82 | 83 | def has(self, cls_name): 84 | """Return True if a class with name cls_name is registered. 85 | 86 | Args: 87 | cls_name (str): Name of class to check. 88 | 89 | See Registry class docstring for example. 90 | """ 91 | return cls_name.lower() in self._lookup 92 | 93 | @property 94 | def entries(self): 95 | """Names of classes in this registry. 96 | 97 | Returns: 98 | A list of strings corresponding to the names of classes registered in 99 | this registry object. 100 | 101 | See Registry class docstring for example. 102 | """ 103 | return sorted(list(self._entries)) 104 | -------------------------------------------------------------------------------- /tutorials/rllib/phase1/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | env: 8 | components: 9 | - Build: 10 | build_labor: 10 11 | payment: 10 12 | payment_max_skill_multiplier: 3 13 | skill_dist: pareto 14 | - ContinuousDoubleAuction: 15 | max_bid_ask: 10 16 | max_num_orders: 5 17 | order_duration: 50 18 | order_labor: 0.25 19 | - Gather: 20 | collect_labor: 1 21 | move_labor: 1 22 | skill_dist: pareto 23 | - PeriodicBracketTax: 24 | bracket_spacing: us-federal 25 | disable_taxes: true 26 | period: 100 27 | tax_annealing_schedule: 28 | - -100 29 | - 0.001 30 | usd_scaling: 1000 31 | dense_log_frequency: 20 32 | energy_cost: 0.21 33 | energy_warmup_constant: 10000 34 | energy_warmup_method: auto 35 | env_layout_file: quadrant_25x25_20each_30clump.txt 36 | episode_length: 1000 37 | fixed_four_skill_and_loc: true 38 | flatten_masks: true 39 | flatten_observations: true 40 | isoelastic_eta: 0.23 41 | multi_action_mode_agents: false 42 | multi_action_mode_planner: true 43 | n_agents: 4 44 | planner_gets_spatial_info: false 45 | planner_reward_type: coin_eq_times_productivity 46 | scenario_name: layout_from_file/simple_wood_and_stone 47 | starting_agent_coin: 0 48 | world_size: 49 | - 25 50 | - 25 51 | general: 52 | ckpt_frequency_steps: 750000 53 | cpus: 15 54 | episodes: 25000 55 | gpus: 0 56 | restore_tf_weights_agents: '' 57 | restore_tf_weights_planner: '' 58 | train_planner: false 59 | agent_policy: 60 | clip_param: 0.3 61 | entropy_coeff: 0.025 62 | entropy_coeff_schedule: null 63 | gamma: 0.998 64 | grad_clip: 10.0 65 | kl_coeff: 0.0 66 | kl_target: 0.01 67 | lambda: 0.98 68 | lr: 0.0003 69 | lr_schedule: null 70 | model: 71 | custom_model: keras_conv_lstm 72 | custom_options: 73 | fc_dim: 128 74 | idx_emb_dim: 4 75 | input_emb_vocab: 100 76 | lstm_cell_size: 128 77 | num_conv: 2 78 | num_fc: 2 79 | max_seq_len: 25 80 | use_gae: true 81 | vf_clip_param: 50.0 82 | vf_loss_coeff: 0.05 83 | vf_share_layers: false 84 | planner_policy: 85 | clip_param: 0.3 86 | entropy_coeff: 0.025 87 | entropy_coeff_schedule: null 88 | gamma: 0.998 89 | grad_clip: 10.0 90 | kl_coeff: 0.0 91 | kl_target: 0.01 92 | lambda: 0.98 93 | lr: 0.0003 94 | lr_schedule: null 95 | model: 96 | custom_model: random 97 | custom_options: {} 98 | max_seq_len: 25 99 | use_gae: true 100 | vf_clip_param: 50.0 101 | vf_loss_coeff: 0.05 102 | vf_share_layers: false 103 | trainer: 104 | batch_mode: truncate_episodes 105 | env_config: null 106 | local_tf_session_args: 107 | inter_op_parallelism_threads: 2 108 | intra_op_parallelism_threads: 24 109 | metrics_smoothing_episodes: null 110 | multiagent: null 111 | no_done_at_end: false 112 | num_envs_per_worker: 2 113 | num_gpus: 0 114 | num_gpus_per_worker: 0 115 | num_sgd_iter: 1 116 | num_workers: 15 117 | observation_filter: NoFilter 118 | rollout_fragment_length: 200 119 | seed: null 120 | sgd_minibatch_size: 1500 121 | shuffle_sequences: true 122 | tf_session_args: 123 | allow_soft_placement: true 124 | device_count: 125 | CPU: 15 126 | GPU: 0 127 | gpu_options: 128 | allow_growth: true 129 | inter_op_parallelism_threads: 2 130 | intra_op_parallelism_threads: 24 131 | log_device_placement: false 132 | train_batch_size: 6000 133 | -------------------------------------------------------------------------------- /tests/run_covid19_cpu_gpu_consistency_checks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | """ 8 | Consistency tests for comparing the cuda (gpu) / no cuda (cpu) version 9 | """ 10 | 11 | import GPUtil 12 | 13 | try: 14 | num_gpus_available = len(GPUtil.getAvailable()) 15 | assert num_gpus_available > 0, "The env consistency checker needs a GPU to run!" 16 | print( 17 | f"Inside env_cpu_gpu_consistency_checker.py: " 18 | f"{num_gpus_available} GPUs are available." 19 | ) 20 | from warp_drive.utils.env_registrar import EnvironmentRegistrar 21 | from warp_drive.env_cpu_gpu_consistency_checker import EnvironmentCPUvsGPU 22 | except ModuleNotFoundError: 23 | raise ModuleNotFoundError( 24 | "The env consistency checker requires the 'WarpDrive' package, please run " 25 | "'pip install rl-warp-drive' first." 26 | ) from None 27 | except ValueError: 28 | raise ValueError("The env consistency checker needs a GPU to run!") from None 29 | 30 | import os 31 | 32 | from ai_economist.foundation.env_wrapper import FoundationEnvWrapper 33 | from ai_economist.foundation.scenarios.covid19.covid19_env import ( 34 | CovidAndEconomyEnvironment, 35 | ) 36 | 37 | env_registrar = EnvironmentRegistrar() 38 | this_file_dir = os.path.dirname(os.path.abspath(__file__)) 39 | env_registrar.add_cuda_env_src_path( 40 | CovidAndEconomyEnvironment.name, 41 | os.path.join(this_file_dir, "../ai_economist/foundation/scenarios/covid19/covid19_build.cu") 42 | ) 43 | env_configs = { 44 | "test1": { 45 | "collate_agent_step_and_reset_data": True, 46 | "components": [ 47 | {"ControlUSStateOpenCloseStatus": {"action_cooldown_period": 28}}, 48 | { 49 | "FederalGovernmentSubsidy": { 50 | "num_subsidy_levels": 20, 51 | "subsidy_interval": 90, 52 | "max_annual_subsidy_per_person": 20000, 53 | } 54 | }, 55 | { 56 | "VaccinationCampaign": { 57 | "daily_vaccines_per_million_people": 3000, 58 | "delivery_interval": 1, 59 | "vaccine_delivery_start_date": "2021-01-12", 60 | } 61 | }, 62 | ], 63 | "economic_reward_crra_eta": 2, 64 | "episode_length": 540, 65 | "flatten_masks": True, 66 | "flatten_observations": False, 67 | "health_priority_scaling_agents": 0.3, 68 | "health_priority_scaling_planner": 0.45, 69 | "infection_too_sick_to_work_rate": 0.1, 70 | "multi_action_mode_agents": False, 71 | "multi_action_mode_planner": False, 72 | "n_agents": 51, 73 | "path_to_data_and_fitted_params": "", 74 | "pop_between_age_18_65": 0.6, 75 | "risk_free_interest_rate": 0.03, 76 | "world_size": [1, 1], 77 | "start_date": "2020-03-22", 78 | "use_real_world_data": False, 79 | "use_real_world_policies": False, 80 | } 81 | } 82 | 83 | num_agents = env_configs["test1"]["n_agents"] 84 | policy_to_agent_ids_mapping = { 85 | "a": [str(agent_id) for agent_id in range(num_agents)], 86 | "p": ["p"], 87 | } 88 | 89 | testing_class = EnvironmentCPUvsGPU( 90 | dual_mode_env_class=CovidAndEconomyEnvironment, 91 | env_configs=env_configs, 92 | num_envs=3, 93 | num_episodes=2, 94 | env_wrapper=FoundationEnvWrapper, 95 | env_registrar=env_registrar, 96 | policy_tag_to_agent_id_map=policy_to_agent_ids_mapping, 97 | create_separate_placeholders_for_each_policy=True, 98 | obs_dim_corresponding_to_num_agents="last" 99 | ) 100 | 101 | testing_class.test_env_reset_and_step() 102 | -------------------------------------------------------------------------------- /tutorials/rllib/phase2/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | env: 8 | components: 9 | - Build: 10 | build_labor: 10 11 | payment: 10 12 | payment_max_skill_multiplier: 3 13 | skill_dist: pareto 14 | - ContinuousDoubleAuction: 15 | max_bid_ask: 10 16 | max_num_orders: 5 17 | order_duration: 50 18 | order_labor: 0.25 19 | - Gather: 20 | collect_labor: 1 21 | move_labor: 1 22 | skill_dist: pareto 23 | - PeriodicBracketTax: 24 | bracket_spacing: us-federal 25 | disable_taxes: false 26 | period: 100 27 | rate_disc: 0.05 28 | tax_annealing_schedule: 29 | - -100 30 | - 0.001 31 | tax_model: model_wrapper 32 | usd_scaling: 1000 33 | dense_log_frequency: 20 34 | energy_cost: 0.21 35 | energy_warmup_constant: 0 36 | env_layout_file: quadrant_25x25_20each_30clump.txt 37 | episode_length: 1000 38 | fixed_four_skill_and_loc: true 39 | flatten_masks: true 40 | flatten_observations: true 41 | isoelastic_eta: 0.23 42 | multi_action_mode_agents: false 43 | multi_action_mode_planner: true 44 | n_agents: 4 45 | planner_gets_spatial_info: false 46 | planner_reward_type: coin_eq_times_productivity 47 | scenario_name: layout_from_file/simple_wood_and_stone 48 | starting_agent_coin: 0 49 | world_size: 50 | - 25 51 | - 25 52 | general: 53 | ckpt_frequency_steps: 20000000 54 | cpus: 15 55 | episodes: 4000000 56 | gpus: 0 57 | restore_tf_weights_agents: phase1/ckpts/agent.tf.weights.global-step-25000000 # path to phase 1 agent policy checkpoint 58 | restore_tf_weights_planner: '' 59 | train_planner: true 60 | agent_policy: 61 | clip_param: 0.3 62 | entropy_coeff: 0.025 63 | entropy_coeff_schedule: null 64 | gamma: 0.998 65 | grad_clip: 10.0 66 | kl_coeff: 0.0 67 | kl_target: 0.01 68 | lambda: 0.98 69 | lr: 0.0003 70 | lr_schedule: null 71 | model: 72 | custom_model: keras_conv_lstm 73 | custom_options: 74 | fc_dim: 128 75 | idx_emb_dim: 4 76 | input_emb_vocab: 100 77 | lstm_cell_size: 128 78 | num_conv: 2 79 | num_fc: 2 80 | max_seq_len: 25 81 | use_gae: true 82 | vf_clip_param: 50.0 83 | vf_loss_coeff: 0.05 84 | vf_share_layers: false 85 | planner_policy: 86 | clip_param: 0.3 87 | entropy_coeff: 0.125 88 | entropy_coeff_schedule: 89 | - - 0 90 | - 2.0 91 | - - 50000000 92 | - 0.125 93 | gamma: 0.998 94 | grad_clip: 10.0 95 | kl_coeff: 0.0 96 | kl_target: 0.01 97 | lambda: 0.98 98 | lr: 0.0001 99 | lr_schedule: null 100 | model: 101 | custom_model: keras_conv_lstm 102 | custom_options: 103 | fc_dim: 256 104 | idx_emb_dim: 4 105 | input_emb_vocab: 100 106 | lstm_cell_size: 256 107 | num_conv: 2 108 | num_fc: 2 109 | max_seq_len: 25 110 | use_gae: true 111 | vf_clip_param: 50.0 112 | vf_loss_coeff: 0.05 113 | vf_share_layers: false 114 | trainer: 115 | batch_mode: truncate_episodes 116 | env_config: null 117 | local_tf_session_args: 118 | inter_op_parallelism_threads: 2 119 | intra_op_parallelism_threads: 24 120 | metrics_smoothing_episodes: null 121 | multiagent: null 122 | no_done_at_end: false 123 | num_envs_per_worker: 2 124 | num_gpus: 0 125 | num_gpus_per_worker: 0 126 | num_sgd_iter: 1 127 | num_workers: 15 128 | observation_filter: NoFilter 129 | rollout_fragment_length: 200 130 | seed: null 131 | sgd_minibatch_size: 1500 132 | shuffle_sequences: true 133 | tf_session_args: 134 | allow_soft_placement: true 135 | device_count: 136 | CPU: 15 137 | GPU: 0 138 | gpu_options: 139 | allow_growth: true 140 | inter_op_parallelism_threads: 2 141 | intra_op_parallelism_threads: 24 142 | log_device_placement: false 143 | train_batch_size: 6000 144 | -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/train_multi_exps.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import argparse 8 | import os 9 | 10 | from experiment_utils import ( 11 | create_job_dir, 12 | run_experiment_batch_parallel, 13 | sweep_cfg_generator, 14 | ) 15 | from rbc.constants import all_agents_short_export_experiment_template 16 | 17 | train_param_sweeps = { 18 | "lr": [0.001], 19 | "entropy": [0.5], 20 | "batch_size": [128], 21 | "clip_grad_norm": [2.0], 22 | "base_seed": [2345], 23 | "should_boost_firm_reward": [False], 24 | "use_ppo": [True], 25 | "ppo_num_updates": [2, 4], 26 | "ppo_clip_param": [0.1], 27 | } 28 | 29 | 30 | agent_param_sweeps = { 31 | "consumer_lr_multiple": [1.0], 32 | "consumer_reward_scale": [5.0], 33 | "government_reward_scale": [5.0 * 100.0 * 2.0], 34 | "firm_reward_scale": [30000], 35 | "government_counts_firm_reward": [1], 36 | "government_lr_multiple": [0.05], 37 | } 38 | 39 | 40 | world_param_sweeps = { 41 | "initial_wages": [0.0], 42 | "interest_rate": [0.0], 43 | "importer_price": [500.0], 44 | "importer_quantity": [100.0], 45 | "use_importer": [1], 46 | } 47 | 48 | 49 | if __name__ == "__main__": 50 | parser = argparse.ArgumentParser() 51 | parser.add_argument("--dry-run", action="store_true") 52 | parser.add_argument("--experiment-dir", type=str, default="experiment/experiment") 53 | parser.add_argument("--group-name", type=str, default="default_group") 54 | parser.add_argument("--job-name-base", type=str, default="rollout") 55 | parser.add_argument("--num-consumers", type=int, default=100) 56 | parser.add_argument("--num-firms", type=int, default=10) 57 | parser.add_argument("--num-governments", type=int, default=1) 58 | parser.add_argument("--run-only", action="store_true") 59 | parser.add_argument("--seed-from-timestamp", action="store_true") 60 | 61 | args = parser.parse_args() 62 | 63 | ( 64 | default_cfg_dict, 65 | consumption_choices, 66 | work_choices, 67 | price_and_wage, 68 | tax_choices, 69 | default_firm_action, 70 | default_government_action, 71 | ) = all_agents_short_export_experiment_template( 72 | args.num_firms, args.num_consumers, args.num_governments 73 | ) 74 | 75 | if args.run_only: 76 | print("Not sweeping over hyperparameter combos...") 77 | else: 78 | for new_cfg in sweep_cfg_generator( 79 | default_cfg_dict, 80 | tr_param_sweeps=train_param_sweeps, 81 | ag_param_sweeps=agent_param_sweeps, 82 | wld_param_sweeps=world_param_sweeps, 83 | seed_from_timestamp=args.seed_from_timestamp, 84 | group_name=args.group_name, 85 | ): 86 | create_job_dir( 87 | args.experiment_dir, 88 | args.job_name_base, 89 | cfg=new_cfg, 90 | action_arrays={ 91 | "consumption_choices": consumption_choices, 92 | "work_choices": work_choices, 93 | "price_and_wage": price_and_wage, 94 | "tax_choices": tax_choices, 95 | }, 96 | ) 97 | 98 | if args.dry_run: 99 | print("Dry-run -> not actually training...") 100 | else: 101 | print("Training multiple experiments locally...") 102 | 103 | # for dirs in experiment dir, run job 104 | experiment_dirs = [ 105 | f.path for f in os.scandir(args.experiment_dir) if f.is_dir() 106 | ] 107 | for experiment in experiment_dirs: 108 | run_experiment_batch_parallel( 109 | experiment, 110 | consumption_choices, 111 | work_choices, 112 | price_and_wage, 113 | tax_choices, 114 | group_name=args.group_name, 115 | consumers_only=False, 116 | no_firms=False, 117 | default_firm_action=default_firm_action, 118 | default_government_action=default_government_action, 119 | ) 120 | -------------------------------------------------------------------------------- /tests/test_env.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | """ 8 | Unit tests for the wood and stone scenario + basic components 9 | """ 10 | 11 | import unittest 12 | 13 | from ai_economist import foundation 14 | 15 | 16 | class CreateEnv: 17 | """ 18 | Create an environment instance based on a configuration 19 | """ 20 | 21 | def __init__(self): 22 | self.env = None 23 | self.set_env_config() 24 | 25 | def set_env_config(self): 26 | """Set up a sample environment config""" 27 | self.env_config = { 28 | # ===== STANDARD ARGUMENTS ====== 29 | "n_agents": 4, # Number of non-planner agents 30 | "world_size": [15, 15], # [Height, Width] of the env world 31 | "episode_length": 1000, # Number of time-steps per episode 32 | # In multi-action-mode, the policy selects an action for each action 33 | # subspace (defined in component code) 34 | # Otherwise, the policy selects only 1 action 35 | "multi_action_mode_agents": False, 36 | "multi_action_mode_planner": True, 37 | # When flattening observations, concatenate scalar & vector observations 38 | # before output 39 | # Otherwise, return observations with minimal processing 40 | "flatten_observations": False, 41 | # When Flattening masks, concatenate each action subspace mask 42 | # into a single array 43 | # Note: flatten_masks = True is recommended for masking action logits 44 | "flatten_masks": True, 45 | # ===== COMPONENTS ===== 46 | # Which components to use 47 | "components": [ 48 | # (1) Building houses 49 | {"Build": {}}, 50 | # (2) Trading collectible resources 51 | {"ContinuousDoubleAuction": {"max_num_orders": 5}}, 52 | # (3) Movement and resource collection 53 | {"Gather": {}}, 54 | ], 55 | # ===== SCENARIO ===== 56 | # Which scenario class to use 57 | "scenario_name": "uniform/simple_wood_and_stone", 58 | # (optional) kwargs of the chosen scenario class 59 | "starting_agent_coin": 10, 60 | "starting_stone_coverage": 0.10, 61 | "starting_wood_coverage": 0.10, 62 | } 63 | 64 | # Create an environment instance from the config 65 | self.env = foundation.make_env_instance(**self.env_config) 66 | 67 | 68 | class TestEnv(unittest.TestCase): 69 | """Unit test to test the env wrapper, reset and step""" 70 | 71 | def test_env_reset_and_step(self): 72 | """ 73 | Unit tests for the reset and step calls 74 | """ 75 | create_env = CreateEnv() 76 | env = create_env.env 77 | 78 | # Assert that the total number of agents matches the sum of the 'n_agents' 79 | # configuration and the number of planners (1 in this case) 80 | num_planners = 1 81 | self.assertEqual( 82 | len(env.all_agents), create_env.env_config["n_agents"] + num_planners 83 | ) 84 | 85 | # Assert that the number of agents created in the world 86 | # matches the configuration specification 87 | self.assertEqual(len(env.world.agents), create_env.env_config["n_agents"]) 88 | 89 | # Assert that the planner's index in the world is 'p' 90 | self.assertEqual(env.world.planner.idx, "p") 91 | 92 | obs = env.reset() 93 | 94 | # Test whether the observation dictionary keys are created as expected 95 | self.assertEqual( 96 | sorted(list(obs.keys())), 97 | [str(i) for i in range(create_env.env_config["n_agents"])] + ["p"], 98 | ) 99 | 100 | obs, reward, done, info = env.step({}) 101 | 102 | # Check that the observation, reward and info keys match 103 | self.assertEqual(obs.keys(), reward.keys()) 104 | self.assertEqual(obs.keys(), info.keys()) 105 | 106 | # Assert that __all__ is in done 107 | assert "__all__" in done 108 | 109 | 110 | if __name__ == "__main__": 111 | unittest.main() 112 | -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/train_bestresponse.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import argparse 8 | import pickle 9 | from collections import defaultdict 10 | from pathlib import Path 11 | 12 | import numpy as np 13 | from experiment_utils import cfg_dict_from_yaml 14 | from rbc.cuda_manager import ConsumerFirmRunManagerBatchParallel 15 | 16 | 17 | def check_if_ep_str_policy_exists(rollout_path, ep_str): 18 | return ( 19 | rollout_path / Path("saved_models") / Path(f"consumer_policy_{ep_str}.pt") 20 | ).is_file() 21 | 22 | 23 | def run_rollout(rollout_path, arguments): 24 | """ 25 | # take in rollout directory 26 | # load latest policies and the action functions and the hparams dict 27 | # make a cudamanager obj and run the job 28 | # this will require initializing everything as before, resetting, 29 | # and running naive policy gradient training at some fixed learning rate 30 | """ 31 | with open(rollout_path / Path("action_arrays.pickle"), "rb") as f: 32 | action_arrays = pickle.load(f) 33 | 34 | consumption_choices, work_choices, price_and_wage, tax_choices = ( 35 | action_arrays["consumption_choices"], 36 | action_arrays["work_choices"], 37 | action_arrays["price_and_wage"], 38 | action_arrays["tax_choices"], 39 | ) 40 | 41 | cfg_dict = cfg_dict_from_yaml( 42 | rollout_path / Path("hparams.yaml"), 43 | consumption_choices, 44 | work_choices, 45 | price_and_wage, 46 | tax_choices, 47 | ) 48 | 49 | print(cfg_dict) 50 | 51 | if arguments.agent_type == "all": 52 | agent_types = ["consumer", "firm", "government"] 53 | else: 54 | agent_types = [arguments.agent_type] 55 | 56 | for agent_type in agent_types: 57 | ep_rewards = defaultdict(list) 58 | for _ in range(arguments.repeat_runs): 59 | for ep_str in arguments.ep_strs: 60 | if not check_if_ep_str_policy_exists(rollout_path, ep_str): 61 | print(f"warning: {rollout_path} {ep_str} policy not found") 62 | ep_rewards[ep_str].append([0.0]) 63 | continue 64 | m = ConsumerFirmRunManagerBatchParallel(cfg_dict) 65 | rewards_start = m.bestresponse_train( 66 | agent_type, 67 | arguments.num_episodes, 68 | rollout_path, 69 | ep_str=ep_str, 70 | checkpoint=arguments.checkpoint_model, 71 | ) 72 | ep_rewards[ep_str].append(rewards_start) 73 | 74 | with open(rollout_path / Path(f"br_{agent_type}_output.txt"), "w") as f: 75 | for ep_str in arguments.ep_strs: 76 | reward_arr = np.array(ep_rewards[ep_str]) 77 | print( 78 | f"mean reward (std) on rollout {ep_str}: " 79 | f"before BR training {reward_arr[:,0].mean()} " 80 | f"({reward_arr[:,0].std()}), " 81 | f"after BR training {reward_arr[:,-1].mean()} " 82 | f"({reward_arr[:,-1].std()}), " 83 | f"mean improvement {(reward_arr[:,-1]-reward_arr[:,0]).mean()} " 84 | f"({(reward_arr[:,-1]-reward_arr[:,0]).std()}", 85 | file=f, 86 | ) 87 | 88 | 89 | if __name__ == "__main__": 90 | parser = argparse.ArgumentParser() 91 | parser.add_argument("rolloutdir", type=str) 92 | parser.add_argument("num_episodes", type=int) 93 | parser.add_argument("--experiment-dir", action="store_true") 94 | parser.add_argument("--ep-strs", nargs="+", default=["0", "latest"]) 95 | parser.add_argument("--agent-type", type=str, default="all") 96 | parser.add_argument("--repeat-runs", type=int, default=1) 97 | parser.add_argument("--checkpoint-model", type=int, default=100) 98 | 99 | args = parser.parse_args() 100 | 101 | if args.experiment_dir: 102 | exp_dir = Path(args.rolloutdir) 103 | for rolloutpath in exp_dir.iterdir(): 104 | if rolloutpath.is_dir(): 105 | run_rollout(rolloutpath, args) 106 | else: 107 | rolloutpath = Path(args.rolloutdir) 108 | run_rollout(rolloutpath, args) 109 | -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/rbc/networks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import torch 8 | import torch.nn.functional as F 9 | from torch import nn 10 | 11 | 12 | class IndependentPolicyNet(nn.Module): 13 | """ 14 | Represents a policy network with separate heads for different types of actions. 15 | Thus, the resulting policy will take the form 16 | $pi(a | s) = pi_1(a_1 | s) pi_2(a_2 | s)...$ 17 | """ 18 | 19 | def __init__(self, state_size, action_size_list, norm_consts=None): 20 | super().__init__() 21 | 22 | self.state_size = state_size 23 | self.action_size_list = action_size_list 24 | if norm_consts is not None: 25 | self.norm_center, self.norm_scale = norm_consts 26 | else: 27 | self.norm_center = torch.zeros(self.state_size).cuda() 28 | self.norm_scale = torch.ones(self.state_size).cuda() 29 | self.fc1 = nn.Linear(state_size, 128) 30 | self.fc2 = nn.Linear(128, 128) 31 | # policy network head 32 | self.action_heads = nn.ModuleList( 33 | [nn.Linear(128, action_size) for action_size in action_size_list] 34 | ) 35 | # value network head 36 | self.fc4 = nn.Linear(128, 1) 37 | 38 | def forward(self, x): 39 | assert x.shape[-1] == self.state_size # Check if the last dimension matches 40 | 41 | # Normalize the model input 42 | new_shape = tuple(1 for _ in x.shape[:-1]) + (x.shape[-1],) 43 | view_center = self.norm_center.view(new_shape) 44 | view_scale = self.norm_scale.view(new_shape) 45 | x = (x - view_center) / view_scale 46 | 47 | # Feed forward 48 | x = F.relu(self.fc1(x)) 49 | x = F.relu(self.fc2(x)) 50 | probs = [F.softmax(action_head(x), dim=-1) for action_head in self.action_heads] 51 | vals = self.fc4(x) 52 | return probs, vals 53 | 54 | 55 | class PolicyNet(nn.Module): 56 | """ 57 | The policy network class to output acton probabilities and the value function. 58 | """ 59 | 60 | def __init__(self, state_size, action_size, norm_consts=None): 61 | super().__init__() 62 | 63 | self.state_size = state_size 64 | self.action_size = action_size 65 | if norm_consts is not None: 66 | self.norm_center, self.norm_scale = norm_consts 67 | else: 68 | self.norm_center = torch.zeros(self.state_size).cuda() 69 | self.norm_scale = torch.ones(self.state_size).cuda() 70 | self.fc1 = nn.Linear(state_size, 128) 71 | self.fc2 = nn.Linear(128, 128) 72 | # policy network head 73 | self.fc3 = nn.Linear(128, action_size) 74 | # value network head 75 | self.fc4 = nn.Linear(128, 1) 76 | 77 | def forward(self, x, actions_mask=None): 78 | # here, the action mask should be large negative constants for actions 79 | # that shouldn't be allowed. 80 | new_shape = tuple(1 for _ in x.shape[:-1]) + (x.shape[-1],) 81 | view_center = self.norm_center.view(new_shape) 82 | view_scale = self.norm_scale.view(new_shape) 83 | x = (x - view_center) / view_scale 84 | x = F.relu(self.fc1(x)) 85 | x = F.relu(self.fc2(x)) 86 | if actions_mask is not None: 87 | probs = F.softmax(self.fc3(x) + actions_mask, dim=-1) 88 | else: 89 | probs = F.softmax(self.fc3(x), dim=-1) 90 | vals = self.fc4(x) 91 | return probs, vals 92 | 93 | 94 | class DeterministicPolicy: 95 | """ 96 | A policy class that outputs deterministic actions. 97 | """ 98 | 99 | def __init__(self, state_size, action_size, action_choice): 100 | self.state_size = state_size 101 | self.action_size = action_size 102 | self.action_choice = action_choice 103 | self.actions_out = torch.zeros(action_size, device="cuda") 104 | self.actions_out[self.action_choice] = 1.0 105 | 106 | def __call__(self, x, actions_mask=None): 107 | return self.forward(x) 108 | 109 | def forward(self, x): 110 | # output enough copies of the delta function 111 | # distribution of the right size given x 112 | x_batch_shapes = x.shape[:-1] 113 | repeat_vals = x_batch_shapes + (1,) 114 | return self.actions_out.repeat(*repeat_vals), None 115 | -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/us_unemployment.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import bz2 8 | import os 9 | import pickle 10 | import queue 11 | import threading 12 | import urllib.request as urllib2 13 | 14 | import pandas as pd 15 | from bs4 import BeautifulSoup 16 | 17 | 18 | class DatasetCovidUnemploymentUS: 19 | """ 20 | Class to load COVID-19 unemployment data for the US states. 21 | Source: https://www.bls.gov/lau/ 22 | """ 23 | 24 | def __init__(self, data_dir="", download_latest_data=True): 25 | if not os.path.exists(data_dir): 26 | print( 27 | "Creating a dynamic data directory to store COVID-19 " 28 | "unemployment data: {}".format(data_dir) 29 | ) 30 | os.makedirs(data_dir) 31 | 32 | filename = "monthly_us_unemployment.bz2" 33 | if download_latest_data or filename not in os.listdir(data_dir): 34 | # Construct the U.S. state to FIPS code mapping 35 | state_fips_df = pd.read_excel( 36 | "https://www2.census.gov/programs-surveys/popest/geographies/2017/" 37 | "state-geocodes-v2017.xlsx", 38 | header=5, 39 | ) 40 | # remove all statistical areas and cities 41 | state_fips_df = state_fips_df.loc[state_fips_df["State (FIPS)"] != 0] 42 | self.us_state_to_fips_dict = pd.Series( 43 | state_fips_df["State (FIPS)"].values, index=state_fips_df.Name 44 | ).to_dict() 45 | 46 | print( 47 | "Fetching the U.S. unemployment data from " 48 | "Bureau of Labor and Statistics, and saving it in {}".format(data_dir) 49 | ) 50 | self.data = self.scrape_bls_data() 51 | fp = bz2.BZ2File(os.path.join(data_dir, filename), "wb") 52 | pickle.dump(self.data, fp) 53 | fp.close() 54 | 55 | else: 56 | print( 57 | "Not fetching the U.S. unemployment data from Bureau of Labor and" 58 | " Statistics. Using whatever was saved earlier in {}!!".format(data_dir) 59 | ) 60 | assert filename in os.listdir(data_dir) 61 | with bz2.BZ2File(os.path.join(data_dir, filename), "rb") as fp: 62 | self.data = pickle.load(fp) 63 | fp.close() 64 | 65 | # Scrape monthly unemployment from the Bureau of Labor Statistics website 66 | def get_monthly_bls_unemployment_rates(self, state_fips): 67 | with urllib2.urlopen( 68 | "https://data.bls.gov/timeseries/LASST{:02d}0000000000003".format( 69 | state_fips 70 | ) 71 | ) as response: 72 | html_doc = response.read() 73 | 74 | soup = BeautifulSoup(html_doc, "html.parser") 75 | table = soup.find_all("table")[1] 76 | table_rows = table.find_all("tr") 77 | 78 | unemployment_dict = {} 79 | 80 | mth2idx = { 81 | "Jan": 1, 82 | "Feb": 2, 83 | "Mar": 3, 84 | "Apr": 4, 85 | "May": 5, 86 | "Jun": 6, 87 | "Jul": 7, 88 | "Aug": 8, 89 | "Sep": 9, 90 | "Oct": 10, 91 | "Nov": 11, 92 | "Dec": 12, 93 | } 94 | 95 | for tr in table_rows[1:-1]: 96 | td = tr.find_all("td")[-1] 97 | unemp = float("".join([c for c in td.text if c.isdigit() or c == "."])) 98 | th = tr.find_all("th") 99 | year = int(th[0].text) 100 | month = mth2idx[th[1].text] 101 | if year not in unemployment_dict: 102 | unemployment_dict[year] = {} 103 | unemployment_dict[year][month] = unemp 104 | 105 | return unemployment_dict 106 | 107 | def scrape_bls_data(self): 108 | def do_scrape(us_state, fips, queue_obj): 109 | out = self.get_monthly_bls_unemployment_rates(fips) 110 | queue_obj.put([us_state, out]) 111 | 112 | print("Getting BLS Data. This might take a minute...") 113 | result = queue.Queue() 114 | threads = [ 115 | threading.Thread(target=do_scrape, args=(us_state, fips, result)) 116 | for us_state, fips in self.us_state_to_fips_dict.items() 117 | ] 118 | for t in threads: 119 | t.start() 120 | for t in threads: 121 | t.join() 122 | 123 | monthly_unemployment = {} 124 | while not result.empty(): 125 | us_state, data = result.get() 126 | monthly_unemployment[us_state] = data 127 | 128 | return monthly_unemployment 129 | -------------------------------------------------------------------------------- /ai_economist/foundation/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import json 8 | import os 9 | import sys 10 | from hashlib import sha512 11 | 12 | import lz4.frame 13 | from Crypto.PublicKey import RSA 14 | 15 | from ai_economist.foundation.base.base_env import BaseEnvironment 16 | 17 | 18 | def save_episode_log(game_object, filepath, compression_level=16): 19 | """Save a lz4 compressed version of the dense log stored 20 | in the provided game object""" 21 | assert isinstance(game_object, BaseEnvironment) 22 | compression_level = int(compression_level) 23 | if compression_level < 0: 24 | compression_level = 0 25 | elif compression_level > 16: 26 | compression_level = 16 27 | 28 | with lz4.frame.open( 29 | filepath, mode="wb", compression_level=compression_level 30 | ) as log_file: 31 | log_bytes = bytes( 32 | json.dumps( 33 | game_object.previous_episode_dense_log, ensure_ascii=False 34 | ).encode("utf-8") 35 | ) 36 | log_file.write(log_bytes) 37 | 38 | 39 | def load_episode_log(filepath): 40 | """Load the dense log saved at provided filepath""" 41 | with lz4.frame.open(filepath, mode="rb") as log_file: 42 | log_bytes = log_file.read() 43 | return json.loads(log_bytes) 44 | 45 | 46 | def verify_activation_code(): 47 | """ 48 | Validate the user's activation code. 49 | If the activation code is valid, also save it in a text file for future reference. 50 | If the activation code is invalid, simply exit the program 51 | """ 52 | path_to_activation_code_dir = os.path.dirname(os.path.abspath(__file__)) 53 | 54 | def validate_activation_code(code, msg=b"covid19 code activation"): 55 | filepath = os.path.abspath( 56 | os.path.join( 57 | path_to_activation_code_dir, 58 | "scenarios/covid19/key_to_check_activation_code_against", 59 | ) 60 | ) 61 | with open(filepath, "r") as fp: 62 | key_pair = RSA.import_key(fp.read()) 63 | 64 | hashed_msg = int.from_bytes(sha512(msg).digest(), byteorder="big") 65 | signature = pow(hashed_msg, key_pair.d, key_pair.n) 66 | try: 67 | exp_from_code = int(code, 16) 68 | hashed_msg_from_signature = pow(signature, exp_from_code, key_pair.n) 69 | 70 | return hashed_msg == hashed_msg_from_signature 71 | except ValueError: 72 | return False 73 | 74 | activation_code_filename = "activation_code.txt" 75 | 76 | filepath = os.path.join(path_to_activation_code_dir, activation_code_filename) 77 | if activation_code_filename in os.listdir(path_to_activation_code_dir): 78 | print("Using the activation code already present in '{}'".format(filepath)) 79 | with open(filepath, "r") as fp: 80 | activation_code = fp.read() 81 | fp.close() 82 | if validate_activation_code(activation_code): 83 | return # already activated 84 | print( 85 | "The activation code saved in '{}' is incorrect! " 86 | "Please correct the activation code and try again.".format(filepath) 87 | ) 88 | sys.exit(0) 89 | else: 90 | print( 91 | "In order to run this simulation, you will need an activation code.\n" 92 | "Please fill out the form at " 93 | "https://forms.gle/dJ2gKDBqLDko1g7m7 and we will send you an " 94 | "activation code to the provided email address.\n" 95 | ) 96 | num_attempts = 5 97 | attempt_num = 0 98 | while attempt_num < num_attempts: 99 | activation_code = input( 100 | f"Whenever you are ready, " 101 | "please enter the activation code: " 102 | f"(attempt {attempt_num + 1} / {num_attempts})" 103 | ) 104 | attempt_num += 1 105 | if validate_activation_code(activation_code): 106 | print( 107 | "Saving the activation code in '{}' for future " 108 | "use.".format(filepath) 109 | ) 110 | with open( 111 | os.path.join(path_to_activation_code_dir, activation_code_filename), 112 | "w", 113 | ) as fp: 114 | fp.write(activation_code) 115 | fp.close() 116 | return 117 | print("Incorrect activation code. Please try again.") 118 | print( 119 | "You have had {} attempts to provide the activate code. Unfortunately, " 120 | "none of the activation code(s) you provided could be validated. " 121 | "Exiting...".format(num_attempts) 122 | ) 123 | sys.exit(0) 124 | -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/rbc/util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import torch 8 | 9 | 10 | def dict_merge(dct, merge_dct): 11 | """Recursive dict merge. Inspired by :meth:``dict.update()``, instead of 12 | updating only top-level keys, dict_merge recurses down into dicts nested 13 | to an arbitrary depth, updating keys. The ``merge_dct`` is merged into 14 | ``dct``. 15 | :param dct: dict onto which the merge is executed 16 | :param merge_dct: dct merged into dct 17 | :return: None 18 | """ 19 | for k, v in merge_dct.items(): 20 | # dct does not have k yet. Add it with value v. 21 | if (k not in dct) and (not isinstance(dct, list)): 22 | dct[k] = v 23 | else: 24 | # dct[k] and merge_dict[k] are both dictionaries. Recurse. 25 | if isinstance(dct[k], (dict, list)) and isinstance(v, dict): 26 | dict_merge(dct[k], merge_dct[k]) 27 | else: 28 | # dct[k] and merge_dict[k] are both tuples or lists. 29 | if isinstance(dct[k], (tuple, list)) and isinstance(v, (tuple, list)): 30 | # They don't match. Overwrite with v. 31 | if len(dct[k]) != len(v): 32 | dct[k] = v 33 | else: 34 | for i, (d_val, v_val) in enumerate(zip(dct[k], v)): 35 | if isinstance(d_val, dict) and isinstance(v_val, dict): 36 | dict_merge(d_val, v_val) 37 | else: 38 | dct[k][i] = v_val 39 | else: 40 | dct[k] = v 41 | 42 | 43 | def min_max_consumer_budget_delta(hparams_dict): 44 | # largest single round changes 45 | max_wage = hparams_dict["agents"]["max_possible_wage"] 46 | max_hours = hparams_dict["agents"]["max_possible_hours_worked"] 47 | max_price = hparams_dict["agents"]["max_possible_price"] 48 | max_singlefirm_consumption = hparams_dict["agents"]["max_possible_consumption"] 49 | num_firms = hparams_dict["agents"]["num_firms"] 50 | 51 | min_budget = ( 52 | -max_singlefirm_consumption * max_price * num_firms 53 | ) # negative budget from consuming only 54 | max_budget = max_hours * max_wage * num_firms # positive budget from only working 55 | return min_budget, max_budget 56 | 57 | 58 | def min_max_stock_delta(hparams_dict): 59 | # for now, assuming 1.0 capital 60 | max_hours = hparams_dict["agents"]["max_possible_hours_worked"] 61 | max_singlefirm_consumption = hparams_dict["agents"]["max_possible_consumption"] 62 | alpha = hparams_dict["world"]["production_alpha"] 63 | if isinstance(alpha, str): 64 | alpha = 0.8 65 | num_consumers = hparams_dict["agents"]["num_consumers"] 66 | max_delta = (max_hours * num_consumers) ** alpha 67 | min_delta = -max_singlefirm_consumption * num_consumers 68 | return min_delta, max_delta 69 | 70 | 71 | def min_max_firm_budget(hparams_dict): 72 | max_wage = hparams_dict["agents"]["max_possible_wage"] 73 | max_hours = hparams_dict["agents"]["max_possible_hours_worked"] 74 | max_singlefirm_consumption = hparams_dict["agents"]["max_possible_consumption"] 75 | num_consumers = hparams_dict["agents"]["num_consumers"] 76 | max_price = hparams_dict["agents"]["max_possible_price"] 77 | max_delta = max_singlefirm_consumption * max_price * num_consumers 78 | min_delta = -max_hours * max_wage * num_consumers 79 | return min_delta, max_delta 80 | 81 | 82 | def expand_to_digit_form(x, dims_to_expand, max_digits): 83 | # first split x up 84 | requires_grad = ( 85 | x.requires_grad 86 | ) # don't want to backprop through these ops, but do want 87 | # gradients if x had them 88 | with torch.no_grad(): 89 | tensor_pieces = [] 90 | expanded_digit_shape = x.shape[:-1] + (max_digits,) 91 | for i in range(x.shape[-1]): 92 | if i not in dims_to_expand: 93 | tensor_pieces.append(x[..., i : i + 1]) 94 | else: 95 | digit_entries = torch.zeros(expanded_digit_shape, device=x.device) 96 | for j in range(max_digits): 97 | digit_entries[..., j] = (x[..., i] % (10 ** (j + 1))) / ( 98 | 10 ** (j + 1) 99 | ) 100 | tensor_pieces.append(digit_entries) 101 | 102 | output = torch.cat(tensor_pieces, dim=-1) 103 | output.requires_grad_(requires_grad) 104 | return output 105 | 106 | 107 | def size_after_digit_expansion(existing_size, dims_to_expand, max_digits): 108 | num_expanded = len(dims_to_expand) 109 | # num non expanded digits, + all the expanded ones 110 | return (existing_size - num_expanded) + (max_digits * num_expanded) 111 | -------------------------------------------------------------------------------- /ai_economist/foundation/components/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | 10 | def annealed_tax_limit(completions, warmup_period, slope, final_max_tax_value=1.0): 11 | """ 12 | Compute the maximum tax rate available at this stage of tax annealing. 13 | 14 | This function uses the number of episode completions and the annealing schedule 15 | (warmup_period, slope, & final_max_tax_value) to determine what the maximum tax 16 | rate can be. 17 | This type of annealing allows for a tax curriculum where earlier episodes are 18 | restricted to lower tax rates. As more episodes are played, higher tax values are 19 | allowed. 20 | 21 | Args: 22 | completions (int): Number of times the environment has completed an episode. 23 | Expected to be >= 0. 24 | warmup_period (int): Until warmup_period completions, only allow 0 tax. Using 25 | a negative value will enable non-0 taxes at 0 environment completions. 26 | slope (float): After warmup_period completions, percentage of full tax value 27 | unmasked with each new completion. 28 | final_max_tax_value (float): The maximum tax value at the end of annealing. 29 | 30 | Returns: 31 | A scalar value indicating the maximum tax at this stage of annealing. 32 | 33 | Example: 34 | >> WARMUP = 100 35 | >> SLOPE = 0.01 36 | >> annealed_tax_limit(0, WARMUP, SLOPE) 37 | 0.0 38 | >> annealed_tax_limit(100, WARMUP, SLOPE) 39 | 0.0 40 | >> annealed_tax_limit(150, WARMUP, SLOPE) 41 | 0.5 42 | >> annealed_tax_limit(200, WARMUP, SLOPE) 43 | 1.0 44 | >> annealed_tax_limit(1000, WARMUP, SLOPE) 45 | 1.0 46 | """ 47 | # What percentage of the full range is currently visible 48 | # (between 0 [only 0 tax] and 1 [all taxes visible]) 49 | percentage_visible = np.maximum( 50 | 0.0, np.minimum(1.0, slope * (completions - warmup_period)) 51 | ) 52 | 53 | # Determine the highest allowable tax, 54 | # given the current position in the annealing schedule 55 | current_max_tax = percentage_visible * final_max_tax_value 56 | 57 | return current_max_tax 58 | 59 | 60 | def annealed_tax_mask(completions, warmup_period, slope, tax_values): 61 | """ 62 | Generate a mask applied to a set of tax values for the purpose of tax annealing. 63 | 64 | This function uses the number of episode completions and the annealing schedule 65 | to determine which of the tax values are considered valid. The most extreme 66 | tax/subsidy values are unmasked last. Zero tax is always unmasked (i.e. always 67 | valid). 68 | This type of annealing allows for a tax curriculum where earlier episodes are 69 | restricted to lower tax rates. As more episodes are played, higher tax values are 70 | allowed. 71 | 72 | Args: 73 | completions (int): Number of times the environment has completed an episode. 74 | Expected to be >= 0. 75 | warmup_period (int): Until warmup_period completions, only allow 0 tax. Using 76 | a negative value will enable non-0 taxes at 0 environment completions. 77 | slope (float): After warmup_period completions, percentage of full tax value 78 | unmasked with each new completion. 79 | tax_values (list): The list of tax values associated with each action to 80 | which this mask will apply. 81 | 82 | Returns: 83 | A binary mask with same shape as tax_values, indicating which tax values are 84 | currently valid. 85 | 86 | Example: 87 | >> WARMUP = 100 88 | >> SLOPE = 0.01 89 | >> TAX_VALUES = [0.0, 0.25, 0.50, 0.75, 1.0] 90 | >> annealed_tax_limit(0, WARMUP, SLOPE, TAX_VALUES) 91 | [0, 0, 0, 0, 0] 92 | >> annealed_tax_limit(100, WARMUP, SLOPE, TAX_VALUES) 93 | [0, 0, 0, 0, 0] 94 | >> annealed_tax_limit(150, WARMUP, SLOPE, TAX_VALUES) 95 | [1, 1, 1, 0, 0] 96 | >> annealed_tax_limit(200, WARMUP, SLOPE, TAX_VALUES) 97 | [1, 1, 1, 1, 1] 98 | >> annealed_tax_limit(1000, WARMUP, SLOPE, TAX_VALUES) 99 | [1, 1, 1, 1, 1] 100 | """ 101 | # Infer the most extreme tax level from the supplied tax values. 102 | abs_tax = np.abs(tax_values) 103 | full_tax_amount = np.max(abs_tax) 104 | 105 | # Determine the highest allowable tax, given the current position 106 | # in the annealing schedule 107 | max_absolute_visible_tax = annealed_tax_limit( 108 | completions, warmup_period, slope, full_tax_amount 109 | ) 110 | 111 | # Return a binary mask to allow for taxes 112 | # at or below the highest absolute visible tax 113 | return np.less_equal(np.abs(tax_values), max_absolute_visible_tax).astype( 114 | np.float32 115 | ) 116 | -------------------------------------------------------------------------------- /ai_economist/datasets/covid19_datasets/us_policies.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | 8 | import os 9 | from datetime import datetime 10 | from io import BytesIO 11 | 12 | import numpy as np 13 | import pandas as pd 14 | import requests 15 | 16 | 17 | class DatasetCovidPoliciesUS: 18 | """ 19 | Class to load COVID-19 government policies for the US states. 20 | Source: https://github.com/OxCGRT/USA-covid-policy 21 | 22 | Other references: 23 | - Codebook: https://github.com/OxCGRT/covid-policy-tracker/blob/master/ 24 | documentation/codebook.md 25 | - Index computation methodology: https://github.com/OxCGRT/covid-policy-tracker/ 26 | blob/master/documentation/index_methodology.md 27 | 28 | Attributes: 29 | df: Timeseries dataframe of state-wide policies 30 | """ 31 | 32 | def __init__(self, data_dir="", download_latest_data=True): 33 | if not os.path.exists(data_dir): 34 | print( 35 | "Creating a dynamic data directory to store COVID-19 " 36 | "policy tracking data: {}".format(data_dir) 37 | ) 38 | os.makedirs(data_dir) 39 | 40 | filename = "daily_us_policies.csv" 41 | if download_latest_data or filename not in os.listdir(data_dir): 42 | print( 43 | "Fetching latest U.S. COVID-19 policies data from OxCGRT, " 44 | "and saving it in {}".format(data_dir) 45 | ) 46 | req = requests.get( 47 | "https://raw.githubusercontent.com/OxCGRT/USA-covid-policy/master/" 48 | "data/OxCGRT_US_latest.csv" 49 | ) 50 | self.df = pd.read_csv(BytesIO(req.content), low_memory=False) 51 | self.df["Date"] = self.df["Date"].apply( 52 | lambda x: datetime.strptime(str(x), "%Y%m%d") 53 | ) 54 | 55 | # Fetch only the state-wide policies 56 | self.df = self.df.loc[self.df["Jurisdiction"] != "NAT_GOV"] 57 | 58 | self.df.to_csv( 59 | os.path.join(data_dir, filename) 60 | ) # Note: performs an overwrite 61 | else: 62 | print( 63 | "Not fetching the latest U.S. COVID-19 policies data from OxCGRT. " 64 | "Using whatever was saved earlier in {}!!".format(data_dir) 65 | ) 66 | assert filename in os.listdir(data_dir) 67 | self.df = pd.read_csv(os.path.join(data_dir, filename), low_memory=False) 68 | 69 | def process_policy_data( 70 | self, stringency_policy_key="StringencyIndex", num_stringency_levels=10 71 | ): 72 | """ 73 | Gather the relevant policy indicator frm the dataframe, 74 | fill in the null values (if any), 75 | and discretize/quantize the policy into num_stringency_levels. 76 | Note: Possible values for stringency_policy_key are 77 | ["StringencyIndex", "Government response index", 78 | "Containment and health index", "Economic Support index".] 79 | Reference: https://github.com/OxCGRT/covid-policy-tracker/blob/master/ 80 | documentation/index_methodology.md 81 | """ 82 | 83 | def discretize(policies, num_indicator_levels=10): 84 | """ 85 | Discretize the policies (a Pandas series) into num_indicator_levels 86 | """ 87 | # Indices are normalized to be in [0, 100] 88 | bins = np.linspace(0, 100, num_indicator_levels) 89 | # Find left and right values of bin and find the nearer edge 90 | bin_index = np.digitize(policies, bins, right=True) 91 | bin_left_edges = bins[bin_index - 1] 92 | bin_right_edges = bins[bin_index] 93 | discretized_policies = bin_index + np.argmin( 94 | np.stack( 95 | ( 96 | np.abs(policies.values - bin_left_edges), 97 | np.abs(policies.values - bin_right_edges), 98 | ) 99 | ), 100 | axis=0, 101 | ) 102 | return discretized_policies 103 | 104 | # Gather just the relevant columns 105 | policy_df = self.df[["RegionName", "Date", stringency_policy_key]].copy() 106 | 107 | # Fill in null values via a "forward fill" 108 | policy_df[stringency_policy_key].fillna(method="ffill", inplace=True) 109 | 110 | # Discretize the stringency indices 111 | discretized_stringency_policies = discretize( 112 | policy_df[stringency_policy_key], num_indicator_levels=num_stringency_levels 113 | ) 114 | policy_df.loc[:, stringency_policy_key] = discretized_stringency_policies 115 | 116 | # Replace Washington DC by District of Columbia to keep consistent 117 | # (with the other data sources) 118 | policy_df = policy_df.replace("Washington DC", "District of Columbia") 119 | 120 | policy_df = policy_df.sort_values(by=["RegionName", "Date"]) 121 | 122 | return policy_df 123 | -------------------------------------------------------------------------------- /ai_economist/foundation/scenarios/utils/rewards.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | from ai_economist.foundation.scenarios.utils import social_metrics 10 | 11 | 12 | def isoelastic_coin_minus_labor( 13 | coin_endowment, total_labor, isoelastic_eta, labor_coefficient 14 | ): 15 | """Agent utility, concave increasing in coin and linearly decreasing in labor. 16 | 17 | Args: 18 | coin_endowment (float, ndarray): The amount of coin owned by the agent(s). 19 | total_labor (float, ndarray): The amount of labor performed by the agent(s). 20 | isoelastic_eta (float): Constant describing the shape of the utility profile 21 | with respect to coin endowment. Must be between 0 and 1. 0 yields utility 22 | that increases linearly with coin. 1 yields utility that increases with 23 | log(coin). Utility from coin uses: 24 | https://en.wikipedia.org/wiki/Isoelastic_utility 25 | labor_coefficient (float): Constant describing the disutility experienced per 26 | unit of labor performed. Disutility from labor equals: 27 | labor_coefficient * total_labor 28 | 29 | Returns: 30 | Agent utility (float) or utilities (ndarray). 31 | """ 32 | # https://en.wikipedia.org/wiki/Isoelastic_utility 33 | assert np.all(coin_endowment >= 0) 34 | assert 0 <= isoelastic_eta <= 1.0 35 | 36 | # Utility from coin endowment 37 | if isoelastic_eta == 1.0: # dangerous 38 | util_c = np.log(np.max(1, coin_endowment)) 39 | else: # isoelastic_eta >= 0 40 | util_c = (coin_endowment ** (1 - isoelastic_eta) - 1) / (1 - isoelastic_eta) 41 | 42 | # disutility from labor 43 | util_l = total_labor * labor_coefficient 44 | 45 | # Net utility 46 | util = util_c - util_l 47 | 48 | return util 49 | 50 | 51 | def coin_minus_labor_cost( 52 | coin_endowment, total_labor, labor_exponent, labor_coefficient 53 | ): 54 | """Agent utility, linearly increasing in coin and decreasing as a power of labor. 55 | 56 | Args: 57 | coin_endowment (float, ndarray): The amount of coin owned by the agent(s). 58 | total_labor (float, ndarray): The amount of labor performed by the agent(s). 59 | labor_exponent (float): Constant describing the shape of the utility profile 60 | with respect to total labor. Must be between >1. 61 | labor_coefficient (float): Constant describing the disutility experienced per 62 | unit of labor performed. Disutility from labor equals: 63 | labor_coefficient * total_labor. 64 | 65 | Returns: 66 | Agent utility (float) or utilities (ndarray). 67 | """ 68 | # https://en.wikipedia.org/wiki/Isoelastic_utility 69 | assert np.all(coin_endowment >= 0) 70 | assert labor_exponent > 1 71 | 72 | # Utility from coin endowment 73 | util_c = coin_endowment 74 | 75 | # Disutility from labor 76 | util_l = (total_labor ** labor_exponent) * labor_coefficient 77 | 78 | # Net utility 79 | util = util_c - util_l 80 | 81 | return util 82 | 83 | 84 | def coin_eq_times_productivity(coin_endowments, equality_weight): 85 | """Social welfare, measured as productivity scaled by the degree of coin equality. 86 | 87 | Args: 88 | coin_endowments (ndarray): The array of coin endowments for each of the 89 | agents in the simulated economy. 90 | equality_weight (float): Constant that determines how productivity is scaled 91 | by coin equality. Must be between 0 (SW = prod) and 1 (SW = prod * eq). 92 | 93 | Returns: 94 | Product of coin equality and productivity (float). 95 | """ 96 | n_agents = len(coin_endowments) 97 | prod = social_metrics.get_productivity(coin_endowments) / n_agents 98 | equality = equality_weight * social_metrics.get_equality(coin_endowments) + ( 99 | 1 - equality_weight 100 | ) 101 | return equality * prod 102 | 103 | 104 | def inv_income_weighted_coin_endowments(coin_endowments): 105 | """Social welfare, as weighted average endowment (weighted by inverse endowment). 106 | 107 | Args: 108 | coin_endowments (ndarray): The array of coin endowments for each of the 109 | agents in the simulated economy. 110 | 111 | Returns: 112 | Weighted average coin endowment (float). 113 | """ 114 | pareto_weights = 1 / np.maximum(coin_endowments, 1) 115 | pareto_weights = pareto_weights / np.sum(pareto_weights) 116 | return np.sum(coin_endowments * pareto_weights) 117 | 118 | 119 | def inv_income_weighted_utility(coin_endowments, utilities): 120 | """Social welfare, as weighted average utility (weighted by inverse endowment). 121 | 122 | Args: 123 | coin_endowments (ndarray): The array of coin endowments for each of the 124 | agents in the simulated economy. 125 | utilities (ndarray): The array of utilities for each of the agents in the 126 | simulated economy. 127 | 128 | Returns: 129 | Weighted average utility (float). 130 | """ 131 | pareto_weights = 1 / np.maximum(coin_endowments, 1) 132 | pareto_weights = pareto_weights / np.sum(pareto_weights) 133 | return np.sum(utilities * pareto_weights) 134 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Salesforce Open Source Community Code of Conduct 2 | 3 | ## About the Code of Conduct 4 | 5 | Equality is a core value at Salesforce. We believe a diverse and inclusive 6 | community fosters innovation and creativity, and are committed to building a 7 | culture where everyone feels included. 8 | 9 | Salesforce open-source projects are committed to providing a friendly, safe, and 10 | welcoming environment for all, regardless of gender identity and expression, 11 | sexual orientation, disability, physical appearance, body size, ethnicity, nationality, 12 | race, age, religion, level of experience, education, socioeconomic status, or 13 | other similar personal characteristics. 14 | 15 | The goal of this code of conduct is to specify a baseline standard of behavior so 16 | that people with different social values and communication styles can work 17 | together effectively, productively, and respectfully in our open source community. 18 | It also establishes a mechanism for reporting issues and resolving conflicts. 19 | 20 | All questions and reports of abusive, harassing, or otherwise unacceptable behavior 21 | in a Salesforce open-source project may be reported by contacting the Salesforce 22 | Open Source Conduct Committee at ossconduct@salesforce.com. 23 | 24 | ## Our Pledge 25 | 26 | In the interest of fostering an open and welcoming environment, we as 27 | contributors and maintainers pledge to making participation in our project and 28 | our community a harassment-free experience for everyone, regardless of gender 29 | identity and expression, sexual orientation, disability, physical appearance, 30 | body size, ethnicity, nationality, race, age, religion, level of experience, education, 31 | socioeconomic status, or other similar personal characteristics. 32 | 33 | ## Our Standards 34 | 35 | Examples of behavior that contributes to creating a positive environment 36 | include: 37 | 38 | * Using welcoming and inclusive language 39 | * Being respectful of differing viewpoints and experiences 40 | * Gracefully accepting constructive criticism 41 | * Focusing on what is best for the community 42 | * Showing empathy toward other community members 43 | 44 | Examples of unacceptable behavior by participants include: 45 | 46 | * The use of sexualized language or imagery and unwelcome sexual attention or 47 | advances 48 | * Personal attacks, insulting/derogatory comments, or trolling 49 | * Public or private harassment 50 | * Publishing, or threatening to publish, others' private information—such as 51 | a physical or electronic address—without explicit permission 52 | * Other conduct which could reasonably be considered inappropriate in a 53 | professional setting 54 | * Advocating for or encouraging any of the above behaviors 55 | 56 | ## Our Responsibilities 57 | 58 | Project maintainers are responsible for clarifying the standards of acceptable 59 | behavior and are expected to take appropriate and fair corrective action in 60 | response to any instances of unacceptable behavior. 61 | 62 | Project maintainers have the right and responsibility to remove, edit, or 63 | reject comments, commits, code, wiki edits, issues, and other contributions 64 | that are not aligned with this Code of Conduct, or to ban temporarily or 65 | permanently any contributor for other behaviors that they deem inappropriate, 66 | threatening, offensive, or harmful. 67 | 68 | ## Scope 69 | 70 | This Code of Conduct applies both within project spaces and in public spaces 71 | when an individual is representing the project or its community. Examples of 72 | representing a project or community include using an official project email 73 | address, posting via an official social media account, or acting as an appointed 74 | representative at an online or offline event. Representation of a project may be 75 | further defined and clarified by project maintainers. 76 | 77 | ## Enforcement 78 | 79 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 80 | reported by contacting the Salesforce Open Source Conduct Committee 81 | at ossconduct@salesforce.com. All complaints will be reviewed and investigated 82 | and will result in a response that is deemed necessary and appropriate to the 83 | circumstances. The committee is obligated to maintain confidentiality with 84 | regard to the reporter of an incident. Further details of specific enforcement 85 | policies may be posted separately. 86 | 87 | Project maintainers who do not follow or enforce the Code of Conduct in good 88 | faith may face temporary or permanent repercussions as determined by other 89 | members of the project's leadership and the Salesforce Open Source Conduct 90 | Committee. 91 | 92 | ## Attribution 93 | 94 | This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant-home], 95 | version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html. 96 | It includes adaptions and additions from [Go Community Code of Conduct][golang-coc], 97 | [CNCF Code of Conduct][cncf-coc], and [Microsoft Open Source Code of Conduct][microsoft-coc]. 98 | 99 | This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us]. 100 | 101 | [contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/) 102 | [golang-coc]: https://golang.org/conduct 103 | [cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md 104 | [microsoft-coc]: https://opensource.microsoft.com/codeofconduct/ 105 | [cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/ -------------------------------------------------------------------------------- /ai_economist/training/training_script.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | """ 8 | Example training script for the grid world and continuous versions of Tag. 9 | Note: This training script only runs on a GPU machine. 10 | You will also need to install WarpDrive (https://github.com/salesforce/warp-drive) 11 | using `pip install rl-warp-drive`, and Pytorch(https://pytorch.org/) 12 | """ 13 | 14 | import argparse 15 | import logging 16 | import os 17 | 18 | import GPUtil 19 | 20 | try: 21 | num_gpus_available = len(GPUtil.getAvailable()) 22 | assert num_gpus_available > 0, "This training script needs a GPU to run!" 23 | print(f"Inside training_script.py: {num_gpus_available} GPUs are available.") 24 | import torch 25 | import yaml 26 | from warp_drive.training.trainer import Trainer 27 | from warp_drive.utils.env_registrar import EnvironmentRegistrar 28 | except ModuleNotFoundError: 29 | raise ModuleNotFoundError( 30 | "This training script requires the 'WarpDrive' package, please run " 31 | "'pip install rl-warp-drive' first." 32 | ) from None 33 | except ValueError: 34 | raise ValueError("This training script needs a GPU to run!") from None 35 | 36 | from ai_economist.foundation.env_wrapper import FoundationEnvWrapper 37 | from ai_economist.foundation.scenarios.covid19.covid19_env import ( 38 | CovidAndEconomyEnvironment, 39 | ) 40 | 41 | logging.getLogger().setLevel(logging.ERROR) 42 | 43 | pytorch_cuda_init_success = torch.cuda.FloatTensor(8) 44 | _COVID_AND_ECONOMY_ENVIRONMENT = "covid_and_economy_environment" 45 | 46 | # Usage: 47 | # >> python ai_economist/training/example_training_script.py 48 | # --env covid_and_economy_environment 49 | 50 | 51 | if __name__ == "__main__": 52 | 53 | parser = argparse.ArgumentParser() 54 | parser.add_argument("--env", "-e", type=str, help="Environment to train.") 55 | 56 | args = parser.parse_args() 57 | 58 | # Read the run configurations specific to each environment. 59 | # Note: The run config yamls are located at warp_drive/training/run_configs 60 | # --------------------------------------------------------------------------- 61 | assert args.env in [_COVID_AND_ECONOMY_ENVIRONMENT], ( 62 | f"Currently, the only environment supported " 63 | f"is {_COVID_AND_ECONOMY_ENVIRONMENT}" 64 | ) 65 | 66 | config_path = os.path.join( 67 | os.path.dirname(os.path.abspath(__file__)), 68 | "run_configs", 69 | f"{args.env}.yaml", 70 | ) 71 | with open(config_path, "r", encoding="utf8") as f: 72 | run_config = yaml.safe_load(f) 73 | 74 | num_envs = run_config["trainer"]["num_envs"] 75 | 76 | # Create a wrapped environment object via the EnvWrapper 77 | # Ensure that use_cuda is set to True (in order to run on the GPU) 78 | # ---------------------------------------------------------------- 79 | if run_config["name"] == _COVID_AND_ECONOMY_ENVIRONMENT: 80 | env_registrar = EnvironmentRegistrar() 81 | this_file_dir = os.path.dirname(os.path.abspath(__file__)) 82 | env_registrar.add_cuda_env_src_path( 83 | CovidAndEconomyEnvironment.name, 84 | os.path.join( 85 | this_file_dir, "../foundation/scenarios/covid19/covid19_build.cu" 86 | ), 87 | ) 88 | env_wrapper = FoundationEnvWrapper( 89 | CovidAndEconomyEnvironment(**run_config["env"]), 90 | num_envs=num_envs, 91 | use_cuda=True, 92 | env_registrar=env_registrar, 93 | ) 94 | else: 95 | raise NotImplementedError 96 | 97 | # The policy_tag_to_agent_id_map dictionary maps 98 | # policy model names to agent ids. 99 | # ---------------------------------------------------- 100 | policy_tag_to_agent_id_map = { 101 | "a": [str(agent_id) for agent_id in range(env_wrapper.env.n_agents)], 102 | "p": ["p"], 103 | } 104 | 105 | # Flag indicating whether separate obs, actions and rewards placeholders 106 | # have to be created for each policy. 107 | # Set "create_separate_placeholders_for_each_policy" to True here 108 | # since the agents and planner have different observation 109 | # and action spaces. 110 | separate_placeholder_per_policy = True 111 | 112 | # Flag indicating the observation dimension corresponding to 113 | # 'num_agents'. 114 | # Note: WarpDrive assumes that all the observation are shaped 115 | # (num_agents, *feature_dim), i.e., the observation dimension 116 | # corresponding to 'num_agents' is the first one. Instead, if the 117 | # observation dimension corresponding to num_agents is the last one, 118 | # we will need to permute the axes to align with WarpDrive's assumption 119 | obs_dim_corresponding_to_num_agents = "last" 120 | 121 | # Trainer object 122 | # -------------- 123 | trainer = Trainer( 124 | env_wrapper=env_wrapper, 125 | config=run_config, 126 | policy_tag_to_agent_id_map=policy_tag_to_agent_id_map, 127 | create_separate_placeholders_for_each_policy=separate_placeholder_per_policy, 128 | obs_dim_corresponding_to_num_agents=obs_dim_corresponding_to_num_agents, 129 | ) 130 | 131 | # Perform training 132 | # ---------------- 133 | trainer.train() 134 | trainer.graceful_close() 135 | -------------------------------------------------------------------------------- /ai_economist/foundation/components/simple_labor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | from ai_economist.foundation.base.base_component import ( 10 | BaseComponent, 11 | component_registry, 12 | ) 13 | 14 | 15 | @component_registry.add 16 | class SimpleLabor(BaseComponent): 17 | """ 18 | Allows Agents to select a level of labor, which earns income based on skill. 19 | 20 | Labor is "simple" because this simplifies labor to a choice along a 1D axis. More 21 | concretely, this component adds 100 labor actions, each representing a choice of 22 | how many hours to work, e.g. action 50 represents doing 50 hours of work; each 23 | Agent earns income proportional to the product of its labor amount (representing 24 | hours worked) and its skill (representing wage), with higher skill and higher labor 25 | yielding higher income. 26 | 27 | This component is intended to be used with the 'PeriodicBracketTax' component and 28 | the 'one-step-economy' scenario. 29 | 30 | Args: 31 | mask_first_step (bool): Defaults to True. If True, masks all non-0 labor 32 | actions on the first step of the environment. When combined with the 33 | intended component/scenario, the first env step is used to set taxes 34 | (via the 'redistribution' component) and the second step is used to 35 | select labor (via this component). 36 | payment_max_skill_multiplier (float): When determining the skill level of 37 | each Agent, sampled skills are clipped to this maximum value. 38 | """ 39 | 40 | name = "SimpleLabor" 41 | required_entities = ["Coin"] 42 | agent_subclasses = ["BasicMobileAgent"] 43 | 44 | def __init__( 45 | self, 46 | *base_component_args, 47 | mask_first_step=True, 48 | payment_max_skill_multiplier=3, 49 | pareto_param=4.0, 50 | **base_component_kwargs 51 | ): 52 | super().__init__(*base_component_args, **base_component_kwargs) 53 | 54 | # This defines the size of the action space (the max # hours an agent can work). 55 | self.num_labor_hours = 100 # max 100 hours 56 | 57 | assert isinstance(mask_first_step, bool) 58 | self.mask_first_step = mask_first_step 59 | 60 | self.is_first_step = True 61 | self.common_mask_on = { 62 | agent.idx: np.ones((self.num_labor_hours,)) for agent in self.world.agents 63 | } 64 | self.common_mask_off = { 65 | agent.idx: np.zeros((self.num_labor_hours,)) for agent in self.world.agents 66 | } 67 | 68 | # Skill distribution 69 | self.pareto_param = float(pareto_param) 70 | assert self.pareto_param > 0 71 | self.payment_max_skill_multiplier = float(payment_max_skill_multiplier) 72 | pmsm = self.payment_max_skill_multiplier 73 | num_agents = len(self.world.agents) 74 | # Generate a batch (1000) of num_agents (sorted/clipped) Pareto samples. 75 | pareto_samples = np.random.pareto(4, size=(1000, num_agents)) 76 | clipped_skills = np.minimum(pmsm, (pmsm - 1) * pareto_samples + 1) 77 | sorted_clipped_skills = np.sort(clipped_skills, axis=1) 78 | # The skill level of the i-th skill-ranked agent is the average of the 79 | # i-th ranked samples throughout the batch. 80 | self.skills = sorted_clipped_skills.mean(axis=0) 81 | 82 | def get_additional_state_fields(self, agent_cls_name): 83 | if agent_cls_name == "BasicMobileAgent": 84 | return {"skill": 0, "production": 0} 85 | return {} 86 | 87 | def additional_reset_steps(self): 88 | self.is_first_step = True 89 | for agent in self.world.agents: 90 | agent.state["skill"] = self.skills[agent.idx] 91 | 92 | def get_n_actions(self, agent_cls_name): 93 | if agent_cls_name == "BasicMobileAgent": 94 | return self.num_labor_hours 95 | return None 96 | 97 | def generate_masks(self, completions=0): 98 | if self.is_first_step: 99 | self.is_first_step = False 100 | if self.mask_first_step: 101 | return self.common_mask_off 102 | 103 | return self.common_mask_on 104 | 105 | def component_step(self): 106 | 107 | for agent in self.world.get_random_order_agents(): 108 | 109 | action = agent.get_component_action(self.name) 110 | 111 | if action == 0: # NO-OP. 112 | # Agent is not interacting with this component. 113 | continue 114 | 115 | if 1 <= action <= self.num_labor_hours: # set reopening phase 116 | 117 | hours_worked = action # NO-OP is 0 hours. 118 | agent.state["endogenous"]["Labor"] = hours_worked 119 | 120 | payoff = hours_worked * agent.state["skill"] 121 | agent.state["production"] += payoff 122 | agent.inventory["Coin"] += payoff 123 | 124 | else: 125 | # If action > num_labor_hours, this is an error. 126 | raise ValueError 127 | 128 | def generate_observations(self): 129 | obs_dict = dict() 130 | for agent in self.world.agents: 131 | obs_dict[str(agent.idx)] = { 132 | "skill": agent.state["skill"] / self.payment_max_skill_multiplier 133 | } 134 | return obs_dict 135 | -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/README.md: -------------------------------------------------------------------------------- 1 | # Real Business Cycle (RBC) 2 | This directory implements a **Real-Business-Cycle** (RBC) simulation with many heterogeneous, interacting strategic agents of various types, such as **consumers, firms, and the government**. For details, please refer to this paper "Finding General Equilibria in Many-Agent Economic Simulations using Deep Reinforcement Learning (ArXiv link forthcoming)". We also provide training code that uses deep multi-agent reinforcement learning to determine optimal economic policies and dynamics in these many agent environments. Below are instructions required to launch the training runs. 3 | 4 | **Note: The experiments require a GPU to run!** 5 | 6 | ## Dependencies 7 | 8 | - torch>=1.9.0 9 | - pycuda==2021.1 10 | - matplotlib==3.2.1 11 | 12 | ## Running Local Jobs 13 | To run a hyperparameter sweep of jobs on a local machine, use (see file for command line arguments and hyperparameter sweep dictionaries) 14 | 15 | ``` 16 | python train_multi_exps.py 17 | ``` 18 | 19 | ## Configuration Dictionaries 20 | 21 | Configuration dictionaries are currently specified in Python code, and then written as `hparams.yaml` in the job directory. For examples, see the file `constants.py`. The dictionaries contain "agents", "world", and "train" dictionaries which contain various hyperparameters. 22 | 23 | ## Hyperparameter Sweeps 24 | 25 | The files `train_multi_exps.py` allow hyperparameter sweeps. These are specified in `*_param_sweeps` dictionaries in the file. For each hyperparameter, specify a list of one or more choices. The Cartesian product of all choices will be used. 26 | 27 | ## Approximate Best Response Training 28 | 29 | To run a single approximate best-response (BR) training job on checkpoint policies, run `python train_bestresponse.py ROLLOUT_DIR NUM_EPISODES_TO_TRAIN --ep-strs ep1 ep2 --agent-type all`. The `--ep-strs` argument specifies which episodes to run on (for example, policies from episode 0, 10000, and 200000). These must be episodes for which policies were saved. It is possible to specify a single agent type. 30 | 31 | 32 | ## What Will Be Saved? 33 | 34 | A large amount of data will be saved -- one can set hyperparamter `train.save_dense_every` in the configuration dictionary (`hparams.yaml`/`constants.py`) to reduce this. 35 | 36 | At the top level, an experiment directory stores the results of many runs in a hyperparameter sweep. Example structure: 37 | 38 | ``` 39 | experiment/experimentname/ 40 | rollout-999999-99999/ 41 | brconsumer/ 42 | ... 43 | brfirm/ 44 | episode_XXXX_consumer.npz 45 | episode_XXXX_government.npz 46 | episode_XXXX_firm.npz 47 | saved_models/ 48 | consumer_policy_XXX.pt 49 | firm_policy_XXX.pt 50 | government_policy_XXX.pt. 51 | brgovernment/ 52 | ... 53 | hparams.yaml 54 | action_arrays.pickle 55 | episode_XXXX_consumer.npz 56 | episode_XXXX_government.npz 57 | episode_XXXX_firm.npz 58 | saved_models/ 59 | consumer_policy_XXX.pt 60 | firm_policy_XXX.pt 61 | government_policy_XXX.pt. 62 | 63 | rollout-777777-77777/ 64 | ... 65 | ``` 66 | 67 | Files: 68 | 69 | `rollout-XXXXXX-XXX`: subdirectory containing all output for a single run. 70 | 71 | `hparams.yaml`: configuration dictionary with hyperparameters 72 | 73 | `action_arrays.pickle`: contains saved action arrays (allowing mapping action indices to the actual action, e.g. index 1 is price 1000.0, etc.) 74 | 75 | `episode_XXXX_AGENTTYPE.npz`: Contains dense rollouts stored as the output of a numpy.savez call. When loaded, can be treated like a dictionary of numpy arrays. Has keys: `['states', 'actions', 'rewards', 'action_array', 'aux_array']` (view keys by using `.files`). `states`, `actions`, `rewards`, and `aux_array` all refer to saved copies of CUDA arrays (described below). `action_array` is a small array mapping action indices to the actual action. 76 | 77 | `saved_models/AGENTTYPE_policy_XXX.pt`: a saved PyTorch state dict of the policy network, after episode XXX. 78 | 79 | ## Structure Of Arrays 80 | 81 | `states` for any given agent type is an array storing observed states. It has shape `batch_size, ep_length, num_agents, agent_total_state_dim`. 82 | 83 | `actions` is an array consisting of the action _indices_ (integers). For firms and government, it is of shape `batch_size, ep_length, num_agents`. For consumers, it is of shape `batch_size, ep_length, num_agents, num_action_heads`. 84 | 85 | `rewards` stores total rewards, and is of shape `batch_size, ep_length, num_agents`. 86 | 87 | The `aux_array` stores additional information and may differ per agent type. The consumer `aux_array` stores _actual_ consumption of each firm's good (as opposed to attempted consumption). The firm `aux_array` stores the amount bought by the export market. 88 | 89 | ## State Array Layout: 90 | 91 | States observed by each agent consist of a global state, plus additional state dimensions per agent. 92 | 93 | Global state: total dimension 4 * num_firms + 2 + 1 94 | - prices: 1 per firm 95 | - wages: 1 per firm 96 | - inventories: 1 per firm 97 | - overdemanded flag: 1 per firm 98 | - time 99 | 100 | Consumer additional state variables: total dimension global state + 2 101 | - budget 102 | - theta 103 | 104 | Firm additional state variables: total dimension global state + 3 + num_firms 105 | - budget 106 | - capital 107 | - production alpha 108 | - one-hot representation identifying which firm 109 | 110 | ## What Gets Loaded And Written By BR Code? 111 | 112 | The best response code loads in the `hparams.yaml` file, and the policies at a given time step (i.e. `saved_models/...policy_XXX.pt`). It then trains one of the policies while keeping the others fixed. Results are written to directories `brfirm`, `brconsumer`, `brgovernment` and contain dense rollouts and saved policy checkpoints, but from the best response training. 113 | 114 | ## Which Hyperparameters Are Managed And Where? 115 | 116 | Initial values of state variables (budgets, initial wages, levels of capital, and so on) are set by the code in the method `__init_cuda_data_structs`. Some of these can be controlled from the hyperparameter dict; others are currently hardcoded. 117 | 118 | Other hyperparameters are specified in the configuration dictionary. 119 | 120 | Finally, the technology parameter (A) of the production function is currently hardcoded in the function call in `rbc/cuda/firm_rbc.cu`. 121 | -------------------------------------------------------------------------------- /tutorials/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /tutorials/rllib/env_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | """ 8 | Wrapper for making the gather-trade-build environment an OpenAI compatible environment. 9 | This can then be used with reinforcement learning frameworks such as RLlib. 10 | """ 11 | 12 | import os 13 | import pickle 14 | import random 15 | import warnings 16 | 17 | import numpy as np 18 | from ai_economist import foundation 19 | from gym import spaces 20 | from gym.utils import seeding 21 | from ray.rllib.env.multi_agent_env import MultiAgentEnv 22 | 23 | _BIG_NUMBER = 1e20 24 | 25 | 26 | def recursive_list_to_np_array(d): 27 | if isinstance(d, dict): 28 | new_d = {} 29 | for k, v in d.items(): 30 | if isinstance(v, list): 31 | new_d[k] = np.array(v) 32 | elif isinstance(v, dict): 33 | new_d[k] = recursive_list_to_np_array(v) 34 | elif isinstance(v, (float, int, np.floating, np.integer)): 35 | new_d[k] = np.array([v]) 36 | elif isinstance(v, np.ndarray): 37 | new_d[k] = v 38 | else: 39 | raise AssertionError 40 | return new_d 41 | raise AssertionError 42 | 43 | 44 | def pretty_print(dictionary): 45 | for key in dictionary: 46 | print("{:15s}: {}".format(key, dictionary[key].shape)) 47 | print("\n") 48 | 49 | 50 | class RLlibEnvWrapper(MultiAgentEnv): 51 | """ 52 | Environment wrapper for RLlib. It sub-classes MultiAgentEnv. 53 | This wrapper adds the action and observation space to the environment, 54 | and adapts the reset and step functions to run with RLlib. 55 | """ 56 | 57 | def __init__(self, env_config, verbose=False): 58 | self.env_config_dict = env_config["env_config_dict"] 59 | 60 | # Adding env id in the case of multiple environments 61 | if hasattr(env_config, "worker_index"): 62 | self.env_id = ( 63 | env_config["num_envs_per_worker"] * (env_config.worker_index - 1) 64 | ) + env_config.vector_index 65 | else: 66 | self.env_id = None 67 | 68 | self.env = foundation.make_env_instance(**self.env_config_dict) 69 | self.verbose = verbose 70 | self.sample_agent_idx = str(self.env.all_agents[0].idx) 71 | 72 | obs = self.env.reset() 73 | 74 | self.observation_space = self._dict_to_spaces_dict(obs["0"]) 75 | self.observation_space_pl = self._dict_to_spaces_dict(obs["p"]) 76 | 77 | if self.env.world.agents[0].multi_action_mode: 78 | self.action_space = spaces.MultiDiscrete( 79 | self.env.get_agent(self.sample_agent_idx).action_spaces 80 | ) 81 | self.action_space.dtype = np.int64 82 | self.action_space.nvec = self.action_space.nvec.astype(np.int64) 83 | 84 | else: 85 | self.action_space = spaces.Discrete( 86 | self.env.get_agent(self.sample_agent_idx).action_spaces 87 | ) 88 | self.action_space.dtype = np.int64 89 | 90 | if self.env.world.planner.multi_action_mode: 91 | self.action_space_pl = spaces.MultiDiscrete( 92 | self.env.get_agent("p").action_spaces 93 | ) 94 | self.action_space_pl.dtype = np.int64 95 | self.action_space_pl.nvec = self.action_space_pl.nvec.astype(np.int64) 96 | 97 | else: 98 | self.action_space_pl = spaces.Discrete( 99 | self.env.get_agent("p").action_spaces 100 | ) 101 | self.action_space_pl.dtype = np.int64 102 | 103 | self._seed = None 104 | if self.verbose: 105 | print("[EnvWrapper] Spaces") 106 | print("[EnvWrapper] Obs (a) ") 107 | pretty_print(self.observation_space) 108 | print("[EnvWrapper] Obs (p) ") 109 | pretty_print(self.observation_space_pl) 110 | print("[EnvWrapper] Action (a)", self.action_space) 111 | print("[EnvWrapper] Action (p)", self.action_space_pl) 112 | 113 | def _dict_to_spaces_dict(self, obs): 114 | dict_of_spaces = {} 115 | for k, v in obs.items(): 116 | 117 | # list of lists are listified np arrays 118 | _v = v 119 | if isinstance(v, list): 120 | _v = np.array(v) 121 | elif isinstance(v, (int, float, np.floating, np.integer)): 122 | _v = np.array([v]) 123 | 124 | # assign Space 125 | if isinstance(_v, np.ndarray): 126 | x = float(_BIG_NUMBER) 127 | # Warnings for extreme values 128 | if np.max(_v) > x: 129 | warnings.warn("Input is too large!") 130 | if np.min(_v) < -x: 131 | warnings.warn("Input is too small!") 132 | box = spaces.Box(low=-x, high=x, shape=_v.shape, dtype=_v.dtype) 133 | low_high_valid = (box.low < 0).all() and (box.high > 0).all() 134 | 135 | # This loop avoids issues with overflow to make sure low/high are good. 136 | while not low_high_valid: 137 | x = x // 2 138 | box = spaces.Box(low=-x, high=x, shape=_v.shape, dtype=_v.dtype) 139 | low_high_valid = (box.low < 0).all() and (box.high > 0).all() 140 | 141 | dict_of_spaces[k] = box 142 | 143 | elif isinstance(_v, dict): 144 | dict_of_spaces[k] = self._dict_to_spaces_dict(_v) 145 | else: 146 | raise TypeError 147 | return spaces.Dict(dict_of_spaces) 148 | 149 | @property 150 | def pickle_file(self): 151 | if self.env_id is None: 152 | return "game_object.pkl" 153 | return "game_object_{:03d}.pkl".format(self.env_id) 154 | 155 | def save_game_object(self, save_dir): 156 | assert os.path.isdir(save_dir) 157 | path = os.path.join(save_dir, self.pickle_file) 158 | with open(path, "wb") as F: 159 | pickle.dump(self.env, F) 160 | 161 | def load_game_object(self, save_dir): 162 | assert os.path.isdir(save_dir) 163 | path = os.path.join(save_dir, self.pickle_file) 164 | with open(path, "rb") as F: 165 | self.env = pickle.load(F) 166 | 167 | @property 168 | def n_agents(self): 169 | return self.env.n_agents 170 | 171 | @property 172 | def summary(self): 173 | last_completion_metrics = self.env.previous_episode_metrics 174 | if last_completion_metrics is None: 175 | return {} 176 | last_completion_metrics["completions"] = int(self.env._completions) 177 | return last_completion_metrics 178 | 179 | def get_seed(self): 180 | return int(self._seed) 181 | 182 | def seed(self, seed): 183 | # Using the seeding utility from OpenAI Gym 184 | # https://github.com/openai/gym/blob/master/gym/utils/seeding.py 185 | _, seed1 = seeding.np_random(seed) 186 | # Derive a random seed. This gets passed as an uint, but gets 187 | # checked as an int elsewhere, so we need to keep it below 188 | # 2**31. 189 | seed2 = seeding.hash_seed(seed1 + 1) % 2 ** 31 190 | 191 | if self.verbose: 192 | print( 193 | "[EnvWrapper] twisting seed {} -> {} -> {} (final)".format( 194 | seed, seed1, seed2 195 | ) 196 | ) 197 | 198 | seed = int(seed2) 199 | np.random.seed(seed2) 200 | random.seed(seed2) 201 | self._seed = seed2 202 | 203 | def reset(self, *args, **kwargs): 204 | obs = self.env.reset(*args, **kwargs) 205 | return recursive_list_to_np_array(obs) 206 | 207 | def step(self, action_dict): 208 | obs, rew, done, info = self.env.step(action_dict) 209 | assert isinstance(obs[self.sample_agent_idx]["action_mask"], np.ndarray) 210 | 211 | return recursive_list_to_np_array(obs), rew, done, info 212 | -------------------------------------------------------------------------------- /ai_economist/real_business_cycle/experiment_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import hashlib 8 | import itertools 9 | import json 10 | import os 11 | import pickle 12 | import struct 13 | import time 14 | from copy import deepcopy 15 | from pathlib import Path 16 | 17 | import numpy as np 18 | import yaml 19 | 20 | # defaults 21 | _NUM_FIRMS = 10 22 | 23 | 24 | def _bigint_from_bytes(num_bytes): 25 | """ 26 | See https://github.com/openai/gym/blob/master/gym/utils/seeding.py. 27 | """ 28 | sizeof_int = 4 29 | padding = sizeof_int - len(num_bytes) % sizeof_int 30 | num_bytes += b"\0" * padding 31 | int_count = int(len(num_bytes) / sizeof_int) 32 | unpacked = struct.unpack("{}I".format(int_count), num_bytes) 33 | accum = 0 34 | for i, val in enumerate(unpacked): 35 | accum += 2 ** (sizeof_int * 8 * i) * val 36 | return accum 37 | 38 | 39 | def seed_from_base_seed(base_seed): 40 | """ 41 | Hash base seed to reduce correlation. 42 | """ 43 | max_bytes = 4 44 | hash_func = hashlib.sha512(str(base_seed).encode("utf8")).digest() 45 | 46 | return _bigint_from_bytes(hash_func[:max_bytes]) 47 | 48 | 49 | def hash_from_dict(d): 50 | d_copy = deepcopy(d) 51 | del (d_copy["train"])["base_seed"] 52 | d_string = json.dumps(d_copy, sort_keys=True) 53 | return int(hashlib.sha256(d_string.encode("utf8")).hexdigest()[:8], 16) 54 | 55 | 56 | def cfg_dict_from_yaml( 57 | hparams_path, 58 | consumption_choices, 59 | work_choices, 60 | price_and_wage, 61 | tax_choices, 62 | group_name=None, 63 | ): 64 | with open(hparams_path) as f: 65 | d = yaml.safe_load(f) 66 | 67 | if group_name is not None: 68 | d["metadata"]["group_name"] = group_name 69 | d["metadata"]["hparamhash"] = hash_from_dict(d) 70 | d["agents"][ 71 | "consumer_consumption_actions_array" 72 | ] = consumption_choices # Note: hardcoded 73 | d["agents"]["consumer_work_actions_array"] = work_choices # Note: hardcoded 74 | d["agents"]["firm_actions_array"] = price_and_wage # Note: hardcoded 75 | d["agents"]["government_actions_array"] = tax_choices 76 | d["train"]["save_dir"] = str(hparams_path.absolute().parent) 77 | d["train"]["seed"] = seed_from_base_seed(d["train"]["base_seed"]) 78 | return d 79 | 80 | 81 | def run_experiment_batch_parallel( 82 | experiment_dir, 83 | consumption_choices, 84 | work_choices, 85 | price_and_wage, 86 | tax_choices, 87 | group_name=None, 88 | consumers_only=False, 89 | no_firms=False, 90 | default_firm_action=None, 91 | default_government_action=None, 92 | ): 93 | hparams_path = Path(experiment_dir) / Path("hparams.yaml") 94 | hparams_dict = cfg_dict_from_yaml( 95 | hparams_path, 96 | consumption_choices, 97 | work_choices, 98 | price_and_wage, 99 | tax_choices, 100 | group_name=group_name, 101 | ) 102 | print(f"hparams_dict {hparams_dict}") 103 | # import this here so rest of file still imports without cuda installed 104 | from rbc.cuda_manager import ConsumerFirmRunManagerBatchParallel 105 | 106 | if consumers_only: 107 | m = ConsumerFirmRunManagerBatchParallel( 108 | hparams_dict, 109 | freeze_firms=default_firm_action, 110 | freeze_govt=default_government_action, 111 | ) 112 | elif no_firms: 113 | m = ConsumerFirmRunManagerBatchParallel( 114 | hparams_dict, 115 | freeze_firms=default_firm_action, 116 | ) 117 | else: 118 | m = ConsumerFirmRunManagerBatchParallel(hparams_dict) 119 | m.train() 120 | 121 | 122 | def compare_global_states_within_type(states, global_state_size): 123 | # every agent within a batch should have the same global state 124 | first_agent_global = states[:, :, :1, :global_state_size] 125 | all_agents_global = states[:, :, :, :global_state_size] 126 | return np.isclose(all_agents_global, first_agent_global).all() 127 | 128 | 129 | def compare_global_states_across_types( 130 | consumer_states, firm_states, government_states, global_state_size 131 | ): 132 | first_agent_global = consumer_states[:, :, :1, :global_state_size] 133 | return ( 134 | np.isclose(firm_states[:, :, :, :global_state_size], first_agent_global).all(), 135 | np.isclose( 136 | government_states[:, :, :, :global_state_size], first_agent_global 137 | ).all(), 138 | np.isclose( 139 | consumer_states[:, :, :, :global_state_size], first_agent_global 140 | ).all(), 141 | ) 142 | 143 | 144 | def check_no_negative_stocks(state, stock_offset, stock_size): 145 | stocks = state[:, :, :, stock_offset : (stock_offset + stock_size)] 146 | return (stocks >= -1.0e-3).all() 147 | 148 | 149 | train_param_sweeps = { 150 | "lr": [0.005, 0.001], 151 | "entropy": [0.01], 152 | "base_seed": [2596], 153 | "batch_size": [64], 154 | "clip_grad_norm": [1.0, 2.0, 5.0], 155 | } 156 | 157 | # Other param sweeps 158 | agent_param_sweeps = { 159 | # "consumer_noponzi_eta": [0.1,0.05] 160 | } 161 | 162 | world_param_sweeps = { 163 | # "interest_rate": [0.02, 0.0] 164 | } 165 | 166 | 167 | def add_all(d, keys_list, target_val): 168 | for k in keys_list: 169 | d[k] = target_val 170 | 171 | 172 | def sweep_cfg_generator( 173 | base_cfg, 174 | tr_param_sweeps=None, 175 | ag_param_sweeps=None, 176 | wld_param_sweeps=None, 177 | seed_from_timestamp=False, 178 | group_name=None, 179 | ): 180 | # train_param_sweeps 181 | if tr_param_sweeps is None: 182 | tr_param_sweeps = {} 183 | # agent_param_sweeps 184 | if ag_param_sweeps is None: 185 | ag_param_sweeps = {} 186 | # world_param_sweeps 187 | if wld_param_sweeps is None: 188 | wld_param_sweeps = {} 189 | 190 | assert isinstance(tr_param_sweeps, dict) 191 | assert isinstance(ag_param_sweeps, dict) 192 | assert isinstance(wld_param_sweeps, dict) 193 | 194 | key_dict = {} # tells which key goes to which dict, e.g. "lr" -> "train", etc. 195 | if len(tr_param_sweeps) > 0: 196 | train_k, train_v = zip(*tr_param_sweeps.items()) 197 | else: 198 | train_k, train_v = (), () 199 | add_all(key_dict, train_k, "train") 200 | if len(ag_param_sweeps) > 0: 201 | agent_k, agent_v = zip(*ag_param_sweeps.items()) 202 | else: 203 | agent_k, agent_v = (), () 204 | add_all(key_dict, agent_k, "agents") 205 | if len(wld_param_sweeps) > 0: 206 | world_k, world_v = zip(*wld_param_sweeps.items()) 207 | else: 208 | world_k, world_v = (), () 209 | add_all(key_dict, world_k, "world") 210 | 211 | k = train_k + agent_k + world_k 212 | v = train_v + agent_v + world_v 213 | 214 | # have a "reverse lookup" dictionary for each key name 215 | for combination in itertools.product(*v): 216 | values_to_substitute = dict(zip(k, combination)) 217 | out = deepcopy(base_cfg) 218 | for key, value in values_to_substitute.items(): 219 | out[key_dict[key]][key] = value 220 | if seed_from_timestamp: 221 | int_timestamp = int( 222 | time.time() * 1000 223 | ) # time.time() returns float, multiply 1000 for higher resolution 224 | out["train"]["base_seed"] += int_timestamp 225 | if group_name is not None: 226 | out["metadata"]["group"] = group_name 227 | yield out 228 | 229 | 230 | def create_job_dir(experiment_dir, job_name_base, cfg=None, action_arrays=None): 231 | unique_id = time.time() 232 | dirname = f"{job_name_base}-{unique_id}".replace(".", "-") 233 | dir_path = Path(experiment_dir) / Path(dirname) 234 | os.makedirs(str(dir_path), exist_ok=True) 235 | cfg["metadata"]["dirname"] = dirname 236 | cfg["metadata"]["group"] = str(Path(experiment_dir).name) 237 | with open(dir_path / Path("hparams.yaml"), "w") as f: 238 | f.write(yaml.dump(cfg)) 239 | 240 | if action_arrays is not None: 241 | with open(dir_path / Path("action_arrays.pickle"), "wb") as f: 242 | pickle.dump(action_arrays, f) 243 | -------------------------------------------------------------------------------- /tutorials/rllib/utils/saving.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import logging 8 | import os 9 | import pickle 10 | import shutil 11 | import sys 12 | 13 | import yaml 14 | from ai_economist import foundation 15 | from .remote import remote_env_fun 16 | 17 | logging.basicConfig(format="%(asctime)s %(message)s") 18 | logger = logging.getLogger("utils") 19 | logger.setLevel(logging.DEBUG) 20 | 21 | 22 | sys.path.append( 23 | os.path.abspath( 24 | os.path.join(os.path.dirname(sys.modules[__name__].__file__), "../../..") 25 | ) 26 | ) 27 | 28 | 29 | def fill_out_run_dir(run_dir): 30 | dense_log_dir = os.path.join(run_dir, "dense_logs") 31 | ckpt_dir = os.path.join(run_dir, "ckpts") 32 | 33 | for sub_dir in [dense_log_dir, ckpt_dir]: 34 | os.makedirs(sub_dir, exist_ok=True) 35 | 36 | latest_filepath = os.path.join(ckpt_dir, "latest_checkpoint.pkl") 37 | restore = bool(os.path.isfile(latest_filepath)) 38 | 39 | return dense_log_dir, ckpt_dir, restore 40 | 41 | 42 | def make_run_dir_path(args, log_group): 43 | assert os.path.isdir(args.top_experiment_dir) 44 | assert args.launch_time 45 | 46 | project_dir = os.path.join(args.top_experiment_dir, args.project) 47 | if not os.path.isdir(project_dir): 48 | try: 49 | os.makedirs(project_dir) 50 | except FileExistsError: 51 | pass 52 | 53 | experiment_dir = os.path.join(project_dir, args.experiment) 54 | if not os.path.isdir(experiment_dir): 55 | try: 56 | os.makedirs(experiment_dir) 57 | except FileExistsError: 58 | pass 59 | 60 | group_dir = os.path.join(experiment_dir, log_group) 61 | if not os.path.isdir(group_dir): 62 | try: 63 | os.makedirs(group_dir) 64 | except FileExistsError: 65 | pass 66 | 67 | run_dir = os.path.join( 68 | group_dir, 69 | "launch_time_{}".format(args.launch_time), 70 | "run_id_{}".format(args.unique_run_id), 71 | ) 72 | debug_dir = os.path.join(run_dir, "debug") 73 | dense_log_dir = os.path.join(run_dir, "dense_logs") 74 | ckpt_dir = os.path.join(run_dir, "ckpts") 75 | latest_filepath = os.path.join(ckpt_dir, "latest_checkpoint.pkl") 76 | 77 | if not os.path.isdir(run_dir): 78 | restore = False 79 | os.makedirs(run_dir) 80 | os.makedirs(debug_dir) 81 | os.makedirs(dense_log_dir) 82 | os.makedirs(ckpt_dir) 83 | 84 | elif os.path.isfile(latest_filepath): 85 | restore = True 86 | 87 | else: 88 | restore = False 89 | 90 | return run_dir, debug_dir, dense_log_dir, ckpt_dir, restore 91 | 92 | 93 | def write_dense_logs(trainer, log_directory, suffix=""): 94 | def save_log(env_wrapper): 95 | if 0 <= env_wrapper.env_id < 4: 96 | my_path = os.path.join( 97 | log_directory, 98 | "env{:03d}{}.lz4".format( 99 | env_wrapper.env_id, "." + suffix if suffix != "" else "" 100 | ), 101 | ) 102 | foundation.utils.save_episode_log(env_wrapper.env, my_path) 103 | 104 | remote_env_fun(trainer, save_log) 105 | 106 | 107 | def save_tf_model_weights(trainer, ckpt_dir, global_step, suffix=""): 108 | if suffix == "agent": 109 | w = trainer.get_weights(["a"]) 110 | pol = trainer.get_policy("a") 111 | model_w_array = pol._sess.run(pol.model.variables()) 112 | elif suffix == "planner": 113 | w = trainer.get_weights(["p"]) 114 | pol = trainer.get_policy("p") 115 | model_w_array = pol._sess.run(pol.model.variables()) 116 | else: 117 | raise NotImplementedError 118 | 119 | fn = os.path.join( 120 | ckpt_dir, "{}.tf.weights.global-step-{}".format(suffix, global_step) 121 | ) 122 | with open(fn, "wb") as f: 123 | pickle.dump(w, f) 124 | 125 | fn = os.path.join( 126 | ckpt_dir, 127 | "{}.policy-model-weight-array.global-step-{}".format(suffix, global_step), 128 | ) 129 | with open(fn, "wb") as f: 130 | pickle.dump(model_w_array, f) 131 | 132 | logger.info("Saved TF weights @ %s", fn) 133 | 134 | 135 | def load_tf_model_weights(trainer, ckpt): 136 | assert os.path.isfile(ckpt) 137 | with open(ckpt, "rb") as f: 138 | weights = pickle.load(f) 139 | trainer.set_weights(weights) 140 | logger.info("loaded tf model weights:\n\t%s\n", ckpt) 141 | 142 | 143 | def save_snapshot(trainer, ckpt_dir, suffix=""): 144 | # Create a new trainer snapshot 145 | filepath = trainer.save(ckpt_dir) 146 | filepath_metadata = filepath + ".tune_metadata" 147 | # Copy this to a standardized name (to only keep the latest) 148 | latest_filepath = os.path.join( 149 | ckpt_dir, "latest_checkpoint{}.pkl".format("." + suffix if suffix != "" else "") 150 | ) 151 | latest_filepath_metadata = latest_filepath + ".tune_metadata" 152 | shutil.copy(filepath, latest_filepath) 153 | shutil.copy(filepath_metadata, latest_filepath_metadata) 154 | # Get rid of the timestamped copy to prevent accumulating too many large files 155 | os.remove(filepath) 156 | os.remove(filepath_metadata) 157 | 158 | # Also take snapshots of each environment object 159 | remote_env_fun(trainer, lambda env_wrapper: env_wrapper.save_game_object(ckpt_dir)) 160 | 161 | logger.info("Saved Trainer snapshot + Env object @ %s", latest_filepath) 162 | 163 | 164 | def load_snapshot(trainer, run_dir, ckpt=None, suffix="", load_latest=False): 165 | 166 | assert ckpt or load_latest 167 | 168 | loaded_ckpt_success = False 169 | 170 | if not ckpt: 171 | if load_latest: 172 | # Restore from the latest checkpoint (pointing to it by path) 173 | ckpt_fp = os.path.join( 174 | run_dir, 175 | "ckpts", 176 | "latest_checkpoint{}.pkl".format("." + suffix if suffix != "" else ""), 177 | ) 178 | if os.path.isfile(ckpt_fp): 179 | trainer.restore(ckpt_fp) 180 | loaded_ckpt_success = True 181 | logger.info( 182 | "load_snapshot -> loading %s SUCCESS for %s %s", 183 | ckpt_fp, 184 | suffix, 185 | trainer, 186 | ) 187 | else: 188 | logger.info( 189 | "load_snapshot -> loading %s FAILED," 190 | " skipping restoring cpkt for %s %s", 191 | ckpt_fp, 192 | suffix, 193 | trainer, 194 | ) 195 | else: 196 | raise NotImplementedError 197 | elif ckpt: 198 | if os.path.isfile(ckpt): 199 | trainer.restore(ckpt) 200 | loaded_ckpt_success = True 201 | logger.info( 202 | "load_snapshot -> loading %s SUCCESS for %s %s", ckpt, suffix, trainer 203 | ) 204 | else: 205 | logger.info( 206 | "load_snapshot -> loading %s FAILED," 207 | " skipping restoring cpkt for %s %s", 208 | ckpt, 209 | suffix, 210 | trainer, 211 | ) 212 | else: 213 | raise AssertionError 214 | 215 | # Also load snapshots of each environment object 216 | remote_env_fun( 217 | trainer, 218 | lambda env_wrapper: env_wrapper.load_game_object( 219 | os.path.join(run_dir, "ckpts") 220 | ), 221 | ) 222 | 223 | return loaded_ckpt_success 224 | 225 | 226 | def dump_dict(obj, run_dir, fn): 227 | 228 | assert isinstance(obj, dict) 229 | 230 | _fn = os.path.join(run_dir, fn) 231 | if _fn[-5:] != ".yaml": 232 | _fn += ".yaml" 233 | 234 | with open(_fn, "w") as f: 235 | yaml.dump(obj, f) 236 | 237 | print(">>> dump_dict", type(obj), run_dir, fn) 238 | 239 | 240 | def dump_as_pkl(obj, run_dir, fn): 241 | _fn = os.path.join(run_dir, fn) 242 | if _fn[-5:] != ".pkl": 243 | _fn += ".pkl" 244 | with open(_fn, "wb") as f: 245 | pickle.dump(obj, f) 246 | 247 | print(">>> dump_as_pkl", type(obj), run_dir, fn) 248 | -------------------------------------------------------------------------------- /ai_economist/foundation/components/move.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | from numpy.random import rand 9 | 10 | from ai_economist.foundation.base.base_component import ( 11 | BaseComponent, 12 | component_registry, 13 | ) 14 | 15 | 16 | @component_registry.add 17 | class Gather(BaseComponent): 18 | """ 19 | Allows mobile agents to move around the world and collect resources and prevents 20 | agents from moving to invalid locations. 21 | 22 | Can be configured to include collection skill, where agents have heterogeneous 23 | probabilities of collecting bonus resources without additional labor cost. 24 | 25 | Args: 26 | move_labor (float): Labor cost associated with movement. Must be >= 0. 27 | Default is 1.0. 28 | collect_labor (float): Labor cost associated with collecting resources. This 29 | cost is added (in addition to any movement cost) when the agent lands on 30 | a tile that is populated with resources (triggering collection). 31 | Must be >= 0. Default is 1.0. 32 | skill_dist (str): Distribution type for sampling skills. Default ("none") 33 | gives all agents identical skill equal to a bonus prob of 0. "pareto" and 34 | "lognormal" sample skills from the associated distributions. 35 | """ 36 | 37 | name = "Gather" 38 | required_entities = ["Coin", "House", "Labor"] 39 | agent_subclasses = ["BasicMobileAgent"] 40 | 41 | def __init__( 42 | self, 43 | *base_component_args, 44 | move_labor=1.0, 45 | collect_labor=1.0, 46 | skill_dist="none", 47 | **base_component_kwargs 48 | ): 49 | super().__init__(*base_component_args, **base_component_kwargs) 50 | 51 | self.move_labor = float(move_labor) 52 | assert self.move_labor >= 0 53 | 54 | self.collect_labor = float(collect_labor) 55 | assert self.collect_labor >= 0 56 | 57 | self.skill_dist = skill_dist.lower() 58 | assert self.skill_dist in ["none", "pareto", "lognormal"] 59 | 60 | self.gathers = [] 61 | 62 | self._aidx = np.arange(self.n_agents)[:, None].repeat(4, axis=1) 63 | self._roff = np.array([[0, 0, -1, 1]]) 64 | self._coff = np.array([[-1, 1, 0, 0]]) 65 | 66 | # Required methods for implementing components 67 | # -------------------------------------------- 68 | 69 | def get_n_actions(self, agent_cls_name): 70 | """ 71 | See base_component.py for detailed description. 72 | 73 | Adds 4 actions (move up, down, left, or right) for mobile agents. 74 | """ 75 | # This component adds 4 action that agents can take: 76 | # move up, down, left, or right 77 | if agent_cls_name == "BasicMobileAgent": 78 | return 4 79 | return None 80 | 81 | def get_additional_state_fields(self, agent_cls_name): 82 | """ 83 | See base_component.py for detailed description. 84 | 85 | For mobile agents, add state field for collection skill. 86 | """ 87 | if agent_cls_name not in self.agent_subclasses: 88 | return {} 89 | if agent_cls_name == "BasicMobileAgent": 90 | return {"bonus_gather_prob": 0.0} 91 | raise NotImplementedError 92 | 93 | def component_step(self): 94 | """ 95 | See base_component.py for detailed description. 96 | 97 | Move to adjacent, unoccupied locations. Collect resources when moving to 98 | populated resource tiles, adding the resource to the agent's inventory and 99 | de-populating it from the tile. 100 | """ 101 | world = self.world 102 | 103 | gathers = [] 104 | for agent in world.get_random_order_agents(): 105 | 106 | if self.name not in agent.action: 107 | return 108 | action = agent.get_component_action(self.name) 109 | 110 | r, c = [int(x) for x in agent.loc] 111 | 112 | if action == 0: # NO-OP! 113 | new_r, new_c = r, c 114 | 115 | elif action <= 4: 116 | if action == 1: # Left 117 | new_r, new_c = r, c - 1 118 | elif action == 2: # Right 119 | new_r, new_c = r, c + 1 120 | elif action == 3: # Up 121 | new_r, new_c = r - 1, c 122 | else: # action == 4, # Down 123 | new_r, new_c = r + 1, c 124 | 125 | # Attempt to move the agent (if the new coordinates aren't accessible, 126 | # nothing will happen) 127 | new_r, new_c = world.set_agent_loc(agent, new_r, new_c) 128 | 129 | # If the agent did move, incur the labor cost of moving 130 | if (new_r != r) or (new_c != c): 131 | agent.state["endogenous"]["Labor"] += self.move_labor 132 | 133 | else: 134 | raise ValueError 135 | 136 | for resource, health in world.location_resources(new_r, new_c).items(): 137 | if health >= 1: 138 | n_gathered = 1 + (rand() < agent.state["bonus_gather_prob"]) 139 | agent.state["inventory"][resource] += n_gathered 140 | world.consume_resource(resource, new_r, new_c) 141 | # Incur the labor cost of collecting a resource 142 | agent.state["endogenous"]["Labor"] += self.collect_labor 143 | # Log the gather 144 | gathers.append( 145 | dict( 146 | agent=agent.idx, 147 | resource=resource, 148 | n=n_gathered, 149 | loc=[new_r, new_c], 150 | ) 151 | ) 152 | 153 | self.gathers.append(gathers) 154 | 155 | def generate_observations(self): 156 | """ 157 | See base_component.py for detailed description. 158 | 159 | Here, agents observe their collection skill. The planner does not observe 160 | anything from this component. 161 | """ 162 | return { 163 | str(agent.idx): {"bonus_gather_prob": agent.state["bonus_gather_prob"]} 164 | for agent in self.world.agents 165 | } 166 | 167 | def generate_masks(self, completions=0): 168 | """ 169 | See base_component.py for detailed description. 170 | 171 | Prevent moving to adjacent tiles that are already occupied (or outside the 172 | boundaries of the world) 173 | """ 174 | world = self.world 175 | 176 | coords = np.array([agent.loc for agent in world.agents])[:, :, None] 177 | ris = coords[:, 0] + self._roff + 1 178 | cis = coords[:, 1] + self._coff + 1 179 | 180 | occ = np.pad(world.maps.unoccupied, ((1, 1), (1, 1))) 181 | acc = np.pad(world.maps.accessibility, ((0, 0), (1, 1), (1, 1))) 182 | mask_array = np.logical_and(occ[ris, cis], acc[self._aidx, ris, cis]).astype( 183 | np.float32 184 | ) 185 | 186 | masks = {agent.idx: mask_array[i] for i, agent in enumerate(world.agents)} 187 | 188 | return masks 189 | 190 | # For non-required customization 191 | # ------------------------------ 192 | 193 | def additional_reset_steps(self): 194 | """ 195 | See base_component.py for detailed description. 196 | 197 | Re-sample agents' collection skills. 198 | """ 199 | for agent in self.world.agents: 200 | if self.skill_dist == "none": 201 | bonus_rate = 0.0 202 | elif self.skill_dist == "pareto": 203 | bonus_rate = np.minimum(2, np.random.pareto(3)) / 2 204 | elif self.skill_dist == "lognormal": 205 | bonus_rate = np.minimum(2, np.random.lognormal(-2.022, 0.938)) / 2 206 | else: 207 | raise NotImplementedError 208 | agent.state["bonus_gather_prob"] = float(bonus_rate) 209 | 210 | self.gathers = [] 211 | 212 | def get_dense_log(self): 213 | """ 214 | Log resource collections. 215 | 216 | Returns: 217 | gathers (list): A list of gather events. Each entry corresponds to a single 218 | timestep and contains a description of any resource gathers that 219 | occurred on that timestep. 220 | 221 | """ 222 | return self.gathers 223 | -------------------------------------------------------------------------------- /ai_economist/foundation/components/build.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, salesforce.com, inc. 2 | # All rights reserved. 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | # For full license text, see the LICENSE file in the repo root 5 | # or https://opensource.org/licenses/BSD-3-Clause 6 | 7 | import numpy as np 8 | 9 | from ai_economist.foundation.base.base_component import ( 10 | BaseComponent, 11 | component_registry, 12 | ) 13 | 14 | 15 | @component_registry.add 16 | class Build(BaseComponent): 17 | """ 18 | Allows mobile agents to build house landmarks in the world using stone and wood, 19 | earning income. 20 | 21 | Can be configured to include heterogeneous building skill where agents earn 22 | different levels of income when building. 23 | 24 | Args: 25 | payment (int): Default amount of coin agents earn from building. 26 | Must be >= 0. Default is 10. 27 | payment_max_skill_multiplier (int): Maximum skill multiplier that an agent 28 | can sample. Must be >= 1. Default is 1. 29 | skill_dist (str): Distribution type for sampling skills. Default ("none") 30 | gives all agents identical skill equal to a multiplier of 1. "pareto" and 31 | "lognormal" sample skills from the associated distributions. 32 | build_labor (float): Labor cost associated with building a house. 33 | Must be >= 0. Default is 10. 34 | """ 35 | 36 | name = "Build" 37 | component_type = "Build" 38 | required_entities = ["Wood", "Stone", "Coin", "House", "Labor"] 39 | agent_subclasses = ["BasicMobileAgent"] 40 | 41 | def __init__( 42 | self, 43 | *base_component_args, 44 | payment=10, 45 | payment_max_skill_multiplier=1, 46 | skill_dist="none", 47 | build_labor=10.0, 48 | **base_component_kwargs 49 | ): 50 | super().__init__(*base_component_args, **base_component_kwargs) 51 | 52 | self.payment = int(payment) 53 | assert self.payment >= 0 54 | 55 | self.payment_max_skill_multiplier = int(payment_max_skill_multiplier) 56 | assert self.payment_max_skill_multiplier >= 1 57 | 58 | self.resource_cost = {"Wood": 1, "Stone": 1} 59 | 60 | self.build_labor = float(build_labor) 61 | assert self.build_labor >= 0 62 | 63 | self.skill_dist = skill_dist.lower() 64 | assert self.skill_dist in ["none", "pareto", "lognormal"] 65 | 66 | self.sampled_skills = {} 67 | 68 | self.builds = [] 69 | 70 | def agent_can_build(self, agent): 71 | """Return True if agent can actually build in its current location.""" 72 | # See if the agent has the resources necessary to complete the action 73 | for resource, cost in self.resource_cost.items(): 74 | if agent.state["inventory"][resource] < cost: 75 | return False 76 | 77 | # Do nothing if this spot is already occupied by a landmark or resource 78 | if self.world.location_resources(*agent.loc): 79 | return False 80 | if self.world.location_landmarks(*agent.loc): 81 | return False 82 | # If we made it here, the agent can build. 83 | return True 84 | 85 | # Required methods for implementing components 86 | # -------------------------------------------- 87 | 88 | def get_n_actions(self, agent_cls_name): 89 | """ 90 | See base_component.py for detailed description. 91 | 92 | Add a single action (build) for mobile agents. 93 | """ 94 | # This component adds 1 action that mobile agents can take: build a house 95 | if agent_cls_name == "BasicMobileAgent": 96 | return 1 97 | 98 | return None 99 | 100 | def get_additional_state_fields(self, agent_cls_name): 101 | """ 102 | See base_component.py for detailed description. 103 | 104 | For mobile agents, add state fields for building skill. 105 | """ 106 | if agent_cls_name not in self.agent_subclasses: 107 | return {} 108 | if agent_cls_name == "BasicMobileAgent": 109 | return {"build_payment": float(self.payment), "build_skill": 1} 110 | raise NotImplementedError 111 | 112 | def component_step(self): 113 | """ 114 | See base_component.py for detailed description. 115 | 116 | Convert stone+wood to house+coin for agents that choose to build and can. 117 | """ 118 | world = self.world 119 | build = [] 120 | # Apply any building actions taken by the mobile agents 121 | for agent in world.get_random_order_agents(): 122 | 123 | action = agent.get_component_action(self.name) 124 | 125 | # This component doesn't apply to this agent! 126 | if action is None: 127 | continue 128 | 129 | # NO-OP! 130 | if action == 0: 131 | pass 132 | 133 | # Build! (If you can.) 134 | elif action == 1: 135 | if self.agent_can_build(agent): 136 | # Remove the resources 137 | for resource, cost in self.resource_cost.items(): 138 | agent.state["inventory"][resource] -= cost 139 | 140 | # Place a house where the agent is standing 141 | loc_r, loc_c = agent.loc 142 | world.create_landmark("House", loc_r, loc_c, agent.idx) 143 | 144 | # Receive payment for the house 145 | agent.state["inventory"]["Coin"] += agent.state["build_payment"] 146 | 147 | # Incur the labor cost for building 148 | agent.state["endogenous"]["Labor"] += self.build_labor 149 | 150 | build.append( 151 | { 152 | "builder": agent.idx, 153 | "loc": np.array(agent.loc), 154 | "income": float(agent.state["build_payment"]), 155 | } 156 | ) 157 | 158 | else: 159 | raise ValueError 160 | 161 | self.builds.append(build) 162 | 163 | def generate_observations(self): 164 | """ 165 | See base_component.py for detailed description. 166 | 167 | Here, agents observe their build skill. The planner does not observe anything 168 | from this component. 169 | """ 170 | 171 | obs_dict = dict() 172 | for agent in self.world.agents: 173 | obs_dict[agent.idx] = { 174 | "build_payment": agent.state["build_payment"] / self.payment, 175 | "build_skill": self.sampled_skills[agent.idx], 176 | } 177 | 178 | return obs_dict 179 | 180 | def generate_masks(self, completions=0): 181 | """ 182 | See base_component.py for detailed description. 183 | 184 | Prevent building only if a landmark already occupies the agent's location. 185 | """ 186 | 187 | masks = {} 188 | # Mobile agents' build action is masked if they cannot build with their 189 | # current location and/or endowment 190 | for agent in self.world.agents: 191 | masks[agent.idx] = np.array([self.agent_can_build(agent)]) 192 | 193 | return masks 194 | 195 | # For non-required customization 196 | # ------------------------------ 197 | 198 | def get_metrics(self): 199 | """ 200 | Metrics that capture what happened through this component. 201 | 202 | Returns: 203 | metrics (dict): A dictionary of {"metric_name": metric_value}, 204 | where metric_value is a scalar. 205 | """ 206 | world = self.world 207 | 208 | build_stats = {a.idx: {"n_builds": 0} for a in world.agents} 209 | for builds in self.builds: 210 | for build in builds: 211 | idx = build["builder"] 212 | build_stats[idx]["n_builds"] += 1 213 | 214 | out_dict = {} 215 | for a in world.agents: 216 | for k, v in build_stats[a.idx].items(): 217 | out_dict["{}/{}".format(a.idx, k)] = v 218 | 219 | num_houses = np.sum(world.maps.get("House") > 0) 220 | out_dict["total_builds"] = num_houses 221 | 222 | return out_dict 223 | 224 | def additional_reset_steps(self): 225 | """ 226 | See base_component.py for detailed description. 227 | 228 | Re-sample agents' building skills. 229 | """ 230 | world = self.world 231 | 232 | self.sampled_skills = {agent.idx: 1 for agent in world.agents} 233 | 234 | PMSM = self.payment_max_skill_multiplier 235 | 236 | for agent in world.agents: 237 | if self.skill_dist == "none": 238 | sampled_skill = 1 239 | pay_rate = 1 240 | elif self.skill_dist == "pareto": 241 | sampled_skill = np.random.pareto(4) 242 | pay_rate = np.minimum(PMSM, (PMSM - 1) * sampled_skill + 1) 243 | elif self.skill_dist == "lognormal": 244 | sampled_skill = np.random.lognormal(-1, 0.5) 245 | pay_rate = np.minimum(PMSM, (PMSM - 1) * sampled_skill + 1) 246 | else: 247 | raise NotImplementedError 248 | 249 | agent.state["build_payment"] = float(pay_rate * self.payment) 250 | agent.state["build_skill"] = float(sampled_skill) 251 | 252 | self.sampled_skills[agent.idx] = sampled_skill 253 | 254 | self.builds = [] 255 | 256 | def get_dense_log(self): 257 | """ 258 | Log builds. 259 | 260 | Returns: 261 | builds (list): A list of build events. Each entry corresponds to a single 262 | timestep and contains a description of any builds that occurred on 263 | that timestep. 264 | 265 | """ 266 | return self.builds 267 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Foundation: An Economic Simulation Framework 2 | 3 | This repo contains an implementation of Foundation, a framework for flexible, modular, and composable environments that **model socio-economic behaviors and dynamics in a society with both agents and governments**. 4 | 5 | Foundation provides a [Gym](https://gym.openai.com/)-style API: 6 | 7 | - `reset`: resets the environment's state and returns the observation. 8 | - `step`: advances the environment by one timestep, and returns the tuple *(observation, reward, done, info)*. 9 | 10 | This simulation can be used in conjunction with reinforcement learning to learn optimal economic policies, as detailed in the following papers: 11 | 12 | **[The AI Economist: Improving Equality and Productivity with AI-Driven Tax Policies](https://arxiv.org/abs/2004.13332)**, 13 | *Stephan Zheng, Alexander Trott, Sunil Srinivasa, Nikhil Naik, Melvin Gruesbeck, David C. Parkes, Richard Socher.* 14 | 15 | **[The AI Economist: Optimal Economic Policy Design via Two-level Deep Reinforcement Learning](https://arxiv.org/abs/2108.02755)** 16 | *Stephan Zheng, Alexander Trott, Sunil Srinivasa, David C. Parkes, Richard Socher.* 17 | 18 | **[Building a Foundation for Data-Driven, Interpretable, and Robust Policy Design using the AI Economist](https://arxiv.org/abs/2108.02904)** 19 | *Alexander Trott, Sunil Srinivasa, Douwe van der Wal, Sebastien Haneuse, Stephan Zheng.* 20 | 21 | If you use this code in your research, please cite us using this BibTeX entry: 22 | 23 | ``` 24 | @misc{2004.13332, 25 | Author = {Stephan Zheng, Alexander Trott, Sunil Srinivasa, Nikhil Naik, Melvin Gruesbeck, David C. Parkes, Richard Socher}, 26 | Title = {The AI Economist: Improving Equality and Productivity with AI-Driven Tax Policies}, 27 | Year = {2020}, 28 | Eprint = {arXiv:2004.13332}, 29 | } 30 | ``` 31 | 32 | For more information and context, check out: 33 | 34 | - [The AI Economist website](https://www.einstein.ai/the-ai-economist) 35 | - [Blog: The AI Economist: Improving Equality and Productivity with AI-Driven Tax Policies](https://blog.einstein.ai/the-ai-economist/) 36 | - [Blog: The AI Economist moonshot](https://blog.einstein.ai/the-ai-economist-moonshot/) 37 | - [Blog: The AI Economist web demo of the COVID-19 case study](https://blog.einstein.ai/ai-economist-covid-case-study-ethics/) 38 | - [Web demo: The AI Economist ethical review of AI policy design and COVID-19 case study](https://einstein.ai/the-ai-economist/ai-policy-foundation-and-covid-case-study) 39 | 40 | ## Simulation Cards: Ethics Review and Intended Use 41 | 42 | Please see our [Simulation Card](https://github.com/salesforce/ai-economist/blob/master/Simulation_Card_Foundation_Economic_Simulation_Framework.pdf) for a review of the intended use and ethical review of our framework. 43 | 44 | Please see our [COVID-19 Simulation Card](https://github.com/salesforce/ai-economist/blob/master/COVID-19_Simulation-Card.pdf) for a review of the ethical aspects of the pandemic simulation (and as fitted for COVID-19). 45 | 46 | ## Join us on Slack 47 | 48 | If you're interested in extending this framework, discussing machine learning for economics, and collaborating on research project: 49 | 50 | - join our Slack channel [aieconomist.slack.com](https://aieconomist.slack.com) using this [invite link](https://join.slack.com/t/aieconomist/shared_invite/zt-g71ajic7-XaMygwNIup~CCzaR1T0wgA), or 51 | - email us @ ai.economist@salesforce.com. 52 | 53 | ## Installation Instructions 54 | 55 | To get started, you'll need to have Python 3.7+ installed. 56 | 57 | ### Using pip 58 | 59 | Simply use the Python package manager: 60 | 61 | ```python 62 | pip install ai-economist 63 | ``` 64 | 65 | ### Installing from Source 66 | 67 | 1. Clone this repository to your local machine: 68 | 69 | ``` 70 | git clone www.github.com/salesforce/ai-economist 71 | ``` 72 | 73 | 2. Create a new conda environment (named "ai-economist" below - replace with anything else) and activate it 74 | 75 | ```pyfunctiontypecomment 76 | conda create --name ai-economist python=3.7 --yes 77 | conda activate ai-economist 78 | ``` 79 | 80 | 3. Either 81 | 82 | a) Edit the PYTHONPATH to include the ai-economist directory 83 | ``` 84 | export PYTHONPATH=:$PYTHONPATH 85 | ``` 86 | 87 | OR 88 | 89 | b) Install as an editable Python package 90 | ```pyfunctiontypecomment 91 | cd ai-economist 92 | pip install -e . 93 | ``` 94 | 95 | Useful tip: for quick access, add the following to your ~/.bashrc or ~/.bash_profile: 96 | 97 | ```pyfunctiontypecomment 98 | alias aiecon="conda activate ai-economist; cd " 99 | ``` 100 | 101 | You can then simply run `aiecon` once to activate the conda environment. 102 | 103 | ### Testing your Install 104 | 105 | To test your installation, try running: 106 | 107 | ``` 108 | conda activate ai-economist 109 | python -c "import ai_economist" 110 | ``` 111 | 112 | ## Getting Started 113 | 114 | To familiarize yourself with Foundation, check out the tutorials in the `tutorials` folder. You can run these notebooks interactively in your browser on Google Colab. 115 | 116 | ### Multi-Agent Simulations 117 | 118 | - [economic_simulation_basic](https://www.github.com/salesforce/ai-economist/blob/master/tutorials/economic_simulation_basic.ipynb) ([Try this on Colab](http://colab.research.google.com/github/salesforce/ai-economist/blob/master/tutorials/economic_simulation_basic.ipynb)!): Shows how to interact with and visualize the simulation. 119 | - [economic_simulation_advanced](https://www.github.com/salesforce/ai-economist/blob/master/tutorials/economic_simulation_advanced.ipynb) ([Try this on Colab](http://colab.research.google.com/github/salesforce/ai-economist/blob/master/tutorials/economic_simulation_advanced.ipynb)!): Explains how Foundation is built up using composable and flexible building blocks. 120 | - [optimal_taxation_theory_and_simulation](https://github.com/salesforce/ai-economist/blob/master/tutorials/optimal_taxation_theory_and_simulation.ipynb) ([Try this on Colab](https://colab.research.google.com/github/salesforce/ai-economist/blob/master/tutorials/optimal_taxation_theory_and_simulation.ipynb)!): Demonstrates how economic simulations can be used to study the problem of optimal taxation. 121 | - [covid19_and_economic_simulation](https://www.github.com/salesforce/ai-economist/blob/master/tutorials/covid19_and_economic_simulation.ipynb) ([Try this on Colab](http://colab.research.google.com/github/salesforce/ai-economist/blob/master/tutorials/covid19_and_economic_simulation.ipynb)!): Introduces a simulation on the COVID-19 pandemic and economy that can be used to study different health and economic policies . 122 | 123 | ### Multi-Agent Training 124 | - [multi_agent_gpu_training_with_warp_drive](https://github.com/salesforce/ai-economist/blob/master/tutorials/multi_agent_gpu_training_with_warp_drive.ipynb) ([Try this on Colab](http://colab.research.google.com/github/salesforce/ai-economist/blob/master/tutorials/multi_agent_gpu_training_with_warp_drive.ipynb)!): Introduces our multi-agent reinforcement learning framework [WarpDrive](https://arxiv.org/abs/2108.13976), which we then use to train the COVID-19 and economic simulation. 125 | - [multi_agent_training_with_rllib](https://github.com/salesforce/ai-economist/blob/master/tutorials/multi_agent_training_with_rllib.ipynb) ([Try this on Colab](http://colab.research.google.com/github/salesforce/ai-economist/blob/master/tutorials/multi_agent_training_with_rllib.ipynb)!): Shows how to perform distributed multi-agent reinforcement learning with [RLlib](https://docs.ray.io/en/latest/rllib/index.html). 126 | - [two_level_curriculum_training_with_rllib](https://github.com/salesforce/ai-economist/blob/master/tutorials/two_level_curriculum_learning_with_rllib.md): Describes how to implement two-level curriculum training with [RLlib](https://docs.ray.io/en/latest/rllib/index.html). 127 | 128 | To run these notebooks *locally*, you need [Jupyter](https://jupyter.org). See [https://jupyter.readthedocs.io/en/latest/install.html](https://jupyter.readthedocs.io/en/latest/install.html) for installation instructions and [(https://jupyter-notebook.readthedocs.io/en/stable/](https://jupyter-notebook.readthedocs.io/en/stable/) for examples of how to work with Jupyter. 129 | 130 | ## Structure of the Code 131 | 132 | - The simulation is located in the `ai_economist/foundation` folder. 133 | 134 | The code repository is organized into the following components: 135 | 136 | | Component | Description | 137 | | --- | --- | 138 | | [base](https://www.github.com/salesforce/ai-economist/blob/master/ai_economist/foundation/base) | Contains base classes to can be extended to define Agents, Components and Scenarios. | 139 | | [agents](https://www.github.com/salesforce/ai-economist/blob/master/ai_economist/foundation/agents) | Agents represent economic actors in the environment. Currently, we have mobile Agents (representing workers) and a social planner (representing a government). | 140 | | [entities](https://www.github.com/salesforce/ai-economist/blob/master/ai_economist/foundation/entities) | Endogenous and exogenous components of the environment. Endogenous entities include labor, while exogenous entity includes landmarks (such as Water and Grass) and collectible Resources (such as Wood and Stone). | 141 | | [components](https://www.github.com/salesforce/ai-economist/blob/master/ai_economist/foundation/components) | Components are used to add some particular dynamics to an environment. They also add action spaces that define how Agents can interact with the environment via the Component. | 142 | | [scenarios](https://www.github.com/salesforce/ai-economist/blob/master/ai_economist/foundation/scenarios) | Scenarios compose Components to define the dynamics of the world. It also computes rewards and exposes states for visualization. | 143 | 144 | - The datasets (including the real-world data on COVID-19) are located in the `ai_economist/datasets` folder. 145 | 146 | ## Releases and Contributing 147 | 148 | - Please let us know if you encounter any bugs by filing a GitHub issue. 149 | - We appreciate all your contributions. If you plan to contribute new Components, Scenarios Entities, or anything else, please see our [contribution guidelines](https://www.github.com/salesforce/ai-economist/blob/master/CONTRIBUTING.md). 150 | 151 | ## Changelog 152 | 153 | For the complete release history, see [CHANGELOG.md](https://www.github.com/salesforce/ai-economist/blob/master/CHANGELOG.md). 154 | 155 | ## License 156 | 157 | Foundation and the AI Economist are released under the [BSD-3 License](LICENSE.txt). 158 | 159 | -------------------------------------------------------------------------------- /tutorials/assets/prod_eq_welfare.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | prod_eq_welfare_ai 5 | Created with Sketch. 6 | 7 | 8 | 9 | Coin Produced 10 | 11 | 12 | 13 | 14 | 15 | 16 | 3285 17 | 18 | 19 | 20 | 21 | 22 | 23 | 2759 24 | 25 | 26 | 27 | 28 | 29 | 30 | 2656 31 | 32 | 33 | 34 | 35 | 36 | 37 | 2907 38 | 39 | 40 | 41 | Economic Productivity 42 | 43 | 44 | 45 | 46 | Coin Equality 47 | 48 | 49 | 50 | 51 | 52 | 53 | 39% 54 | 55 | 56 | 57 | 58 | 59 | 60 | 52% 61 | 62 | 63 | 64 | 65 | 66 | 67 | 48% 68 | 69 | 70 | 71 | 72 | 73 | 74 | 57% 75 | 76 | 77 | 78 | Income Equality 79 | 80 | 81 | 82 | 83 | Eq / Prod Tradeoff 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 1278 92 | 93 | 94 | 95 | 96 | 97 | 98 | 1435 99 | 100 | 101 | 102 | 103 | 104 | 105 | 1261 106 | 107 | 108 | 109 | 110 | 111 | 1664 112 | 113 | 114 | 115 | 116 | 117 | Equality x Productivity 118 | 119 | 120 | 121 | 122 | 123 | Free Market 124 | 125 | 126 | 127 | 128 | 129 | US Federal 130 | 131 | 132 | 133 | 134 | 135 | Saez Formula 136 | 137 | 138 | 139 | 140 | 141 | AI Economist 142 | 143 | 144 | 145 | 146 | 147 | --------------------------------------------------------------------------------