├── .flooignore ├── jarbas ├── core │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ └── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0020_rename_supplier_to_company.py │ │ ├── 0016_add_custom_ordering_to_reimbursement.py │ │ ├── 0031_add_index_together_for_reimbursement.py │ │ ├── 0018_make_issue_date_required.py │ │ ├── 0012_use_date_not_datetime.py │ │ ├── 0004_add_receipt_url.py │ │ ├── 0036_alter_tweet_status_to_decimal.py │ │ ├── 0026_order_reimbursements_by_year_and_issue_date.py │ │ ├── 0028_auto_20170601_1701.py │ │ ├── 0021_make_reciept_fetched_a_db_index.py │ │ ├── 0023_add_last_update_field_to_reimbursements.py │ │ ├── 0032_auto_20170613_0641.py │ │ ├── 0040_create_gin_index_with_search_vector.py │ │ ├── 0019_cleanup_remove_old_api.py │ │ ├── 0024_add_available_in_latest_dataset_field_to_reimbursement.py │ │ ├── 0005_add_receipt_url_last_update.py │ │ ├── 0022_remove_unique_together_from_reimbursement.py │ │ ├── 0029_make_issue_date_an_index.py │ │ ├── 0037_auto_20170727_1624.py │ │ ├── 0011_subquota_description_length.py │ │ ├── 0039_add_search_vector_to_reimbursement.py │ │ ├── 0009_add_latitude_and_longitude.py │ │ ├── 0015_add_receipt_to_reimbursement.py │ │ ├── 0014_add_suspicions_and_probability_to_reimbursements.py │ │ ├── 0035_create_model_tweet.py │ │ ├── 0006_lazy_backend_receipt_url.py │ │ ├── 0038_auto_20170728_1748.py │ │ ├── 0033_add_index_for_subquota_description.py │ │ ├── 0041_migrate_data_to_chamber_of_deputies_app.py │ │ ├── 0003_remove_some_indexes.py │ │ ├── 0010_extract_receipt.py │ │ ├── 0017_make_some_fields_optional.py │ │ ├── 0030_remove_unused_indexes.py │ │ └── 0034_auto_20170629_2150.py │ ├── app.py │ ├── context_processors.py │ ├── urls.py │ ├── tests │ │ ├── test_home_view.py │ │ ├── test_healthcheck_url.py │ │ ├── fixtures │ │ │ └── reimbursements.csv │ │ ├── test_company_model.py │ │ ├── __init__.py │ │ └── test_company_view.py │ ├── serializers.py │ ├── views.py │ └── models.py ├── dashboard │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ └── test_dashboard_view.py │ ├── templatetags │ │ ├── __init__.py │ │ └── dashboard.py │ ├── urls.py │ ├── templates │ │ └── admin │ │ │ ├── base.html │ │ │ └── base_site.html │ ├── admin │ │ ├── paginators.py │ │ └── widgets.py │ └── static │ │ ├── dashboard.js │ │ └── dashboard.css ├── layers │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_static_files.py │ │ └── test_home_view.py │ ├── static │ │ ├── digitalocean.png │ │ ├── favicon │ │ │ ├── favicon.ico │ │ │ ├── apple-icon.png │ │ │ ├── favicon-16x16.png │ │ │ ├── favicon-32x32.png │ │ │ ├── favicon-96x96.png │ │ │ ├── ms-icon-70x70.png │ │ │ ├── apple-icon-57x57.png │ │ │ ├── apple-icon-60x60.png │ │ │ ├── apple-icon-72x72.png │ │ │ ├── apple-icon-76x76.png │ │ │ ├── ms-icon-144x144.png │ │ │ ├── ms-icon-150x150.png │ │ │ ├── ms-icon-310x310.png │ │ │ ├── android-icon-36x36.png │ │ │ ├── android-icon-48x48.png │ │ │ ├── android-icon-72x72.png │ │ │ ├── android-icon-96x96.png │ │ │ ├── apple-icon-114x114.png │ │ │ ├── apple-icon-120x120.png │ │ │ ├── apple-icon-144x144.png │ │ │ ├── apple-icon-152x152.png │ │ │ ├── apple-icon-180x180.png │ │ │ ├── android-icon-144x144.png │ │ │ ├── android-icon-192x192.png │ │ │ ├── apple-icon-precomposed.png │ │ │ ├── browserconfig.xml │ │ │ └── manifest.json │ │ └── image │ │ │ ├── facebook-icon.png │ │ │ ├── receipt_icon.png │ │ │ └── twitter-icon.png │ ├── urls.py │ ├── views.py │ └── elm │ │ ├── Internationalization │ │ ├── Reimbursement │ │ │ ├── Tweet.elm │ │ │ ├── Receipt.elm │ │ │ ├── Search.elm │ │ │ ├── Fieldset.elm │ │ │ └── Common.elm │ │ ├── DocumentType.elm │ │ ├── Common.elm │ │ └── Suspicion.elm │ │ ├── Reimbursement │ │ ├── Map │ │ │ ├── Update.elm │ │ │ ├── Model.elm │ │ │ └── View.elm │ │ ├── Tweet │ │ │ ├── Update.elm │ │ │ ├── Model.elm │ │ │ └── View.elm │ │ ├── Search │ │ │ └── Model.elm │ │ ├── SameDay │ │ │ ├── View.elm │ │ │ └── Update.elm │ │ ├── SameSubquota │ │ │ ├── View.elm │ │ │ └── Update.elm │ │ ├── Receipt │ │ │ ├── Model.elm │ │ │ ├── Decoder.elm │ │ │ ├── Update.elm │ │ │ └── View.elm │ │ ├── RelatedTable │ │ │ ├── Model.elm │ │ │ └── Decoder.elm │ │ └── Company │ │ │ ├── Model.elm │ │ │ ├── Decoder.elm │ │ │ └── Update.elm │ │ ├── Main.elm │ │ ├── Model.elm │ │ ├── Format │ │ ├── Date.elm │ │ ├── Url.elm │ │ ├── Price.elm │ │ └── CnpjCpf.elm │ │ ├── View.elm │ │ └── Layout.elm ├── public_admin │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_dummy_user.py │ │ └── test_public_admin_site.py │ ├── admin.py │ └── sites.py ├── chamber_of_deputies │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── tweet.py │ │ │ ├── socialmedia.py │ │ │ └── reimbursements.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0011_make_issue_date_required.py │ │ ├── 0012_make_party_field_longer.py │ │ ├── 0003_remove_available_in_latest_dataset_field.py │ │ ├── 0009_add_index_to_term.py │ │ ├── 0006_change_on_delete_social_media_to_set_null.py │ │ ├── 0010_remove_null_issue_date_rows.py │ │ ├── 0008_remove_related_field_social_media.py │ │ ├── 0002_remove_django_simple_history.py │ │ ├── 0007_auto_20180913_0209.py │ │ ├── 0005_create_social_media_model.py │ │ └── 0004_alter_field_names_following_toolbox_renamings.py │ ├── app.py │ ├── tests │ │ ├── test_serializers.py │ │ ├── test_searchvector_command.py │ │ ├── test_tweet_model.py │ │ └── test_receipt_class.py │ ├── urls.py │ ├── tasks.py │ └── fields.py ├── __init__.py ├── wsgi.py ├── storages.py ├── celery.py └── urls.py ├── rosie ├── rosie │ ├── __init__.py │ ├── core │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── fixtures │ │ │ │ └── invalid_cnpj_cpf_classifier.csv │ │ │ └── test_invalid_cnpj_cpf_classifier.py │ │ └── classifiers │ │ │ ├── __init__.py │ │ │ └── invalid_cnpj_cpf_classifier.py │ ├── federal_senate │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_federal_senate.py │ │ │ ├── fixtures │ │ │ │ └── federal_senate_reimbursements.xz │ │ │ └── test_adapter.py │ │ ├── settings.py │ │ ├── __init__.py │ │ └── adapter.py │ └── chamber_of_deputies │ │ ├── tests │ │ ├── __init__.py │ │ ├── fixtures │ │ │ ├── companies.xz │ │ │ ├── reimbursements-2010.csv │ │ │ ├── reimbursements-2016.csv │ │ │ ├── reimbursements-2009.csv │ │ │ ├── reimbursements-2012.csv │ │ │ ├── reimbursements-2011.csv │ │ │ └── traveled_speeds_classifier.csv │ │ ├── test_election_expenses_classifier.py │ │ ├── test_chamber_of_deputies.py │ │ └── test_irregular_companies_classifier.py │ │ ├── __init__.py │ │ ├── classifiers │ │ ├── __init__.py │ │ ├── election_expenses_classifier.py │ │ └── irregular_companies_classifier.py │ │ └── settings.py ├── .coveragerc ├── .gitignore ├── requirements.txt ├── Dockerfile ├── .github │ ├── PULL_REQUEST_TEMPLATE.md │ └── ISSUE_TEMPLATE.md ├── rosie.py └── README.md ├── .pyup.yml ├── setup.cfg ├── docs ├── logo.png ├── okbr.png └── digitalocean.png ├── contrib ├── update │ ├── ansible.cfg │ ├── .env.sample │ ├── Pipfile │ ├── cleanup.py │ └── README.md ├── crontab │ ├── cleanup │ ├── update │ ├── whistleblower │ ├── receipts │ └── crontab ├── data │ ├── companies_sample.xz │ ├── suspicions_sample.xz │ └── README.md ├── deploy.sh ├── README.md └── .env.sample ├── requirements-dev.txt ├── research ├── src │ ├── table_config.json │ ├── backup_data.py │ ├── fetch_inbox.py │ └── fetch_purchase_suppliers.py ├── requirements.txt ├── Dockerfile └── setup ├── .coveragerc ├── Dockerfile-elm ├── .gitignore ├── package.json ├── .codeclimate.yml ├── requirements.txt ├── .github ├── PULL_REQUEST_TEMPLATE.md └── ISSUE_TEMPLATE.md ├── gulpfile.js ├── .editorconfig ├── manage.py ├── docker-compose.override.yml ├── elm-package.json ├── LICENSE ├── Dockerfile ├── docker-compose.prod.yml ├── .travis.yml └── docker-compose.yml /.flooignore: -------------------------------------------------------------------------------- 1 | .gitignore -------------------------------------------------------------------------------- /jarbas/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rosie/rosie/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/dashboard/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/layers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/layers/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/public_admin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | schedule: "every month" 2 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/core/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/core/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/dashboard/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/public_admin/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rosie/rosie/core/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/dashboard/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rosie/rosie/federal_senate/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pep8] 2 | max-line-length = 120 3 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rosie/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = rosie 3 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rosie/rosie/federal_senate/tests/test_federal_senate.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/docs/logo.png -------------------------------------------------------------------------------- /docs/okbr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/docs/okbr.png -------------------------------------------------------------------------------- /contrib/update/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking = False 3 | ServerAliveInterval=30 4 | -------------------------------------------------------------------------------- /jarbas/__init__.py: -------------------------------------------------------------------------------- 1 | from jarbas.celery import app as celery_app 2 | 3 | __all__ = ['celery_app'] 4 | -------------------------------------------------------------------------------- /rosie/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .coverage 3 | .env 4 | .python-version 5 | __pycache__/ 6 | htmlcov/ 7 | -------------------------------------------------------------------------------- /docs/digitalocean.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/docs/digitalocean.png -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | django-test-without-migrations==0.6 3 | ipdb==0.11 4 | mixer==6.1.3 5 | -------------------------------------------------------------------------------- /contrib/crontab/cleanup: -------------------------------------------------------------------------------- 1 | cd /opt/serenata-de-amor/contrib/update 2 | pipenv install 3 | pipenv run python cleanup.py 4 | -------------------------------------------------------------------------------- /contrib/data/companies_sample.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/contrib/data/companies_sample.xz -------------------------------------------------------------------------------- /rosie/rosie/core/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | from rosie.core.classifiers.invalid_cnpj_cpf_classifier import InvalidCnpjCpfClassifier -------------------------------------------------------------------------------- /contrib/crontab/update: -------------------------------------------------------------------------------- 1 | cd /opt/serenata-de-amor/contrib/update 2 | pipenv install 3 | pipenv run ansible-playbook update.yml -vv 4 | -------------------------------------------------------------------------------- /contrib/data/suspicions_sample.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/contrib/data/suspicions_sample.xz -------------------------------------------------------------------------------- /jarbas/layers/static/digitalocean.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/digitalocean.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/favicon.ico -------------------------------------------------------------------------------- /contrib/update/.env.sample: -------------------------------------------------------------------------------- 1 | DO_API_TOKEN=get it from DigitalOcean 2 | DO_SSH_KEY_NAME=rosie 3 | DATABASE_URL=postgres://user:password@server/database 4 | -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon.png -------------------------------------------------------------------------------- /jarbas/layers/static/image/facebook-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/image/facebook-icon.png -------------------------------------------------------------------------------- /jarbas/layers/static/image/receipt_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/image/receipt_icon.png -------------------------------------------------------------------------------- /jarbas/layers/static/image/twitter-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/image/twitter-icon.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/favicon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/favicon-96x96.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/ms-icon-70x70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/ms-icon-70x70.png -------------------------------------------------------------------------------- /jarbas/core/app.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class CoreConfig(AppConfig): 5 | name = 'jarbas.core' 6 | verbose_name = 'Jarbas' 7 | -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-57x57.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-57x57.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-60x60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-60x60.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-72x72.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-76x76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-76x76.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/ms-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/ms-icon-144x144.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/ms-icon-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/ms-icon-150x150.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/ms-icon-310x310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/ms-icon-310x310.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/android-icon-36x36.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/android-icon-36x36.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/android-icon-48x48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/android-icon-48x48.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/android-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/android-icon-72x72.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/android-icon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/android-icon-96x96.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-114x114.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-114x114.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-120x120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-120x120.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-144x144.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-152x152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-152x152.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-180x180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-180x180.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/android-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/android-icon-144x144.png -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/android-icon-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/android-icon-192x192.png -------------------------------------------------------------------------------- /jarbas/core/context_processors.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | def google_analytics(request): 5 | return {'google_analytics': settings.GOOGLE_ANALYTICS} 6 | -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/apple-icon-precomposed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/jarbas/layers/static/favicon/apple-icon-precomposed.png -------------------------------------------------------------------------------- /contrib/crontab/whistleblower: -------------------------------------------------------------------------------- 1 | cd /opt/serenata-de-amor 2 | /usr/local/bin/docker-compose -f docker-compose.yml -f docker-compose.prod.yml run --rm django python manage.py tweet 3 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/fixtures/companies.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/rosie/rosie/chamber_of_deputies/tests/fixtures/companies.xz -------------------------------------------------------------------------------- /jarbas/dashboard/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from jarbas.public_admin.sites import public_admin 4 | 5 | 6 | urlpatterns = [ 7 | path('', public_admin.urls) 8 | ] 9 | -------------------------------------------------------------------------------- /contrib/crontab/receipts: -------------------------------------------------------------------------------- 1 | cd /opt/serenata-de-amor 2 | /usr/local/bin/docker-compose -f docker-compose.yml -f docker-compose.prod.yml run --rm django python manage.py receipts --pause 15 --batch-size 128 3 | -------------------------------------------------------------------------------- /jarbas/layers/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from jarbas.layers.views import home 4 | 5 | 6 | app_name = 'layers' 7 | 8 | urlpatterns = [ 9 | path('', home, name='home') 10 | ] 11 | -------------------------------------------------------------------------------- /rosie/rosie/federal_senate/settings.py: -------------------------------------------------------------------------------- 1 | from rosie.core.classifiers import InvalidCnpjCpfClassifier 2 | 3 | CLASSIFIERS = { 4 | 'invalid_cnpj_cpf': InvalidCnpjCpfClassifier, 5 | } 6 | 7 | UNIQUE_IDS = None -------------------------------------------------------------------------------- /rosie/rosie/federal_senate/tests/fixtures/federal_senate_reimbursements.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okfn-brasil/serenata-de-amor/HEAD/rosie/rosie/federal_senate/tests/fixtures/federal_senate_reimbursements.xz -------------------------------------------------------------------------------- /contrib/update/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | ansible = "==2.7.5" 8 | dopy = "==0.3.7" 9 | 10 | [requires] 11 | python_version = "2.7" 12 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/app.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ChamberOfDeputiesConfig(AppConfig): 5 | name = 'jarbas.chamber_of_deputies' 6 | verbose_name = 'Câmara dos Deputados - Cota para Exercício da Atividade Parlamentar' 7 | -------------------------------------------------------------------------------- /jarbas/layers/tests/test_static_files.py: -------------------------------------------------------------------------------- 1 | from django.contrib.staticfiles import finders 2 | from django.test import TestCase 3 | 4 | 5 | class TestStatic(TestCase): 6 | 7 | def test_digitalocean(self): 8 | self.assertTrue(finders.find('digitalocean.png')) 9 | -------------------------------------------------------------------------------- /research/src/table_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "cnpj_cpf": { 3 | "reimbursements": "cnpj_cpf", 4 | "current-year": "cnpj_cpf", 5 | "last-year": "cnpj_cpf", 6 | "previous-years": "cnpj_cpf", 7 | "amendments": "amendment_beneficiary" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = jarbas 3 | omit = 4 | jarbas/chamber_of_deputies/tests/*.py 5 | jarbas/chamber_of_deputies/migrations/*.py 6 | jarbas/core/migrations/*.py 7 | jarbas/core/tests/*.py 8 | jarbas/frontend/tests/*.py 9 | jarbas/wsgi.py 10 | -------------------------------------------------------------------------------- /research/requirements.txt: -------------------------------------------------------------------------------- 1 | aiofiles==0.4.0 2 | aiohttp==3.5.4 3 | beautifulsoup4==4.7.1 4 | geopy==1.18.1 5 | grequests==0.3.0 6 | humanize==0.5.1 7 | numpy==1.16.2 8 | pandas==0.24.1 9 | python-decouple==3.1 10 | requests==2.21.0 11 | serenata-toolbox # pyup: ignore 12 | tqdm==4.31.1 13 | -------------------------------------------------------------------------------- /rosie/requirements.txt: -------------------------------------------------------------------------------- 1 | brutils==1.0.1 2 | docopt==0.6.2 3 | freezegun==0.3.11 4 | geopy==1.18.1 5 | ipdb==0.11 6 | numpy==1.16.2 7 | # scipy must come before scikit-learn in order to build the wheel in docker image 8 | scipy==1.2.1 9 | scikit-learn==0.20.2 10 | serenata-toolbox # pyup: ignore 11 | -------------------------------------------------------------------------------- /jarbas/layers/views.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.shortcuts import render 3 | 4 | 5 | def home(request): 6 | context = { 7 | 'google_street_view_api_key': settings.GOOGLE_STREET_VIEW_API_KEY 8 | } 9 | return render(request, 'layers/home.html', context=context) 10 | -------------------------------------------------------------------------------- /rosie/rosie/federal_senate/__init__.py: -------------------------------------------------------------------------------- 1 | from rosie.federal_senate import settings 2 | from rosie.federal_senate.adapter import Adapter 3 | from rosie.core import Core 4 | 5 | 6 | def main(target_directory='/tmp/serenata-data'): 7 | adapter = Adapter(target_directory) 8 | core = Core(settings, adapter) 9 | core() -------------------------------------------------------------------------------- /Dockerfile-elm: -------------------------------------------------------------------------------- 1 | FROM node:9.7.1-slim 2 | 3 | WORKDIR /code 4 | 5 | COPY package.json package.json 6 | COPY package-lock.json package-lock.json 7 | COPY elm-package.json elm-package.json 8 | COPY gulpfile.js gulpfile.js 9 | 10 | RUN npm install 11 | 12 | COPY ./jarbas /code/jarbas 13 | 14 | CMD ["npm", "run", "assets"] 15 | -------------------------------------------------------------------------------- /jarbas/core/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import re_path 2 | 3 | from jarbas.core.views import CompanyDetailView 4 | 5 | 6 | app_name = 'core' 7 | 8 | urlpatterns = [ 9 | re_path( 10 | r'^company/(?P\d{14})/$', 11 | CompanyDetailView.as_view(), 12 | name='company-detail' 13 | ), 14 | ] 15 | -------------------------------------------------------------------------------- /rosie/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM conda/miniconda3 2 | 3 | ARG AMAZON_BUCKET=serenata-de-amor-data 4 | ARG AMAZON_ENDPOINT=https://nyc3.digitaloceanspaces.com 5 | ARG AMAZON_REGION=nyc3 6 | 7 | WORKDIR /code 8 | COPY requirements.txt ./ 9 | COPY rosie.py ./ 10 | 11 | RUN pip install -r requirements.txt 12 | 13 | COPY rosie ./rosie 14 | -------------------------------------------------------------------------------- /jarbas/core/tests/test_home_view.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.shortcuts import resolve_url 3 | from django.test import TestCase 4 | 5 | 6 | class TestHome(TestCase): 7 | 8 | def test_redirects(self): 9 | resp = self.client.get('/') 10 | self.assertRedirects(resp, settings.HOMES_REDIRECTS_TO) 11 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/Reimbursement/Tweet.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.Reimbursement.Tweet exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | rosiesTweet : TranslationSet 7 | rosiesTweet = 8 | TranslationSet 9 | " Rosie's tweet" 10 | " Tweet da Rosie" 11 | -------------------------------------------------------------------------------- /research/src/backup_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from serenata_toolbox.datasets import Datasets 4 | 5 | datasets = Datasets('data') 6 | 7 | for file_name in datasets.pending: 8 | file_path = os.path.join(datasets.local.directory, file_name) 9 | print('Uploading {}…'.format(file_path)) 10 | datasets.remote.upload(file_path) 11 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/__init__.py: -------------------------------------------------------------------------------- 1 | from rosie.chamber_of_deputies import settings 2 | from rosie.chamber_of_deputies.adapter import Adapter 3 | from rosie.core import Core 4 | 5 | 6 | def main(target_directory='/tmp/serenata-data'): 7 | adapter = Adapter(target_directory) 8 | core = Core(settings, adapter) 9 | core() 10 | -------------------------------------------------------------------------------- /rosie/rosie/core/tests/fixtures/invalid_cnpj_cpf_classifier.csv: -------------------------------------------------------------------------------- 1 | recipient_id,document_type 2 | 22472225000183,bill_of_sale 3 | 22472225000180,bill_of_sale 4 | ,bill_of_sale 5 | ,expense_made_abroad 6 | 22472225000183,expense_made_abroad 7 | 22472225000180,expense_made_abroad 8 | 57725723501,bill_of_sale 9 | 11111111111,bill_of_sale 10 | 22472225000180, 11 | -------------------------------------------------------------------------------- /jarbas/core/tests/test_healthcheck_url.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.shortcuts import resolve_url 3 | from django.test import TestCase 4 | 5 | 6 | class TestHealthCheck(TestCase): 7 | 8 | def test_status(self): 9 | resp = self.client.get(resolve_url('healthcheck')) 10 | self.assertEqual(200, resp.status_code) 11 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Map/Update.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Map.Update exposing (Msg(..), update) 2 | 3 | import Material 4 | import Reimbursement.Map.Model exposing (Model) 5 | 6 | 7 | type Msg 8 | = Mdl (Material.Msg Msg) 9 | 10 | 11 | update : Msg -> Model -> ( Model, Cmd Msg ) 12 | update msg model = 13 | case msg of 14 | Mdl mdlMsg -> 15 | Material.update mdlMsg model 16 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Tweet/Update.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Tweet.Update exposing (Msg(..), update) 2 | 3 | import Material 4 | import Reimbursement.Tweet.Model exposing (Model) 5 | 6 | 7 | type Msg 8 | = Mdl (Material.Msg Msg) 9 | 10 | 11 | update : Msg -> Model -> ( Model, Cmd Msg ) 12 | update msg model = 13 | case msg of 14 | Mdl mdlMsg -> 15 | Material.update mdlMsg model 16 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Search/Model.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Search.Model exposing (Model, model) 2 | 3 | import Reimbursement.Fields as Fields exposing (Field(..), Label(..), searchableLabels) 4 | 5 | 6 | type alias Model = 7 | List Field 8 | 9 | 10 | model : Model 11 | model = 12 | let 13 | toFormField label = 14 | Field (Tuple.first label) "" 15 | in 16 | List.map toFormField searchableLabels 17 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Tweet/Model.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Tweet.Model exposing (Model, modelFrom) 2 | 3 | import Internationalization.Types exposing (Language(..)) 4 | import Material 5 | 6 | 7 | type alias Model = 8 | { url : Maybe String 9 | , lang : Language 10 | , mdl : Material.Model 11 | } 12 | 13 | 14 | modelFrom : Language -> Maybe String -> Model 15 | modelFrom lang url = 16 | Model url lang Material.model 17 | -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | #ffffff 9 | 10 | 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # development environment 2 | .coverage 3 | .env 4 | .floo 5 | *.sqlite3 6 | db/ 7 | htmlcov/ 8 | npm-debug.log 9 | 10 | # cache, compiled & downloaded files 11 | *.pyc 12 | .webassets-cache/ 13 | __pycache__/ 14 | app.css 15 | app.js 16 | ceap-datasets.html 17 | elm-stuff/ 18 | jarbas/frontend/tests/Doc/ 19 | jarbas/frontend/tests/elm-package.json 20 | node_modules/ 21 | staticfiles/ 22 | 23 | # vim stuff 24 | *.swp 25 | 26 | # ansible stuff 27 | contrib/update/update.retry 28 | -------------------------------------------------------------------------------- /contrib/crontab/crontab: -------------------------------------------------------------------------------- 1 | # m h dom mon dow command 2 | 0 22 * * 1,3 /usr/local/bin/whistleblower > /opt/crontab.log 2>&1 3 | 0 17 * * 2,4 /usr/local/bin/whistleblower > /opt/crontab.log 2>&1 4 | 0 13 * * 5 /usr/local/bin/whistleblower > /opt/crontab.log 2>&1 5 | 0 1 * * 1 /usr/local/bin/update > /opt/crontab.log 2>&1 6 | 0 4 * * * /usr/local/bin/cleanup > /opt/crontab.log 2>&1 7 | 0 4 * * 1 /usr/local/bin/receipts > /opt/crontab.log 2>&1 8 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0020_rename_supplier_to_company.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-21 22:05 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0019_cleanup_remove_old_api'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RenameModel( 16 | old_name='Supplier', 17 | new_name='Company', 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /jarbas/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for Jarbas project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jarbas.settings") 15 | 16 | 17 | import newrelic.agent 18 | newrelic.agent.initialize() 19 | 20 | application = get_wsgi_application() 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "MIT", 3 | "repository": { 4 | "type": "git", 5 | "url": "git://github.com/okfn-brasil/jarbas.git" 6 | }, 7 | "scripts": { 8 | "postinstall": "elm-package install --yes", 9 | "assets": "gulp elm", 10 | "watch": "gulp watch" 11 | }, 12 | "dependencies": { 13 | "elm": "^0.18", 14 | "gulp": "^3.9.1", 15 | "gulp-cli": "^2.0.1", 16 | "gulp-elm": "^0.7.2", 17 | "gulp-rename": "^1.2.2", 18 | "gulp-uglify": "^3.0.0", 19 | "gulp-watch": "^5.0.1" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | pep8: 3 | enabled: true 4 | 5 | csslint: 6 | enabled: true 7 | 8 | duplication: 9 | enabled: true 10 | config: 11 | languages: 12 | - javascript: 13 | - python: 14 | 15 | eslint: 16 | enabled: true 17 | 18 | fixme: 19 | enabled: true 20 | 21 | markdownlint: 22 | enabled: true 23 | 24 | ratings: 25 | paths: 26 | - "**.css" 27 | - "**.js" 28 | - "**.md" 29 | - "**.py" 30 | - "**.sh" 31 | 32 | exclude_paths: 33 | - "**/migrations/" 34 | - "**/tests/*.py" 35 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0016_add_custom_ordering_to_reimbursement.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-10 15:23 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0015_add_receipt_to_reimbursement'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterModelOptions( 16 | name='reimbursement', 17 | options={'ordering': ['-issue_date']}, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/SameDay/View.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.SameDay.View exposing (view) 2 | 3 | import Html 4 | import Internationalization exposing (translate) 5 | import Internationalization.Types exposing (TranslationId(..)) 6 | import Reimbursement.RelatedTable.Model exposing (Model) 7 | import Reimbursement.RelatedTable.Update exposing (Msg) 8 | import Reimbursement.RelatedTable.View as RelatedTable 9 | 10 | 11 | view : Model -> Html.Html Msg 12 | view model = 13 | SameDayTitle 14 | |> translate model.lang 15 | |> RelatedTable.view model 16 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/tests/test_serializers.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from jarbas.chamber_of_deputies.serializers import clean_cnpj_cpf 4 | 5 | 6 | class TestCleanCnpjCpf(TestCase): 7 | def test_should_return_cnpj_cpf_without_mask(self): 8 | self.assertEqual('12345678901234', clean_cnpj_cpf('12.345.678/9012-34')) 9 | self.assertEqual('12345678901234', clean_cnpj_cpf('12345678901234')) 10 | self.assertEqual('02002002002', clean_cnpj_cpf('020.020.020-02')) 11 | self.assertEqual('02002002002', clean_cnpj_cpf('02002002002')) 12 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0031_add_index_together_for_reimbursement.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-06-05 19:45 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0030_remove_unused_indexes'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterIndexTogether( 16 | name='reimbursement', 17 | index_together=set([('year', 'issue_date', 'id')]), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /jarbas/storages.py: -------------------------------------------------------------------------------- 1 | from whitenoise.storage import CompressedManifestStaticFilesStorage 2 | 3 | 4 | class WhiteNoiseStaticFilesStorage(CompressedManifestStaticFilesStorage): 5 | manifest_strict = False 6 | 7 | def hashed_name(self, *args, **kwargs): 8 | """Skip hashing app.js because it is included in the container volume 9 | after collectstatic runs.""" 10 | name, *_ = args 11 | if name.endswith('/static/app.js'): 12 | return name 13 | 14 | name = super(WhiteNoiseStaticFilesStorage, self).hashed_name(*args, **kwargs) 15 | -------------------------------------------------------------------------------- /research/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM conda/miniconda3 2 | 3 | USER root 4 | ARG AMAZON_BUCKET=serenata-de-amor-data 5 | ARG AMAZON_ENDPOINT=https://nyc3.digitaloceanspaces.com 6 | ARG AMAZON_REGION=nyc3 7 | 8 | RUN apt-get update \ 9 | && apt-get install -y --no-install-recommends \ 10 | unzip \ 11 | libxml2-dev \ 12 | libxslt-dev \ 13 | && apt-get clean \ 14 | && rm -rf /var/lib/apt/lists/* 15 | 16 | WORKDIR /mnt/code 17 | 18 | COPY ./requirements.txt ./requirements.txt 19 | COPY ./setup ./setup 20 | 21 | RUN pip install -r requirements.txt 22 | RUN ./setup 23 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0011_make_issue_date_required.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.8 on 2018-09-13 13:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('chamber_of_deputies', '0010_remove_null_issue_date_rows'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='reimbursement', 15 | name='issue_date', 16 | field=models.DateField(db_index=True, verbose_name='Data de Emissão'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/SameSubquota/View.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.SameSubquota.View exposing (view) 2 | 3 | import Html 4 | import Internationalization exposing (translate) 5 | import Internationalization.Types exposing (TranslationId(..)) 6 | import Reimbursement.RelatedTable.Model exposing (Model) 7 | import Reimbursement.RelatedTable.Update exposing (Msg) 8 | import Reimbursement.RelatedTable.View as RelatedTable 9 | 10 | 11 | view : Model -> Html.Html Msg 12 | view model = 13 | SameSubquotaTitle 14 | |> translate model.lang 15 | |> RelatedTable.view model 16 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | from rosie.chamber_of_deputies.classifiers.election_expenses_classifier import ElectionExpensesClassifier 2 | from rosie.chamber_of_deputies.classifiers.irregular_companies_classifier import IrregularCompaniesClassifier 3 | from rosie.chamber_of_deputies.classifiers.meal_price_outlier_classifier import MealPriceOutlierClassifier 4 | from rosie.chamber_of_deputies.classifiers.monthly_subquota_limit_classifier import MonthlySubquotaLimitClassifier 5 | from rosie.chamber_of_deputies.classifiers.traveled_speeds_classifier import TraveledSpeedsClassifier -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0012_make_party_field_longer.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.5 on 2019-01-31 18:15 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('chamber_of_deputies', '0011_make_issue_date_required'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='reimbursement', 15 | name='party', 16 | field=models.CharField(blank=True, max_length=14, null=True, verbose_name='Partido'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /rosie/.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | This is just a template to help you make your point clear with this PR. :) 2 | 3 | **What is the purpose of this Pull Request?** 4 | __Tell here how this can help this project__ 5 | 6 | **What was done to achieve this purpose?** 7 | __Explain what was done to make it work__ 8 | 9 | **How to test if it really works?** 10 | __Write how could it be tested and checked here__ 11 | 12 | **Who can help reviewing it?** 13 | __Who are the best people to help reviews it?__ 14 | 15 | **TODO** 16 | __Here goes the todo list with some missing step to complete it__ 17 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0003_remove_available_in_latest_dataset_field.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.10 on 2018-02-16 13:59 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('chamber_of_deputies', '0002_remove_django_simple_history'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name='reimbursement', 17 | name='available_in_latest_dataset', 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==2.1.7 2 | brazilnum==0.8.8 3 | celery==4.2.1 4 | dj-database-url==0.5.0 5 | django-assets==0.12 6 | django-bulk-update==2.2.0 7 | django-cors-middleware==1.3.1 8 | django-debug-toolbar==1.11 9 | django-extensions==2.1.6 10 | django-test-without-migrations==0.6 11 | djangorestframework==3.9.1 12 | freezegun==0.3.11 13 | gunicorn==19.9.0 14 | newrelic==4.14.0.115 15 | psycopg2-binary==2.7.7 16 | python-decouple==3.1 17 | python-memcached==1.59 18 | python-twitter==3.5 19 | reprint==0.5.1 # pyup: ignore 20 | requests==2.21.0 21 | rows==0.4.1 22 | tqdm==4.31.1 23 | whitenoise==4.1.2 24 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0009_add_index_to_term.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.8 on 2018-09-13 09:50 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('chamber_of_deputies', '0008_remove_related_field_social_media'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='reimbursement', 15 | name='term', 16 | field=models.IntegerField(blank=True, db_index=True, null=True, verbose_name='Número da Legislatura'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0018_make_issue_date_required.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-18 16:25 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0017_make_some_fields_optional'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='reimbursement', 17 | name='issue_date', 18 | field=models.DateField(verbose_name='Issue date'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/fixtures/reimbursements-2010.csv: -------------------------------------------------------------------------------- 1 | year,applicant_id,document_id,total_net_value,numbers,congressperson_name,congressperson_id,congressperson_document,term,state,party,term_id,subquota_number,subquota_description,subquota_group_id,subquota_group_description,supplier,cnpj_cpf,document_number,document_type,issue_date,document_value,remark_value,month,installment,passenger,leg_of_the_trip,batch_number 2 | 2010,1455,1779403,30.0,['3244'],LUIS CARLOS HEINZE,73483.0,500.0,2015.0,RS,PP,55.0,13,Congressperson meal,0,,RESTAURANTE CAPILE LTDA,01947426000110,16457,2,2010-01-02,30.0,0.0,1,0,,,454614 3 | -------------------------------------------------------------------------------- /jarbas/core/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from jarbas.core.models import Activity, Company 4 | 5 | 6 | class ActivitySerializer(serializers.ModelSerializer): 7 | 8 | class Meta: 9 | model = Activity 10 | fields = ('code', 'description') 11 | 12 | 13 | class CompanySerializer(serializers.ModelSerializer): 14 | 15 | main_activity = ActivitySerializer(many=True, read_only=True) 16 | secondary_activity = ActivitySerializer(many=True, read_only=True) 17 | 18 | class Meta: 19 | model = Company 20 | exclude = ('id',) 21 | depth = 1 22 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0012_use_date_not_datetime.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.3 on 2016-11-25 15:10 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0011_subquota_description_length'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='document', 17 | name='issue_date', 18 | field=models.DateField(blank=True, null=True, verbose_name='Issue date'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/fixtures/reimbursements-2016.csv: -------------------------------------------------------------------------------- 1 | year,applicant_id,document_id,total_net_value,numbers,congressperson_name,congressperson_id,congressperson_document,term,state,party,term_id,subquota_number,subquota_description,subquota_group_id,subquota_group_description,supplier,cnpj_cpf,document_number,document_type,issue_date,document_value,remark_value,month,installment,passenger,leg_of_the_trip,batch_number 2 | 2016,773,6078354,143.01,['5529'],NELSON MARQUEZELLI,73553.0,381.0,2015.0,SP,PTB,55.0,3,Fuels and lubricants,1,Veículos Automotores,POSTO ALVORADA,14665711000190,126451,0,2016-08-03,143.01,0.0,8,0,,,1316031 3 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Main.elm: -------------------------------------------------------------------------------- 1 | module Main exposing (..) 2 | 3 | import Model exposing (Model, model) 4 | import Navigation 5 | import Update exposing (Flags, Msg(..), update, updateFromFlags, urlUpdate) 6 | import View exposing (view) 7 | 8 | 9 | init : Flags -> Navigation.Location -> ( Model, Cmd Msg ) 10 | init flags location = 11 | urlUpdate location (updateFromFlags flags model) 12 | 13 | 14 | main : Program Flags Model Msg 15 | main = 16 | Navigation.programWithFlags ChangeUrl 17 | { init = init 18 | , update = update 19 | , view = view 20 | , subscriptions = (\_ -> Sub.none) 21 | } 22 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0004_add_receipt_url.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.1 on 2016-09-21 17:55 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0003_remove_some_indexes'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='document', 17 | name='receipt_url', 18 | field=models.URLField(blank=True, default=None, max_length=16, null=True, verbose_name='Receipt URL'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/Reimbursement/Receipt.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.Reimbursement.Receipt exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | notAvailable : TranslationSet 7 | notAvailable = 8 | TranslationSet 9 | " Digitalized receipt not available." 10 | " Recibo não disponível." 11 | 12 | 13 | available : TranslationSet 14 | available = 15 | TranslationSet 16 | " View receipt" 17 | " Ver recibo" 18 | 19 | 20 | fetch : TranslationSet 21 | fetch = 22 | TranslationSet 23 | " Fetch receipt" 24 | " Buscar recibo" 25 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/fixtures/reimbursements-2009.csv: -------------------------------------------------------------------------------- 1 | year,applicant_id,document_id,total_net_value,numbers,congressperson_name,congressperson_id,congressperson_document,term,state,party,term_id,subquota_number,subquota_description,subquota_group_id,subquota_group_description,supplier,cnpj_cpf,document_number,document_type,issue_date,document_value,remark_value,month,installment,passenger,leg_of_the_trip,batch_number 2 | 2009,1828,1614713,16.73,['2954'],EUGÊNIO RABELO,141426.0,93.0,2007.0,CE,PP,53.0,1,Maintenance of office supporting parliamentary activity,0,,TELEMAR NORTE LESTE S/A,33000118001574,29220,3,2009-06-01,32.6,15.87,6,0,,,406104 3 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0036_alter_tweet_status_to_decimal.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-07-07 16:13 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0035_create_model_tweet'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='tweet', 17 | name='status', 18 | field=models.DecimalField(db_index=True, decimal_places=0, max_digits=25, verbose_name='Tweet ID'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/fixtures/reimbursements-2012.csv: -------------------------------------------------------------------------------- 1 | year,applicant_id,document_id,total_net_value,numbers,congressperson_name,congressperson_id,congressperson_document,term,state,party,term_id,subquota_number,subquota_description,subquota_group_id,subquota_group_description,supplier,cnpj_cpf,document_number,document_type,issue_date,document_value,remark_value,month,installment,passenger,leg_of_the_trip,batch_number 2 | 2012,2258,2349132,210.0,['4005'],ANTÔNIA LÚCIA,123756.0,53.0,2011.0,AC,PSC,54.0,1,Maintenance of office supporting parliamentary activity,0,,AMORETTO CAFÉS EXPRESSO LTDA,08532429000131,6406,1,2012-06-04,210.0,0.0,6,0,,,621955 3 | -------------------------------------------------------------------------------- /jarbas/core/tests/fixtures/reimbursements.csv: -------------------------------------------------------------------------------- 1 | applicant_id,batch_number,cnpj_cpf,congressperson_document,congressperson_id,congressperson_name,document_id,document_number,document_type,document_value,installment,issue_date,leg_of_the_trip,month,numbers,party,passenger,remark_value,state,subquota_description,subquota_group_description,subquota_group_id,subquota_number,supplier,term,term_id,total_net_value,total_value,year 2 | 3052,1524175,05634562000100,365,178982,LUIZ LAURO FILHO,6657248,827719,4,195.47,0,2018-08-15 00:00:00,,8,['6369'],PSB,,0,SP,Fuels and lubricants,Veículos Automotores,1,3,POSTO AVENIDA NOSSA SENHORA DE FÁTIMA CAMPINAS LTDA,2015,55,195.47,0,2018 3 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/tests/test_searchvector_command.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from mixer.backend.django import mixer 3 | 4 | from jarbas.chamber_of_deputies.management.commands.searchvector import Command 5 | from jarbas.chamber_of_deputies.models import Reimbursement 6 | 7 | 8 | class TestCommandHandler(TestCase): 9 | 10 | def test_handler(self): 11 | mixer.cycle(3).blend(Reimbursement, search_vector=None) 12 | command = Command() 13 | command.handle(batch_size=2, silent=True) 14 | 15 | queryset = Reimbursement.objects.exclude(search_vector=None) 16 | self.assertEqual(3, queryset.count()) 17 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0026_order_reimbursements_by_year_and_issue_date.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-06-01 11:08 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0025_historicalreimbursement'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterModelOptions( 16 | name='reimbursement', 17 | options={'ordering': ('-year', '-issue_date'), 'verbose_name': 'reimbursement', 'verbose_name_plural': 'reimbursements'}, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | **What is the purpose of this Pull Request?** 5 | 6 | 7 | **What was done to achieve this purpose?** 8 | 9 | 10 | **How to test if it really works?** 11 | 12 | 13 | **Who can help reviewing it?** 14 | 15 | 16 | **TODO** 17 | 18 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0028_auto_20170601_1701.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-06-01 20:01 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0027_translate_verbose_names'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='historicalreimbursement', 17 | name='history_type', 18 | field=models.CharField(choices=[('+', 'Criado'), ('~', 'Modificado'), ('-', 'Excluído')], max_length=1), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0021_make_reciept_fetched_a_db_index.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-22 18:22 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0020_rename_supplier_to_company'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='reimbursement', 17 | name='receipt_fetched', 18 | field=models.BooleanField(db_index=True, default=False, verbose_name='Was the receipt URL fetched?'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/dashboard/templates/admin/base.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base.html" %} 2 | 3 | {% block footer %} 4 | 5 | {% if google_analytics %} 6 | 14 | {% endif %} 15 | {% endblock %} 16 | -------------------------------------------------------------------------------- /contrib/deploy.sh: -------------------------------------------------------------------------------- 1 | export CURRENT_DIR=$(pwd) 2 | cd /opt/serenata-de-amor 3 | git pull origin main 4 | docker-compose -f docker-compose.yml -f docker-compose.prod.yml pull 5 | docker-compose -f docker-compose.yml -f docker-compose.prod.yml stop 6 | docker-compose -f docker-compose.yml -f docker-compose.prod.yml run --rm django python manage.py collectstatic --no-input 7 | docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d 8 | docker-compose -f docker-compose.yml -f docker-compose.prod.yml run --rm django python manage.py migrate 9 | docker-compose -f docker-compose.yml -f docker-compose.prod.yml run --rm django python manage.py clear_cache 10 | cd $CURRENT_DIR 11 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0023_add_last_update_field_to_reimbursements.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-25 19:53 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0022_remove_unique_together_from_reimbursement'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='reimbursement', 17 | name='last_update', 18 | field=models.DateTimeField(auto_now=True, db_index=True, verbose_name='Last update'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0032_auto_20170613_0641.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-06-13 09:41 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0031_add_index_together_for_reimbursement'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='historicalreimbursement', 17 | name='history_type', 18 | field=models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Model.elm: -------------------------------------------------------------------------------- 1 | module Model exposing (Model, model) 2 | 3 | import Internationalization.Types exposing (Language(..)) 4 | import Layout 5 | import Material 6 | import Reimbursement.Model as Reimbursement 7 | 8 | 9 | type alias Model = 10 | { reimbursements : Reimbursement.Model 11 | , layout : Layout.Model 12 | , googleStreetViewApiKey : Maybe String 13 | , lang : Language 14 | , mdl : Material.Model 15 | } 16 | 17 | 18 | model : Model 19 | model = 20 | { reimbursements = Reimbursement.model 21 | , layout = Layout.model 22 | , googleStreetViewApiKey = Nothing 23 | , lang = English 24 | , mdl = Material.model 25 | } 26 | -------------------------------------------------------------------------------- /jarbas/dashboard/admin/paginators.py: -------------------------------------------------------------------------------- 1 | from hashlib import md5 2 | 3 | from django.core.cache import cache 4 | from django.core.paginator import Paginator 5 | 6 | 7 | class CachedCountPaginator(Paginator): 8 | """Cached the paginator count (for performance)""" 9 | 10 | @property 11 | def count(self): 12 | query = self.object_list.query.__str__() 13 | hashed = md5(query.encode('utf-8')).hexdigest() 14 | key = f'dashboard_count_{hashed}' 15 | count = cache.get(key) 16 | 17 | if count is None: 18 | count = super(CachedCountPaginator, self).count 19 | cache.set(key, count, 60 * 60 * 6) 20 | 21 | return count 22 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0040_create_gin_index_with_search_vector.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2017-09-28 02:37 3 | from __future__ import unicode_literals 4 | 5 | import django.contrib.postgres.indexes 6 | from django.db import migrations 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('core', '0039_add_search_vector_to_reimbursement'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddIndex( 17 | model_name='reimbursement', 18 | index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='core_reimbu_search__ba9b2f_gin'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0019_cleanup_remove_old_api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-21 22:00 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0018_make_issue_date_required'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name='receipt', 17 | name='document', 18 | ), 19 | migrations.DeleteModel( 20 | name='Document', 21 | ), 22 | migrations.DeleteModel( 23 | name='Receipt', 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /contrib/README.md: -------------------------------------------------------------------------------- 1 | # Contrib: scripts to help you maintain Serenata de Amor 2 | 3 | ## `.env.sample` 4 | 5 | Sample environment variables to run Serenata de Amor. 6 | 7 | ## `crontab/` 8 | 9 | This is a reference of the `crontab` we keep in our servers and the scripts/executables we schedule for maintenance. 10 | 11 | ## `data/` 12 | 13 | Sample data to run Jarbas, check Jarba's `README.md` to know how load it. 14 | 15 | ## `deploy.sh` 16 | 17 | This is a script we use for deploy, running locally `ssh user@serever ./deploy.sh` to launch new versions of our applications. 18 | 19 | ## `update/` 20 | 21 | This is a pair of Ansible Playbooks and environment settings to automatically run Rosie and update Jarbas. -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | var gulp = require('gulp'); 2 | var elm = require('gulp-elm'); 3 | var watch = require('gulp-watch'); 4 | var uglify = require('gulp-uglify'); 5 | var rename = require('gulp-rename'); 6 | 7 | gulp.task('elm', function () { 8 | return gulp.src('./jarbas/layers/elm/Main.elm') 9 | .pipe(elm({warn: true})) 10 | .on('error', onError) 11 | .pipe(uglify()) 12 | .pipe(rename('./jarbas/layers/static/app.js')) 13 | .pipe(gulp.dest('.')); 14 | }); 15 | 16 | gulp.task('watch', ['elm'], function () { 17 | watch('./**/*.elm', function () { 18 | gulp.start('elm'); 19 | }); 20 | }); 21 | 22 | function onError(err) { 23 | console.log(err); 24 | this.emit('end'); 25 | } 26 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0024_add_available_in_latest_dataset_field_to_reimbursement.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-25 21:27 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0023_add_last_update_field_to_reimbursements'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='reimbursement', 17 | name='available_in_latest_dataset', 18 | field=models.BooleanField(default=True, verbose_name='Available in the latest dataset'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Format/Date.elm: -------------------------------------------------------------------------------- 1 | module Format.Date exposing (formatDate) 2 | 3 | import Date 4 | import Date.Extra.Config.Config_en_us as Config_en_us 5 | import Date.Extra.Config.Config_pt_br as Config_pt_br 6 | import Date.Extra.Format exposing (formatUtc) 7 | import Internationalization.Types exposing (Language(..)) 8 | 9 | 10 | formatDate : Language -> Date.Date -> String 11 | formatDate lang date = 12 | case lang of 13 | Portuguese -> 14 | formatUtc 15 | Config_pt_br.config 16 | "%d/%m/%Y" 17 | date 18 | 19 | _ -> 20 | formatUtc 21 | Config_en_us.config 22 | "%b %-@d, %Y" 23 | date 24 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0006_change_on_delete_social_media_to_set_null.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.8 on 2018-09-13 04:42 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('chamber_of_deputies', '0005_create_social_media_model'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='reimbursement', 17 | name='social_media', 18 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='chamber_of_deputies.SocialMedia'), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Receipt/Model.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Receipt.Model exposing (Model, ReimbursementId, model) 2 | 3 | import Http 4 | import Internationalization.Types exposing (Language(..)) 5 | import Material 6 | 7 | 8 | type alias ReimbursementId = 9 | { year : Int 10 | , applicantId : Int 11 | , documentId : Int 12 | } 13 | 14 | 15 | type alias Model = 16 | { reimbursement : Maybe ReimbursementId 17 | , url : Maybe String 18 | , fetched : Bool 19 | , loading : Bool 20 | , error : Maybe Http.Error 21 | , lang : Language 22 | , mdl : Material.Model 23 | } 24 | 25 | 26 | model : Model 27 | model = 28 | Model Nothing Nothing False False Nothing English Material.model 29 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0010_remove_null_issue_date_rows.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.8 on 2018-09-13 13:40 2 | 3 | from django.db import migrations 4 | 5 | 6 | def forwards_func(apps, schema_editor): 7 | Reimbursement = apps.get_model("chamber_of_deputies", "Reimbursement") 8 | db_alias = schema_editor.connection.alias 9 | Reimbursement.objects.using(db_alias).filter(issue_date=None).delete() 10 | 11 | 12 | def reverse_func(*args, **kwargs): 13 | pass 14 | 15 | 16 | class Migration(migrations.Migration): 17 | 18 | dependencies = [ 19 | ('chamber_of_deputies', '0009_add_index_to_term'), 20 | ] 21 | 22 | operations = [ 23 | migrations.RunPython(forwards_func, reverse_func), 24 | ] 25 | -------------------------------------------------------------------------------- /research/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import argparse 5 | 6 | DATA_DIR = os.path.join('research', 'data') 7 | 8 | parser = argparse.ArgumentParser( 9 | description='Setup serenata-de-amor using serenata-toolbox.') 10 | 11 | parser.add_argument('--all', dest='all', action='store_true', 12 | help='Force all datasets from serenata-toolbox \ 13 | to be downloaded') 14 | 15 | args = parser.parse_args() 16 | 17 | print('Downloading datasets (this might take several minutes \ 18 | depending on your internet connection)') 19 | 20 | from serenata_toolbox.datasets import fetch_latest_backup 21 | os.makedirs(DATA_DIR, exist_ok=True) 22 | fetch_latest_backup(DATA_DIR, force_all=args.all) 23 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0005_add_receipt_url_last_update.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.1 on 2016-09-23 10:01 3 | from __future__ import unicode_literals 4 | 5 | import datetime 6 | from django.db import migrations, models 7 | from django.utils.timezone import utc 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | ('core', '0004_add_receipt_url'), 14 | ] 15 | 16 | operations = [ 17 | migrations.AddField( 18 | model_name='document', 19 | name='receipt_url_last_update', 20 | field=models.URLField(db_index=True, default=datetime.datetime(1970, 1, 1, 0, 0, tzinfo=utc), verbose_name='Receipt URL Last Update'), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /contrib/update/cleanup.py: -------------------------------------------------------------------------------- 1 | from os import getenv 2 | 3 | from dopy.manager import DoManager 4 | 5 | 6 | NAME = "serenata-update" 7 | 8 | 9 | def destroy_droplet(manager): 10 | droplet_id = None 11 | for droplet in manager.all_active_droplets(): 12 | if droplet["name"] == NAME: 13 | droplet_id = droplet["id"] 14 | break 15 | 16 | if not droplet_id: 17 | print("Droplet {} not found.".format(NAME)) 18 | return 19 | 20 | output = manager.destroy_droplet(droplet_id) 21 | print("Droplet {} ({}) deleted.".format(NAME, droplet_id)) 22 | return output 23 | 24 | 25 | if __name__ == "__main__": 26 | manager = DoManager(None, getenv("DO_API_TOKEN"), api_version=2) 27 | destroy_droplet(manager) 28 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/SameDay/Update.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.SameDay.Update exposing (..) 2 | 3 | import Reimbursement.RelatedTable.Update exposing (Msg, loadUrl) 4 | import String 5 | 6 | 7 | {-| Creates an URL from an UniqueId: 8 | 9 | getUrl 42 --> "/api/chamber_of_deputies/reimbursement/42/same_day/?format=json" 10 | 11 | -} 12 | getUrl : Int -> String 13 | getUrl documentId = 14 | String.join 15 | "/" 16 | [ "/api" 17 | , "chamber_of_deputies" 18 | , "reimbursement" 19 | , toString documentId 20 | , "same_day/?format=json" 21 | ] 22 | |> Debug.log "url" 23 | 24 | 25 | load : Int -> Cmd Msg 26 | load documentId = 27 | documentId 28 | |> getUrl 29 | |> loadUrl 30 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | 11 | # Matches multiple files with brace expansion notation 12 | # Set default charset 13 | [*.{py,elm,sh,txt,html,gitignore,md}] 14 | charset = utf-8 15 | 16 | # 4 space indentation 17 | [*.{py,elm}] 18 | indent_style = space 19 | indent_size = 4 20 | 21 | # Tab indentation (no size specified) 22 | [Makefile] 23 | indent_style = tab 24 | 25 | # Matches the exact files either package.json or .travis.yml 26 | [{package.json,.travis.yml,.codeclimate.yml,.coveragerc,elm-package.json}] 27 | indent_style = space 28 | indent_size = 2 29 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0008_remove_related_field_social_media.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.8 on 2018-09-13 05:46 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('chamber_of_deputies', '0007_auto_20180913_0209'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='reimbursement', 15 | name='social_media', 16 | ), 17 | migrations.AlterField( 18 | model_name='reimbursement', 19 | name='congressperson_id', 20 | field=models.IntegerField(blank=True, db_index=True, null=True, verbose_name='Identificador Único do Parlamentar'), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/DocumentType.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.DocumentType exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | billOfSale : TranslationSet 7 | billOfSale = 8 | TranslationSet 9 | "Bill of sale" 10 | "Nota fiscal" 11 | 12 | 13 | simpleReceipt : TranslationSet 14 | simpleReceipt = 15 | TranslationSet 16 | "Simple receipt" 17 | "Recibo simples" 18 | 19 | 20 | expenseMadeAbroad : TranslationSet 21 | expenseMadeAbroad = 22 | TranslationSet 23 | "Expense made abroad" 24 | "Despesa no exterior" 25 | 26 | electronicReceipt : TranslationSet 27 | electronicReceipt = 28 | TranslationSet 29 | "Electronic receipt" 30 | "Nota fiscal eletrônica" 31 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0022_remove_unique_together_from_reimbursement.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-11 20:23 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0021_make_reciept_fetched_a_db_index'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='reimbursement', 17 | name='document_id', 18 | field=models.IntegerField(db_index=True, unique=True, verbose_name='Document ID'), 19 | ), 20 | migrations.AlterUniqueTogether( 21 | name='reimbursement', 22 | unique_together=set([]), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /rosie/.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | This template is here to help you to write your issue ;) 2 | 3 | **What is the problem?** 4 | __Try to give as many details about the bug/new feature/enhancement as possible, but keep it friendly for new comers__ 5 | 6 | **How can this be addressed?** 7 | __New comers might need help on how to approach or build a solution, maybe a step-by-step on how to address the issue can help__ 8 | 9 | **Who could help with this issue?** 10 | __We know sometimes a good session of pair programming can do wonders, maybe indicate someone that can be reached to help out and discuss ideas. If you don't know someone but is up for a pair programming session use this space to tell us o/__ 11 | 12 | **Labels** 13 | __Feel free to write up labels that you think might apply here. e.g. [first-timers-only] [bug] __ -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | This template is here to help you to write your issue ;) 2 | 3 | **What is the problem?** 4 | Try to give as many details about the bug/new feature/enhancement as possible, 5 | but keep it friendly for new comers 6 | 7 | **How can this be addressed?** 8 | New comers might need help on how to approach or build a solution, maybe a 9 | step-by-step on how to address the issue can help 10 | 11 | **Who could help with this issue?** 12 | We know sometimes a good session of pair programming can do wonders, maybe 13 | indicate someone that can be reached to help out and discuss ideas. If you 14 | don't know someone but is up for a pair programming session use this space to 15 | tell us o/ 16 | 17 | **Labels** 18 | Feel free to write up labels that you think might apply here. e.g. 19 | [first-timers-only] [bug] 20 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/management/commands/tweet.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from jarbas.chamber_of_deputies.twitter import Twitter 4 | 5 | 6 | class Command(BaseCommand): 7 | help = 'Tweet the next suspicion at @RosieDaSerenata account' 8 | 9 | def add_arguments(self, parser): 10 | parser.add_argument( 11 | '--fake', 12 | action='store_true', 13 | help='Do not tweet, just show the tweet message' 14 | ) 15 | 16 | def handle(self, *args, **options): 17 | twitter = Twitter() 18 | if not twitter.reimbursement: 19 | print('No suspicion to tweet') 20 | return None 21 | 22 | if options.get('fake'): 23 | print(twitter.message) 24 | return None 25 | 26 | twitter.publish() 27 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0029_make_issue_date_an_index.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-06-02 16:47 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0028_auto_20170601_1701'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='historicalreimbursement', 17 | name='issue_date', 18 | field=models.DateField(db_index=True, verbose_name='Data de Emissão'), 19 | ), 20 | migrations.AlterField( 21 | model_name='reimbursement', 22 | name='issue_date', 23 | field=models.DateField(db_index=True, verbose_name='Data de Emissão'), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/fixtures/reimbursements-2011.csv: -------------------------------------------------------------------------------- 1 | year,applicant_id,document_id,total_net_value,numbers,congressperson_name,congressperson_id,congressperson_document,term,state,party,term_id,subquota_number,subquota_description,subquota_group_id,subquota_group_description,supplier,cnpj_cpf,document_number,document_type,issue_date,document_value,remark_value,month,installment,passenger,leg_of_the_trip,batch_number 2 | 2011,2277,2109866,143.06,['3687'],SANDRO ALEX,160621.0,465.0,2015.0,PR,PSD,55.0,3,Fuels and lubricants,1,Veículos Automotores,B.P. COMÉRCIO DE COMBUSTÍVEIS S/A,82686114000100,302028,0,2011-06-20,143.06,0.0,6,0,,,549586 3 | 2011,1,2109866,143.06,['3687'],SANDRO ALEX,,465.0,2015.0,PR,PSD,55.0,3,Fuels and lubricants,1,Veículos Automotores,B.P. COMÉRCIO DE COMBUSTÍVEIS S/A,82686114000100,302028,0,2011-06-20,143.06,0.0,6,0,,,549586 4 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0037_auto_20170727_1624.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-07-27 19:24 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0036_alter_tweet_status_to_decimal'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='historicalreimbursement', 17 | name='reimbursement_text', 18 | field=models.TextField(blank=True, null=True, verbose_name='OCR'), 19 | ), 20 | migrations.AddField( 21 | model_name='reimbursement', 22 | name='reimbursement_text', 23 | field=models.TextField(blank=True, null=True, verbose_name='OCR'), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/classifiers/election_expenses_classifier.py: -------------------------------------------------------------------------------- 1 | from sklearn.base import TransformerMixin 2 | 3 | 4 | ELECTION_LEGAL_ENTITY = '409-0 - CANDIDATO A CARGO POLITICO ELETIVO' 5 | 6 | class ElectionExpensesClassifier(TransformerMixin): 7 | """ 8 | Election Expenses classifier. 9 | 10 | Check a `legal_entity` field for the presency of the political candidacy 11 | category in the Brazilian Federal Revenue. 12 | 13 | Dataset 14 | ------- 15 | legal_entity : string column 16 | Brazilian Federal Revenue category of companies, preceded by its code. 17 | """ 18 | 19 | def fit(self, dataframe): 20 | pass 21 | 22 | def transform(self, dataframe=None): 23 | pass 24 | 25 | def predict(self, dataframe): 26 | return dataframe['legal_entity'] == ELECTION_LEGAL_ENTITY 27 | -------------------------------------------------------------------------------- /jarbas/core/tests/test_company_model.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from jarbas.core.models import Activity, Company 3 | from jarbas.core.tests import sample_activity_data, sample_company_data 4 | 5 | 6 | class TestCreate(TestCase): 7 | 8 | def setUp(self): 9 | self.activity = Activity.objects.create(**sample_activity_data) 10 | self.data = sample_company_data 11 | 12 | def test_create(self): 13 | self.assertEqual(0, Company.objects.count()) 14 | company = Company.objects.create(**self.data) 15 | company.main_activity.add(self.activity) 16 | company.secondary_activity.add(self.activity) 17 | company.save() 18 | self.assertEqual(1, Company.objects.count()) 19 | self.assertEqual(1, company.main_activity.count()) 20 | self.assertEqual(1, company.secondary_activity.count()) 21 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0011_subquota_description_length.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.3 on 2016-11-24 07:38 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0010_extract_receipt'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name='document', 17 | name='line', 18 | ), 19 | migrations.RemoveField( 20 | model_name='document', 21 | name='source', 22 | ), 23 | migrations.AlterField( 24 | model_name='document', 25 | name='subquota_description', 26 | field=models.CharField(max_length=128, verbose_name='Subquota descrition'), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /jarbas/dashboard/templates/admin/base_site.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base.html" %} 2 | {% load dashboard %} 3 | {% load static %} 4 | 5 | {% block title %}{{ title|rename_title }} | {{ site_title|default:_('Jarbas Dashboard') }}{% endblock %} 6 | 7 | {% block extrahead %} 8 | 9 | 10 | {% endblock %} 11 | 12 | {% block blockbots %}{% endblock %} 13 | 14 | {% block branding %} 15 |

{{ site_header|default:_('Jarbas Dashboard') }}

16 | {% endblock %} 17 | 18 | {% block usertools %}{% endblock %} 19 | {% block content_title %}{% if title %}

{{ title|rename_title }}

{% endif %}{% endblock %} 20 | 21 | {% block nav-global %}{% endblock %} 22 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/management/commands/socialmedia.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | 4 | from jarbas.core.management.commands import LoadCommand 5 | from jarbas.chamber_of_deputies.models import SocialMedia 6 | 7 | 8 | class Command(LoadCommand): 9 | help = 'Load congresspeople social media accounts' 10 | count = 0 11 | 12 | def handle(self, *args, **options): 13 | self.path = options['dataset'] 14 | if not os.path.exists(self.path): 15 | raise FileNotFoundError(os.path.abspath(self.path)) 16 | 17 | if options.get('drop', False): 18 | self.drop_all(SocialMedia) 19 | 20 | print('Saving social media accounts') 21 | with open(self.path) as fobj: 22 | bulk = (SocialMedia(**line) for line in csv.DictReader(fobj)) 23 | SocialMedia.objects.bulk_create(bulk) 24 | print('Done!') 25 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0039_add_search_vector_to_reimbursement.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2017-09-27 20:19 3 | from __future__ import unicode_literals 4 | 5 | import django.contrib.postgres.search 6 | from django.db import migrations 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('core', '0038_auto_20170728_1748'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name='historicalreimbursement', 18 | name='search_vector', 19 | field=django.contrib.postgres.search.SearchVectorField(null=True), 20 | ), 21 | migrations.AddField( 22 | model_name='reimbursement', 23 | name='search_vector', 24 | field=django.contrib.postgres.search.SearchVectorField(null=True), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Receipt/Decoder.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Receipt.Decoder exposing (decoder, urlDecoder) 2 | 3 | import Internationalization.Types exposing (Language(..)) 4 | import Json.Decode exposing (at, bool, nullable, string) 5 | import Json.Decode.Pipeline exposing (decode, hardcoded, required) 6 | import Material 7 | import Reimbursement.Receipt.Model exposing (Model, ReimbursementId) 8 | 9 | 10 | urlDecoder : Json.Decode.Decoder (Maybe String) 11 | urlDecoder = 12 | at [ "url" ] (nullable string) 13 | 14 | 15 | decoder : Language -> Json.Decode.Decoder Model 16 | decoder lang = 17 | decode Model 18 | |> hardcoded Nothing 19 | |> required "url" (nullable string) 20 | |> required "fetched" bool 21 | |> hardcoded False 22 | |> hardcoded Nothing 23 | |> hardcoded lang 24 | |> hardcoded Material.model 25 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jarbas.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0002_remove_django_simple_history.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.10 on 2018-02-16 13:20 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('chamber_of_deputies', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name='historicalreimbursement', 17 | name='history_user', 18 | ), 19 | migrations.AlterModelTable( 20 | name='reimbursement', 21 | table=None, 22 | ), 23 | migrations.AlterModelTable( 24 | name='tweet', 25 | table=None, 26 | ), 27 | migrations.DeleteModel( 28 | name='HistoricalReimbursement', 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0009_add_latitude_and_longitude.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.1 on 2016-09-28 21:00 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0008_optimize_char_field_lengths'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='supplier', 17 | name='latitude', 18 | field=models.DecimalField(blank=True, decimal_places=7, max_digits=10, null=True, verbose_name='Latitude'), 19 | ), 20 | migrations.AddField( 21 | model_name='supplier', 22 | name='longitude', 23 | field=models.DecimalField(blank=True, decimal_places=7, max_digits=10, null=True, verbose_name='Longitude'), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0015_add_receipt_to_reimbursement.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-09 21:17 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0014_add_suspicions_and_probability_to_reimbursements'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='reimbursement', 17 | name='receipt_fetched', 18 | field=models.BooleanField(default=False, verbose_name='Was the receipt URL fetched?'), 19 | ), 20 | migrations.AddField( 21 | model_name='reimbursement', 22 | name='receipt_url', 23 | field=models.CharField(blank=True, max_length=140, null=True, verbose_name='Receipt URL'), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /jarbas/layers/elm/View.elm: -------------------------------------------------------------------------------- 1 | module View exposing (view) 2 | 3 | import Html 4 | import Layout 5 | import Material.Layout 6 | import Model exposing (Model) 7 | import Reimbursement.View 8 | import Update exposing (Msg(..)) 9 | 10 | 11 | view : Model -> Html.Html Msg 12 | view model = 13 | let 14 | header = 15 | Html.map LayoutMsg <| Layout.header model.layout 16 | 17 | drawer = 18 | List.map (\x -> Html.map LayoutMsg x) (Layout.drawer model.layout) 19 | 20 | reimbursements = 21 | Html.map ReimbursementMsg <| Reimbursement.View.view model.reimbursements 22 | in 23 | Material.Layout.render 24 | Mdl 25 | model.mdl 26 | [ Material.Layout.fixedHeader ] 27 | { header = [ header ] 28 | , drawer = drawer 29 | , tabs = ( [], [] ) 30 | , main = [ reimbursements ] 31 | } 32 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/tests/test_tweet_model.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from mixer.backend.django import mixer 3 | 4 | from jarbas.chamber_of_deputies.models import Tweet 5 | 6 | 7 | class TestTweet(TestCase): 8 | 9 | def setUp(self): 10 | self.tweet = mixer.blend(Tweet, reimbursement__search_vector=None, status=42) 11 | 12 | def test_ordering(self): 13 | mixer.blend(Tweet, reimbursement__search_vector=None, status=1) 14 | self.assertEqual(42, Tweet.objects.first().status) 15 | 16 | def test_get_url(self): 17 | expected = 'https://twitter.com/RosieDaSerenata/status/42' 18 | self.assertEqual(expected, self.tweet.get_url()) 19 | 20 | def test_repr(self): 21 | expected = '' 22 | self.assertEqual(expected, self.tweet.__repr__()) 23 | 24 | def test_str(self): 25 | self.assertEqual(self.tweet.get_url(), str(self.tweet)) 26 | -------------------------------------------------------------------------------- /jarbas/public_admin/tests/test_dummy_user.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from jarbas.public_admin.sites import DummyUser 4 | 5 | 6 | class TestDummyUser(TestCase): 7 | 8 | def setUp(self): 9 | self.user = DummyUser() 10 | 11 | def test_has_module_perms(self): 12 | self.assertTrue(self.user.has_module_perms('chamber_of_deputies')) 13 | self.assertFalse(self.user.has_module_perms('core')) 14 | self.assertFalse(self.user.has_module_perms('api')) 15 | self.assertFalse(self.user.has_module_perms('dashboard')) 16 | self.assertFalse(self.user.has_module_perms('layers')) 17 | 18 | def test_has_perm(self): 19 | self.assertTrue(self.user.has_perm('chamber_of_deputies.change_reimbursement')) 20 | self.assertFalse(self.user.has_perm('chamber_of_deputies.add_reimbursement')) 21 | self.assertFalse(self.user.has_perm('chamber_of_deputies.delete_reimbursement')) 22 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/Reimbursement/Search.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.Reimbursement.Search exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | fieldsetReimbursement : TranslationSet 7 | fieldsetReimbursement = 8 | TranslationSet 9 | "Reimbursement data" 10 | "Dados do reembolso" 11 | 12 | 13 | fieldsetCongressperson : TranslationSet 14 | fieldsetCongressperson = 15 | TranslationSet 16 | "Congressperson & expense data" 17 | "Dados do(a) deputado(a) e da despesa" 18 | 19 | 20 | search : TranslationSet 21 | search = 22 | TranslationSet 23 | "Search" 24 | "Buscar" 25 | 26 | 27 | newSearch : TranslationSet 28 | newSearch = 29 | TranslationSet 30 | "New search" 31 | "Nova busca" 32 | 33 | 34 | loading : TranslationSet 35 | loading = 36 | TranslationSet 37 | "Loading…" 38 | "Carregando…" 39 | -------------------------------------------------------------------------------- /jarbas/public_admin/admin.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.contrib.admin import ModelAdmin 4 | 5 | from jarbas.public_admin.sites import public_admin 6 | 7 | 8 | class PublicAdminModelAdmin(ModelAdmin): 9 | 10 | def has_add_permission(self, request): 11 | return False 12 | 13 | def has_change_permission(self, request, obj=None): 14 | return request.method == 'GET' 15 | 16 | def has_delete_permission(self, request, obj=None): 17 | return False 18 | 19 | @staticmethod 20 | def rename_change_url(url): 21 | if 'change' in url.pattern.regex.pattern: 22 | new_re = url.pattern.regex.pattern.replace('change', 'details') 23 | url.regex = re.compile(new_re, re.UNICODE) 24 | return url 25 | 26 | def get_urls(self): 27 | return [ 28 | self.rename_change_url(url) for url in super().get_urls() 29 | if public_admin.valid_url(url) 30 | ] 31 | -------------------------------------------------------------------------------- /docker-compose.override.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | 4 | django: 5 | command: ["python", "manage.py", "runserver", "0.0.0.0:8000"] 6 | depends_on: 7 | - postgres 8 | ports: 9 | - "8000:8000" 10 | restart: "no" 11 | volumes: 12 | - .:/code 13 | - ./contrib/data:/mnt/data 14 | - /tmp/serenata-data:/mnt/rosie-output 15 | 16 | elm: 17 | command: ["npm", "run", "watch"] 18 | restart: "no" 19 | volumes: 20 | - ./jarbas:/code/jarbas 21 | 22 | postgres: 23 | env_file: 24 | - .env 25 | image: postgres:10.3-alpine 26 | ports: 27 | - "5432:5432" 28 | volumes: 29 | - ./db:/var/lib/postgresql 30 | 31 | rosie: 32 | volumes: 33 | - /tmp/serenata-data:/tmp/serenata-data 34 | - ./rosie:/code 35 | 36 | tasks: 37 | restart: "no" 38 | volumes: 39 | - .:/code 40 | 41 | beat: 42 | restart: "no" 43 | volumes: 44 | - .:/code 45 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/settings.py: -------------------------------------------------------------------------------- 1 | from rosie.chamber_of_deputies.classifiers import ElectionExpensesClassifier 2 | from rosie.chamber_of_deputies.classifiers import IrregularCompaniesClassifier 3 | from rosie.chamber_of_deputies.classifiers import MealPriceOutlierClassifier 4 | from rosie.chamber_of_deputies.classifiers import MonthlySubquotaLimitClassifier 5 | from rosie.chamber_of_deputies.classifiers import TraveledSpeedsClassifier 6 | from rosie.core.classifiers import InvalidCnpjCpfClassifier 7 | 8 | CLASSIFIERS = { 9 | 'meal_price_outlier': MealPriceOutlierClassifier, 10 | 'over_monthly_subquota_limit': MonthlySubquotaLimitClassifier, 11 | 'suspicious_traveled_speed_day': TraveledSpeedsClassifier, 12 | 'invalid_cnpj_cpf': InvalidCnpjCpfClassifier, 13 | 'election_expenses': ElectionExpensesClassifier, 14 | 'irregular_companies_classifier': IrregularCompaniesClassifier 15 | } 16 | 17 | UNIQUE_IDS = ['applicant_id', 'year', 'document_id'] 18 | -------------------------------------------------------------------------------- /elm-package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0.0", 3 | "summary": "Jarbas", 4 | "repository": "https://github.com/okfn-brasil/jarbas.git", 5 | "license": "BSD3", 6 | "source-directories": [ 7 | "./jarbas/layers/elm", 8 | "./jarbas/layers/tests" 9 | ], 10 | "exposed-modules": [], 11 | "dependencies": { 12 | "MichaelCombs28/elm-mdl": "1.0.1 <= v < 2.0.0", 13 | "NoRedInk/elm-decode-pipeline": "3.0.0 <= v < 4.0.0", 14 | "cuducos/elm-format-number": "2.0.0 <= v < 3.0.0", 15 | "elm-community/json-extra": "2.1.0 <= v < 3.0.0", 16 | "elm-community/list-extra": "5.0.1 <= v < 6.0.0", 17 | "elm-lang/core": "5.0.0 <= v < 6.0.0", 18 | "elm-lang/html": "2.0.0 <= v < 3.0.0", 19 | "elm-lang/http": "1.0.0 <= v < 2.0.0", 20 | "elm-lang/navigation": "2.0.1 <= v < 3.0.0", 21 | "rluiten/elm-date-extra": "8.2.0 <= v < 9.0.0" 22 | }, 23 | "elm-version": "0.18.0 <= v < 0.19.0" 24 | } 25 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0014_add_suspicions_and_probability_to_reimbursements.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-08 13:03 3 | from __future__ import unicode_literals 4 | 5 | import django.contrib.postgres.fields.jsonb 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('core', '0013_create_model_reimbursement'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name='reimbursement', 18 | name='probability', 19 | field=models.DecimalField(blank=True, decimal_places=5, max_digits=6, null=True, verbose_name='Probability'), 20 | ), 21 | migrations.AddField( 22 | model_name='reimbursement', 23 | name='suspicions', 24 | field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='Suspicions'), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /jarbas/core/views.py: -------------------------------------------------------------------------------- 1 | from django.db import connection 2 | from django.http import HttpResponse 3 | from django.shortcuts import get_object_or_404 4 | from rest_framework.generics import RetrieveAPIView 5 | 6 | from jarbas.core.models import Company 7 | from jarbas.core.serializers import CompanySerializer 8 | from jarbas.chamber_of_deputies.serializers import format_cnpj 9 | 10 | 11 | class CompanyDetailView(RetrieveAPIView): 12 | 13 | lookup_field = 'cnpj' 14 | queryset = Company.objects.all() 15 | serializer_class = CompanySerializer 16 | 17 | def get_object(self): 18 | cnpj = self.kwargs.get(self.lookup_field, '00000000000000') 19 | return get_object_or_404(Company, cnpj=format_cnpj(cnpj)) 20 | 21 | 22 | def healthcheck(request): 23 | """A simple view to run a health check in Django and in the database""" 24 | with connection.cursor() as cursor: 25 | cursor.execute('SELECT 1') 26 | cursor.fetchone() 27 | return HttpResponse() 28 | -------------------------------------------------------------------------------- /jarbas/layers/tests/test_home_view.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import resolve_url 2 | from django.test import TestCase, override_settings 3 | 4 | 5 | class TestGet(TestCase): 6 | 7 | def setUp(self): 8 | self.resp = self.client.get(resolve_url('layers:home')) 9 | 10 | def test_status_code(self): 11 | self.assertEqual(200, self.resp.status_code) 12 | 13 | def test_template(self): 14 | resp = self.client.get(resolve_url('layers:home')) 15 | self.assertTemplateUsed(resp, 'layers/home.html') 16 | 17 | def test_contents(self): 18 | expected = 'Jarbas | Serenata de Amor' 19 | self.assertIn(expected, self.resp.content.decode('utf-8')) 20 | 21 | @override_settings(GOOGLE_STREET_VIEW_API_KEY=42) 22 | def test_google_api_key(self): 23 | resp = self.client.get(resolve_url('layers:home')) 24 | expected = "googleStreetViewApiKey: '42'" 25 | self.assertIn(expected, resp.content.decode('utf-8')) 26 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0035_create_model_tweet.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-06-24 04:03 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('core', '0034_auto_20170629_2150'), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Tweet', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('status', models.IntegerField(db_index=True, verbose_name='Tweet ID')), 21 | ('reimbursement', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='core.Reimbursement')), 22 | ], 23 | options={ 24 | 'ordering': ('-status',), 25 | }, 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0006_lazy_backend_receipt_url.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.1 on 2016-09-24 03:28 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0005_add_receipt_url_last_update'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name='document', 17 | name='receipt_url_last_update', 18 | ), 19 | migrations.AddField( 20 | model_name='document', 21 | name='receipt_fetched', 22 | field=models.BooleanField(default=False, verbose_name='Was receipt fetched?'), 23 | ), 24 | migrations.AlterField( 25 | model_name='document', 26 | name='receipt_url', 27 | field=models.URLField(blank=True, default=None, max_length=128, null=True, verbose_name='Receipt URL'), 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /jarbas/dashboard/static/dashboard.js: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * This JS assures backward compatibility for URLs aiming at the Layers app 4 | * version os a receipt, i.e. the Elm interface. 5 | * 6 | * Old URLs such as: 7 | * https://domain.tld/#/documentId/42 8 | * 9 | * will be redirected by the backend to: 10 | * https://domain.tld//dashboard/chamber_of_deputies/reimbursement/#/documentId/42 11 | * 12 | * and then this JS will correct it to: 13 | * https://domain.tld/layers/#/documentId/42 14 | * 15 | * */ 16 | 17 | var redirectedPath = '/dashboard/chamber_of_deputies/reimbursement/'; 18 | var layersPath = '/layers/'; 19 | var hash = 'documentId'; // we look only for fragments containing this word 20 | 21 | var isLayersUrl = function () { 22 | if (redirectedPath !== window.location.pathname) return false; 23 | return window.location.hash.split('/').includes(hash); 24 | }; 25 | 26 | var redirectToLayers = function () { 27 | if (isLayersUrl()) window.location.pathname =layersPath; 28 | }; 29 | 30 | redirectToLayers(); 31 | -------------------------------------------------------------------------------- /jarbas/layers/static/favicon/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Jarbas", 3 | "icons": [ 4 | { 5 | "src": "\/static\/favicon\/android-icon-36x36.png", 6 | "sizes": "36x36", 7 | "type": "image\/png", 8 | "density": "0.75" 9 | }, 10 | { 11 | "src": "\/static\/favicon\/android-icon-48x48.png", 12 | "sizes": "48x48", 13 | "type": "image\/png", 14 | "density": "1.0" 15 | }, 16 | { 17 | "src": "\/static\/favicon\/android-icon-72x72.png", 18 | "sizes": "72x72", 19 | "type": "image\/png", 20 | "density": "1.5" 21 | }, 22 | { 23 | "src": "\/static\/favicon\/android-icon-96x96.png", 24 | "sizes": "96x96", 25 | "type": "image\/png", 26 | "density": "2.0" 27 | }, 28 | { 29 | "src": "\/static\/favicon\/android-icon-144x144.png", 30 | "sizes": "144x144", 31 | "type": "image\/png", 32 | "density": "3.0" 33 | }, 34 | { 35 | "src": "\/static\/favicon\/android-icon-192x192.png", 36 | "sizes": "192x192", 37 | "type": "image\/png", 38 | "density": "4.0" 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Map/Model.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Map.Model exposing (Model, modelFrom) 2 | 3 | import Internationalization.Types exposing (Language(..)) 4 | import Material 5 | import Reimbursement.Company.Model as Company 6 | 7 | 8 | type alias GeoCoord = 9 | { latitude : Maybe String 10 | , longitude : Maybe String 11 | } 12 | 13 | 14 | type alias Model = 15 | { geoCoord : GeoCoord 16 | , lang : Language 17 | , mdl : Material.Model 18 | } 19 | 20 | 21 | modelFrom : Language -> Company.Model -> Model 22 | modelFrom lang model = 23 | case model.company of 24 | Just company -> 25 | Model 26 | { latitude = company.latitude 27 | , longitude = company.longitude 28 | } 29 | lang 30 | Material.model 31 | 32 | Nothing -> 33 | Model 34 | { latitude = Nothing 35 | , longitude = Nothing 36 | } 37 | lang 38 | Material.model 39 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/RelatedTable/Model.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.RelatedTable.Model exposing (ReimbursementSummary, Model, Results, model) 2 | 3 | import Array exposing (Array) 4 | import Internationalization.Types exposing (Language(..)) 5 | import Material 6 | 7 | 8 | type alias ReimbursementSummary = 9 | { applicantId : Int 10 | , city : Maybe String 11 | , documentId : Int 12 | , subquotaDescription : String 13 | , subquotaNumber : Int 14 | , supplier : String 15 | , totalNetValue : Float 16 | , year : Int 17 | , over : Bool 18 | } 19 | 20 | 21 | type alias Results = 22 | { reimbursements : Array ReimbursementSummary 23 | , nextPageUrl : Maybe String 24 | } 25 | 26 | 27 | type alias Model = 28 | { results : Results 29 | , parentId : Maybe Int 30 | , lang : Language 31 | , mdl : Material.Model 32 | } 33 | 34 | 35 | results : Results 36 | results = 37 | Results Array.empty Nothing 38 | 39 | 40 | model : Model 41 | model = 42 | Model results Nothing English Material.model 43 | -------------------------------------------------------------------------------- /jarbas/core/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from datetime import date 2 | 3 | from django.utils import timezone 4 | 5 | 6 | sample_activity_data = dict( 7 | code='42', 8 | description='So long, so long, and thanks for all the fish' 9 | ) 10 | 11 | sample_company_data = dict( 12 | cnpj='12.345.678/9012-34', 13 | opening=date(1995, 9, 27), 14 | legal_entity='42 - The answer to life, the universe, and everything', 15 | trade_name="Don't panic", 16 | name='Do not panic, sir', 17 | type='BOOK', 18 | status='OK', 19 | situation='EXISTS', 20 | situation_reason='Douglas Adams wrote it', 21 | situation_date=date(2016, 9, 25), 22 | special_situation='WE LOVE IT', 23 | special_situation_date=date(1997, 9, 28), 24 | responsible_federative_entity='Vogons', 25 | address='Earth', 26 | number='', 27 | additional_address_details='', 28 | neighborhood='', 29 | zip_code='', 30 | city='', 31 | state='', 32 | email='', 33 | phone='', 34 | last_updated=timezone.now(), 35 | latitude=None, 36 | longitude=None 37 | ) 38 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Tweet/View.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Tweet.View exposing (view) 2 | 3 | import Html exposing (a, text) 4 | import Html.Attributes exposing (class, href, target) 5 | import Internationalization exposing (translate) 6 | import Internationalization.Types exposing (Language(..), TranslationId(..)) 7 | import Material.Button as Button 8 | import Material.Icon as Icon 9 | import Reimbursement.Tweet.Model exposing (Model) 10 | import Reimbursement.Tweet.Update exposing (Msg(Mdl)) 11 | 12 | 13 | view : Model -> Html.Html Msg 14 | view model = 15 | case model.url of 16 | Nothing -> 17 | text "" 18 | 19 | Just url -> 20 | a 21 | [ href url, target "_blank", class "tweet" ] 22 | [ Button.render 23 | Mdl 24 | [ 1 ] 25 | model.mdl 26 | [ Button.minifab, Button.primary ] 27 | [ Icon.i "share" 28 | , text (translate model.lang RosiesTweet) 29 | ] 30 | ] 31 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/RelatedTable/Decoder.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.RelatedTable.Decoder exposing (decoder) 2 | 3 | import Json.Decode exposing (array, float, int, nullable, string) 4 | import Json.Decode.Pipeline exposing (decode, hardcoded, optional, required) 5 | import Reimbursement.RelatedTable.Model exposing (ReimbursementSummary, Results) 6 | 7 | 8 | reimbursementSummaryDecoder : Json.Decode.Decoder ReimbursementSummary 9 | reimbursementSummaryDecoder = 10 | decode ReimbursementSummary 11 | |> required "applicant_id" int 12 | |> optional "city" (nullable string) Nothing 13 | |> required "document_id" int 14 | |> required "subquota_description" string 15 | |> required "subquota_number" int 16 | |> required "supplier" string 17 | |> required "total_net_value" float 18 | |> required "year" int 19 | |> hardcoded False 20 | 21 | 22 | decoder : Json.Decode.Decoder Results 23 | decoder = 24 | decode Results 25 | |> required "results" (array reimbursementSummaryDecoder) 26 | |> required "next" (nullable string) 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Data Science Brigade 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/Common.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.Common exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | about : TranslationSet 7 | about = 8 | TranslationSet 9 | "About" 10 | "Sobre" 11 | 12 | 13 | aboutJarbas : TranslationSet 14 | aboutJarbas = 15 | TranslationSet 16 | "About Jarbas" 17 | "Sobre o Jarbas" 18 | 19 | 20 | aboutSerenata : TranslationSet 21 | aboutSerenata = 22 | TranslationSet 23 | "About Serenata de Amor" 24 | "Sobre a Serenata de Amor" 25 | 26 | 27 | thousandSeparator : TranslationSet 28 | thousandSeparator = 29 | TranslationSet 30 | "," 31 | "." 32 | 33 | 34 | decimalSeparator : TranslationSet 35 | decimalSeparator = 36 | TranslationSet 37 | "." 38 | "," 39 | 40 | 41 | brazilianCurrency : String -> TranslationSet 42 | brazilianCurrency value = 43 | TranslationSet 44 | (value ++ " BRL") 45 | ("R$ " ++ value) 46 | 47 | 48 | empty : TranslationSet 49 | empty = 50 | TranslationSet 51 | "" 52 | "" 53 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/SameSubquota/Update.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.SameSubquota.Update exposing (..) 2 | 3 | import Format.Url exposing (url) 4 | import Reimbursement.RelatedTable.Update exposing (Msg, loadUrl) 5 | 6 | 7 | type alias Filter = 8 | { applicantId : Int 9 | , year : Int 10 | , month : Int 11 | , subquotaNumber : Int 12 | } 13 | 14 | 15 | {-| Creates an URL from a Filter: 16 | 17 | getUrl { year = 2016, applicantId = 13, subquotaNumber = 42, month = 2 } 18 | --> "/api/chamber_of_deputies/reimbursement/?applicant_id=13&year=2016&month=2&subquota_number=42&format=json" 19 | 20 | -} 21 | getUrl : Filter -> String 22 | getUrl filter = 23 | url "/api/chamber_of_deputies/reimbursement/" 24 | [ ( "applicant_id", toString filter.applicantId ) 25 | , ( "year", toString filter.year ) 26 | , ( "month", toString filter.month ) 27 | , ( "subquota_number", toString filter.subquotaNumber ) 28 | , ( "format", "json" ) 29 | ] 30 | 31 | 32 | load : Filter -> Cmd Msg 33 | load filter = 34 | filter 35 | |> getUrl 36 | |> loadUrl 37 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0038_auto_20170728_1748.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-07-28 20:48 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0037_auto_20170727_1624'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name='historicalreimbursement', 17 | name='reimbursement_text', 18 | ), 19 | migrations.RemoveField( 20 | model_name='reimbursement', 21 | name='reimbursement_text', 22 | ), 23 | migrations.AddField( 24 | model_name='historicalreimbursement', 25 | name='receipt_text', 26 | field=models.TextField(blank=True, null=True, verbose_name='Texto do Recibo'), 27 | ), 28 | migrations.AddField( 29 | model_name='reimbursement', 30 | name='receipt_text', 31 | field=models.TextField(blank=True, null=True, verbose_name='Texto do Recibo'), 32 | ), 33 | ] 34 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0033_add_index_for_subquota_description.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-06-21 18:37 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0032_auto_20170613_0641'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='historicalreimbursement', 17 | name='history_type', 18 | field=models.CharField(choices=[('+', 'Criado'), ('~', 'Modificado'), ('-', 'Excluído')], max_length=1), 19 | ), 20 | migrations.AlterField( 21 | model_name='historicalreimbursement', 22 | name='subquota_description', 23 | field=models.CharField(db_index=True, max_length=140, verbose_name='Descrição da Subcota'), 24 | ), 25 | migrations.AlterField( 26 | model_name='reimbursement', 27 | name='subquota_description', 28 | field=models.CharField(db_index=True, max_length=140, verbose_name='Descrição da Subcota'), 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6.4-alpine3.7 2 | 3 | ENV AMAZON_BUCKET=serenata-de-amor-data \ 4 | AMAZON_REGION=sa-east-1 \ 5 | PYTHONUNBUFFERED=1 \ 6 | SECRET_KEY=${SECRET_KEY:-my-secret} 7 | 8 | COPY ./requirements.txt /code/requirements.txt 9 | COPY ./requirements-dev.txt /code/requirements-dev.txt 10 | COPY manage.py /code/manage.py 11 | COPY jarbas /code/jarbas 12 | 13 | WORKDIR /code 14 | 15 | RUN set -ex && \ 16 | apk update && apk add --no-cache curl tzdata libpq && \ 17 | cp /usr/share/zoneinfo/America/Sao_Paulo /etc/localtime && \ 18 | echo "America/Sao_Paulo" > /etc/timezone && \ 19 | apk update && apk add --no-cache \ 20 | --virtual=.build-dependencies \ 21 | gcc \ 22 | musl-dev \ 23 | postgresql-dev \ 24 | git \ 25 | python3-dev && \ 26 | python -m pip --no-cache install -U pip && \ 27 | python -m pip --no-cache install -r requirements-dev.txt && \ 28 | python manage.py collectstatic --no-input && \ 29 | apk del --purge .build-dependencies 30 | 31 | HEALTHCHECK --interval=1m --timeout=2m CMD curl 0.0.0.0:8000/healthcheck/ 32 | EXPOSE 8000 33 | CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] 34 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from jarbas.chamber_of_deputies.views import ( 4 | ReimbursementListView, 5 | ReimbursementDetailView, 6 | ReceiptDetailView, 7 | SameDayReimbursementListView, 8 | ApplicantListView, 9 | SubquotaListView, 10 | ) 11 | 12 | 13 | app_name = 'chamber_of_deputies' 14 | 15 | urlpatterns = [ 16 | path( 17 | 'reimbursement/', 18 | ReimbursementListView.as_view(), 19 | name='reimbursement-list' 20 | ), 21 | path( 22 | 'reimbursement//', 23 | ReimbursementDetailView.as_view(), 24 | name='reimbursement-detail' 25 | ), 26 | path( 27 | 'reimbursement//receipt/', 28 | ReceiptDetailView.as_view(), 29 | name='reimbursement-receipt' 30 | ), 31 | path( 32 | 'reimbursement//same_day/', 33 | SameDayReimbursementListView.as_view(), 34 | name='reimbursement-same-day' 35 | ), 36 | path('applicant/', ApplicantListView.as_view(), name='applicant-list'), 37 | path('subquota/', SubquotaListView.as_view(), name='subquota-list') 38 | ] 39 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Format/Url.elm: -------------------------------------------------------------------------------- 1 | module Format.Url exposing (queryEscape, queryPair, url) 2 | 3 | import Http exposing (encodeUri) 4 | 5 | 6 | -- 7 | -- Quick replacement for the missing Http.url function 8 | -- 9 | 10 | 11 | {-| Escapes a URL with query parameters: 12 | 13 | url "http://jarbas.dsbr.com/" [ ( "format", "json" ), ( "another", "foo bar" ) ] 14 | --> "http://jarbas.dsbr.com/?format=json&another=foo+bar" 15 | 16 | -} 17 | url : String -> List ( String, String ) -> String 18 | url baseUrl args = 19 | case args of 20 | [] -> 21 | baseUrl 22 | 23 | _ -> 24 | baseUrl ++ "?" ++ String.join "&" (List.map queryPair args) 25 | 26 | 27 | {-| Generates an encoded URL parameter: 28 | 29 | queryPair ( "another", "foo bar" ) --> "another=foo+bar" 30 | 31 | -} 32 | queryPair : ( String, String ) -> String 33 | queryPair ( key, value ) = 34 | queryEscape key ++ "=" ++ queryEscape value 35 | 36 | 37 | {-| Generates an encoded URL value: 38 | 39 | queryEscape "foo bar" --> "foo+bar" 40 | 41 | -} 42 | queryEscape : String -> String 43 | queryEscape string = 44 | string 45 | |> encodeUri 46 | |> String.split "%20" 47 | |> String.join "+" 48 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0041_migrate_data_to_chamber_of_deputies_app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.5 on 2017-10-14 07:45 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0040_create_gin_index_with_search_vector'), 12 | ] 13 | 14 | database_operations = [ 15 | migrations.AlterModelTable('Reimbursement', 'chamber_of_deputies_reimbursement'), 16 | migrations.AlterModelTable('HistoricalReimbursement', 'chamber_of_deputies_historicalreimbursement'), 17 | migrations.AlterModelTable('Tweet', 'chamber_of_deputies_tweet'), 18 | ] 19 | 20 | state_operations = [ 21 | migrations.DeleteModel( 22 | name='Reimbursement', 23 | ), 24 | migrations.DeleteModel( 25 | name='HistoricalReimbursement', 26 | ), 27 | migrations.DeleteModel( 28 | name='Tweet', 29 | ), 30 | ] 31 | 32 | operations = [ 33 | migrations.SeparateDatabaseAndState( 34 | database_operations=database_operations, 35 | state_operations=state_operations) 36 | ] 37 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/classifiers/irregular_companies_classifier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.base import TransformerMixin 3 | 4 | 5 | class IrregularCompaniesClassifier(TransformerMixin): 6 | """ 7 | Irregular Companies classifier. 8 | 9 | Check for the official state of the company in the 10 | Brazilian Federal Revenue and reports for rows with companies unauthorized 11 | to sell products or services. 12 | 13 | Dataset 14 | ------- 15 | issue_date : datetime column 16 | Date when the expense was made. 17 | 18 | situation : string column 19 | Situation of the company according to the Brazilian Federal Revenue. 20 | 21 | situation_date : datetime column 22 | Date when the situation was last updated. 23 | """ 24 | 25 | def fit(self, X): 26 | return self 27 | 28 | def transform(self, X=None): 29 | return self 30 | 31 | def predict(self, X): 32 | statuses = ['BAIXADA', 'NULA', 'SUSPENSA', 'INAPTA'] 33 | self._X = X.apply(self.__compare_date, axis=1) 34 | return np.r_[self._X & X['situation'].isin(statuses)] 35 | 36 | def __compare_date(self, row): 37 | return (row['situation_date'] < row['issue_date']) 38 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/Suspicion.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.Suspicion exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | mealPriceOutlier : TranslationSet 7 | mealPriceOutlier = 8 | TranslationSet 9 | "Meal price is an outlier" 10 | "Preço de refeição muito incomum" 11 | 12 | 13 | overMonthlySubquoteLimit : TranslationSet 14 | overMonthlySubquoteLimit = 15 | TranslationSet 16 | "Expenses over the (sub)quota limit" 17 | "Extrapolou limita da (sub)quota" 18 | 19 | 20 | suspiciousTraveledSpeedDay : TranslationSet 21 | suspiciousTraveledSpeedDay = 22 | TranslationSet 23 | "Many expenses in different cities at the same day" 24 | "Muitas despesas em diferentes cidades no mesmo dia" 25 | 26 | 27 | invalidCpfCnpj : TranslationSet 28 | invalidCpfCnpj = 29 | TranslationSet 30 | "Invalid CNPJ or CPF" 31 | "CPF ou CNPJ inválidos" 32 | 33 | 34 | electionExpenses : TranslationSet 35 | electionExpenses = 36 | TranslationSet 37 | "Expense in electoral campaign" 38 | "Gasto com campanha eleitoral" 39 | 40 | 41 | irregularCompany : TranslationSet 42 | irregularCompany = 43 | TranslationSet 44 | "Irregular company" 45 | "CNPJ irregular" 46 | -------------------------------------------------------------------------------- /docker-compose.prod.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | 4 | django: 5 | volumes: 6 | - assets:/code/staticfiles 7 | 8 | elm: 9 | volumes: 10 | - assets:/code/jarbas/layers/static 11 | 12 | logging: 13 | command: ["syslog+tls://logs2.papertrailapp.com:54868"] 14 | image: gliderlabs/logspout 15 | restart: always 16 | volumes: 17 | - /var/run/docker.sock:/var/run/docker.sock 18 | 19 | nginx-letsencrypt: 20 | image: jrcs/letsencrypt-nginx-proxy-companion 21 | restart: always 22 | volumes: 23 | - /var/run/certs:/etc/nginx/certs:rw 24 | - /var/run/conf.d:/etc/nginx/conf.d 25 | - /var/run/docker.sock:/var/run/docker.sock:ro 26 | - /var/run/html:/usr/share/nginx/html 27 | - /var/run/vhost.d:/etc/nginx/vhost.d 28 | 29 | nginx-proxy: 30 | image: jwilder/nginx-proxy 31 | labels: 32 | - "com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy" 33 | ports: 34 | - "80:80" 35 | - "443:443" 36 | restart: always 37 | volumes: 38 | - /var/run/certs:/etc/nginx/certs:rw 39 | - /var/run/conf.d:/etc/nginx/conf.d 40 | - /var/run/docker.sock:/tmp/docker.sock:ro 41 | - /var/run/html:/usr/share/nginx/html 42 | - /var/run/vhost.d:/etc/nginx/vhost.d 43 | 44 | volumes: 45 | assets: 46 | -------------------------------------------------------------------------------- /jarbas/celery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from celery import Celery 4 | from celery.schedules import crontab 5 | 6 | from django.conf import settings 7 | 8 | logger = logging.getLogger('celery') 9 | 10 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'jarbas.settings') 11 | 12 | app = Celery('jarbas') 13 | app.config_from_object('django.conf:settings', namespace='CELERY') 14 | app.autodiscover_tasks() 15 | 16 | 17 | @app.on_after_configure.connect 18 | def setup_periodic_tasks(sender, **kwargs): 19 | from django.core import management 20 | 21 | @app.task(ignore_result=True) 22 | def searchvector(): 23 | logger.info('Running searchvector...') 24 | management.call_command('searchvector') 25 | logger.info('Searchvector is done') 26 | 27 | if not settings.SCHEDULE_SEARCHVECTOR: 28 | return 29 | 30 | sender.add_periodic_task( 31 | crontab( 32 | minute=settings.SCHEDULE_SEARCHVECTOR_CRON_MINUTE, 33 | hour=settings.SCHEDULE_SEARCHVECTOR_CRON_HOUR, 34 | day_of_week=settings.SCHEDULE_SEARCHVECTOR_CRON_DAY_OF_WEEK, 35 | day_of_month=settings.SCHEDULE_SEARCHVECTOR_CRON_DAY_OF_MONTH, 36 | month_of_year=settings.SCHEDULE_SEARCHVECTOR_CRON_MONTH_OF_YEAR, 37 | ), 38 | searchvector.s(), 39 | ) 40 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0003_remove_some_indexes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.1 on 2016-09-14 13:44 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0002_add_indexes'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='document', 17 | name='document_value', 18 | field=models.DecimalField(decimal_places=3, max_digits=10, verbose_name='Document value'), 19 | ), 20 | migrations.AlterField( 21 | model_name='document', 22 | name='net_value', 23 | field=models.DecimalField(decimal_places=3, max_digits=10, verbose_name='Net value'), 24 | ), 25 | migrations.AlterField( 26 | model_name='document', 27 | name='reimbursement_value', 28 | field=models.DecimalField(decimal_places=3, max_digits=10, verbose_name='Reimbusrsement value'), 29 | ), 30 | migrations.AlterField( 31 | model_name='document', 32 | name='remark_value', 33 | field=models.DecimalField(decimal_places=3, max_digits=10, verbose_name='Remark value'), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/tasks.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | from jarbas.chamber_of_deputies.fields import ArrayField, DateAsStringField, FloatField, IntegerField 3 | from jarbas.chamber_of_deputies.models import Reimbursement 4 | 5 | 6 | INTEGERS = ( 7 | 'applicant_id', 8 | 'batch_number', 9 | 'congressperson_document', 10 | 'congressperson_id', 11 | 'document_id', 12 | 'document_type', 13 | 'installment', 14 | 'month', 15 | 'subquota_group_id', 16 | 'subquota_number', 17 | 'term', 18 | 'term_id', 19 | 'year' 20 | ) 21 | 22 | FLOATS = ( 23 | 'document_value', 24 | 'remark_value', 25 | 'total_net_value', 26 | 'total_value' 27 | ) 28 | 29 | TYPES = tuple(chain( 30 | ((field, IntegerField) for field in INTEGERS), 31 | ((field, FloatField) for field in FLOATS), 32 | (('issue_date', DateAsStringField),), 33 | (('numbers', ArrayField),), 34 | )) 35 | 36 | 37 | def serialize(row): 38 | """Read the dict generated by the reimbursement command and returns a 39 | Reimbursement model instance.""" 40 | for key, type_ in TYPES: 41 | value = row.get(key) 42 | row[key] = type_.deserialize(value) 43 | 44 | for field in FLOATS: 45 | row[field] = row[field] if row[field] else 0.0 46 | 47 | if row['issue_date']: 48 | return Reimbursement(**row) 49 | -------------------------------------------------------------------------------- /rosie/rosie/core/classifiers/invalid_cnpj_cpf_classifier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.base import TransformerMixin 3 | from brutils import cpf, cnpj 4 | 5 | 6 | class InvalidCnpjCpfClassifier(TransformerMixin): 7 | """ 8 | Invalid CNPJ/CPF classifier. 9 | 10 | Validate a `recipient_id` field by calculating its expected check digit 11 | and verifying the authenticity of the provided ones. 12 | 13 | Dataset 14 | ------- 15 | document_type : category column 16 | Validate rows with values 'bill_of_sale' or 'simple_receipt' or 'unknown'. 17 | 'unknown' value is used on Federal Senate data. 18 | 19 | recipient_id : string column 20 | A CNPJ (Brazilian company ID) or CPF (Brazilian personal tax ID). 21 | """ 22 | def fit(self, dataframe): 23 | return self 24 | 25 | def transform(self, dataframe=None): 26 | return self 27 | 28 | def predict(self, dataframe): 29 | def is_invalid(row): 30 | valid_cpf = cpf.validate(str(row['recipient_id']).zfill(11)) 31 | valid_cnpj = cnpj.validate(str(row['recipient_id']).zfill(14)) 32 | good_doctype = row['document_type'] in ('bill_of_sale', 'simple_receipt', 'unknown') 33 | return good_doctype and (not (valid_cpf or valid_cnpj)) 34 | return np.r_[dataframe.apply(is_invalid, axis=1)] 35 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Format/Price.elm: -------------------------------------------------------------------------------- 1 | module Format.Price exposing (..) 2 | 3 | import FormatNumber exposing (Locale, formatFloat) 4 | import Internationalization exposing (translate) 5 | import Internationalization.Types exposing (Language(..), TranslationId(..)) 6 | import String 7 | 8 | 9 | formatPrice : Language -> Float -> String 10 | formatPrice lang price = 11 | let 12 | locale : Locale 13 | locale = 14 | Locale 15 | 2 16 | (translate lang ThousandSeparator) 17 | (translate lang DecimalSeparator) 18 | in 19 | formatFloat locale price 20 | |> BrazilianCurrency 21 | |> translate lang 22 | 23 | 24 | formatPrices : Language -> List Float -> String 25 | formatPrices lang prices = 26 | List.map (formatPrice lang) prices 27 | |> String.join ", " 28 | 29 | 30 | maybeFormatPrice : Language -> Maybe Float -> String 31 | maybeFormatPrice lang maybePrice = 32 | case maybePrice of 33 | Just price -> 34 | formatPrice lang price 35 | 36 | Nothing -> 37 | "" 38 | 39 | 40 | maybeFormatPrices : Language -> Maybe (List Float) -> String 41 | maybeFormatPrices lang maybePrices = 42 | case maybePrices of 43 | Just prices -> 44 | formatPrices lang prices 45 | 46 | Nothing -> 47 | "" 48 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0010_extract_receipt.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.1 on 2016-09-29 16:07 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('core', '0009_add_latitude_and_longitude'), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Receipt', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('url', models.URLField(blank=True, default=None, max_length=128, null=True, verbose_name='URL')), 21 | ('fetched', models.BooleanField(default=False, verbose_name='Was fetched?')), 22 | ], 23 | ), 24 | migrations.RemoveField( 25 | model_name='document', 26 | name='receipt_fetched', 27 | ), 28 | migrations.RemoveField( 29 | model_name='document', 30 | name='receipt_url', 31 | ), 32 | migrations.AddField( 33 | model_name='receipt', 34 | name='document', 35 | field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='core.Document'), 36 | ), 37 | ] 38 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | dist: trusty 3 | 4 | language: python 5 | python: 3.6 6 | cache: pip 7 | 8 | addons: 9 | postgresql: "9.6" 10 | 11 | services: 12 | - postgresql 13 | 14 | env: 15 | global: 16 | - AMAZON_BUCKET=serenata-de-amor-data 17 | - AMAZON_REGION=sa-east-1 18 | - DATABASE_URL=postgres://postgres@localhost/jarbas 19 | - CACHE_BACKEND=django.core.cache.backends.dummy.DummyCache 20 | - CC_TEST_REPORTER_ID=351fd43955e403ad2d8ef0c653d4238d1afafdf55396622d3631d3dc1e06d635 21 | 22 | install: 23 | - cp contrib/.env.sample .env 24 | - python -m pip install -U pip coverage 25 | - psql -U postgres -c 'create database "jarbas";' 26 | - python -m pip install -r requirements-dev.txt 27 | - python -m pip install -r rosie/requirements.txt 28 | 29 | before_script: 30 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 31 | - chmod +x ./cc-test-reporter 32 | - ./cc-test-reporter before-build 33 | - python manage.py migrate 34 | - python manage.py collectstatic --no-input 35 | 36 | script: 37 | - coverage run -a manage.py test 38 | - cd rosie 39 | - coverage run -a rosie.py test 40 | - coverage run -a rosie.py run federal_senate 41 | 42 | after_success: 43 | - coverage xml 44 | - ./cc-test-reporter after-build --coverage-input-type coverage.py --exit-code $TRAVIS_TEST_RESULT 45 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0007_auto_20180913_0209.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.8 on 2018-09-13 05:09 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('chamber_of_deputies', '0006_change_on_delete_social_media_to_set_null'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='reimbursement', 16 | name='document_id', 17 | field=models.IntegerField(db_index=True, verbose_name='Número do Reembolso'), 18 | ), 19 | migrations.AlterField( 20 | model_name='reimbursement', 21 | name='issue_date', 22 | field=models.DateField(blank=True, db_index=True, null=True, verbose_name='Data de Emissão'), 23 | ), 24 | migrations.AlterField( 25 | model_name='reimbursement', 26 | name='numbers', 27 | field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128, verbose_name='Números dos Ressarcimentos'), default=list, size=None), 28 | ), 29 | migrations.AlterField( 30 | model_name='reimbursement', 31 | name='supplier', 32 | field=models.CharField(max_length=256, verbose_name='Fornecedor'), 33 | ), 34 | ] 35 | -------------------------------------------------------------------------------- /contrib/data/README.md: -------------------------------------------------------------------------------- 1 | This directory contains sample data, useful if you just want to setup a developement environment. 2 | 3 | This is **not** the full dataset. 4 | 5 | These samples were created with a script like this (`pandas` and `numpy` required and **not** included in this repository): 6 | 7 | 8 | ```python 9 | import re 10 | 11 | import pandas as pd 12 | import numpy as np 13 | 14 | 15 | CSV_PARAMS = { 16 | 'compression': 'xz', 17 | 'encoding': 'utf-8', 18 | 'index': False 19 | } 20 | 21 | # read all datasets 22 | reimbursements = pd.read_csv('reimbursements.xz', dtype={ 'cnpj_cpf': np.str, 'reimbursement_numbers': np.str }) 23 | companies = pd.read_csv('companies.xz', dtype={ 'cnpj': np.str }) 24 | suspicions = pd.read_csv('suspicions.xz') 25 | 26 | # get a sample of the reimbursements 27 | sample = reimbursements.sample(1000) 28 | sample.to_csv('reimbursements_sample.xz', **CSV_PARAMS) 29 | 30 | # filter companies present in the sample 31 | companies['cnpj_'] = companies.cnpj.apply(lambda x: re.sub(r'\D', '', x)) 32 | companies_sample = companies[companies.cnpj_.isin(sample.cnpj_cpf)] 33 | del companies_sample['cnpj_'] 34 | companies_sample.to_csv('companies_sample.xz', **CSV_PARAMS) 35 | 36 | # filter suspicions present in the sample 37 | suspicions_sample = suspicions[suspicions.document_id.isin(sample.document_id)] 38 | suspicions_sample.to_csv('suspicions_sample.xz', **CSV_PARAMS) 39 | ``` 40 | -------------------------------------------------------------------------------- /jarbas/dashboard/static/dashboard.css: -------------------------------------------------------------------------------- 1 | #content-main .changelink, 2 | #content-main .submit-row, 3 | #content-main p, 4 | #historicalreimbursement_form p, 5 | #recent-actions-module { 6 | display: none 7 | } 8 | 9 | #content-main p { 10 | display: block 11 | } 12 | 13 | td.field-value { 14 | white-space: nowrap 15 | } 16 | 17 | .required label, 18 | label.required { 19 | color: #666; 20 | font-weight: normal 21 | } 22 | 23 | #changelist-form .results { 24 | width: calc(calc(100% - 240px) - 1rem); /* 240px is the filter tab width */ 25 | } 26 | 27 | .bar-chart { 28 | display: flex; 29 | height: 160px; 30 | justify-content: space-around; 31 | overflow: hidden; 32 | padding-top: 60px; /* csslint allow: box-model */ 33 | } 34 | 35 | .bar-chart .bar { 36 | align-self: flex-end; 37 | background-color: #79aec8; 38 | flex: 100%; 39 | margin-right: 2px; 40 | position: relative; 41 | } 42 | 43 | .bar-chart .bar:last-child { 44 | margin: 0; 45 | } 46 | 47 | .bar-chart .bar:hover { 48 | background-color: #417690; 49 | } 50 | 51 | .bar-chart .bar .bar-tooltip { 52 | position: relative; 53 | z-index: 999; 54 | } 55 | 56 | .bar-chart .bar .bar-tooltip { 57 | font-size: 0.68rem; 58 | left: 50%; 59 | opacity: 0; 60 | position: absolute; 61 | text-align: center; 62 | top: -3rem; 63 | transform: translateX(-50%); 64 | } 65 | 66 | .bar-chart .bar:hover .bar-tooltip { 67 | opacity: 1; 68 | } 69 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0005_create_social_media_model.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.8 on 2018-09-13 03:25 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('chamber_of_deputies', '0004_alter_field_names_following_toolbox_renamings'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='SocialMedia', 17 | fields=[ 18 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 19 | ('congressperson_name', models.CharField(blank=True, default='', max_length=255)), 20 | ('congressperson_id', models.IntegerField(blank=True, db_index=True, null=True)), 21 | ('twitter_profile', models.CharField(blank=True, default='', max_length=255)), 22 | ('secondary_twitter_profile', models.CharField(blank=True, default='', max_length=255)), 23 | ('facebook_page', models.CharField(blank=True, default='', max_length=255)), 24 | ], 25 | ), 26 | migrations.AddField( 27 | model_name='reimbursement', 28 | name='social_media', 29 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='chamber_of_deputies.SocialMedia'), 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /contrib/.env.sample: -------------------------------------------------------------------------------- 1 | ALLOWED_HOSTS=* 2 | CACHE_BACKEND=django.core.cache.backends.memcached.MemcachedCache 3 | ENVIRONMENT=development 4 | HTTPS_METHOD=redirect 5 | LOG_LEVEL=debug 6 | SECRET_KEY=my-secret 7 | USE_X_FORWARDED_HOST=False 8 | WEB_TIMEOUT=60 9 | WEB_WORKERS=2 10 | 11 | VIRTUAL_HOST=jarbas.serenata.ai 12 | VIRTUAL_PROTO=http 13 | LETSENCRYPT_EMAIL=contato@serenata.ai 14 | 15 | POSTGRES_PASSWORD=mysecretpassword 16 | POSTGRES_USER=jarbas 17 | POSTGRES_DB=jarbas 18 | 19 | DATABASE_URL=postgres://jarbas:mysecretpassword@postgres/jarbas 20 | CELERY_BROKER_URL=amqp://guest:guest@queue/ 21 | CACHE_LOCATION=localhost:11211 22 | LOGGING_URL= 23 | 24 | AMAZON_ACCESS_KEY= 25 | AMAZON_BUCKET=serenata-de-amor-data 26 | AMAZON_REGION=sa-east-1 27 | AMAZON_SECRET_KEY= 28 | 29 | GOOGLE_API_KEY= 30 | GOOGLE_STREET_VIEW_API_KEY= 31 | 32 | FOURSQUARE_CLIENT_ID= 33 | FOURSQUARE_CLIENT_SECRET= 34 | 35 | NEW_RELIC_DEVELOPER_MODE=true 36 | NEW_RELIC_ENVIRONMENT=development 37 | NEW_RELIC_LICENSE_KEY= 38 | 39 | TWITTER_CONSUMER_KEY= 40 | TWITTER_CONSUMER_SECRET= 41 | TWITTER_ACCESS_TOKEN= 42 | TWITTER_ACCESS_SECRET= 43 | 44 | YELP_ACCESS_TOKEN= 45 | 46 | NEW_RELIC_DEVELOPER_MODE=true 47 | NEW_RELIC_ENVIRONMENT=development 48 | NEW_RELIC_LICENSE_KEY= 49 | 50 | INBOX_PASSWORD= 51 | 52 | SCHEDULE_SEARCHVECTOR=True 53 | SCHEDULE_SEARCHVECTOR_CRON_MINUTE=0 54 | SCHEDULE_SEARCHVECTOR_CRON_HOUR=2 55 | SCHEDULE_SEARCHVECTOR_CRON_DAY_OF_WEEK=* 56 | SCHEDULE_SEARCHVECTOR_CRON_DAY_OF_MONTH=*/2 57 | SCHEDULE_SEARCHVECTOR_CRON_MONTH_OF_YEAR=* 58 | 59 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/fields.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from rows import fields 4 | 5 | 6 | class FloatField(fields.FloatField): 7 | 8 | @classmethod 9 | def deserialize(cls, value, *args, **kwargs): 10 | try: # Rows cannot convert values such as '14,96' to float 11 | value = float(value.replace(',', '.')) 12 | except: 13 | pass 14 | return super(FloatField, cls).deserialize(value) 15 | 16 | 17 | class IntegerField(fields.IntegerField): 18 | 19 | @classmethod 20 | def deserialize(cls, value, *args, **kwargs): 21 | try: # Rows cannot convert values such as '2011.0' to integer 22 | value = int(float(value)) 23 | except: 24 | pass 25 | return super(IntegerField, cls).deserialize(value) 26 | 27 | 28 | class DateAsStringField(fields.DateField): 29 | INPUT_FORMAT = '%Y-%m-%d %H:%M:%S' 30 | OUTPUT_FORMAT = '%Y-%m-%d' 31 | 32 | @classmethod 33 | def deserialize(cls, value, *args, **kwargs): 34 | value = value.replace('T', ' ') # normalize date/time separator 35 | return super(DateAsStringField, cls).deserialize(value) 36 | 37 | 38 | class ArrayField(fields.JSONField): 39 | TYPE = (list,) 40 | 41 | @classmethod 42 | def deserialize(cls, value, *args, **kwargs): 43 | value = value.replace('\'', '"').replace('nan', 'null') 44 | if value is None or isinstance(value, cls.TYPE): 45 | return value 46 | else: 47 | return json.loads(value) 48 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | 4 | cache: 5 | image: memcached:1.5.8-alpine 6 | restart: always 7 | 8 | elm: 9 | hostname: elm 10 | image: serenata/elm 11 | restart: on-failure 12 | 13 | django: 14 | command: ["gunicorn", "jarbas.wsgi:application", "--reload", "--bind", "0.0.0.0:8000", "--workers", $WEB_WORKERS, "--log-level", $LOG_LEVEL, "--timeout", $WEB_TIMEOUT] 15 | depends_on: 16 | - cache 17 | - elm 18 | - tasks 19 | env_file: 20 | - .env 21 | environment: 22 | - NEW_RELIC_APP_NAME=Jarbas (Django); Jarbas (Combined) 23 | hostname: django 24 | image: serenata/django 25 | restart: always 26 | 27 | queue: 28 | hostname: queue 29 | image: rabbitmq:3.7.3-alpine 30 | restart: on-failure 31 | 32 | rosie: 33 | image: serenata/rosie 34 | 35 | tasks: 36 | command: ["newrelic-admin", "run-program", "celery", "worker", "--app", "jarbas"] 37 | depends_on: 38 | - queue 39 | env_file: 40 | - .env 41 | environment: 42 | - NEW_RELIC_APP_NAME=Jarbas (Celery); Jarbas (Combined) 43 | hostname: tasks 44 | image: serenata/django 45 | restart: always 46 | 47 | beat: 48 | command: ["newrelic-admin", "run-program", "celery", "beat", "--app", "jarbas"] 49 | depends_on: 50 | - queue 51 | env_file: 52 | - .env 53 | environment: 54 | - NEW_RELIC_APP_NAME=Jarbas (Beat); Jarbas (Combined) 55 | hostname: beat 56 | image: serenata/django 57 | restart: always 58 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Map/View.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Map.View exposing (view) 2 | 3 | import Html exposing (a, text) 4 | import Html.Attributes exposing (href, target) 5 | import Internationalization exposing (translate) 6 | import Internationalization.Types exposing (Language(..), TranslationId(..)) 7 | import Material.Button as Button 8 | import Material.Icon as Icon 9 | import Reimbursement.Map.Model exposing (Model) 10 | import Reimbursement.Map.Update exposing (Msg(Mdl)) 11 | import String 12 | 13 | 14 | view : Model -> Html.Html Msg 15 | view model = 16 | case model.geoCoord.latitude of 17 | Just lat -> 18 | case model.geoCoord.longitude of 19 | Just long -> 20 | let 21 | mapUrl = 22 | String.concat [ "https://ddg.gg/?q=!gm+", lat, ",", long ] 23 | in 24 | a 25 | [ href mapUrl, target "_blank" ] 26 | [ Button.render 27 | Mdl 28 | [ 0 ] 29 | model.mdl 30 | [ Button.minifab, Button.primary ] 31 | [ Icon.i "place" 32 | , text (translate model.lang Map) 33 | ] 34 | ] 35 | 36 | Nothing -> 37 | text "" 38 | 39 | Nothing -> 40 | text "" 41 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/tests/test_receipt_class.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from django.test import TestCase 4 | from requests.exceptions import ConnectionError 5 | 6 | from jarbas.chamber_of_deputies.models import Receipt 7 | 8 | 9 | class TestReceipt(TestCase): 10 | 11 | def setUp(self): 12 | self.receipt = Receipt(1970, 13, 42, 1) 13 | self.electronic_receipt = Receipt(1970, 13, 42, 4) 14 | 15 | def test_url(self): 16 | expected = 'http://www.camara.gov.br/cota-parlamentar/documentos/publ/13/1970/42.pdf' 17 | self.assertEqual(expected, self.receipt.url) 18 | 19 | def test_electronic_url(self): 20 | expected = ('https://www.camara.leg.br/cota-parlamentar/nota-fiscal-eletronica?' 21 | 'ideDocumentoFiscal=42') 22 | self.assertEqual(expected, self.electronic_receipt.url) 23 | 24 | @patch('jarbas.chamber_of_deputies.models.head') 25 | def test_existing_url(self, mocked_head): 26 | mocked_head.return_value.status_code = 200 27 | self.assertTrue(self.receipt.exists) 28 | 29 | @patch('jarbas.chamber_of_deputies.models.head') 30 | def test_no_existing_url(self, mocked_head): 31 | mocked_head.return_value.status_code = 404 32 | self.assertFalse(self.receipt.exists) 33 | 34 | @patch('jarbas.chamber_of_deputies.models.head') 35 | def test_connection_error(self, mocked_head): 36 | mocked_head.side_effect = ConnectionError 37 | with self.assertRaises(ConnectionError): 38 | self.receipt.exists 39 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0017_make_some_fields_optional.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2016-12-11 09:58 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0016_add_custom_ordering_to_reimbursement'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='reimbursement', 17 | name='document_number', 18 | field=models.CharField(blank=True, max_length=140, null=True, verbose_name='Document number'), 19 | ), 20 | migrations.AlterField( 21 | model_name='reimbursement', 22 | name='party', 23 | field=models.CharField(blank=True, db_index=True, max_length=7, null=True, verbose_name='Party'), 24 | ), 25 | migrations.AlterField( 26 | model_name='reimbursement', 27 | name='state', 28 | field=models.CharField(blank=True, db_index=True, max_length=2, null=True, verbose_name='State'), 29 | ), 30 | migrations.AlterField( 31 | model_name='reimbursement', 32 | name='term', 33 | field=models.IntegerField(blank=True, null=True, verbose_name='Term'), 34 | ), 35 | migrations.AlterField( 36 | model_name='reimbursement', 37 | name='term_id', 38 | field=models.IntegerField(blank=True, null=True, verbose_name='Term ID'), 39 | ), 40 | ] 41 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/Reimbursement/Fieldset.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.Reimbursement.Fieldset exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | currencyDetails : TranslationSet 7 | currencyDetails = 8 | TranslationSet 9 | "Expense made abroad: " 10 | "Despesa feita no exterior " 11 | 12 | 13 | detailsLink : TranslationSet 14 | detailsLink = 15 | TranslationSet 16 | "check the currency rate on " 17 | "veja a cotação em " 18 | 19 | 20 | companyDetails : TranslationSet 21 | companyDetails = 22 | TranslationSet 23 | "If we can find the CNPJ of this supplier in our database more info will be available in the sidebar." 24 | "Se o CNPJ estiver no nosso banco de dados mais detalhes sobre o fornecedor aparecerão ao lado." 25 | 26 | 27 | summary : TranslationSet 28 | summary = 29 | TranslationSet 30 | "Summary" 31 | "Resumo" 32 | 33 | 34 | reimbursement : TranslationSet 35 | reimbursement = 36 | TranslationSet 37 | "Reimbursement details" 38 | "Detalhes do reembolso" 39 | 40 | 41 | congressperson : TranslationSet 42 | congressperson = 43 | TranslationSet 44 | "Congressperson details" 45 | "Detalhes do(a) deputado(a)" 46 | 47 | 48 | congresspersonProfile : TranslationSet 49 | congresspersonProfile = 50 | TranslationSet 51 | "Congressperson profile" 52 | "Perfil do(a) deputado(a)" 53 | 54 | 55 | trip : TranslationSet 56 | trip = 57 | TranslationSet 58 | "Ticket details" 59 | "Detalhes da passagem" 60 | -------------------------------------------------------------------------------- /jarbas/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serenata URL Configuration 3 | 4 | The `urlpatterns` list routes URLs to views. For more information please see: 5 | https://docs.djangoproject.com/en/2.0/ref/urls/#django.urls.path 6 | Examples: 7 | Function views 8 | 1. Add an import: from my_app import views 9 | 2. Add a URL to urlpatterns: path('home/', views.home, name='home') 10 | Class-based views 11 | 1. Add an import: from other_app.views import Home 12 | 2. Add a URL to urlpatterns: path('home/', Home.as_view(), name='home') 13 | Including another URLconf 14 | 1. Import the include() function: from django.conf.urls import url, include 15 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 16 | """ 17 | from django.conf import settings 18 | from django.conf.urls import include 19 | from django.shortcuts import redirect 20 | from django.urls import path 21 | 22 | from jarbas.core.views import healthcheck 23 | 24 | 25 | urlpatterns = [ 26 | path('', lambda _: redirect(settings.HOMES_REDIRECTS_TO), name='home'), 27 | path('dashboard/', include('jarbas.dashboard.urls')), 28 | path('layers/', include('jarbas.layers.urls', namespace='layers')), 29 | path('api/', include('jarbas.core.urls', namespace='core')), 30 | path('api/chamber_of_deputies/', 31 | include( 32 | 'jarbas.chamber_of_deputies.urls', 33 | namespace='chamber_of_deputies')), 34 | path('healthcheck/', healthcheck, name='healthcheck'), 35 | ] 36 | 37 | if settings.LOG_LEVEL == 'debug': 38 | import debug_toolbar 39 | urlpatterns = [ 40 | path('__debug__/', include(debug_toolbar.urls)), 41 | ] + urlpatterns 42 | -------------------------------------------------------------------------------- /jarbas/public_admin/tests/test_public_admin_site.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch 2 | 3 | from django.contrib.auth import get_user_model 4 | from django.test import TestCase 5 | 6 | from jarbas.public_admin.sites import PublicAdminSite, public_admin 7 | 8 | User = get_user_model() 9 | 10 | 11 | class TestPublicAdminSite(TestCase): 12 | 13 | def setUp(self): 14 | self.site = public_admin 15 | 16 | def test_init(self): 17 | self.assertEqual({}, dict(self.site.actions)) 18 | self.assertEqual({}, dict(self.site._global_actions)) 19 | self.assertEqual('dashboard', self.site.name) 20 | 21 | def test_valid_url(self): 22 | valid, invalid = MagicMock(), MagicMock() 23 | valid.pattern.regex.pattern = '/whatever/' 24 | invalid.pattern.regex.pattern = '/whatever/add/' 25 | self.assertTrue(self.site.valid_url(valid)) 26 | self.assertFalse(self.site.valid_url(invalid)) 27 | 28 | @patch.object(PublicAdminSite, 'get_urls') 29 | @patch.object(PublicAdminSite, 'valid_url') 30 | def test_urls(self, valid_url, get_urls): 31 | valid_url.side_effect = (True, False, True) 32 | get_urls.return_value = range(3) 33 | expected = [0, 2], 'admin', 'dashboard' 34 | self.assertEqual(expected, self.site.urls) 35 | 36 | def test_has_permission_get(self): 37 | request = MagicMock() 38 | request.method = 'GET' 39 | self.assertTrue(self.site.has_permission(request)) 40 | 41 | def test_has_permission_post(self): 42 | request = MagicMock() 43 | request.method = 'POST' 44 | self.assertFalse(self.site.has_permission(request)) 45 | -------------------------------------------------------------------------------- /rosie/rosie/federal_senate/adapter.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import pandas as pd 5 | 6 | from serenata_toolbox.federal_senate.dataset import Dataset 7 | 8 | COLUMNS = { 9 | 'net_value': 'reimbursement_value', 10 | 'recipient_id': 'cnpj_cpf', 11 | 'recipient': 'supplier', 12 | } 13 | 14 | 15 | class Adapter: 16 | 17 | def __init__(self, path): 18 | self.path = path 19 | 20 | @property 21 | def dataset(self): 22 | path = self.update_datasets() 23 | self._dataset = pd.read_csv(path, dtype={'cnpj_cpf': np.str}, encoding='utf-8') 24 | self.prepare_dataset() 25 | return self._dataset 26 | 27 | def prepare_dataset(self): 28 | self.drop_null_cnpj_cpf() 29 | self.rename_columns() 30 | self.create_columns() 31 | 32 | def drop_null_cnpj_cpf(self): 33 | self._dataset[self._dataset['cnpj_cpf'].notnull()] 34 | 35 | def rename_columns(self): 36 | columns = {v: k for k, v in COLUMNS.items()} 37 | self._dataset.rename(columns=columns, inplace=True) 38 | 39 | def create_columns(self): 40 | # Federate Senate Reimbursments do not have document_type column which 41 | # is required by Rosie's core module, so we add all of them as 'unknown' 42 | self._dataset['document_type'] = 'unknown' 43 | 44 | def update_datasets(self): 45 | os.makedirs(self.path, exist_ok=True) 46 | federal_senate = Dataset(self.path) 47 | federal_senate.fetch() 48 | federal_senate.translate() 49 | federal_senate_reimbursements_path = federal_senate.clean() 50 | 51 | return federal_senate_reimbursements_path 52 | -------------------------------------------------------------------------------- /contrib/update/README.md: -------------------------------------------------------------------------------- 1 | # Serenata Update 2 | 3 | This directory contains files to automatically run Rosie and update Jarbas. 4 | 5 | ## Requirements 6 | 7 | * [Python](https://python.org) 2 binary available (see [Warning](#warning)) 8 | * [Pipenv](https://pipenv.readthedocs.io/) 9 | 10 | Then install the required Python packages: 11 | 12 | ```console 13 | $ pipenv install 14 | ``` 15 | 16 | ## Settings 17 | 18 | Copy `.env.sample` as `.env` and set the following variables: 19 | 20 | | Name | Description | 21 | |:-----|:------------| 22 | | `DO_API_TOKEN` | DigitalOcean' API token | 23 | | `DO_SSH_KEY_NAME` | Name of a SSH key registered at DigitalOcean | 24 | | `DATABASE_URL`| Credentials for Jarba's production database | 25 | 26 | ## Running 27 | 28 | To run Rosie and automatically update Jarbas: 29 | 30 | ```console 31 | $ pipenv run ansible-playbook update.yml 32 | ``` 33 | 34 | ```console 35 | $ pipenv run python cleanup.py 36 | ``` 37 | 38 | ## Warning 39 | 40 | This module is based on Python 2 since: 41 | 42 | * The [`dopy`](https://pypi.org/project/dopy/) Python package Ansible depends on is 43 | [only available in Python 2](https://github.com/Wiredcraft/dopy/issues/61), and is not updated in ages 44 | * We could use a fork, but we 45 | [would need to trust the fork owner](https://github.com/okfn-brasil/serenata-de-amor/pull/449#discussion_r253397600) 46 | * Maintaining a fork is out of our scope at the moment 47 | 48 | However, Ansible is already 49 | [migrating for an alternative package](https://github.com/ansible/ansible/pull/33984) 50 | and soon (before the end of life of Python 2) we will be able to update. This new module is already merged, but not released yet. -------------------------------------------------------------------------------- /jarbas/core/tests/test_company_view.py: -------------------------------------------------------------------------------- 1 | import re 2 | import json 3 | 4 | from django.shortcuts import resolve_url 5 | from django.test import TestCase 6 | 7 | from jarbas.core.models import Activity, Company 8 | from jarbas.core.tests import sample_activity_data, sample_company_data 9 | 10 | 11 | class TestApi(TestCase): 12 | 13 | def setUp(self): 14 | activity = Activity.objects.create(**sample_activity_data) 15 | self.company = Company.objects.create(**sample_company_data) 16 | self.company.main_activity.add(activity) 17 | self.company.save() 18 | cnpj = re.compile(r'\D').sub('', self.company.cnpj) 19 | self.url = resolve_url('core:company-detail', cnpj) 20 | 21 | 22 | class TestGet(TestApi): 23 | 24 | def setUp(self): 25 | super().setUp() 26 | self.resp = self.client.get(self.url) 27 | 28 | def test_status_code(self): 29 | self.assertEqual(200, self.resp.status_code) 30 | 31 | def test_content(self): 32 | response = json.loads(self.resp.content.decode('utf-8')) 33 | self.assertEqual( 34 | '42 - The answer to life, the universe, and everything', 35 | response['legal_entity'] 36 | ) 37 | self.assertEqual('42', response['main_activity'][0]['code']) 38 | self.assertIn('latitude', response) 39 | self.assertIn('longitude', response) 40 | 41 | 42 | class TestGetNonExistentCompany(TestApi): 43 | 44 | def setUp(self): 45 | url = resolve_url('core:company-detail', '42424242424242') 46 | self.resp = self.client.get(url) 47 | 48 | def test_status_code(self): 49 | self.assertEqual(404, self.resp.status_code) 50 | -------------------------------------------------------------------------------- /rosie/rosie.py: -------------------------------------------------------------------------------- 1 | """ 2 | Hi, I am Rosie from Operação Serenata de Amor! 🤖 3 | 4 | I'm a proof-of-concept for the usage of artificial intelligence for social 5 | control of public administration. 6 | 7 | Usage: 8 | rosie.py run (chamber_of_deputies|federal_senate) [--output=] 9 | rosie.py test [chamber_of_deputies|federal_senate|core] 10 | 11 | Options: 12 | --help Show this screen 13 | --output= Output directory [default: /tmp/serenata-data] 14 | """ 15 | import os 16 | import unittest 17 | 18 | from docopt import docopt 19 | 20 | import rosie 21 | import rosie.chamber_of_deputies 22 | import rosie.federal_senate 23 | 24 | 25 | def get_module(arguments): 26 | modules = ('chamber_of_deputies', 'federal_senate', 'core') 27 | for module in modules: 28 | if arguments[module]: 29 | return module 30 | 31 | 32 | def run(module, directory): 33 | module = getattr(rosie, module) 34 | module.main(directory) 35 | 36 | 37 | def test(module=None): 38 | loader = unittest.TestLoader() 39 | tests_path = 'rosie' 40 | 41 | if module: 42 | tests_path = os.path.join(tests_path, module) 43 | 44 | tests = loader.discover(tests_path) 45 | testRunner = unittest.runner.TextTestRunner() 46 | result = testRunner.run(tests) 47 | if not result.wasSuccessful(): 48 | exit(1) 49 | 50 | 51 | def main(): 52 | arguments = docopt(__doc__) 53 | module = get_module(arguments) 54 | 55 | if arguments['test']: 56 | test(module) 57 | 58 | if arguments['run']: 59 | module = module if module != 'core' else None 60 | run(module, arguments['--output']) 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /jarbas/dashboard/admin/widgets.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.forms.widgets import Widget 4 | 5 | from jarbas.dashboard.admin.subquotas import Subquotas 6 | 7 | 8 | class ReceiptUrlWidget(Widget): 9 | 10 | def render(self, name, value, attrs=None, renderer=None): 11 | if not value: 12 | return '' 13 | 14 | url = '
{}
' 15 | return url.format(value, value) 16 | 17 | 18 | class SubquotaWidget(Widget, Subquotas): 19 | 20 | def render(self, name, value, attrs=None, renderer=None): 21 | value = self.pt_br(value) or value 22 | return '
{}
'.format(value) 23 | 24 | 25 | class SuspiciousWidget(Widget): 26 | 27 | SUSPICIONS = ( 28 | 'meal_price_outlier', 29 | 'over_monthly_subquota_limit', 30 | 'suspicious_traveled_speed_day', 31 | 'invalid_cnpj_cpf', 32 | 'election_expenses', 33 | 'irregular_companies_classifier' 34 | ) 35 | 36 | HUMAN_NAMES = ( 37 | 'Preço de refeição muito incomum', 38 | 'Extrapolou limita da (sub)quota', 39 | 'Muitas despesas em diferentes cidades no mesmo dia', 40 | 'CPF ou CNPJ inválidos', 41 | 'Gasto com campanha eleitoral', 42 | 'CNPJ irregular' 43 | ) 44 | 45 | MAP = dict(zip(SUSPICIONS, HUMAN_NAMES)) 46 | 47 | def render(self, name, value, attrs=None, renderer=None): 48 | value_as_dict = json.loads(value) 49 | if not value_as_dict: 50 | return '' 51 | 52 | values = (self.MAP.get(k, k) for k in value_as_dict.keys()) 53 | suspicions = '
'.join(values) 54 | return '
{}
'.format(suspicions) 55 | -------------------------------------------------------------------------------- /rosie/rosie/federal_senate/tests/test_adapter.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from tempfile import mkdtemp 4 | from unittest import TestCase 5 | from unittest.mock import patch 6 | 7 | import pandas as pd 8 | 9 | from rosie.federal_senate.adapter import COLUMNS as ADAPTER_COLUMNS 10 | from rosie.federal_senate.adapter import Adapter as subject_class 11 | 12 | FIXTURE_PATH = os.path.join('rosie', 13 | 'federal_senate', 14 | 'tests', 15 | 'fixtures', 16 | 'federal_senate_reimbursements.xz') 17 | 18 | 19 | class TestAdapter(TestCase): 20 | 21 | def setUp(self): 22 | self.temp_path = mkdtemp() 23 | subject = subject_class(self.temp_path) 24 | with patch.object(subject_class, 'update_datasets') as mocked_update: 25 | mocked_update.return_value = FIXTURE_PATH 26 | self.dataset = subject.dataset 27 | 28 | def tearDown(self): 29 | shutil.rmtree(self.temp_path) 30 | 31 | def test_renamed_columns(self): 32 | adapter_keys = ADAPTER_COLUMNS.keys() 33 | dataset_keys = self.dataset.columns 34 | self.assertTrue(set(adapter_keys).issubset(set(dataset_keys))) 35 | 36 | def test_created_document_type_column_successfully(self): 37 | self.assertIn('document_type', self.dataset.columns) 38 | 39 | def test_document_type_value_is_simple_receipt(self): 40 | self.assertEqual(self.dataset['document_type'].all(), 'unknown') 41 | 42 | def test_dataset_is_a_pandas_DataFrame(self): 43 | self.assertIsInstance(self.dataset, pd.core.frame.DataFrame) 44 | 45 | def test_droped_all_null_values(self): 46 | self.assertTrue(self.dataset['recipient_id'].all()) 47 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Company/Model.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Company.Model exposing (Model, Company, Activity, model) 2 | 3 | import Date 4 | import Http 5 | import Internationalization.Types exposing (Language(..)) 6 | import Material 7 | 8 | 9 | type alias Activity = 10 | { code : String 11 | , description : String 12 | } 13 | 14 | 15 | type alias Company = 16 | { main_activity : List Activity 17 | , secondary_activity : List Activity 18 | , cnpj : String 19 | , opening : Maybe Date.Date 20 | , legal_entity : Maybe String 21 | , trade_name : Maybe String 22 | , name : Maybe String 23 | , company_type : Maybe String 24 | , status : Maybe String 25 | , situation : Maybe String 26 | , situation_reason : Maybe String 27 | , situation_date : Maybe Date.Date 28 | , special_situation : Maybe String 29 | , special_situation_date : Maybe Date.Date 30 | , responsible_federative_entity : Maybe String 31 | , address : Maybe String 32 | , address_number : Maybe String 33 | , additional_address_details : Maybe String 34 | , neighborhood : Maybe String 35 | , zip_code : Maybe String 36 | , city : Maybe String 37 | , state : Maybe String 38 | , email : Maybe String 39 | , phone : Maybe String 40 | , latitude : Maybe String 41 | , longitude : Maybe String 42 | , last_updated : Maybe Date.Date 43 | } 44 | 45 | 46 | type alias Model = 47 | { company : Maybe Company 48 | , loading : Bool 49 | , loaded : Bool 50 | , error : Maybe Http.Error 51 | , googleStreetViewApiKey : Maybe String 52 | , lang : Language 53 | , mdl : Material.Model 54 | } 55 | 56 | 57 | model : Model 58 | model = 59 | Model Nothing False False Nothing Nothing English Material.model 60 | -------------------------------------------------------------------------------- /jarbas/dashboard/templatetags/dashboard.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from django import template 3 | from django.template.defaultfilters import stringfilter 4 | 5 | from jarbas.dashboard.admin.subquotas import Subquotas 6 | 7 | 8 | BR_NUMBER_TRANSLATION = str.maketrans(',.', '.,') 9 | register = template.Library() 10 | 11 | 12 | @register.filter 13 | @stringfilter 14 | def rename_title(title): 15 | title = title.replace('modificar', 'visualizar') 16 | title = title.replace('Modificar', 'Visualizar') 17 | return title 18 | 19 | 20 | @register.filter() 21 | def percentof(amount, total): 22 | try: 23 | return f'{brazilian_float(amount * 100 / total)}%' 24 | except ZeroDivisionError: 25 | return None 26 | 27 | 28 | @register.filter() 29 | def brazilian_reais(value): 30 | return f'R$ {brazilian_float(value)}' 31 | 32 | 33 | @register.filter() 34 | def brazilian_float(value): 35 | value = value or 0 36 | value = f'{value:,.2f}' 37 | return value.translate(BR_NUMBER_TRANSLATION) 38 | 39 | 40 | @register.filter() 41 | def brazilian_integer(value): 42 | value = value or 0 43 | value = f'{value:,.0f}' 44 | return value.translate(BR_NUMBER_TRANSLATION) 45 | 46 | 47 | @register.filter() 48 | def translate_subquota(value): 49 | return Subquotas.pt_br(value) or value 50 | 51 | 52 | @register.filter() 53 | def translate_chart_grouping(value): 54 | translation = {'month': 'mês', 'year': 'ano'} 55 | return translation.get(value, value) 56 | 57 | 58 | @register.filter() 59 | def chart_grouping_as_date(value): 60 | """Transforms a string YYYYMM or YYYY in a date object""" 61 | value = str(value) 62 | for format in ('%Y', '%Y%m'): 63 | try: 64 | return datetime.strptime(str(value), format).date() 65 | except ValueError: 66 | pass 67 | -------------------------------------------------------------------------------- /rosie/rosie/core/tests/test_invalid_cnpj_cpf_classifier.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | import pandas as pd 5 | 6 | from rosie.core.classifiers import InvalidCnpjCpfClassifier 7 | 8 | 9 | class TestInvalidCnpjCpfClassifier(TestCase): 10 | 11 | def setUp(self): 12 | self.dataset = pd.read_csv('rosie/core/tests/fixtures/invalid_cnpj_cpf_classifier.csv', 13 | dtype={'recipient_id': np.str}) 14 | self.subject = InvalidCnpjCpfClassifier() 15 | 16 | def test_is_valid_cnpj(self): 17 | self.assertEqual(self.subject.predict(self.dataset)[0], False) 18 | 19 | def test_is_invalid_cnpj(self): 20 | self.assertEqual(self.subject.predict(self.dataset)[1], True) 21 | 22 | def test_is_none(self): 23 | self.assertEqual(self.subject.predict(self.dataset)[2], True) 24 | 25 | def test_none_cnpj_cpf_abroad_is_valid(self): 26 | self.assertEqual(self.subject.predict(self.dataset)[3], False) 27 | 28 | def test_valid_cnpj_cpf_abroad_is_valid(self): 29 | self.assertEqual(self.subject.predict(self.dataset)[4], False) 30 | 31 | def test_invalid_cnpj_cpf_abroad_is_valid(self): 32 | self.assertEqual(self.subject.predict(self.dataset)[5], False) 33 | 34 | def test_is_valid_cpf(self): 35 | self.assertEqual(self.subject.predict(self.dataset)[6], False) 36 | 37 | def test_is_invalid_cpf(self): 38 | self.assertEqual(self.subject.predict(self.dataset)[7], True) 39 | 40 | def test_invalid_document_type(self): 41 | self.assertEqual(self.subject.predict(self.dataset)[8], False) 42 | 43 | def test_fit(self): 44 | self.assertEqual(self.subject.fit(self.dataset), self.subject) 45 | 46 | def test_transform(self): 47 | self.assertEqual(self.subject.transform(), self.subject) 48 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0030_remove_unused_indexes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-06-02 18:08 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0029_make_issue_date_an_index'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='historicalreimbursement', 17 | name='congressperson_id', 18 | field=models.IntegerField(blank=True, null=True, verbose_name='Identificador Único do Parlamentar'), 19 | ), 20 | migrations.AlterField( 21 | model_name='historicalreimbursement', 22 | name='party', 23 | field=models.CharField(blank=True, max_length=7, null=True, verbose_name='Partido'), 24 | ), 25 | migrations.AlterField( 26 | model_name='historicalreimbursement', 27 | name='total_net_value', 28 | field=models.DecimalField(decimal_places=3, max_digits=10, verbose_name='Valor Líquido'), 29 | ), 30 | migrations.AlterField( 31 | model_name='reimbursement', 32 | name='congressperson_id', 33 | field=models.IntegerField(blank=True, null=True, verbose_name='Identificador Único do Parlamentar'), 34 | ), 35 | migrations.AlterField( 36 | model_name='reimbursement', 37 | name='party', 38 | field=models.CharField(blank=True, max_length=7, null=True, verbose_name='Partido'), 39 | ), 40 | migrations.AlterField( 41 | model_name='reimbursement', 42 | name='total_net_value', 43 | field=models.DecimalField(decimal_places=3, max_digits=10, verbose_name='Valor Líquido'), 44 | ), 45 | ] 46 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/test_election_expenses_classifier.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | import pandas as pd 5 | 6 | from rosie.chamber_of_deputies.classifiers import ElectionExpensesClassifier 7 | 8 | 9 | class TestElectionExpensesClassifier(TestCase): 10 | 11 | def setUp(self): 12 | self.election_expenser_classifier = ElectionExpensesClassifier() 13 | 14 | def test_legal_entity_is_a_election_company(self): 15 | self.dataframe = self._create_dataframe([[ 16 | 'CARLOS ALBERTO DA SILVA', 17 | 'ELEICAO 2006 CARLOS ALBERTO DA SILVA DEPUTADO', 18 | '409-0 - CANDIDATO A CARGO POLITICO ELETIVO' 19 | ]]) 20 | 21 | prediction_result = self.election_expenser_classifier.predict(self.dataframe) 22 | 23 | self.assertEqual(prediction_result[0], True) 24 | 25 | def test_legal_entity_is_not_election_company(self): 26 | self.dataframe = self._create_dataframe([[ 27 | 'PAULO ROGERIO ROSSETO DE MELO', 28 | 'POSTO ROTA 116 DERIVADOS DE PETROLEO LTDA', 29 | '401-4 - EMPRESA INDIVIDUAL IMOBILIARIA' 30 | ]]) 31 | 32 | prediction_result = self.election_expenser_classifier.predict(self.dataframe) 33 | 34 | self.assertEqual(prediction_result[0], False) 35 | 36 | def test_fit_just_for_formality_because_its_never_used(self): 37 | empty_dataframe = pd.DataFrame() 38 | self.assertTrue(self.election_expenser_classifier.fit(empty_dataframe) is None) 39 | 40 | def test_transform_just_for_formality_because_its_never_used(self): 41 | self.assertTrue(self.election_expenser_classifier.transform() is None) 42 | 43 | def _create_dataframe(self, dataframe_data): 44 | return pd.DataFrame(data=dataframe_data, columns=['congressperson_name', 'name', 'legal_entity']) 45 | -------------------------------------------------------------------------------- /jarbas/core/migrations/0034_auto_20170629_2150.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.2 on 2017-06-30 00:50 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('core', '0033_add_index_for_subquota_description'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='historicalreimbursement', 17 | name='congressperson_document', 18 | field=models.IntegerField(blank=True, null=True, verbose_name='Número da Carteira Parlamentar'), 19 | ), 20 | migrations.AlterField( 21 | model_name='historicalreimbursement', 22 | name='receipt_fetched', 23 | field=models.BooleanField(db_index=True, default=False, verbose_name='Tentamos acessar a URL do documento fiscal?'), 24 | ), 25 | migrations.AlterField( 26 | model_name='historicalreimbursement', 27 | name='term', 28 | field=models.IntegerField(blank=True, null=True, verbose_name='Número da Legislatura'), 29 | ), 30 | migrations.AlterField( 31 | model_name='reimbursement', 32 | name='congressperson_document', 33 | field=models.IntegerField(blank=True, null=True, verbose_name='Número da Carteira Parlamentar'), 34 | ), 35 | migrations.AlterField( 36 | model_name='reimbursement', 37 | name='receipt_fetched', 38 | field=models.BooleanField(db_index=True, default=False, verbose_name='Tentamos acessar a URL do documento fiscal?'), 39 | ), 40 | migrations.AlterField( 41 | model_name='reimbursement', 42 | name='term', 43 | field=models.IntegerField(blank=True, null=True, verbose_name='Número da Legislatura'), 44 | ), 45 | ] 46 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/test_chamber_of_deputies.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from pathlib import Path 4 | from tempfile import mkdtemp 5 | from unittest import TestCase 6 | from unittest.mock import MagicMock, PropertyMock, patch 7 | 8 | import pandas as pd 9 | 10 | from rosie.chamber_of_deputies import settings 11 | from rosie.chamber_of_deputies.adapter import Adapter 12 | from rosie.core import Core 13 | 14 | 15 | class TestChamberOfDeputies(TestCase): 16 | 17 | def setUp(self): 18 | row = pd.Series({'applicant_id': 444, 19 | 'document_id': 999, 20 | 'year': 2016}) 21 | self.dataset = pd.DataFrame().append(row, ignore_index=True) 22 | self.temp_dir = mkdtemp() 23 | self.classifier = MagicMock() 24 | self.classifier.__name__ = 'MockedClassifier' 25 | 26 | def tearDown(self): 27 | shutil.rmtree(self.temp_dir) 28 | 29 | @patch.object(Adapter, 'dataset', new_callable=PropertyMock) 30 | @patch('rosie.core.joblib') 31 | def test_load_trained_model_trains_model_when_not_persisted(self, _, dataset): 32 | dataset.return_value = self.dataset 33 | adapter = Adapter(self.temp_dir) 34 | subject = Core(settings, adapter) 35 | subject.load_trained_model(self.classifier) 36 | self.classifier.return_value.fit.assert_called_once_with(self.dataset) 37 | 38 | @patch.object(Adapter, 'dataset', new_callable=PropertyMock) 39 | @patch('rosie.core.joblib') 40 | def test_load_trained_model_doesnt_train_model_when_already_persisted(self, _, dataset): 41 | dataset.return_value = self.dataset 42 | adapter = Adapter(self.temp_dir) 43 | subject = Core(settings, adapter) 44 | Path(os.path.join(subject.data_path, 'mockedclassifier.pkl')).touch() 45 | model = subject.load_trained_model(self.classifier) 46 | model.fit.assert_not_called() 47 | -------------------------------------------------------------------------------- /jarbas/public_admin/sites.py: -------------------------------------------------------------------------------- 1 | from functools import update_wrapper 2 | 3 | from django.contrib.admin.sites import AdminSite 4 | from django.contrib.auth.models import AnonymousUser 5 | from django.http import HttpResponseForbidden 6 | from django.views.decorators.csrf import csrf_protect 7 | 8 | 9 | class DummyUser(AnonymousUser): 10 | def has_module_perms(self, app_label): 11 | return app_label == 'chamber_of_deputies' 12 | 13 | def has_perm(self, permission, obj=None): 14 | return permission == 'chamber_of_deputies.change_reimbursement' 15 | 16 | 17 | class PublicAdminSite(AdminSite): 18 | 19 | site_title = 'Dashboard' 20 | site_header = 'Jarbas Dashboard' 21 | index_title = 'Jarbas' 22 | 23 | def __init__(self): 24 | super().__init__('dashboard') 25 | self._actions, self._global_actions = {}, {} 26 | 27 | @staticmethod 28 | def valid_url(url): 29 | forbidden = ( 30 | 'auth', 31 | 'login', 32 | 'logout', 33 | 'password', 34 | 'add', 35 | 'delete', 36 | ) 37 | return all( 38 | label not in url.pattern.regex.pattern for label in forbidden) 39 | 40 | @property 41 | def urls(self): 42 | urls = (url for url in self.get_urls() if self.valid_url(url)) 43 | return list(urls), 'admin', self.name 44 | 45 | def has_permission(self, request): 46 | return request.method == 'GET' 47 | 48 | def admin_view(self, view, cacheable=False): 49 | def inner(request, *args, **kwargs): 50 | request.user = DummyUser() 51 | if not self.has_permission(request): 52 | return HttpResponseForbidden() 53 | return view(request, *args, **kwargs) 54 | 55 | if not getattr(view, 'csrf_exempt', False): 56 | inner = csrf_protect(inner) 57 | 58 | return update_wrapper(inner, view) 59 | 60 | 61 | public_admin = PublicAdminSite() 62 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Internationalization/Reimbursement/Common.elm: -------------------------------------------------------------------------------- 1 | module Internationalization.Reimbursement.Common exposing (..) 2 | 3 | import Internationalization.Types exposing (TranslationSet) 4 | 5 | 6 | paginationPage : TranslationSet 7 | paginationPage = 8 | TranslationSet 9 | "Page " 10 | "Página " 11 | 12 | 13 | paginationOf : TranslationSet 14 | paginationOf = 15 | TranslationSet 16 | " of " 17 | " de " 18 | 19 | 20 | reimbursementNotFound : TranslationSet 21 | reimbursementNotFound = 22 | TranslationSet 23 | "Document not found." 24 | "Documento não encontrado." 25 | 26 | 27 | reimbursementTitle : TranslationSet 28 | reimbursementTitle = 29 | TranslationSet 30 | "Document #" 31 | "Documento nº" 32 | 33 | 34 | reimbursementSource : TranslationSet 35 | reimbursementSource = 36 | TranslationSet 37 | "Source: " 38 | "Fonte: " 39 | 40 | 41 | reimbursementChamberOfDeputies : TranslationSet 42 | reimbursementChamberOfDeputies = 43 | TranslationSet 44 | "Chamber of Deputies" 45 | "Câmara dos Deputados" 46 | 47 | 48 | resultTitleSingular : TranslationSet 49 | resultTitleSingular = 50 | TranslationSet 51 | " document found." 52 | " documento encontrado." 53 | 54 | 55 | resultTitlePlural : TranslationSet 56 | resultTitlePlural = 57 | TranslationSet 58 | " documents found." 59 | " documentos encontrados." 60 | 61 | 62 | map : TranslationSet 63 | map = 64 | TranslationSet 65 | " Company on Maps" 66 | " Ver no Google Maps" 67 | 68 | 69 | sameSubquoteTitle : TranslationSet 70 | sameSubquoteTitle = 71 | TranslationSet 72 | "Other reimbursements from the same month & subquota" 73 | "Outros reembolsos do mesmo mês e subquota" 74 | 75 | 76 | sameDayTitle : TranslationSet 77 | sameDayTitle = 78 | TranslationSet 79 | "Other reimbursements from the same day" 80 | "Outros reembolsos do mesmo dia" 81 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Format/CnpjCpf.elm: -------------------------------------------------------------------------------- 1 | module Format.CnpjCpf exposing (formatCnpj, formatCnpjCpf, formatCpf) 2 | 3 | import String 4 | 5 | 6 | {-| Format a CPF number: 7 | 8 | formatCpf "12345678901" --> "123.456.789-01" 9 | 10 | -} 11 | formatCpf : String -> String 12 | formatCpf cpf = 13 | let 14 | part1 = 15 | String.slice 0 3 cpf 16 | 17 | part2 = 18 | String.slice 3 6 cpf 19 | 20 | part3 = 21 | String.slice 6 9 cpf 22 | 23 | part4 = 24 | String.slice 9 11 cpf 25 | in 26 | String.concat 27 | [ part1 28 | , "." 29 | , part2 30 | , "." 31 | , part3 32 | , "-" 33 | , part4 34 | ] 35 | 36 | 37 | {-| Format a CNPJ number: 38 | 39 | formatCnpj "12345678901234" --> "12.345.678/9012-34" 40 | 41 | -} 42 | formatCnpj : String -> String 43 | formatCnpj cnpj = 44 | let 45 | part1 = 46 | String.slice 0 2 cnpj 47 | 48 | part2 = 49 | String.slice 2 5 cnpj 50 | 51 | part3 = 52 | String.slice 5 8 cnpj 53 | 54 | part4 = 55 | String.slice 8 12 cnpj 56 | 57 | part5 = 58 | String.slice 12 14 cnpj 59 | in 60 | String.concat 61 | [ part1 62 | , "." 63 | , part2 64 | , "." 65 | , part3 66 | , "/" 67 | , part4 68 | , "-" 69 | , part5 70 | ] 71 | 72 | 73 | {-| Format a CNPJ or CPF number: 74 | 75 | formatCnpjCpf "12345678901" --> "123.456.789-01" 76 | 77 | formatCnpjCpf "12345678901234" --> "12.345.678/9012-34" 78 | 79 | formatCnpjCpf "42" --> "42" 80 | 81 | -} 82 | formatCnpjCpf : String -> String 83 | formatCnpjCpf value = 84 | case String.length value of 85 | 11 -> 86 | formatCpf value 87 | 88 | 14 -> 89 | formatCnpj value 90 | 91 | _ -> 92 | value 93 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/management/commands/reimbursements.py: -------------------------------------------------------------------------------- 1 | from csv import DictReader 2 | 3 | from jarbas.core.management.commands import LoadCommand 4 | from jarbas.chamber_of_deputies.models import Reimbursement 5 | from jarbas.chamber_of_deputies.tasks import serialize 6 | 7 | 8 | class Command(LoadCommand): 9 | help = 'Load Serenata de Amor reimbursements dataset' 10 | BATCH_SIZE = 4096 11 | 12 | def add_arguments(self, parser): 13 | super().add_arguments(parser) 14 | parser.add_argument( 15 | '--batch-size', '-b', dest='batch_size', type=int, 16 | default=self.BATCH_SIZE, 17 | help='Batch size for bulk update (default: 4096)' 18 | ) 19 | 20 | def handle(self, *args, **options): 21 | self.path = options['dataset'] 22 | self.batch_size = options.get('batch_size', self.BATCH_SIZE) 23 | self.batch, self.count = [], 0 24 | 25 | if options.get('drop', False): 26 | self.drop_all(Reimbursement) 27 | 28 | self.create_batches() 29 | 30 | @property 31 | def reimbursements(self): 32 | """Returns a Generator with a Reimbursement instance for each row.""" 33 | with open(self.path, 'rt') as file_handler: 34 | for row in DictReader(file_handler): 35 | obj = serialize(row) 36 | if obj: 37 | yield obj 38 | 39 | def create_batches(self): 40 | for count, reimbursement in enumerate(self.reimbursements, 1): 41 | self.count = count 42 | self.batch.append(reimbursement) 43 | if len(self.batch) >= self.batch_size: 44 | self.persist_batch() 45 | self.persist_batch(print_permanent=True) 46 | 47 | def persist_batch(self, print_permanent=False): 48 | Reimbursement.objects.bulk_create(self.batch) 49 | self.batch = [] 50 | self.print_count( 51 | Reimbursement, 52 | count=self.count, 53 | permanent=print_permanent 54 | ) 55 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/fixtures/traveled_speeds_classifier.csv: -------------------------------------------------------------------------------- 1 | applicant_id,is_party_expense,issue_date,category,recipient_id,latitude,longitude 2 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 3 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 4 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 5 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 6 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 7 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 8 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 9 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 10 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 11 | 999,false,2016-01-01,Flight ticket issue,08378940000120,-29.2310464,-51.1597365 12 | 999,true,2016-01-01,Meal,08378940000120,-9.9753770,-67.8248977 13 | 999,false,2016-01-01,Meal,08378940000120,,-67.8248977 14 | 999,false,2016-01-01,Meal,08378940000120,-9.9753770, 15 | 999,false,2016-01-02,Meal,08378940000120,-9.9753770,-67.8248977 16 | 999,false,2016-01-03,Meal,08378940000120,-9.9753770,-67.8248977 17 | 999,false,2016-01-03,Meal,08378940000120,-9.9753770,-67.8248977 18 | 999,false,2016-01-04,Meal,08378940000120,-9.9753770,-67.8248977 19 | 999,false,2016-01-04,Meal,08378940000120,-9.9753770,-67.8248977 20 | 999,false,2016-01-04,Meal,08378940000120,-9.9753770,-67.8248977 21 | 999,false,2016-01-04,Meal,08378940000120,-9.9753770,-67.8248977 22 | 999,false,2016-01-05,Meal,08378940000120,-9.9753770,-67.8248977 23 | 999,false,2016-01-05,Meal,08378940000120,-9.9753770,-67.8248977 24 | 999,false,2016-01-05,Meal,08378940000120,-9.9753770,-67.8248977 25 | 999,false,2016-01-05,Meal,14047033000100,-10.6519807,-68.4995996 26 | 999,false,2016-01-05,Meal,08378940000120,-9.9753770,-67.8248977 27 | 999,false,2016-01-05,Meal,08378940000120,-9.9753770,-67.8248977 28 | 999,false,2016-01-05,Meal,08378940000120,-9.9753770,-67.8248977 29 | 999,false,2016-01-05,Meal,08378940000120,-9.9753770,-67.8248977 30 | 999,false,2016-01-01,Meal,08378940000120,6.9753770,-33.8248977 31 | -------------------------------------------------------------------------------- /rosie/README.md: -------------------------------------------------------------------------------- 1 | # Rosie, the robot 2 | 3 | A Python application reading receipts from the Quota for Exercising Parliamentary Activity from Brazilian's Chamber of Deputies and Federal Senate. Rosie flag suspicious reimbursements and, in that case, offer a list of reasons why it was considered suspicious. 4 | 5 | ## Running 6 | 7 | ### With Docker 8 | 9 | #### Running 10 | 11 | ```console 12 | $ docker run --rm -v /tmp/serenata-data:/tmp/serenata-data serenata/rosie python rosie.py run 13 | ``` 14 | 15 | `` might be either `chamber_of_deputies` or `federal_senate`. After running it, check your `/tmp/serenata-data/` directory in you host machine for `suspicions.xz`. It's a compacted CSV with all the irregularities Rosie was able to find. 16 | 17 | #### Testing 18 | 19 | ```console 20 | $ docker run --rm -v /tmp/serenata-data:/tmp/serenata-data serenata/rosie python rosie.py test 21 | ``` 22 | 23 | ### Without Docker 24 | 25 | #### Setup 26 | 27 | There are a few options to setup your environment and download dependencies. The simplest way is [installing Anaconda](https://docs.anaconda.com/anaconda/install/) then run: 28 | 29 | ```console 30 | $ conda update conda 31 | $ conda create --name serenata python=3 32 | $ conda activate serenata 33 | $ pip install -r requirements.txt 34 | ``` 35 | 36 | #### Running 37 | 38 | 39 | ```console 40 | $ python rosie.py run 41 | ``` 42 | 43 | `` might be either `chamber_of_deputies` or `federal_senate`. 44 | 45 | A `/tmp/serenata-data/suspicions.xz` file will be created. It's a compacted CSV with all the irregularities Rosie was able to find. 46 | 47 | You can choose a custom a target directory: 48 | 49 | ```console 50 | $ python rosie.py run chamber_of_deputies --output /my/serenata/directory/ 51 | ``` 52 | 53 | #### Testing 54 | 55 | You can either run all tests with: 56 | 57 | ```console 58 | $ python rosie.py test 59 | ``` 60 | 61 | Or test each submodule a time by passing a name: 62 | 63 | ```console 64 | $ python rosie.py test core 65 | $ python rosie.py test chamber_of_deputies 66 | $ python rosie.py test federal_senate 67 | ``` -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Company/Decoder.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Company.Decoder exposing (decoder) 2 | 3 | import Json.Decode exposing (nullable, string) 4 | import Json.Decode.Extra exposing (date) 5 | import Json.Decode.Pipeline exposing (decode, required) 6 | import Reimbursement.Company.Model exposing (Activity, Company) 7 | 8 | 9 | decoder : Json.Decode.Decoder Company 10 | decoder = 11 | decode Company 12 | |> required "main_activity" decodeActivities 13 | |> required "secondary_activity" decodeActivities 14 | |> required "cnpj" string 15 | |> required "opening" (nullable date) 16 | |> required "legal_entity" (nullable string) 17 | |> required "trade_name" (nullable string) 18 | |> required "name" (nullable string) 19 | |> required "type" (nullable string) 20 | |> required "status" (nullable string) 21 | |> required "situation" (nullable string) 22 | |> required "situation_reason" (nullable string) 23 | |> required "situation_date" (nullable date) 24 | |> required "special_situation" (nullable string) 25 | |> required "special_situation_date" (nullable date) 26 | |> required "responsible_federative_entity" (nullable string) 27 | |> required "address" (nullable string) 28 | |> required "number" (nullable string) 29 | |> required "additional_address_details" (nullable string) 30 | |> required "neighborhood" (nullable string) 31 | |> required "zip_code" (nullable string) 32 | |> required "city" (nullable string) 33 | |> required "state" (nullable string) 34 | |> required "email" (nullable string) 35 | |> required "phone" (nullable string) 36 | |> required "latitude" (nullable string) 37 | |> required "longitude" (nullable string) 38 | |> required "last_updated" (nullable date) 39 | 40 | 41 | decodeActivities : Json.Decode.Decoder (List Activity) 42 | decodeActivities = 43 | Json.Decode.list <| 44 | Json.Decode.map2 Activity 45 | (Json.Decode.at [ "code" ] string) 46 | (Json.Decode.at [ "description" ] string) 47 | -------------------------------------------------------------------------------- /research/src/fetch_inbox.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import dateutil.parser 3 | import email 4 | import imaplib 5 | import os 6 | import quopri 7 | import re 8 | import sys 9 | import unicodedata 10 | 11 | 12 | def normalize_string(string): 13 | if isinstance(string, bytes): 14 | string = string.decode('utf-8', 'replace') 15 | if isinstance(string, str): 16 | nfkd_form = unicodedata.normalize('NFKD', string.lower()) 17 | return nfkd_form.encode('ASCII', 'ignore').decode('utf-8') 18 | 19 | 20 | EMAIL_ACCOUNT = 'op.serenatadeamor@gmail.com' 21 | EMAIL_PASSWORD = config('INBOX_PASSWORD') 22 | EMAIL_FOLDER = '"{}"'.format(sys.argv[1]) 23 | EMAIL_REGEX = r'op\.serenatadeamor?\+(\w+)@gmail\.com' 24 | PATH = os.path.join('data/email_inbox', EMAIL_FOLDER[1:-1]) 25 | 26 | 27 | M = imaplib.IMAP4_SSL('imap.gmail.com') 28 | rv, data = M.login(EMAIL_ACCOUNT, EMAIL_PASSWORD) 29 | rv, mailboxes = M.list() 30 | rv, data = M.select(EMAIL_FOLDER) 31 | rv, data = M.search(None, 'ALL') 32 | 33 | for code in data[0].split(): 34 | rv, data = M.fetch(code, '(RFC822)') 35 | message = email.message_from_bytes(data[0][1]) 36 | body = quopri.decodestring(str(message)) 37 | date = dateutil.parser.parse(message['Date']) \ 38 | .astimezone(datetime.timezone.utc).isoformat()[:-6] 39 | subject = email.header.decode_header(message['Subject'])[0][0] 40 | subject = normalize_string(subject) 41 | filename = re.sub(r'[:\+\. ]', '_', '{} {}'.format(date, subject)) 42 | mailpath = os.path.join(PATH, filename) 43 | mailtextpath = os.path.join(mailpath, 'message.txt') 44 | 45 | if os.path.exists(mailpath): 46 | continue 47 | 48 | os.makedirs(mailpath) 49 | 50 | with open(mailtextpath, 'w') as file_: 51 | file_.write(body.decode('utf-8', 'replace')) 52 | if message.get_content_maintype() == 'multipart': 53 | for part in message.walk(): 54 | if part.get_content_maintype() != 'multipart' and part.get('Content-Disposition') is not None: 55 | attachmentpath = os.path.join(mailpath, part.get_filename()) 56 | open(attachmentpath, 'wb').write(part.get_payload(decode=True)) 57 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Receipt/Update.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Receipt.Update exposing (Msg(..), update) 2 | 3 | import Format.Url exposing (url) 4 | import Http 5 | import Material 6 | import Reimbursement.Receipt.Decoder exposing (urlDecoder) 7 | import Reimbursement.Receipt.Model exposing (Model, ReimbursementId) 8 | import String 9 | 10 | 11 | type Msg 12 | = UpdateReimbursementId ReimbursementId 13 | | SearchReceipt (Maybe ReimbursementId) 14 | | LoadReceipt (Result Http.Error (Maybe String)) 15 | | Mdl (Material.Msg Msg) 16 | 17 | 18 | update : Msg -> Model -> ( Model, Cmd Msg ) 19 | update msg model = 20 | case msg of 21 | UpdateReimbursementId reimbursement -> 22 | ( { model | reimbursement = Just reimbursement }, Cmd.none ) 23 | 24 | SearchReceipt (Just reimbursement) -> 25 | ( { model | loading = True }, loadUrl reimbursement ) 26 | 27 | SearchReceipt Nothing -> 28 | ( model, Cmd.none ) 29 | 30 | LoadReceipt (Ok maybeUrl) -> 31 | ( { model | url = maybeUrl, loading = False, fetched = True }, Cmd.none ) 32 | 33 | LoadReceipt (Err error) -> 34 | let 35 | err = 36 | Debug.log "ApiFail" (toString error) 37 | in 38 | ( { model | loading = False, fetched = True, error = Just error }, Cmd.none ) 39 | 40 | Mdl mdlMsg -> 41 | Material.update mdlMsg model 42 | 43 | 44 | loadUrl : ReimbursementId -> Cmd Msg 45 | loadUrl reimbursement = 46 | let 47 | query = 48 | [ ( "format", "json" ) 49 | , ( "force", "true" ) 50 | ] 51 | 52 | path = 53 | String.join "/" 54 | [ "/api" 55 | , "chamber_of_deputies" 56 | , "reimbursement" 57 | , toString reimbursement.year 58 | , toString reimbursement.applicantId 59 | , toString reimbursement.documentId 60 | , "receipt/" 61 | ] 62 | in 63 | urlDecoder 64 | |> Http.get (url path query) 65 | |> Http.send LoadReceipt 66 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Receipt/View.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Receipt.View exposing (view) 2 | 3 | import Html exposing (a, div, text) 4 | import Html.Attributes exposing (class, href, target) 5 | import Internationalization exposing (translate) 6 | import Internationalization.Types exposing (Language(..), TranslationId(..)) 7 | import Material.Button as Button 8 | import Material.Icon as Icon 9 | import Material.Options as Options 10 | import Material.Spinner as Spinner 11 | import Reimbursement.Receipt.Model exposing (Model) 12 | import Reimbursement.Receipt.Update exposing (Msg(Mdl, SearchReceipt)) 13 | 14 | 15 | view : Model -> Html.Html Msg 16 | view model = 17 | case model.url of 18 | Just url -> 19 | a 20 | [ href url, target "_blank", class "receipt view-receipt" ] 21 | [ Button.render 22 | Mdl 23 | [ 1 ] 24 | model.mdl 25 | [ Button.minifab, Button.primary ] 26 | [ Icon.i "receipt" 27 | , text (translate model.lang ReceiptAvailable) 28 | ] 29 | ] 30 | 31 | Nothing -> 32 | if model.fetched then 33 | Button.render Mdl 34 | [ 2 ] 35 | model.mdl 36 | [ Button.minifab 37 | , Button.disabled 38 | ] 39 | [ Icon.i "receipt" 40 | , text (translate model.lang ReceiptNotAvailable) 41 | ] 42 | else if model.loading then 43 | Spinner.spinner [ Spinner.active True ] 44 | else 45 | Button.render Mdl 46 | [ 0 ] 47 | model.mdl 48 | [ Button.minifab 49 | , Button.primary 50 | , Button.onClick (SearchReceipt model.reimbursement) 51 | , Options.cs "receipt fetch-receipt" 52 | ] 53 | [ Icon.i "search" 54 | , text (translate model.lang ReceiptFetch) 55 | ] 56 | -------------------------------------------------------------------------------- /rosie/rosie/chamber_of_deputies/tests/test_irregular_companies_classifier.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from datetime import date 3 | from itertools import chain 4 | from unittest import TestCase 5 | 6 | import pandas as pd 7 | 8 | from rosie.chamber_of_deputies.classifiers import IrregularCompaniesClassifier 9 | 10 | 11 | Status = namedtuple('Status', ('situation_date', 'issue_date', 'expected')) 12 | 13 | 14 | class TestIrregularCompaniesClassifier(TestCase): 15 | 16 | SUSPICIOUS_SITUATIONS = ( 17 | 'BAIXADA', 18 | 'NULA', 19 | 'INAPTA', 20 | 'SUSPENSA', 21 | ) 22 | 23 | SITUATIONS = chain(SUSPICIOUS_SITUATIONS, ('ABERTA',)) 24 | 25 | STATUS = ( 26 | Status(date(2013, 1, 30), date(2013, 1, 1), False), 27 | Status(date(2013, 1, 1), date(2013, 1, 30), True) 28 | ) 29 | 30 | def setUp(self): 31 | self.subject = IrregularCompaniesClassifier() 32 | 33 | def _get_company_dataset(self, **kwargs): 34 | base_company = { 35 | 'recipient_id': '02989654001197', 36 | 'situation_date': date(2013, 1, 3), 37 | 'issue_date': date(2013, 1, 30), 38 | 'situation': '', 39 | } 40 | base_company.update(kwargs) 41 | dataset = pd.DataFrame([base_company]) 42 | return dataset 43 | 44 | def test_is_regular_company(self): 45 | for situation in self.SITUATIONS: 46 | company = self._get_company_dataset(situation=situation) 47 | expected = situation in self.SUSPICIOUS_SITUATIONS 48 | result, *_ = self.subject.predict(company) 49 | with self.subTest(): 50 | self.assertEqual(result, expected, msg=company) 51 | 52 | def test_if_company_is_suspended(self): 53 | for status in self.STATUS: 54 | company = self._get_company_dataset( 55 | situation='SUSPENSA', 56 | situation_date=status.situation_date, 57 | issue_date=status.issue_date, 58 | ) 59 | result, *_ = self.subject.predict(company) 60 | with self.subTest(): 61 | self.assertEqual(result, status.expected, msg=company) 62 | -------------------------------------------------------------------------------- /jarbas/dashboard/tests/test_dashboard_view.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | 3 | from django.shortcuts import resolve_url 4 | from django.test import TestCase 5 | from mixer.backend.django import mixer 6 | 7 | from jarbas.chamber_of_deputies.models import Reimbursement 8 | 9 | 10 | class TestDashboard(TestCase): 11 | 12 | def setUp(self): 13 | obj = mixer.blend(Reimbursement, search_vector=None) 14 | self.urls = ( 15 | resolve_url('dashboard:index'), 16 | resolve_url('dashboard:chamber_of_deputies_reimbursement_changelist'), 17 | resolve_url('dashboard:chamber_of_deputies_reimbursement_change', obj.pk), 18 | resolve_url('dashboard:chamber_of_deputies_reimbursement_history', obj.pk), 19 | ) 20 | self.forbidden = ( 21 | '/login/', 22 | '/logout/', 23 | '/password_change/', 24 | '/password_change/done/', 25 | '/auth/group/,', 26 | '/auth/user/,', 27 | '/auth/' 28 | ) 29 | 30 | 31 | class TestGet(TestDashboard): 32 | 33 | def test_successful_get(self): 34 | for url in self.urls: 35 | resp = self.client.get(url) 36 | self.assertEqual(200, resp.status_code, url) 37 | 38 | def test_forbidden_get(self): 39 | for url in self.forbidden: 40 | resp = self.client.get(url) 41 | self.assertEqual(404, resp.status_code, url) 42 | 43 | 44 | class TestPostPutDelete(TestDashboard): 45 | 46 | def get_responses(self, url): 47 | methods = ('post', 'put', 'patch', 'delete', 'head') 48 | for method in methods: 49 | reference = '{} {}'.format(method.upper(), url) 50 | request = getattr(self.client, method) 51 | yield request(url, follow=False), reference 52 | 53 | def test_forbidden_methods(self): 54 | responses = map(self.get_responses, self.urls) 55 | for resp, reference in chain(*responses): 56 | self.assertEqual(403, resp.status_code, reference) 57 | 58 | def test_forbidden_urls(self): 59 | responses = map(self.get_responses, self.forbidden) 60 | for resp, reference in chain(*responses): 61 | self.assertEqual(404, resp.status_code, reference) 62 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Layout.elm: -------------------------------------------------------------------------------- 1 | module Layout exposing (Model, drawer, header, model) 2 | 3 | import Html exposing (a, div, img, text) 4 | import Html.Attributes exposing (alt, href, src, style) 5 | import Internationalization exposing (translate) 6 | import Internationalization.Types exposing (Language(..), TranslationId(..)) 7 | import Material 8 | import Material.Layout as Layout 9 | 10 | 11 | -- 12 | -- Model 13 | -- 14 | 15 | 16 | type alias Model = 17 | { lang : Language 18 | , mdl : Material.Model 19 | } 20 | 21 | 22 | model : Model 23 | model = 24 | Model English Material.model 25 | 26 | 27 | 28 | -- 29 | -- View 30 | -- 31 | 32 | 33 | header : Model -> Html.Html a 34 | header model = 35 | let 36 | digitalOcean = 37 | Layout.link 38 | [ Layout.href "http://digitalocean.com" ] 39 | [ img 40 | [ src "/static/digitalocean.png" 41 | , alt "Powered by Digital Ocean" 42 | , style [ ( "height", "1.5rem" ), ( "opacity", "0.5" ) ] 43 | ] 44 | [] 45 | ] 46 | 47 | title = 48 | a 49 | [ href "/" 50 | , style 51 | [ ( "color", "white" ) 52 | , ( "text-decoration", "none" ) 53 | , ( "font-weight", "normal" ) 54 | ] 55 | ] 56 | [ text "Jarbas" ] 57 | in 58 | Layout.row 59 | [] 60 | [ Layout.title [] [ title ] 61 | , Layout.spacer 62 | , Layout.navigation [] [ digitalOcean ] 63 | ] 64 | 65 | 66 | drawerLinks : ( String, String ) -> Html.Html a 67 | drawerLinks ( url, content ) = 68 | Layout.link [ Layout.href url ] [ text content ] 69 | 70 | 71 | drawer : Model -> List (Html.Html a) 72 | drawer model = 73 | [ Layout.title [] [ text (translate model.lang AboutJarbas) ] 74 | , Layout.navigation [] <| 75 | List.map 76 | drawerLinks 77 | [ ( "http://github.com/okfn-brasil/serenata-de-amor", translate model.lang AboutJarbas ) 78 | , ( "https://serenata.ai", translate model.lang AboutSerenata ) 79 | ] 80 | ] 81 | -------------------------------------------------------------------------------- /jarbas/layers/elm/Reimbursement/Company/Update.elm: -------------------------------------------------------------------------------- 1 | module Reimbursement.Company.Update exposing (..) 2 | 3 | import Char 4 | import Format.Url exposing (url) 5 | import Http 6 | import Material 7 | import Reimbursement.Company.Decoder exposing (decoder) 8 | import Reimbursement.Company.Model exposing (Company, Model) 9 | import String 10 | 11 | 12 | type Msg 13 | = LoadCompany (Result Http.Error Company) 14 | | Mdl (Material.Msg Msg) 15 | 16 | 17 | {-| Cleans up a CNPJ field allowing numbers only: 18 | 19 | cleanUp (Just "12.345.678/9012-34") --> "12345678901234" 20 | 21 | -} 22 | cleanUp : Maybe String -> String 23 | cleanUp cnpj = 24 | cnpj 25 | |> Maybe.withDefault "" 26 | |> String.filter Char.isDigit 27 | 28 | 29 | {-| CNPJ validator: 30 | 31 | isValid (Just "12.345.678/9012-34") --> True 32 | 33 | isValid (Just "12345678901234") --> True 34 | 35 | isValid (Just "123.456.789-01") --> False 36 | 37 | isValid Nothing --> False 38 | 39 | -} 40 | isValid : Maybe String -> Bool 41 | isValid cnpj = 42 | if String.length (cleanUp cnpj) == 14 then 43 | True 44 | else 45 | False 46 | 47 | 48 | update : Msg -> Model -> ( Model, Cmd Msg ) 49 | update msg model = 50 | case msg of 51 | LoadCompany (Ok company) -> 52 | ( { model | company = Just company, loading = False, loaded = True }, Cmd.none ) 53 | 54 | LoadCompany (Err error) -> 55 | let 56 | err = 57 | Debug.log "ApiError" (toString error) 58 | in 59 | ( { model | loaded = True, error = Just error }, Cmd.none ) 60 | 61 | Mdl mdlMsg -> 62 | Material.update mdlMsg model 63 | 64 | 65 | load : Maybe String -> Cmd Msg 66 | load cnpj = 67 | if isValid cnpj then 68 | let 69 | path : String 70 | path = 71 | String.concat 72 | [ "/api/company/" 73 | , cleanUp cnpj 74 | , "/" 75 | ] 76 | 77 | query : List ( String, String ) 78 | query = 79 | [ ( "format", "json" ) ] 80 | in 81 | decoder 82 | |> Http.get (url path query) 83 | |> Http.send LoadCompany 84 | else 85 | Cmd.none 86 | -------------------------------------------------------------------------------- /jarbas/core/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | from datetime import date 2 | from re import match 3 | 4 | from django.core.management.base import BaseCommand 5 | 6 | 7 | class LoadCommand(BaseCommand): 8 | 9 | def add_arguments(self, parser, add_drop_all=True): 10 | parser.add_argument('dataset', help='Path to the .xz dataset') 11 | if add_drop_all: 12 | parser.add_argument( 13 | '--drop-all', '-d', dest='drop', action='store_true', 14 | help='Drop all existing records before loading the datasets' 15 | ) 16 | 17 | @staticmethod 18 | def to_number(value, cast=None): 19 | if value.lower() in ('nan', ''): 20 | return None 21 | 22 | number = float(value) 23 | if cast: 24 | return cast(number) 25 | return number 26 | 27 | @staticmethod 28 | def to_date(text): 29 | 30 | ddmmyyyy = match(r'^[\d]{1,2}/[\d]{1,2}/[\d]{2,4}$', text) 31 | yyyymmdd = match(r'^[\d]{2,4}-[\d]{1,2}-[\d]{2,4}', text) 32 | 33 | if ddmmyyyy: 34 | day, month, year = map(int, ddmmyyyy.group().split('/')) 35 | elif yyyymmdd: 36 | year, month, day = map(int, yyyymmdd.group().split('-')) 37 | else: 38 | return None 39 | 40 | try: 41 | if 0 <= year <= 50: 42 | year += 2000 43 | elif 50 < year <= 99: 44 | year += 1900 45 | return date(year, month, day) 46 | 47 | except ValueError: 48 | return None 49 | 50 | def drop_all(self, model): 51 | if model.objects.count() != 0: 52 | msg = 'Deleting all existing records from {} model' 53 | print(msg.format(self.get_model_name(model))) 54 | model.objects.all().delete() 55 | self.print_count(model, permanent=True) 56 | 57 | def print_count(self, model, **kwargs): 58 | count = kwargs.get('count', model.objects.count()) 59 | raw_msg = 'Current count: {:,} {}s ' 60 | msg = raw_msg.format(count, self.get_model_name(model)) 61 | end = '\n' if kwargs.get('permanent', False) else '\r' 62 | print(msg, end=end) 63 | return count 64 | 65 | @staticmethod 66 | def get_model_name(model): 67 | return model._meta.label.split('.')[-1] 68 | -------------------------------------------------------------------------------- /jarbas/chamber_of_deputies/migrations/0004_alter_field_names_following_toolbox_renamings.py: -------------------------------------------------------------------------------- 1 | from django.contrib.postgres.fields import ArrayField 2 | from django.db import migrations, models 3 | 4 | 5 | def convert_reimbursement_numbers_to_array(apps, schema_editor): 6 | Reimbursement = apps.get_model("chamber_of_deputies", "Reimbursement") 7 | for record in Reimbursement.objects.all(): 8 | record.numbers = record.reimbursement_numbers.split(", ") 9 | record.save() 10 | 11 | 12 | def convert_reimbursement_numbers_to_array_rollback(apps, schema_editor): 13 | Reimbursement = apps.get_model("chamber_of_deputies", "Reimbursement") 14 | for record in Reimbursement.objects.all(): 15 | record.reimbursement_numbers = ", ".join(record.numbers) 16 | record.save() 17 | 18 | 19 | class Migration(migrations.Migration): 20 | 21 | dependencies = [ 22 | ("chamber_of_deputies", "0003_remove_available_in_latest_dataset_field") 23 | ] 24 | 25 | operations = [ 26 | migrations.AlterField( 27 | model_name="reimbursement", 28 | name="document_id", 29 | field=models.IntegerField(db_index=True), 30 | ), 31 | migrations.AlterField( 32 | model_name="reimbursement", 33 | name="supplier", 34 | field=models.CharField(max_length=256), 35 | ), 36 | migrations.AlterField( 37 | model_name="reimbursement", 38 | name="issue_date", 39 | field=models.DateField(null=True), 40 | ), 41 | migrations.RenameField( 42 | model_name="reimbursement", 43 | old_name="total_reimbursement_value", 44 | new_name="total_value", 45 | ), 46 | migrations.RenameField( 47 | model_name="reimbursement", 48 | old_name="subquota_id", 49 | new_name="subquota_number", 50 | ), 51 | migrations.AddField( 52 | model_name="reimbursement", 53 | name="numbers", 54 | field=ArrayField(models.CharField(max_length=128), default=list), 55 | ), 56 | migrations.RunPython( 57 | convert_reimbursement_numbers_to_array, 58 | convert_reimbursement_numbers_to_array_rollback, 59 | ), 60 | migrations.RemoveField(model_name="reimbursement", name="net_values"), 61 | migrations.RemoveField( 62 | model_name="reimbursement", name="reimbursement_numbers" 63 | ), 64 | migrations.RemoveField(model_name="reimbursement", name="reimbursement_values"), 65 | ] 66 | -------------------------------------------------------------------------------- /jarbas/core/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Activity(models.Model): 5 | 6 | code = models.CharField('Code', max_length=10) 7 | description = models.CharField('Description', max_length=167) 8 | 9 | 10 | class Company(models.Model): 11 | 12 | cnpj = models.CharField('CNPJ', db_index=True, max_length=18) 13 | opening = models.DateField('Opening date', blank=True, null=True) 14 | 15 | legal_entity = models.CharField('Legal entity', blank=True, null=True, max_length=72) 16 | trade_name = models.CharField('Trade name', blank=True, null=True, max_length=55) 17 | name = models.CharField('Name', blank=True, null=True, max_length=144) 18 | type = models.CharField('Type', blank=True, null=True, max_length=6) 19 | 20 | main_activity = models.ManyToManyField(Activity, related_name='main') 21 | secondary_activity = models.ManyToManyField(Activity, related_name='secondary') 22 | 23 | status = models.CharField('Status', blank=True, null=True, max_length=5) 24 | situation = models.CharField('Situation', blank=True, null=True, max_length=8) 25 | situation_reason = models.CharField('Situation reason', blank=True, null=True, max_length=44) 26 | situation_date = models.DateField('Situation date', blank=True, null=True) 27 | special_situation = models.CharField('Special situation', blank=True, null=True, max_length=51) 28 | special_situation_date = models.DateField('Special situation date', blank=True, null=True) 29 | responsible_federative_entity = models.CharField('Responsible federative entity', blank=True, null=True, max_length=38) 30 | 31 | address = models.CharField('Address', blank=True, null=True, max_length=64) 32 | number = models.CharField('Number', blank=True, null=True, max_length=6) 33 | additional_address_details = models.CharField('Additional address details', blank=True, null=True, max_length=143) 34 | neighborhood = models.CharField('Neighborhood', blank=True, null=True, max_length=50) 35 | zip_code = models.CharField('Zip code', blank=True, null=True, max_length=10) 36 | city = models.CharField('City', blank=True, null=True, max_length=32) 37 | state = models.CharField('State', blank=True, null=True, max_length=2) 38 | email = models.EmailField('Email', blank=True, null=True, max_length=58) 39 | phone = models.CharField('Phone', blank=True, null=True, max_length=32) 40 | 41 | latitude = models.DecimalField('Latitude', decimal_places=7, max_digits=10, blank=True, null=True) 42 | longitude = models.DecimalField('Longitude', decimal_places=7, max_digits=10, blank=True, null=True) 43 | 44 | last_updated = models.DateTimeField('Last updated', blank=True, null=True) 45 | -------------------------------------------------------------------------------- /research/src/fetch_purchase_suppliers.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import date 3 | 4 | import pandas as pd 5 | import requests 6 | from tqdm import tqdm 7 | 8 | API_SERVER = 'http://compras.dados.gov.br' 9 | API_ENDPOINT = API_SERVER + '/fornecedores/v1/fornecedores.json' 10 | 11 | 12 | class Suppliers: 13 | 14 | def __init__(self): 15 | self.response = requests.get(API_ENDPOINT).json() 16 | self.total = self.response.get('count', 0) 17 | 18 | @property 19 | def next(self): 20 | path = self.response.get('_links', {}).get('next', {}).get('href') 21 | if path: 22 | return API_SERVER + path 23 | return False 24 | 25 | def pages(self): 26 | yield self.response 27 | while self.next: 28 | 29 | response = requests.get(self.next) 30 | if 200 <= response.status_code < 400: 31 | self.response = response.json() 32 | yield self.response 33 | 34 | else: 35 | msg = 'Server responded with a {} HTTP Status' 36 | print(msg.format(response.status_code)) 37 | break 38 | 39 | def details(self): 40 | for page in self.pages(): 41 | for supplier in page.get('_embedded', {}).get('fornecedores', []): 42 | yield supplier 43 | 44 | def fetch(self): 45 | with tqdm(total=self.total) as progress: 46 | for supplier in self.details(): 47 | progress.update(1) 48 | yield supplier 49 | 50 | 51 | def retrieve_data(): 52 | columns = { 53 | 'id': 'id', 54 | 'cnpj': 'cnpj', 55 | 'nome': 'name', 56 | 'ativo': 'active', 57 | 'recadastrado': 'relisted', 58 | 'id_municipio': 'city_id', 59 | 'uf': 'state', 60 | 'id_natureza_juridica': 'legal_nature_id', 61 | 'id_porte_empresa': 'company_size_id', 62 | 'id_ramo_negocio': 'business_id', 63 | 'id_unidade_cadastradora': 'responsible_entity', 64 | 'id_cnae': 'cnae_id', 65 | 'habilitado_licitar': 'allowed_to_bid' 66 | } 67 | 68 | suppliers = Suppliers() 69 | df = pd.DataFrame(suppliers.fetch(), columns=columns) 70 | df.rename(columns=columns, inplace=True) 71 | 72 | return df 73 | 74 | 75 | def save_csv(df): 76 | dataset_name = date.today().strftime('%Y-%m-%d') + '-purchase-suppliers.xz' 77 | dataset_path = os.path.join('data', dataset_name) 78 | df.to_csv(dataset_path, compression='xz', encoding='utf-8', index=False) 79 | 80 | 81 | def main(): 82 | suppliers = retrieve_data() 83 | save_csv(suppliers) 84 | 85 | 86 | if __name__ == '__main__': 87 | main() 88 | --------------------------------------------------------------------------------