├── .github ├── CODEOWNERS ├── README.md ├── pull_request_template.md ├── workflow.md └── workflows │ └── build-and-deploy.yml ├── .gitignore ├── DQ Measures Specifications_V4.0.xlsx ├── LICENSE ├── README_Public_Repo.md ├── State_DQ_Missingness_Measures_v4.0.xlsx ├── Thresholds.xlsx ├── create_setup_local.py ├── databricks ├── notebooks │ ├── DQM UAT.py │ ├── DQM v3.1 UAT.py │ └── VAL │ │ ├── DQM UAT.ipynb │ │ └── DQM v3.1 UAT.ipynb ├── reset_all_jobs_PROD.bat └── reset_all_jobs_VAL.bat ├── deploy.sh ├── dqm ├── DQClosure.py ├── DQM_Metadata.py ├── DQMeasures.py ├── DQPrepETL.py ├── Module.py ├── __init__.py ├── batch │ ├── reverse_lookup.pkl │ ├── reverse_lookup.py │ ├── run_101.pkl │ ├── run_101.py │ ├── run_102.pkl │ ├── run_102.py │ ├── run_103.pkl │ ├── run_103.py │ ├── run_104.pkl │ ├── run_104.py │ ├── run_105.pkl │ ├── run_105.py │ ├── run_106.pkl │ ├── run_106.py │ ├── run_107.pkl │ ├── run_107.py │ ├── run_108.pkl │ ├── run_108.py │ ├── run_109.pkl │ ├── run_109.py │ ├── run_110.pkl │ ├── run_110.py │ ├── run_201.pkl │ ├── run_201.py │ ├── run_202.pkl │ ├── run_202.py │ ├── run_204.pkl │ ├── run_204.py │ ├── run_205.pkl │ ├── run_205.py │ ├── run_206.pkl │ ├── run_206.py │ ├── run_501.pkl │ ├── run_501.py │ ├── run_502.pkl │ ├── run_502.py │ ├── run_503.pkl │ ├── run_503.py │ ├── run_504.pkl │ ├── run_504.py │ ├── run_601.pkl │ ├── run_601.py │ ├── run_602.pkl │ ├── run_602.py │ ├── run_603.pkl │ ├── run_603.py │ ├── run_701.pkl │ ├── run_701.py │ ├── run_702.pkl │ ├── run_702.py │ ├── run_703.pkl │ ├── run_703.py │ ├── run_704.pkl │ ├── run_704.py │ ├── run_705.pkl │ ├── run_705.py │ ├── run_706.pkl │ ├── run_706.py │ ├── run_707.pkl │ ├── run_707.py │ ├── run_708.pkl │ ├── run_708.py │ ├── run_709.pkl │ ├── run_709.py │ ├── run_710.pkl │ ├── run_710.py │ ├── run_711.pkl │ ├── run_711.py │ ├── run_712.pkl │ ├── run_712.py │ ├── run_713.pkl │ ├── run_713.py │ ├── run_714.pkl │ ├── run_714.py │ ├── run_715.py │ ├── run_716.py │ ├── run_801.pkl │ ├── run_801.py │ ├── run_801_miss_claims.pkl │ ├── run_801_miss_non_claims.pkl │ ├── run_801_missvar.pkl │ ├── run_802.pkl │ ├── run_803.pkl │ ├── run_901.pkl │ ├── run_901.py │ ├── run_902.pkl │ ├── run_902.py │ ├── run_903.pkl │ ├── run_903.py │ ├── run_904.pkl │ ├── run_904.py │ ├── run_905.pkl │ ├── run_905.py │ ├── run_906.pkl │ ├── run_906.py │ ├── run_907.pkl │ ├── run_907.py │ ├── run_909.pkl │ ├── run_909.py │ ├── run_910.pkl │ ├── run_910.py │ ├── run_911.pkl │ ├── run_911.py │ ├── run_912.pkl │ ├── run_912.py │ ├── run_913.pkl │ ├── run_913.py │ ├── run_914.pkl │ ├── run_914.py │ ├── run_915.pkl │ ├── run_915.py │ ├── run_916.pkl │ ├── run_916.py │ ├── run_917.pkl │ ├── run_917.py │ ├── run_918.pkl │ ├── run_918.py │ ├── run_919.pkl │ ├── run_919.py │ ├── run_920.pkl │ └── run_920.py ├── cfg │ ├── Provider Classification Lookup.csv │ ├── State_DQ_Missingness_Measures_final_list.csv │ ├── abd.pkl │ ├── apdxc.pkl │ ├── atypical_provider_table.pkl │ ├── countystate_lookup.pkl │ ├── fmg.pkl │ ├── missVar.pkl │ ├── prgncy.pkl │ ├── provider_classification_lookup.pkl │ ├── prvtxnmy.pkl │ ├── sauths.pkl │ ├── schip.pkl │ ├── splans.pkl │ ├── st2_name.pkl │ ├── st_fips.pkl │ ├── st_name.pkl │ ├── st_usps.pkl │ ├── stabr.pkl │ ├── stc_cd.pkl │ ├── thresh.pkl │ ├── thresh_measures.pkl │ ├── thresholds.csv │ ├── thresholds.pkl │ ├── zcc.pkl │ └── zipstate_lookup.pkl ├── submodules │ ├── Runner_101.py │ ├── Runner_102.py │ ├── Runner_103.py │ ├── Runner_104.py │ ├── Runner_105.py │ ├── Runner_106.py │ ├── Runner_107.py │ ├── Runner_108.py │ ├── Runner_109.py │ ├── Runner_110.py │ ├── Runner_201.py │ ├── Runner_202.py │ ├── Runner_204.py │ ├── Runner_205.py │ ├── Runner_206.py │ ├── Runner_501.py │ ├── Runner_502.py │ ├── Runner_503.py │ ├── Runner_504.py │ ├── Runner_601.py │ ├── Runner_602.py │ ├── Runner_603.py │ ├── Runner_701.py │ ├── Runner_702.py │ ├── Runner_703.py │ ├── Runner_704.py │ ├── Runner_705.py │ ├── Runner_706.py │ ├── Runner_707.py │ ├── Runner_708.py │ ├── Runner_709.py │ ├── Runner_710.py │ ├── Runner_711.py │ ├── Runner_712.py │ ├── Runner_713.py │ ├── Runner_714.py │ ├── Runner_715.py │ ├── Runner_716.py │ ├── Runner_802.py │ ├── Runner_803.py │ ├── Runner_901.py │ ├── Runner_902.py │ ├── Runner_903.py │ ├── Runner_904.py │ ├── Runner_905.py │ ├── Runner_906.py │ ├── Runner_907.py │ ├── Runner_909.py │ ├── Runner_910.py │ ├── Runner_911.py │ ├── Runner_912.py │ ├── Runner_913.py │ ├── Runner_914.py │ ├── Runner_915.py │ ├── Runner_916.py │ ├── Runner_917.py │ ├── Runner_918.py │ ├── Runner_919.py │ ├── Runner_920.py │ └── __init__.py └── thresholds.py ├── notebooks └── val │ └── Shared │ └── DQ-Measures │ ├── DQ Measures (submodule - prepare).py │ └── DQ Measures (submodule).py ├── requirements.txt ├── setup.py └── static ├── csv ├── abd.csv ├── apdxc.csv ├── atypical_provider_table.csv ├── countystate_lookup.csv ├── fmg.csv ├── missvar.csv ├── prgncy.csv ├── provider_classification_lookup.csv ├── prvtxnmy.csv ├── sauths.csv ├── schip.csv ├── splans.csv ├── st2_name.csv ├── st_fips.csv ├── st_name.csv ├── st_usps.csv ├── stabr.csv ├── type_of_service.csv ├── zip_county_crosswalk.csv └── zipstate_lookup.csv ├── csv2pkl.py └── pkl2csv.py /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # List of default reviewers for changes to all files 2 | * @S8G5-Mathematica @XiaoBarry @V2I8-Mathematica @BX9I-Mathematica @C1KB-tensileai 3 | -------------------------------------------------------------------------------- /.github/README.md: -------------------------------------------------------------------------------- 1 | # dq_measures_python 2 | Python Implementation of Data Quality Measures for Databricks 3 | 4 | This is a Python Library for the maintenance and processing of Data Quality (DQ) Measures with distributed computing framework using Databricks. Measures and analytic files may be independently run within Notebooks, allowing them to be grouped into parallel processes based on state, data dependency, time interval, or any custom business rules. Each process can be calibrated to optimally meet demand and deliverables. Custom Python libraries will be created to facilitate consistent management and execution of processes as well as simplify the creation of new analyses. This design is ideal for imposing best practices amongst distributed services which are appropriately granted resources and permit focus on test-driven development. 5 | 6 | 7 | - [ ] Create or modify a Runner Class 8 | - [ ] Create or modify a Runner Manifest 9 | - [ ] New modules are added to the Registry and Reverse Lookup 10 | - [ ] Process the reverse lookup and all manifest files 11 | - [ ] Encapsulating the _Thresholds_ File 12 | - [ ] Increment the library version number(s) 13 | - [ ] Build the library 14 | - [ ] Upload the WHL file to the Databricks environment 15 | - [ ] Deploy the library to the Databricks cluster 16 | 17 | # Creating a New Measure 18 | 19 | To author and 20 | 21 | ```python 22 | from dqm.DQM_Metadata import DQM_Metadata 23 | from dqm.DQMeasures import DQMeasures 24 | 25 | class Runner_n(): 26 | ``` 27 | 28 | ## Create or modify a Runner Class 29 | Measure semantic processing is implemented within a Runner class object. Measures methods may be re-used driven by parameters in the manifest or distinct functions may be implemented. 30 | ```python 31 | def my_measure(spark, dqm: DQMeasures, measure_id, x) : 32 | 33 | z = f""" 34 | select 35 | '{dqm.state}' as submtg_state_cd 36 | ,'{measure_id}' as measure_id 37 | ,'' as submodule 38 | , ... as numer 39 | , ... as denom 40 | , ... as mvalue 41 | , ... as valid_value 42 | from 43 | . 44 | . 45 | """ 46 | 47 | dqm.logger.debug(z) 48 | return spark.sql(z) 49 | ``` 50 | Measure functions must be included in the _v-table_ within the Runner class by name. 51 | ```python 52 | v_table = { '': my_measure } 53 | ``` 54 | 55 | ## Create or modify a Runner Manifest 56 | The runner manifest includes the metadata and parameters ( _if any_ ). The manifest must be compiled with the process of building the distributable library. 57 | ```python 58 | from pandas import pandas as pd 59 | from pandas import DataFrame 60 | 61 | run_n =[ 62 | . 63 | . 64 | ['', '', '', [param 1], .. [parameter n] ], 65 | . 66 | . 67 | ] 68 | 69 | 70 | df = DataFrame(run_105, columns=['series', 'cb', 'measure_id', ... ]) 71 | df.to_pickle('./run_n.pkl') 72 | ``` 73 | ## New modules are added to the Registry and Reverse Lookup 74 | ### Registry 75 | Single instances of each _runner_ are contained within _Module_ for dispatching measures. 76 | ```python 77 | . 78 | . 79 | from dqm.submodules import Runner_n as n 80 | . 81 | . 82 | class Module(): 83 | 84 | def __init__(self): 85 | . 86 | . 87 | self.run = r.Runner_n 88 | . 89 | . 90 | self.runners = { 91 | . 92 | . 93 | '': self.run, 94 | . 95 | . 96 | } 97 | ``` 98 | 99 | ### Reverse Lookup 100 | The reverse lookup compiles all of the _runner manifests_ and links individual measures by their ID. The _reverse lookup_ must be compiled with the process of building the distributable library. 101 | ```python 102 | . 103 | . 104 | series = [ 105 | '901', '902', '903', '904', '905', '906', ... 106 | '201', '202', '204', '205', '206', ... 107 | '801', ... 108 | '101', '102', '103', '104', '105', '106', '107', ... 109 | '701', '702', '703', '704', ... 110 | '501', '502', '503', '504', ... 111 | '601', '602', ... 112 | ] 113 | . 114 | . 115 | ``` 116 | 117 | ## Encapsulating the _Thresholds_ File 118 | 119 | 120 | ## Process the thresholds file 121 | 122 | Copy the latest thresholds xlsx file produced by the researchers from SharePoint to the ```dqm\cfg``` folder. Delete any previous versions or an assertion will fail. 123 | 124 | from within the ```dq_measures_python``` folder: 125 | 126 | - > ```python .\dqm\thresholds.py``` 127 | 128 | ## Process the reverse lookup and all manifest files 129 | 130 | from within the ```dqm\batch``` folder: 131 | 132 | - > ```python .\reverse_lookup.py``` 133 | 134 | ## Increment the library version number(s) 135 | 136 | The library version is included the source code. It can be updated in ```_init_()``` method of the DQMeasures module. 137 | When running or deploying code in development, version should be set through the `VERSION` variable in your environment. 138 | 139 | ### Examples 140 | 141 | > .env 142 | ```bash 143 | export VERSION=2.6.10 144 | ``` 145 | * note, if the environment variable hasn't been picked up after you edited this file, try restarting the terminal or running `source .env` from the same directory. 146 | 147 | > DQMeasures.py ( ~ line 96 ) 148 | ```python 149 | self.version = '2.6.10' # internal library version 150 | ``` 151 | 152 | > DQMeasures.py ( ~ line 99 ) 153 | ```python 154 | self.specvrsn = 'V2.6' # specification version 155 | ``` 156 | > setup.py ( ~ line 8 ) 157 | ```python 158 | version="2.6.10", # deployed library version 159 | ``` 160 | 161 | ## Update VAL job metdata 162 | > from the ```.\databricks\``` folder: 163 | run ```.\reset_all_jobs_VAL.bat``` 164 | 165 | # Workflow 166 | 167 | For automated and manual workflows, see [workflow.md](workflow.md). 168 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## What is this and why are we doing it? 2 | 3 | 4 | * Link to the Jira ticket for this change: https://jiraent.cms.gov/browse/DBDAART-#### 5 | 6 | ## Briefly describe any new measures and modifications to existing measures. 7 | 8 | 9 | ## Provide the URLs to Notebooks that were used in development testing. 10 | 11 | 12 | ## What are the security implications from this change? 13 | 14 | 15 | ## What version of Thresholds does this apply? 16 | 17 | 18 | ## List (if any) base table view definitions that were modified: 19 | 20 | 21 | ## List (if any) medata .pkl files that were modified: 22 | 23 | 24 | ## How did I test this? 25 | 26 | 27 | ## Should there be new or updated documentation for this change? (Be specific.) 28 | 29 | 30 | ## PR Checklist 31 | - [ ] The JIRA ticket number and a short description is in the subject line 32 | - [ ] My code follows any applicable [style guides](https://cms-dataconnect.atlassian.net/wiki/search?text=style%20guide) 33 | - [ ] I have performed a self-review of my own code 34 | - [ ] I have commented my code, particularly in hard-to-understand areas 35 | - [ ] I have made corresponding changes to the documentation 36 | -------------------------------------------------------------------------------- /.github/workflow.md: -------------------------------------------------------------------------------- 1 | # Workflow 2 | 3 | This documents how to build and deploy the wheel file. 4 | 5 | ## Automated Workflow 6 | 7 | Automation is still being built and this document will need to be updated as we go. 8 | 9 | ### Current 10 | 11 | When a Pull Request is merged to the master branch, GitHub is configured to trigger a [Jenkins job](https://jenkins.macbisdw.cmscloud.local/job/build-and-deploy-dq-measures-python/) that automatically builds the Jenkinsfile that is in the root directory of this repository. 12 | 13 | This job will: 14 | 15 | 1. Launch a Docker container using the Dockerfile to run the Jenkins job in. 16 | 1. Build the wheel and version it according to the version number parameter configured in setup.py. 17 | 1. Upload the wheel file to s3 bucket attached to databricks clusters: 18 | - s3://dataconnect-dbx-rootbucket-val/nvirginia-prod/955724715920583/shared_uploads/dqm/ 19 | - s3://dataconnect-dbx-rootbucket-prod/nvirginia-prod/667862292061707/shared_uploads/dqm/ 20 | 21 | ### To Be Done 22 | 23 | These are ideas that haven't been executed yet 24 | 25 | Deploying wheels to clusters should be triggered manually. We can't assume anything about what particular version of the library people want on particular clusters at a particular time. 26 | 27 | #### Deploy to All-purpose Clusters 28 | 29 | Deploying a wheel to all-purpose clusters can be done by running a different Jenkins job than the automatically triggered one. It will edit the json files by using a terraform variable to update the location and version of the wheel library. Those json files go into a notebook? Then the Jenkins job will run the notebook? 30 | 31 | - This job should give the user an option to choose the version of the library via a text box. If the version they choose doesn't exist in s3 then the job will just fail. 32 | - Another option to choose the environment (val/state-prod/prod). 33 | 34 | We would need to have a way to give Mathmatic 35 | 36 | #### Deploy to Interactive Clusters 37 | 38 | Deploying wheels to interactive clusters can be done via notebooks. Maybe using the %pip command? 39 | 40 | #### Cleanup 41 | 42 | We could create an automated cleanup job that runs nightly. It could keep the last 5 versions of the wheel? 43 | 44 | We could clean up the s3 folders. 45 | We could cleanup the clusters too because the installed library wheels will accumulate. 46 | 47 | 48 | ## Local Workflow 49 | 50 | DQ Measures is deployed as distributable WHL ("Wheel") file. WHL files are built using [_setuptools_](https://pypi.org/project/setuptools/) 51 | 52 | Building and deploying the WHL can be done automatically through the `deploy.sh` script in the root of the repository, as long as your VERSION is set in your environment. One way to do this is to place `export VERSION=1.2.3` as a line in a file named `.env`, also in the root of the repository. Then you can either restart your terminal, or run `source .env` from the command line, and the variable should get picked up. This must be done every time the `.env` file is updated. 53 | 54 | To run the deployment, you must a `bash` terminal, *not* a `powershell` terminal. The command to run it is `bash deploy.sh`. 55 | 56 | The script will default to uploading the wheel file to the `VAL` environment, but if deploying for production add another variable to the `.env` file to set `export ENVIRON=PROD`. 57 | 58 | ### Manually build the library 59 | 60 | If not done so already, run these commands: 61 | 62 | 1. > ```python -m venv .venv``` 63 | 2. > ```.venv/Scripts/Activate.ps1``` 64 | 3. > ```python -m pip install --upgrade pip``` 65 | 4. > ```python -m pip install -r requirements.txt``` 66 | 67 | If there have been any updates to measures, run the reverse lookup as described in the README file. 68 | 69 | From the top level folder, run these commands: 70 | 71 | 5. > ```rm -r -fo .\build; rm -r -fo .\*.egg-info``` (only if you have created a wheel file before) 72 | 6. > ```python setup.py bdist_wheel``` 73 | 74 | #### Upload the WHL file to the Databricks environment 75 | 76 | 7. > ```databricks --profile val fs cp ./dist/dqm-2.6.10-py3-none-any.whl dbfs:/FileStore/shared_uploads/akira/lib/dqm-2.6.10-py3-none-any.whl --overwrite``` 77 | 78 | ### Deploy the library to the Databricks cluster 79 | 80 | Deploy the library WHL file to Databricks clusters using [these](https://docs.databricks.com/libraries/cluster-libraries.html) instructions where applicable. 81 | 82 | ## Testing 83 | 84 | (there is a testing automation effort that needs to be described here) 85 | -------------------------------------------------------------------------------- /.github/workflows/build-and-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Build and Deploy 2 | 3 | # CD runs on merge to main. 4 | on: 5 | push: 6 | branches: 7 | - main 8 | 9 | # Disable default permissions for all of the available scopes in this workflow. 10 | permissions: {} 11 | 12 | jobs: 13 | Build-And-Deploy: 14 | runs-on: ubuntu-latest 15 | 16 | # 5 minutes should be plenty of time 17 | timeout-minutes: 5 18 | 19 | # Set the permission scope of this particular job 20 | permissions: 21 | # id-token: write permits an action to fetch an OpenID Connect (OIDC) token 22 | id-token: write 23 | # contents: read permits an action to list the commits 24 | contents: read 25 | 26 | steps: 27 | - uses: actions/checkout@v4 28 | 29 | # Assume the appropriate AWS role 30 | - name: Configure AWS Credentials 31 | uses: aws-actions/configure-aws-credentials@v2 32 | with: 33 | aws-region: us-east-1 34 | role-to-assume: arn:aws:iam::810638926762:role/dqm_github_actions_cd_role 35 | role-session-name: DQM_Build_Deploy 36 | 37 | # Set python version 38 | - name: Set up Python 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: 3.10.12 42 | 43 | # Install requirements 44 | - name: Install Python requirements 45 | run: | 46 | pip install -r requirements.txt 47 | 48 | # Build the wheel 49 | - name: Build wheel 50 | run: | 51 | python setup.py bdist_wheel 52 | 53 | # # Deploy the wheel to S3 54 | # - name: Deploy wheel 55 | # run: | 56 | # aws s3 cp dist s3://dataconnect-dbx-rootbucket-val/nvirginia-prod/955724715920583/shared_uploads/dqm/ --recursive' 57 | # aws s3 cp dist s3://dataconnect-dbx-rootbucket-prod/nvirginia-prod/667862292061707/shared_uploads/dqm/ --recursive' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dqmenv/* 2 | DQMenv/* 3 | .venv/ 4 | .flake8 5 | .vscode/ 6 | build/ 7 | dist/ 8 | dqm.egg-info/ 9 | dqm/batch/*.pkl 10 | .env 11 | setup_local.py -------------------------------------------------------------------------------- /DQ Measures Specifications_V4.0.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/DQ Measures Specifications_V4.0.xlsx -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | CC0 1.0 Universal 2 | 3 | Statement of Purpose 4 | 5 | The laws of most jurisdictions throughout the world automatically confer 6 | exclusive Copyright and Related Rights (defined below) upon the creator and 7 | subsequent owner(s) (each and all, an "owner") of an original work of 8 | authorship and/or a database (each, a "Work"). 9 | 10 | Certain owners wish to permanently relinquish those rights to a Work for the 11 | purpose of contributing to a commons of creative, cultural and scientific 12 | works ("Commons") that the public can reliably and without fear of later 13 | claims of infringement build upon, modify, incorporate in other works, reuse 14 | and redistribute as freely as possible in any form whatsoever and for any 15 | purposes, including without limitation commercial purposes. These owners may 16 | contribute to the Commons to promote the ideal of a free culture and the 17 | further production of creative, cultural and scientific works, or to gain 18 | reputation or greater distribution for their Work in part through the use and 19 | efforts of others. 20 | 21 | For these and/or other purposes and motivations, and without any expectation 22 | of additional consideration or compensation, the person associating CC0 with a 23 | Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 24 | and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 25 | and publicly distribute the Work under its terms, with knowledge of his or her 26 | Copyright and Related Rights in the Work and the meaning and intended legal 27 | effect of CC0 on those rights. 28 | 29 | 1. Copyright and Related Rights. A Work made available under CC0 may be 30 | protected by copyright and related or neighboring rights ("Copyright and 31 | Related Rights"). Copyright and Related Rights include, but are not limited 32 | to, the following: 33 | 34 | i. the right to reproduce, adapt, distribute, perform, display, communicate, 35 | and translate a Work; 36 | 37 | ii. moral rights retained by the original author(s) and/or performer(s); 38 | 39 | iii. publicity and privacy rights pertaining to a person's image or likeness 40 | depicted in a Work; 41 | 42 | iv. rights protecting against unfair competition in regards to a Work, 43 | subject to the limitations in paragraph 4(a), below; 44 | 45 | v. rights protecting the extraction, dissemination, use and reuse of data in 46 | a Work; 47 | 48 | vi. database rights (such as those arising under Directive 96/9/EC of the 49 | European Parliament and of the Council of 11 March 1996 on the legal 50 | protection of databases, and under any national implementation thereof, 51 | including any amended or successor version of such directive); and 52 | 53 | vii. other similar, equivalent or corresponding rights throughout the world 54 | based on applicable law or treaty, and any national implementations thereof. 55 | 56 | 2. Waiver. To the greatest extent permitted by, but not in contravention of, 57 | applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 58 | unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 59 | and Related Rights and associated claims and causes of action, whether now 60 | known or unknown (including existing as well as future claims and causes of 61 | action), in the Work (i) in all territories worldwide, (ii) for the maximum 62 | duration provided by applicable law or treaty (including future time 63 | extensions), (iii) in any current or future medium and for any number of 64 | copies, and (iv) for any purpose whatsoever, including without limitation 65 | commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 66 | the Waiver for the benefit of each member of the public at large and to the 67 | detriment of Affirmer's heirs and successors, fully intending that such Waiver 68 | shall not be subject to revocation, rescission, cancellation, termination, or 69 | any other legal or equitable action to disrupt the quiet enjoyment of the Work 70 | by the public as contemplated by Affirmer's express Statement of Purpose. 71 | 72 | 3. Public License Fallback. Should any part of the Waiver for any reason be 73 | judged legally invalid or ineffective under applicable law, then the Waiver 74 | shall be preserved to the maximum extent permitted taking into account 75 | Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 76 | is so judged Affirmer hereby grants to each affected person a royalty-free, 77 | non transferable, non sublicensable, non exclusive, irrevocable and 78 | unconditional license to exercise Affirmer's Copyright and Related Rights in 79 | the Work (i) in all territories worldwide, (ii) for the maximum duration 80 | provided by applicable law or treaty (including future time extensions), (iii) 81 | in any current or future medium and for any number of copies, and (iv) for any 82 | purpose whatsoever, including without limitation commercial, advertising or 83 | promotional purposes (the "License"). The License shall be deemed effective as 84 | of the date CC0 was applied by Affirmer to the Work. Should any part of the 85 | License for any reason be judged legally invalid or ineffective under 86 | applicable law, such partial invalidity or ineffectiveness shall not 87 | invalidate the remainder of the License, and in such case Affirmer hereby 88 | affirms that he or she will not (i) exercise any of his or her remaining 89 | Copyright and Related Rights in the Work or (ii) assert any associated claims 90 | and causes of action with respect to the Work, in either case contrary to 91 | Affirmer's express Statement of Purpose. 92 | 93 | 4. Limitations and Disclaimers. 94 | 95 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 96 | surrendered, licensed or otherwise affected by this document. 97 | 98 | b. Affirmer offers the Work as-is and makes no representations or warranties 99 | of any kind concerning the Work, express, implied, statutory or otherwise, 100 | including without limitation warranties of title, merchantability, fitness 101 | for a particular purpose, non infringement, or the absence of latent or 102 | other defects, accuracy, or the present or absence of errors, whether or not 103 | discoverable, all to the greatest extent permissible under applicable law. 104 | 105 | c. Affirmer disclaims responsibility for clearing rights of other persons 106 | that may apply to the Work or any use thereof, including without limitation 107 | any person's Copyright and Related Rights in the Work. Further, Affirmer 108 | disclaims responsibility for obtaining any necessary consents, permissions 109 | or other rights required for any use of the Work. 110 | 111 | d. Affirmer understands and acknowledges that Creative Commons is not a 112 | party to this document and has no duty or obligation with respect to this 113 | CC0 or use of the Work. 114 | 115 | For more information, please see 116 | 117 | -------------------------------------------------------------------------------- /README_Public_Repo.md: -------------------------------------------------------------------------------- 1 | # T-MSIS-Data-Quality-Measures-Generation-Code 2 | 3 | This project aims to provide resources and tools that help the research community and other stakeholders (the end user community) obtain clear, concise information about the overall data quality, as well as granular detail that can inform specific research projects or information needs in the Transformed Medicaid Statistical Information System (T-MSIS). 4 | 5 | **Getting the code** 6 | 7 | The easiest way to obtain the code is to clone it with git. If you're not familiar with git, a tool like Github Desktop or SourceTree can help make the experience easier. The HTTPS link is https://github.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code 8 | If you're familiar with git and just want to work from the command line, you just need to run: 9 | git clone https://github.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code.git 10 | If you would prefer not to use git, you can also download the most recent code as a ZIP file. 11 | 12 | **How the code is used** 13 | 14 | The DQ measures code is written in Databricks SQL executed through explicit SQL passthrough embedded within a python code wrapper. The code is written for and is executed on T-MSIS data that is stored in DataConnect, a Center for Medicaid and CHIP Services (CMCS) data warehouse. The code is executed on new monthly state T-MSIS submissions as soon as possible after the submissions have successfully been processed and the resulting data have been loaded into DataConnect for analytic use, or on historical T-MSIS data that is available in DataConnect on an as-needed basis. The code and related documentation are available in this public repository to provide explicit information about the calculation of measures in case there are questions about how to interpret the measure specifications. It is also available to provide insights into the code if states are attempting to replicate any DQ measures on T-MSIS data within their own systems. The code available in this GitHub repository can only be successfully executed on T-MSIS data in DataConnect. Modifications would be necessary to execute the code in any other environment. 15 | 16 | **Lookup Tables** 17 | 18 | Lookup tables that are part of the measure calculation can be found in the repository in csv form https://github.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/tree/main/static/csv and pkl form https://github.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/tree/main/dqm/cfg. 19 | 20 | **Contributing** 21 | 22 | We would be happy to receive suggestions on how to fix bugs or make improvements, though we will not support changes made through this repository. Instead, please send your suggestions to MACBISData@cms.hhs.gov. 23 | 24 | **Public domain** 25 | 26 | This project is in the worldwide public domain. 27 | This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through the CC0 1.0 Universal public domain dedication. 28 | 29 | **Background on data quality measures** 30 | 31 | A measure is a calculated statistic. There are several types of data quality measures: percentage, count, sum, ratio, average number of occurrences, average, ratio of averages, index of dissimilarity, and frequency. 32 | 33 | There are several types of data quality validations users can apply to the measures. Two types are most common: 34 | 1. General inferential validations check that a monthly statistic generated for a measure falls within a range of expected values for one point in time. 35 | 2. Longitudinal inferential validations compare the most recent month’s statistic for a measure against the average of the prior six months’ statistics for the same measure to see whether the most recent statistic deviates more than expected. 36 | 37 | Both general inferential validations and longitudinal inferential validations rely on data quality standards, which are either tolerance thresholds or minimum-maximum ranges, depending upon the type of validation for the measure. The ranges and tolerances are based on historical and expected data patterns. Currently, CMS does not have state-specific thresholds, so the thresholds are the same for all states, regardless of the states’ programs or population size. In addition, some measures do not have any data quality standards because a threshold has not yet been established. Measure specific data quality standards are available in the https://github.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/blob/main/Thresholds.xlsx. 38 | Note that a data quality measure statistic falling outside of the data quality standard is not the same as an error. Errors indicate a business rule violation (i.e. something is wrong with the data) and are based on the validation rules in the T-MSIS Data Dictionary. The data quality standards are based on expected patterns in the data, and a statistic outside of the expected range may or may not reflect actual errors in the data. 39 | 40 | Sometimes, the data are obviously wrong. But often, a measure statistic outside of the expected range indicates only a possible data problem. For example, if the data showed 100 percent of Medicaid-eligible beneficiaries died in a month, the data would be wrong. However, if the percentage of Medicaid eligible beneficiaries who died in a month was 5 percent, this would indicate a possible data problem because the measure statistic of 5 percent is outside the maximum threshold (i.e. the expected maximum of the percentage of Medicaid eligible beneficiaries who would die in one month) of 4 percent. It is not certain there is an issue with the data in that month; it would be important to consider both the value of the statistic and the expected range or longitudinal tolerance. In some cases, it may also be important to consider policy or operational factors in a state’s Medicaid or CHIP program. 41 | There are two other less common types of validation users can apply to the measures: 42 | 1. Authoritative source validation compares reported values with the values that should have been reported based on state program characteristics—for example, the managed care operating authorities that a state has. 43 | 2. Index of dissimilarity validation checks that the frequency distribution of a data element does not change significantly compared with the prior month. 44 | -------------------------------------------------------------------------------- /State_DQ_Missingness_Measures_v4.0.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/State_DQ_Missingness_Measures_v4.0.xlsx -------------------------------------------------------------------------------- /Thresholds.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/Thresholds.xlsx -------------------------------------------------------------------------------- /create_setup_local.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | # read in setup.py as text 4 | with open('setup.py', 'r') as f: 5 | text = f.read() 6 | 7 | # update the version to use the .env variable 8 | text_out = re.sub('version="\d{1,2}\.\d{1,2}\.\d{1,2}"', "version=os.environ['VERSION']", text, flags=re.I) 9 | text_out2 = text_out.replace("import setuptools\n", 10 | 'import setuptools\nimport os\nif \'VERSION\' not in os.environ:\n print("Please set VERSION environment variable, see workflow.md.")\n quit(1)\n\n') 11 | 12 | # write out new script 13 | with open('setup_local.py', 'w') as f: 14 | f.write(text_out2) 15 | -------------------------------------------------------------------------------- /databricks/notebooks/DQM UAT.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | # initialize variables with parameter values passed from job 3 | _reportMonth = dbutils.widgets.get("reportMonth") 4 | _stateCode = dbutils.widgets.get("stateCode") 5 | _entity = dbutils.widgets.get("entity") 6 | _runID = dbutils.widgets.get("runID") 7 | 8 | # COMMAND ---------- 9 | 10 | # import library and instantiate dqm object 11 | import logging 12 | from dqm import DQMeasures as s 13 | 14 | 15 | if (len(_runID)): 16 | dqm = s.DQMeasures(report_month = _reportMonth 17 | , rpt_state = _stateCode 18 | , separate_entity = _entity 19 | , run_id = _runID) 20 | else: 21 | dqm = s.DQMeasures(report_month = _reportMonth 22 | , rpt_state = _stateCode 23 | , separate_entity = _entity) 24 | 25 | # COMMAND ---------- 26 | 27 | # set level of verbosity for logging 28 | dqm.logger.setLevel(logging.DEBUG) 29 | 30 | # COMMAND ---------- 31 | 32 | # create base tables and views 33 | dqm.init() 34 | 35 | # COMMAND ---------- 36 | 37 | #display(dqm.thresholds) 38 | 39 | # COMMAND ---------- 40 | 41 | # configure spark with the task prefix for this run 42 | spark.conf.set('dqm.taskprefix', dqm.taskprefix) 43 | 44 | # COMMAND ---------- 45 | 46 | import re 47 | ver = re.sub('\..{1,2}$', '', dqm.version) 48 | # redirect s3 bucket folder path 49 | # conditionally handle seperate entities for medicaid and chip 50 | if (_entity == '0'): 51 | dqm.setS3Bucket('macbis-dw-dqm-val') 52 | dqm.s3folder = 'sas-dqm/uat/' + ver + '/' + dqm.rpt_state + '/' + dqm.rpt_fldr + '/' + dqm.z_run_id 53 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 54 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 55 | dqm.s3path 56 | elif (_entity == '1'): 57 | dqm.setS3Bucket('macbis-dw-dqm-val') 58 | dqm.s3folder = 'sas-dqm/uat/' + ver + '/' + dqm.rpt_state + '-M/' + dqm.rpt_fldr + '/' + dqm.z_run_id 59 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 60 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 61 | dqm.s3path 62 | elif (_entity == '2'): 63 | dqm.setS3Bucket('macbis-dw-dqm-val') 64 | dqm.s3folder = 'sas-dqm/uat/' + ver + '/' + dqm.rpt_state + '-C/' + dqm.rpt_fldr + '/' + dqm.z_run_id 65 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 66 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 67 | dqm.s3path 68 | 69 | # COMMAND ---------- 70 | 71 | # display useful information about this run 72 | dqm.print() 73 | 74 | # COMMAND ---------- 75 | 76 | # run all measures 77 | dqm.run(spark, interactive=False) 78 | 79 | # COMMAND ---------- 80 | 81 | # excel file output 82 | dqm.xlsx('summary') 83 | dqm.xlsx('plan') 84 | dqm.xlsx('waiver') -------------------------------------------------------------------------------- /databricks/notebooks/DQM v3.1 UAT.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | # initialize variables with parameter values passed from job 3 | _reportMonth = dbutils.widgets.get("reportMonth") 4 | _stateCode = dbutils.widgets.get("stateCode") 5 | _entity = dbutils.widgets.get("entity") 6 | _runID = dbutils.widgets.get("runID") 7 | 8 | # COMMAND ---------- 9 | 10 | # import library and instantiate dqm object 11 | import logging 12 | from dqm import DQMeasures as s 13 | 14 | 15 | if (len(_runID)): 16 | dqm = s.DQMeasures(report_month = _reportMonth 17 | , rpt_state = _stateCode 18 | , separate_entity = _entity 19 | , run_id = _runID) 20 | else: 21 | dqm = s.DQMeasures(report_month = _reportMonth 22 | , rpt_state = _stateCode 23 | , separate_entity = _entity) 24 | 25 | # COMMAND ---------- 26 | 27 | # set level of verbosity for logging 28 | dqm.logger.setLevel(logging.DEBUG) 29 | 30 | # COMMAND ---------- 31 | 32 | # create base tables and views 33 | dqm.init() 34 | 35 | # COMMAND ---------- 36 | 37 | # dqm.thresholds(spark, 's3a://macbis-dw-dqm-val/sas-dqm/thresholds/Thresholds_Document_v2.5.3_CMS-OD.xlsx') 38 | # dqm.setThresholds(spark, 's3a://macbis-dw-dqm-val/sas-dqm/thresholds/Thresholds Document v2.6.xlsx') 39 | #dqm.setThresholds(spark, 's3a://macbis-dw-dqm-val/sas-dqm/thresholds/Thresholds Document v2.6.1_CMS-OD.xlsx') 40 | 41 | # COMMAND ---------- 42 | 43 | #display(dqm.thresholds) 44 | 45 | # COMMAND ---------- 46 | 47 | # configure spark with the task prefix for this run 48 | spark.conf.set('dqm.taskprefix', dqm.taskprefix) 49 | 50 | # COMMAND ---------- 51 | 52 | # redirect s3 bucket folder path 53 | # conditionally handle seperate entities for medicaid and chip 54 | if (_entity == '0'): 55 | dqm.setS3Bucket('macbis-dw-dqm-val') 56 | dqm.s3folder = 'sas-dqm/uat/3.1/' + dqm.rpt_state + '/' + dqm.rpt_fldr + '/' + dqm.z_run_id 57 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 58 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 59 | dqm.s3path 60 | elif (_entity == '1'): 61 | dqm.setS3Bucket('macbis-dw-dqm-val') 62 | dqm.s3folder = 'sas-dqm/uat/3.1/' + dqm.rpt_state + '-M/' + dqm.rpt_fldr + '/' + dqm.z_run_id 63 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 64 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 65 | dqm.s3path 66 | elif (_entity == '2'): 67 | dqm.setS3Bucket('macbis-dw-dqm-val') 68 | dqm.s3folder = 'sas-dqm/uat/3.1/' + dqm.rpt_state + '-C/' + dqm.rpt_fldr + '/' + dqm.z_run_id 69 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 70 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 71 | dqm.s3path 72 | 73 | # COMMAND ---------- 74 | 75 | # display useful information about this run 76 | dqm.print() 77 | 78 | # COMMAND ---------- 79 | 80 | # run all measures 81 | dqm.run(spark, interactive=False) 82 | 83 | # COMMAND ---------- 84 | 85 | # excel file output 86 | dqm.xlsx('summary') 87 | dqm.xlsx('plan') 88 | dqm.xlsx('waiver') -------------------------------------------------------------------------------- /databricks/notebooks/VAL/DQM UAT.ipynb: -------------------------------------------------------------------------------- 1 | {"cells":[{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3283ed31-e386-4314-bed7-11313ee7b60a","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["# initialize variables with parameter values passed from job\n","_reportMonth = dbutils.widgets.get(\"reportMonth\")\n","_stateCode = dbutils.widgets.get(\"stateCode\")\n","_entity = dbutils.widgets.get(\"entity\")\n","_runID = dbutils.widgets.get(\"runID\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7f5d75c2-a25e-4db3-b783-28ee0c42714a","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["# set level of verbosity for logging\n","dqm.logger.setLevel(logging.DEBUG)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1344a6f3-477d-44fe-8545-1133ea593ef1","showTitle":false,"title":""}},"outputs":[],"source":["display(dqm.thresholds)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"578f508b-7d1c-47e9-a31b-f170507f8f04","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["# configure spark with the task prefix for this run\n","spark.conf.set('dqm.taskprefix', dqm.taskprefix)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"43fcdaed-daf6-4f40-93eb-b7950bd66a69","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["import re\n","ver = re.sub('\\..{1,2}$', '', dqm.version)\n","# redirect s3 bucket folder path\n","# conditionally handle seperate entities for medicaid and chip\n","if (_entity == '0'):\n"," dqm.setS3Bucket('macbis-dw-dqm-val')\n"," dqm.s3folder = 'sas-dqm/uat/' + ver + '/' + dqm.rpt_state + '/' + dqm.rpt_fldr + '/' + dqm.z_run_id\n"," dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder\n"," dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart\n"," dqm.s3path\n","elif (_entity == '1'):\n"," dqm.setS3Bucket('macbis-dw-dqm-val')\n"," dqm.s3folder = 'sas-dqm/uat/' + ver + '/' + dqm.rpt_state + '-M/' + dqm.rpt_fldr + '/' + dqm.z_run_id\n"," dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder\n"," dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart\n"," dqm.s3path\n","elif (_entity == '2'):\n"," dqm.setS3Bucket('macbis-dw-dqm-val')\n"," dqm.s3folder = 'sas-dqm/uat/' + ver + '/' + dqm.rpt_state + '-C/' + dqm.rpt_fldr + '/' + dqm.z_run_id\n"," dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder\n"," dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart\n"," dqm.s3path"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a3a54ef-11b4-480e-8bdd-bdd84247d3a6","showTitle":false,"title":""}},"outputs":[],"source":["# display useful information about this run\n","dqm.print()"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2db4b637-ff85-441e-bbab-12c961c135ef","showTitle":false,"title":""}},"outputs":[],"source":["# run all measures\n","dqm.run(spark, interactive=False)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"65d1ae6c-0fff-4a89-85e6-586ceae93f80","showTitle":false,"title":""}},"outputs":[],"source":["# excel file output\n","dqm.xlsx('summary')\n","dqm.xlsx('plan')\n","dqm.xlsx('waiver')"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":2},"notebookName":"DQM UAT","notebookOrigID":437900,"widgets":{}},"kernelspec":{"display_name":"DQMenv","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.9.13 (tags/v3.9.13:6de2ca5, May 17 2022, 16:36:42) [MSC v.1929 64 bit (AMD64)]"},"vscode":{"interpreter":{"hash":"96c372f71c040d05cf2a20db150d7d899ff817dd3a5df5ba9005f6ca61858cf9"}}},"nbformat":4,"nbformat_minor":0} 2 | -------------------------------------------------------------------------------- /databricks/notebooks/VAL/DQM v3.1 UAT.ipynb: -------------------------------------------------------------------------------- 1 | {"cells":[{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3283ed31-e386-4314-bed7-11313ee7b60a","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["# initialize variables with parameter values passed from job\n","_reportMonth = dbutils.widgets.get(\"reportMonth\")\n","_stateCode = dbutils.widgets.get(\"stateCode\")\n","_entity = dbutils.widgets.get(\"entity\")\n","_runID = dbutils.widgets.get(\"runID\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7f5d75c2-a25e-4db3-b783-28ee0c42714a","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["# set level of verbosity for logging\n","dqm.logger.setLevel(logging.DEBUG)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3a153793-2c73-453d-9009-006b4ea30f9a","showTitle":false,"title":""}},"outputs":[],"source":["# dqm.thresholds(spark, 's3a://macbis-dw-dqm-val/sas-dqm/thresholds/Thresholds_Document_v2.5.3_CMS-OD.xlsx')\n","# dqm.setThresholds(spark, 's3a://macbis-dw-dqm-val/sas-dqm/thresholds/Thresholds Document v2.6.xlsx')\n","# dqm.setThresholds(spark, 's3a://macbis-dw-dqm-val/sas-dqm/thresholds/Thresholds Document v2.6.1_CMS-OD.xlsx')"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1344a6f3-477d-44fe-8545-1133ea593ef1","showTitle":false,"title":""}},"outputs":[],"source":["#display(dqm.thresholds)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"578f508b-7d1c-47e9-a31b-f170507f8f04","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["# configure spark with the task prefix for this run\n","spark.conf.set('dqm.taskprefix', dqm.taskprefix)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"43fcdaed-daf6-4f40-93eb-b7950bd66a69","showTitle":false,"title":""}},"outputs":[{"data":{"text/html":["\n","
"]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":"
","datasetInfos":[],"metadata":{},"removedWidgets":[],"type":"html"}},"output_type":"display_data"}],"source":["# redirect s3 bucket folder path\n","# conditionally handle seperate entities for medicaid and chip\n","if (_entity == '0'):\n"," dqm.setS3Bucket('macbis-dw-dqm-val')\n"," dqm.s3folder = 'sas-dqm/uat/3.1/' + dqm.rpt_state + '/' + dqm.rpt_fldr + '/' + dqm.z_run_id\n"," dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder\n"," dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart\n"," dqm.s3path\n","elif (_entity == '1'):\n"," dqm.setS3Bucket('macbis-dw-dqm-val')\n"," dqm.s3folder = 'sas-dqm/uat/3.1/' + dqm.rpt_state + '-M/' + dqm.rpt_fldr + '/' + dqm.z_run_id\n"," dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder\n"," dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart\n"," dqm.s3path\n","elif (_entity == '2'):\n"," dqm.setS3Bucket('macbis-dw-dqm-val')\n"," dqm.s3folder = 'sas-dqm/uat/3.1/' + dqm.rpt_state + '-C/' + dqm.rpt_fldr + '/' + dqm.z_run_id\n"," dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder\n"," dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart\n"," dqm.s3path"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a3a54ef-11b4-480e-8bdd-bdd84247d3a6","showTitle":false,"title":""}},"outputs":[],"source":["# display useful information about this run\n","dqm.print()"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2db4b637-ff85-441e-bbab-12c961c135ef","showTitle":false,"title":""}},"outputs":[],"source":["# run all measures\n","dqm.run(spark, interactive=False)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"65d1ae6c-0fff-4a89-85e6-586ceae93f80","showTitle":false,"title":""}},"outputs":[],"source":["# excel file output\n","dqm.xlsx('summary')\n","dqm.xlsx('plan')\n","dqm.xlsx('waiver')"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":2},"notebookName":"DQM v3.1 UAT","notebookOrigID":437900,"widgets":{}},"kernelspec":{"display_name":"Python 3.9.6 ('.venv': venv)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.9.6"},"vscode":{"interpreter":{"hash":"0adc807b1c48ec2ccc96381ae40cb77b48d18e1ceda510d785437b846f22ab03"}}},"nbformat":4,"nbformat_minor":0} 2 | -------------------------------------------------------------------------------- /databricks/reset_all_jobs_PROD.bat: -------------------------------------------------------------------------------- 1 | databricks --profile prod jobs reset --job-id 3499 --json-file ./jobs/PROD/alabama.json 2 | databricks --profile prod jobs reset --job-id 3680 --json-file ./jobs/PROD/alaska.json 3 | databricks --profile prod jobs reset --job-id 934 --json-file ./jobs/PROD/arizona.json 4 | databricks --profile prod jobs reset --job-id 3402 --json-file ./jobs/PROD/arkansas.json 5 | 6 | databricks --profile prod jobs reset --job-id 314 --json-file ./jobs/PROD/california.json 7 | databricks --profile prod jobs reset --job-id 12718 --json-file ./jobs/PROD/california_900.json 8 | databricks --profile prod jobs reset --job-id 12757 --json-file ./jobs/PROD/california_800.json 9 | databricks --profile prod jobs reset --job-id 12781 --json-file ./jobs/PROD/california_700.json 10 | databricks --profile prod jobs reset --job-id 12818 --json-file ./jobs/PROD/california_600.json 11 | databricks --profile prod jobs reset --job-id 12877 --json-file ./jobs/PROD/california_500.json 12 | databricks --profile prod jobs reset --job-id 12896 --json-file ./jobs/PROD/california_200.json 13 | databricks --profile prod jobs reset --job-id 12964 --json-file ./jobs/PROD/california_100.json 14 | databricks --profile prod jobs reset --job-id 13006 --json-file ./jobs/PROD/california_reports.json 15 | 16 | databricks --profile prod jobs reset --job-id 3727 --json-file ./jobs/PROD/colorado.json 17 | databricks --profile prod jobs reset --job-id 3824 --json-file ./jobs/PROD/connecticut.json 18 | databricks --profile prod jobs reset --job-id 1071 --json-file ./jobs/PROD/district_of_columbia.json 19 | databricks --profile prod jobs reset --job-id 3854 --json-file ./jobs/PROD/delaware.json 20 | databricks --profile prod jobs reset --job-id 740 --json-file ./jobs/PROD/florida.json 21 | databricks --profile prod jobs reset --job-id 3915 --json-file ./jobs/PROD/georgia.json 22 | databricks --profile prod jobs reset --job-id 999 --json-file ./jobs/PROD/hawaii.json 23 | databricks --profile prod jobs reset --job-id 3949 --json-file ./jobs/PROD/idaho.json 24 | databricks --profile prod jobs reset --job-id 3951 --json-file ./jobs/PROD/illinois.json 25 | databricks --profile prod jobs reset --job-id 787 --json-file ./jobs/PROD/indiana.json 26 | databricks --profile prod jobs reset --job-id 3935 --json-file ./jobs/PROD/iowa.json 27 | databricks --profile prod jobs reset --job-id 3980 --json-file ./jobs/PROD/kansas.json 28 | databricks --profile prod jobs reset --job-id 4003 --json-file ./jobs/PROD/kentucky.json 29 | databricks --profile prod jobs reset --job-id 4008 --json-file ./jobs/PROD/louisiana.json 30 | databricks --profile prod jobs reset --job-id 4180 --json-file ./jobs/PROD/maine.json 31 | databricks --profile prod jobs reset --job-id 4016 --json-file ./jobs/PROD/maryland.json 32 | databricks --profile prod jobs reset --job-id 4100 --json-file ./jobs/PROD/massachusetts.json 33 | databricks --profile prod jobs reset --job-id 4258 --json-file ./jobs/PROD/michigan.json 34 | databricks --profile prod jobs reset --job-id 4261 --json-file ./jobs/PROD/minnesota.json 35 | databricks --profile prod jobs reset --job-id 4314 --json-file ./jobs/PROD/mississippi.json 36 | databricks --profile prod jobs reset --job-id 4264 --json-file ./jobs/PROD/missouri.json 37 | databricks --profile prod jobs reset --job-id 4322 --json-file ./jobs/PROD/montana.json 38 | databricks --profile prod jobs reset --job-id 887 --json-file ./jobs/PROD/nebraska.json 39 | databricks --profile prod jobs reset --job-id 564 --json-file ./jobs/PROD/nevada.json 40 | databricks --profile prod jobs reset --job-id 4485 --json-file ./jobs/PROD/new_hampshire.json 41 | databricks --profile prod jobs reset --job-id 4576 --json-file ./jobs/PROD/new_jersey.json 42 | databricks --profile prod jobs reset --job-id 4590 --json-file ./jobs/PROD/new_mexico.json 43 | databricks --profile prod jobs reset --job-id 464 --json-file ./jobs/PROD/new_york.json 44 | databricks --profile prod jobs reset --job-id 4346 --json-file ./jobs/PROD/north_carolina.json 45 | databricks --profile prod jobs reset --job-id 4427 --json-file ./jobs/PROD/north_dakota.json 46 | databricks --profile prod jobs reset --job-id 868 --json-file ./jobs/PROD/ohio.json 47 | databricks --profile prod jobs reset --job-id 4606 --json-file ./jobs/PROD/oklahoma.json 48 | databricks --profile prod jobs reset --job-id 31041 --json-file ./jobs/PROD/oregon.json 49 | databricks --profile prod jobs reset --job-id 4740 --json-file ./jobs/PROD/pennsylvania_medicaid.json 50 | databricks --profile prod jobs reset --job-id 4673 --json-file ./jobs/PROD/pennsylvania_chip.json 51 | databricks --profile prod jobs reset --job-id 4806 --json-file ./jobs/PROD/puerto_rico.json 52 | databricks --profile prod jobs reset --job-id 4849 --json-file ./jobs/PROD/rhode_island.json 53 | databricks --profile prod jobs reset --job-id 4933 --json-file ./jobs/PROD/south_carolina.json 54 | databricks --profile prod jobs reset --job-id 4971 --json-file ./jobs/PROD/south_dakota.json 55 | databricks --profile prod jobs reset --job-id 4974 --json-file ./jobs/PROD/tennessee.json 56 | databricks --profile prod jobs reset --job-id 1188 --json-file ./jobs/PROD/texas.json 57 | databricks --profile prod jobs reset --job-id 5034 --json-file ./jobs/PROD/utah.json 58 | databricks --profile prod jobs reset --job-id 5148 --json-file ./jobs/PROD/vermont.json 59 | databricks --profile prod jobs reset --job-id 1126 --json-file ./jobs/PROD/us_virgin_islands.json 60 | databricks --profile prod jobs reset --job-id 3586 --json-file ./jobs/PROD/virginia.json 61 | databricks --profile prod jobs reset --job-id 987 --json-file ./jobs/PROD/washington.json 62 | databricks --profile prod jobs reset --job-id 5055 --json-file ./jobs/PROD/west_virginia.json 63 | databricks --profile prod jobs reset --job-id 3549 --json-file ./jobs/PROD/wisconsin.json 64 | databricks --profile prod jobs reset --job-id 5086 --json-file ./jobs/PROD/wyoming_medicaid.json 65 | -------------------------------------------------------------------------------- /databricks/reset_all_jobs_VAL.bat: -------------------------------------------------------------------------------- 1 | databricks --profile val jobs reset --job-id 77011 --json-file .\jobs\VAL\alabama.json 2 | databricks --profile val jobs reset --job-id 77101 --json-file .\jobs\VAL\alaska.json 3 | databricks --profile val jobs reset --job-id 77140 --json-file .\jobs\VAL\arizona.json 4 | databricks --profile val jobs reset --job-id 77185 --json-file .\jobs\VAL\arkansas.json 5 | 6 | databricks --profile val jobs reset --job-id 77251 --json-file .\jobs\VAL\california.json 7 | databricks --profile val jobs reset --job-id 598923 --json-file .\jobs\VAL\california_prep.json 8 | databricks --profile val jobs reset --job-id 598736 --json-file .\jobs\VAL\california_900.json 9 | databricks --profile val jobs reset --job-id 598725 --json-file .\jobs\VAL\california_800.json 10 | databricks --profile val jobs reset --job-id 598691 --json-file .\jobs\VAL\california_700.json 11 | databricks --profile val jobs reset --job-id 598637 --json-file .\jobs\VAL\california_600.json 12 | databricks --profile val jobs reset --job-id 598619 --json-file .\jobs\VAL\california_500.json 13 | databricks --profile val jobs reset --job-id 598578 --json-file .\jobs\VAL\california_200.json 14 | databricks --profile val jobs reset --job-id 598532 --json-file .\jobs\VAL\california_100.json 15 | databricks --profile val jobs reset --job-id 598747 --json-file .\jobs\VAL\california_reports.json 16 | 17 | databricks --profile val jobs reset --job-id 77643 --json-file .\jobs\VAL\colorado.json 18 | databricks --profile val jobs reset --job-id 77712 --json-file .\jobs\VAL\connecticut.json 19 | databricks --profile val jobs reset --job-id 77842 --json-file .\jobs\VAL\district_of_columbia.json 20 | databricks --profile val jobs reset --job-id 77761 --json-file .\jobs\VAL\delaware.json 21 | databricks --profile val jobs reset --job-id 77844 --json-file .\jobs\VAL\florida.json 22 | databricks --profile val jobs reset --job-id 77886 --json-file .\jobs\VAL\georgia.json 23 | databricks --profile val jobs reset --job-id 77893 --json-file .\jobs\VAL\hawaii.json 24 | databricks --profile val jobs reset --job-id 77979 --json-file .\jobs\VAL\idaho.json 25 | databricks --profile val jobs reset --job-id 78047 --json-file .\jobs\VAL\illinois.json 26 | databricks --profile val jobs reset --job-id 78114 --json-file .\jobs\VAL\indiana.json 27 | databricks --profile val jobs reset --job-id 78209 --json-file .\jobs\VAL\iowa.json 28 | databricks --profile val jobs reset --job-id 78280 --json-file .\jobs\VAL\kansas.json 29 | databricks --profile val jobs reset --job-id 78331 --json-file .\jobs\VAL\kentucky.json 30 | databricks --profile val jobs reset --job-id 78377 --json-file .\jobs\VAL\louisiana.json 31 | databricks --profile val jobs reset --job-id 78441 --json-file .\jobs\VAL\maine.json 32 | databricks --profile val jobs reset --job-id 78509 --json-file .\jobs\VAL\maryland.json 33 | databricks --profile val jobs reset --job-id 78565 --json-file .\jobs\VAL\massachusetts.json 34 | databricks --profile val jobs reset --job-id 78614 --json-file .\jobs\VAL\michigan.json 35 | databricks --profile val jobs reset --job-id 78684 --json-file .\jobs\VAL\minnesota.json 36 | databricks --profile val jobs reset --job-id 78777 --json-file .\jobs\VAL\mississippi.json 37 | databricks --profile val jobs reset --job-id 78788 --json-file .\jobs\VAL\missouri.json 38 | databricks --profile val jobs reset --job-id 78887 --json-file .\jobs\VAL\montana.json 39 | databricks --profile val jobs reset --job-id 78977 --json-file .\jobs\VAL\nebraska.json 40 | databricks --profile val jobs reset --job-id 79063 --json-file .\jobs\VAL\nevada.json 41 | databricks --profile val jobs reset --job-id 79143 --json-file .\jobs\VAL\new_hampshire.json 42 | databricks --profile val jobs reset --job-id 79221 --json-file .\jobs\VAL\new_jersey.json 43 | databricks --profile val jobs reset --job-id 79312 --json-file .\jobs\VAL\new_mexico.json 44 | databricks --profile val jobs reset --job-id 79374 --json-file .\jobs\VAL\new_york.json 45 | databricks --profile val jobs reset --job-id 79399 --json-file .\jobs\VAL\north_carolina.json 46 | databricks --profile val jobs reset --job-id 79424 --json-file .\jobs\VAL\north_dakota.json 47 | databricks --profile val jobs reset --job-id 79472 --json-file .\jobs\VAL\ohio.json 48 | databricks --profile val jobs reset --job-id 79550 --json-file .\jobs\VAL\oklahoma.json 49 | databricks --profile val jobs reset --job-id 79595 --json-file .\jobs\VAL\oregon.json 50 | databricks --profile val jobs reset --job-id 79672 --json-file .\jobs\VAL\pennsylvania_chip.json 51 | databricks --profile val jobs reset --job-id 79717 --json-file .\jobs\VAL\pennsylvania_medicaid.json 52 | databricks --profile val jobs reset --job-id 79773 --json-file .\jobs\VAL\puerto_rico.json 53 | databricks --profile val jobs reset --job-id 79811 --json-file .\jobs\VAL\rhode_island.json 54 | databricks --profile val jobs reset --job-id 79823 --json-file .\jobs\VAL\south_carolina.json 55 | databricks --profile val jobs reset --job-id 79831 --json-file .\jobs\VAL\south_dakota.json 56 | databricks --profile val jobs reset --job-id 79913 --json-file .\jobs\VAL\tennessee.json 57 | databricks --profile val jobs reset --job-id 80006 --json-file .\jobs\VAL\texas.json 58 | databricks --profile val jobs reset --job-id 80078 --json-file .\jobs\VAL\utah.json 59 | databricks --profile val jobs reset --job-id 80140 --json-file .\jobs\VAL\vermont.json 60 | databricks --profile val jobs reset --job-id 80053 --json-file .\jobs\VAL\us_virgin_islands.json 61 | databricks --profile val jobs reset --job-id 80201 --json-file .\jobs\VAL\virginia.json 62 | databricks --profile val jobs reset --job-id 80263 --json-file .\jobs\VAL\washington.json 63 | databricks --profile val jobs reset --job-id 80264 --json-file .\jobs\VAL\west_virginia.json 64 | databricks --profile val jobs reset --job-id 80341 --json-file .\jobs\VAL\wisconsin.json 65 | databricks --profile val jobs reset --job-id 80397 --json-file .\jobs\VAL\wyoming_medicaid.json 66 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | shopt -s nocasematch 2 | # if environment is not specified in .env, default to VAL 3 | if [[ -z "${ENVIRON}" ]]; then 4 | ENVIRON=e2val 5 | echo environment=${ENVIRON} 6 | else if ! [[ "${ENVIRON}" =~ ^(e2val)$ ]]; then 7 | # user must confirm they want to deploy to an environment that is not VAL 8 | read -p "Are you sure you want to deploy to ${ENVIRON}? (enter y or n) " -n 1 -r 9 | if [[ ! $REPLY =~ ^[Yy]$ ]] 10 | then 11 | exit 1 12 | else 13 | echo "" 14 | fi 15 | fi 16 | fi 17 | # if deploying to PROD, check that the version name does not have the user's initials from testing and is on the master branch 18 | if [[ "${ENVIRON}" =~ ^(e2prod)$ ]]; then 19 | if ! [[ "${VERSION}" =~ ^[1-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then 20 | echo "ERROR: Prod deployment must not include initials/characters in version (or must be specified)" ; 21 | exit 1 22 | fi 23 | if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then 24 | echo "ERROR: Prod deployment must be on master branch"; 25 | exit 1; 26 | fi 27 | fi 28 | # if the environment specified is not VAL or PROD, do not deploy 29 | if ! [[ "${ENVIRON}" =~ ^(e2val|e2prod)$ ]]; then 30 | echo "ERROR: Environment is not valid (must be val (default) or prod)" ; 31 | else 32 | # confirm that the version is specified 33 | if [[ -z "${VERSION}" ]]; then 34 | echo "ERROR: Please set `VERSION` variable in .env file." 35 | # if all checks pass, execute deployment 36 | # create temporary local version of setup.py and delete after creating wheel 37 | else 38 | # set upload location 39 | if [[ "${ENVIRON}" =~ ^(e2val)$ ]]; then 40 | UPLOAD=uat_val_catalog/dqm_conv/dqm_package_volume_val; 41 | elif [[ "${ENVIRON}" =~ ^(e2prod)$ ]]; then 42 | UPLOAD=cms_prod_catalog/dqm_conv/dqm_package_volume_prod; 43 | fi 44 | echo "Deploying (VERSION=${VERSION}) ..." 45 | cd dqm/batch 46 | python reverse_lookup.py 47 | cd ../.. 48 | rm -rf build/* 49 | rm -rf *.egg-info/* 50 | python create_setup_local.py 51 | python setup_local.py bdist_wheel 52 | rm -rf setup_local.py 53 | echo "databricks --profile ${ENVIRON} fs cp ./dist/dqm-${VERSION}-py3-none-any.whl dbfs:/Volumes/${UPLOAD}/dqm-${VERSION}-py3-none-any.whl --overwrite" 54 | databricks --profile ${ENVIRON} fs cp ./dist/dqm-${VERSION}-py3-none-any.whl dbfs:/Volumes/${UPLOAD}/dqm-${VERSION}-py3-none-any.whl --overwrite 55 | fi 56 | fi -------------------------------------------------------------------------------- /dqm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/__init__.py -------------------------------------------------------------------------------- /dqm/batch/reverse_lookup.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/reverse_lookup.pkl -------------------------------------------------------------------------------- /dqm/batch/run_101.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_101.pkl -------------------------------------------------------------------------------- /dqm/batch/run_102.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_102.pkl -------------------------------------------------------------------------------- /dqm/batch/run_103.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_103.pkl -------------------------------------------------------------------------------- /dqm/batch/run_104.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_104.pkl -------------------------------------------------------------------------------- /dqm/batch/run_105.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_105.pkl -------------------------------------------------------------------------------- /dqm/batch/run_105.py: -------------------------------------------------------------------------------- 1 | from pandas import pandas as pd 2 | from pandas import DataFrame 3 | 4 | run_105 =[ 5 | ['105', 'el1003', 'el10_3', 'el10.3'], 6 | ['105', 'el1004', 'el10_4', 'el10.4'], 7 | ['105', 'el120t', 'el1_20', 'el1.20'], 8 | ['105', 'el314t', 'el3_14', 'el3.14'] 9 | ] 10 | 11 | 12 | df = DataFrame(run_105, columns=['series', 'cb', 'measure_id', 'id']) 13 | df['measure_id'] = df['measure_id'].str.upper() 14 | 15 | 16 | with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 2000): 17 | print (df) 18 | 19 | df.to_pickle('./run_105.pkl') 20 | 21 | # CC0 1.0 Universal 22 | 23 | # Statement of Purpose 24 | 25 | # The laws of most jurisdictions throughout the world automatically confer 26 | # exclusive Copyright and Related Rights (defined below) upon the creator and 27 | # subsequent owner(s) (each and all, an "owner") of an original work of 28 | # authorship and/or a database (each, a "Work"). 29 | 30 | # Certain owners wish to permanently relinquish those rights to a Work for the 31 | # purpose of contributing to a commons of creative, cultural and scientific 32 | # works ("Commons") that the public can reliably and without fear of later 33 | # claims of infringement build upon, modify, incorporate in other works, reuse 34 | # and redistribute as freely as possible in any form whatsoever and for any 35 | # purposes, including without limitation commercial purposes. These owners may 36 | # contribute to the Commons to promote the ideal of a free culture and the 37 | # further production of creative, cultural and scientific works, or to gain 38 | # reputation or greater distribution for their Work in part through the use and 39 | # efforts of others. 40 | 41 | # For these and/or other purposes and motivations, and without any expectation 42 | # of additional consideration or compensation, the person associating CC0 with a 43 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 44 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 45 | # and publicly distribute the Work under its terms, with knowledge of his or her 46 | # Copyright and Related Rights in the Work and the meaning and intended legal 47 | # effect of CC0 on those rights. 48 | 49 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 50 | # protected by copyright and related or neighboring rights ("Copyright and 51 | # Related Rights"). Copyright and Related Rights include, but are not limited 52 | # to, the following: 53 | 54 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 55 | # and translate a Work; 56 | 57 | # ii. moral rights retained by the original author(s) and/or performer(s); 58 | 59 | # iii. publicity and privacy rights pertaining to a person's image or likeness 60 | # depicted in a Work; 61 | 62 | # iv. rights protecting against unfair competition in regards to a Work, 63 | # subject to the limitations in paragraph 4(a), below; 64 | 65 | # v. rights protecting the extraction, dissemination, use and reuse of data in 66 | # a Work; 67 | 68 | # vi. database rights (such as those arising under Directive 96/9/EC of the 69 | # European Parliament and of the Council of 11 March 1996 on the legal 70 | # protection of databases, and under any national implementation thereof, 71 | # including any amended or successor version of such directive); and 72 | 73 | # vii. other similar, equivalent or corresponding rights throughout the world 74 | # based on applicable law or treaty, and any national implementations thereof. 75 | 76 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 77 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 78 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 79 | # and Related Rights and associated claims and causes of action, whether now 80 | # known or unknown (including existing as well as future claims and causes of 81 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 82 | # duration provided by applicable law or treaty (including future time 83 | # extensions), (iii) in any current or future medium and for any number of 84 | # copies, and (iv) for any purpose whatsoever, including without limitation 85 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 86 | # the Waiver for the benefit of each member of the public at large and to the 87 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 88 | # shall not be subject to revocation, rescission, cancellation, termination, or 89 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 90 | # by the public as contemplated by Affirmer's express Statement of Purpose. 91 | 92 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 93 | # judged legally invalid or ineffective under applicable law, then the Waiver 94 | # shall be preserved to the maximum extent permitted taking into account 95 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 96 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 97 | # non transferable, non sublicensable, non exclusive, irrevocable and 98 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 99 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 100 | # provided by applicable law or treaty (including future time extensions), (iii) 101 | # in any current or future medium and for any number of copies, and (iv) for any 102 | # purpose whatsoever, including without limitation commercial, advertising or 103 | # promotional purposes (the "License"). The License shall be deemed effective as 104 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 105 | # License for any reason be judged legally invalid or ineffective under 106 | # applicable law, such partial invalidity or ineffectiveness shall not 107 | # invalidate the remainder of the License, and in such case Affirmer hereby 108 | # affirms that he or she will not (i) exercise any of his or her remaining 109 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 110 | # and causes of action with respect to the Work, in either case contrary to 111 | # Affirmer's express Statement of Purpose. 112 | 113 | # 4. Limitations and Disclaimers. 114 | 115 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 116 | # surrendered, licensed or otherwise affected by this document. 117 | 118 | # b. Affirmer offers the Work as-is and makes no representations or warranties 119 | # of any kind concerning the Work, express, implied, statutory or otherwise, 120 | # including without limitation warranties of title, merchantability, fitness 121 | # for a particular purpose, non infringement, or the absence of latent or 122 | # other defects, accuracy, or the present or absence of errors, whether or not 123 | # discoverable, all to the greatest extent permissible under applicable law. 124 | 125 | # c. Affirmer disclaims responsibility for clearing rights of other persons 126 | # that may apply to the Work or any use thereof, including without limitation 127 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 128 | # disclaims responsibility for obtaining any necessary consents, permissions 129 | # or other rights required for any use of the Work. 130 | 131 | # d. Affirmer understands and acknowledges that Creative Commons is not a 132 | # party to this document and has no duty or obligation with respect to this 133 | # CC0 or use of the Work. 134 | 135 | # For more information, please see 136 | # 137 | -------------------------------------------------------------------------------- /dqm/batch/run_106.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_106.pkl -------------------------------------------------------------------------------- /dqm/batch/run_106.py: -------------------------------------------------------------------------------- 1 | from pandas import pandas as pd 2 | from pandas import DataFrame 3 | 4 | run_106 =[ 5 | ['106', 'el103', 'el1_3', 'el1.3'], 6 | ['106', 'el901', 'el9_1', 'el9.1'], 7 | ['106', 'el701', 'el7_1', 'el7.1'] 8 | ] 9 | 10 | 11 | df = DataFrame(run_106, columns=['series', 'cb', 'measure_id', 'id']) 12 | df['measure_id'] = df['measure_id'].str.upper() 13 | 14 | 15 | with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 2000): 16 | print (df) 17 | 18 | df.to_pickle('./run_106.pkl') 19 | 20 | # CC0 1.0 Universal 21 | 22 | # Statement of Purpose 23 | 24 | # The laws of most jurisdictions throughout the world automatically confer 25 | # exclusive Copyright and Related Rights (defined below) upon the creator and 26 | # subsequent owner(s) (each and all, an "owner") of an original work of 27 | # authorship and/or a database (each, a "Work"). 28 | 29 | # Certain owners wish to permanently relinquish those rights to a Work for the 30 | # purpose of contributing to a commons of creative, cultural and scientific 31 | # works ("Commons") that the public can reliably and without fear of later 32 | # claims of infringement build upon, modify, incorporate in other works, reuse 33 | # and redistribute as freely as possible in any form whatsoever and for any 34 | # purposes, including without limitation commercial purposes. These owners may 35 | # contribute to the Commons to promote the ideal of a free culture and the 36 | # further production of creative, cultural and scientific works, or to gain 37 | # reputation or greater distribution for their Work in part through the use and 38 | # efforts of others. 39 | 40 | # For these and/or other purposes and motivations, and without any expectation 41 | # of additional consideration or compensation, the person associating CC0 with a 42 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 43 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 44 | # and publicly distribute the Work under its terms, with knowledge of his or her 45 | # Copyright and Related Rights in the Work and the meaning and intended legal 46 | # effect of CC0 on those rights. 47 | 48 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 49 | # protected by copyright and related or neighboring rights ("Copyright and 50 | # Related Rights"). Copyright and Related Rights include, but are not limited 51 | # to, the following: 52 | 53 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 54 | # and translate a Work; 55 | 56 | # ii. moral rights retained by the original author(s) and/or performer(s); 57 | 58 | # iii. publicity and privacy rights pertaining to a person's image or likeness 59 | # depicted in a Work; 60 | 61 | # iv. rights protecting against unfair competition in regards to a Work, 62 | # subject to the limitations in paragraph 4(a), below; 63 | 64 | # v. rights protecting the extraction, dissemination, use and reuse of data in 65 | # a Work; 66 | 67 | # vi. database rights (such as those arising under Directive 96/9/EC of the 68 | # European Parliament and of the Council of 11 March 1996 on the legal 69 | # protection of databases, and under any national implementation thereof, 70 | # including any amended or successor version of such directive); and 71 | 72 | # vii. other similar, equivalent or corresponding rights throughout the world 73 | # based on applicable law or treaty, and any national implementations thereof. 74 | 75 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 76 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 77 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 78 | # and Related Rights and associated claims and causes of action, whether now 79 | # known or unknown (including existing as well as future claims and causes of 80 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 81 | # duration provided by applicable law or treaty (including future time 82 | # extensions), (iii) in any current or future medium and for any number of 83 | # copies, and (iv) for any purpose whatsoever, including without limitation 84 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 85 | # the Waiver for the benefit of each member of the public at large and to the 86 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 87 | # shall not be subject to revocation, rescission, cancellation, termination, or 88 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 89 | # by the public as contemplated by Affirmer's express Statement of Purpose. 90 | 91 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 92 | # judged legally invalid or ineffective under applicable law, then the Waiver 93 | # shall be preserved to the maximum extent permitted taking into account 94 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 95 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 96 | # non transferable, non sublicensable, non exclusive, irrevocable and 97 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 98 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 99 | # provided by applicable law or treaty (including future time extensions), (iii) 100 | # in any current or future medium and for any number of copies, and (iv) for any 101 | # purpose whatsoever, including without limitation commercial, advertising or 102 | # promotional purposes (the "License"). The License shall be deemed effective as 103 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 104 | # License for any reason be judged legally invalid or ineffective under 105 | # applicable law, such partial invalidity or ineffectiveness shall not 106 | # invalidate the remainder of the License, and in such case Affirmer hereby 107 | # affirms that he or she will not (i) exercise any of his or her remaining 108 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 109 | # and causes of action with respect to the Work, in either case contrary to 110 | # Affirmer's express Statement of Purpose. 111 | 112 | # 4. Limitations and Disclaimers. 113 | 114 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 115 | # surrendered, licensed or otherwise affected by this document. 116 | 117 | # b. Affirmer offers the Work as-is and makes no representations or warranties 118 | # of any kind concerning the Work, express, implied, statutory or otherwise, 119 | # including without limitation warranties of title, merchantability, fitness 120 | # for a particular purpose, non infringement, or the absence of latent or 121 | # other defects, accuracy, or the present or absence of errors, whether or not 122 | # discoverable, all to the greatest extent permissible under applicable law. 123 | 124 | # c. Affirmer disclaims responsibility for clearing rights of other persons 125 | # that may apply to the Work or any use thereof, including without limitation 126 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 127 | # disclaims responsibility for obtaining any necessary consents, permissions 128 | # or other rights required for any use of the Work. 129 | 130 | # d. Affirmer understands and acknowledges that Creative Commons is not a 131 | # party to this document and has no duty or obligation with respect to this 132 | # CC0 or use of the Work. 133 | 134 | # For more information, please see 135 | # 136 | -------------------------------------------------------------------------------- /dqm/batch/run_107.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_107.pkl -------------------------------------------------------------------------------- /dqm/batch/run_107.py: -------------------------------------------------------------------------------- 1 | from pandas import pandas as pd 2 | from pandas import DataFrame 3 | 4 | run_107 =[ 5 | ['107', 'el_802', 'EL8_2', 'EL8.2'] 6 | ] 7 | 8 | df = DataFrame(run_107, columns=['series', 'cb', 'measure_id', 'id']) 9 | df['measure_id'] = df['measure_id'].str.upper() 10 | 11 | 12 | with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 2000): 13 | print (df) 14 | 15 | df.to_pickle('./run_107.pkl') 16 | 17 | # CC0 1.0 Universal 18 | 19 | # Statement of Purpose 20 | 21 | # The laws of most jurisdictions throughout the world automatically confer 22 | # exclusive Copyright and Related Rights (defined below) upon the creator and 23 | # subsequent owner(s) (each and all, an "owner") of an original work of 24 | # authorship and/or a database (each, a "Work"). 25 | 26 | # Certain owners wish to permanently relinquish those rights to a Work for the 27 | # purpose of contributing to a commons of creative, cultural and scientific 28 | # works ("Commons") that the public can reliably and without fear of later 29 | # claims of infringement build upon, modify, incorporate in other works, reuse 30 | # and redistribute as freely as possible in any form whatsoever and for any 31 | # purposes, including without limitation commercial purposes. These owners may 32 | # contribute to the Commons to promote the ideal of a free culture and the 33 | # further production of creative, cultural and scientific works, or to gain 34 | # reputation or greater distribution for their Work in part through the use and 35 | # efforts of others. 36 | 37 | # For these and/or other purposes and motivations, and without any expectation 38 | # of additional consideration or compensation, the person associating CC0 with a 39 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 40 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 41 | # and publicly distribute the Work under its terms, with knowledge of his or her 42 | # Copyright and Related Rights in the Work and the meaning and intended legal 43 | # effect of CC0 on those rights. 44 | 45 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 46 | # protected by copyright and related or neighboring rights ("Copyright and 47 | # Related Rights"). Copyright and Related Rights include, but are not limited 48 | # to, the following: 49 | 50 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 51 | # and translate a Work; 52 | 53 | # ii. moral rights retained by the original author(s) and/or performer(s); 54 | 55 | # iii. publicity and privacy rights pertaining to a person's image or likeness 56 | # depicted in a Work; 57 | 58 | # iv. rights protecting against unfair competition in regards to a Work, 59 | # subject to the limitations in paragraph 4(a), below; 60 | 61 | # v. rights protecting the extraction, dissemination, use and reuse of data in 62 | # a Work; 63 | 64 | # vi. database rights (such as those arising under Directive 96/9/EC of the 65 | # European Parliament and of the Council of 11 March 1996 on the legal 66 | # protection of databases, and under any national implementation thereof, 67 | # including any amended or successor version of such directive); and 68 | 69 | # vii. other similar, equivalent or corresponding rights throughout the world 70 | # based on applicable law or treaty, and any national implementations thereof. 71 | 72 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 73 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 74 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 75 | # and Related Rights and associated claims and causes of action, whether now 76 | # known or unknown (including existing as well as future claims and causes of 77 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 78 | # duration provided by applicable law or treaty (including future time 79 | # extensions), (iii) in any current or future medium and for any number of 80 | # copies, and (iv) for any purpose whatsoever, including without limitation 81 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 82 | # the Waiver for the benefit of each member of the public at large and to the 83 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 84 | # shall not be subject to revocation, rescission, cancellation, termination, or 85 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 86 | # by the public as contemplated by Affirmer's express Statement of Purpose. 87 | 88 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 89 | # judged legally invalid or ineffective under applicable law, then the Waiver 90 | # shall be preserved to the maximum extent permitted taking into account 91 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 92 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 93 | # non transferable, non sublicensable, non exclusive, irrevocable and 94 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 95 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 96 | # provided by applicable law or treaty (including future time extensions), (iii) 97 | # in any current or future medium and for any number of copies, and (iv) for any 98 | # purpose whatsoever, including without limitation commercial, advertising or 99 | # promotional purposes (the "License"). The License shall be deemed effective as 100 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 101 | # License for any reason be judged legally invalid or ineffective under 102 | # applicable law, such partial invalidity or ineffectiveness shall not 103 | # invalidate the remainder of the License, and in such case Affirmer hereby 104 | # affirms that he or she will not (i) exercise any of his or her remaining 105 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 106 | # and causes of action with respect to the Work, in either case contrary to 107 | # Affirmer's express Statement of Purpose. 108 | 109 | # 4. Limitations and Disclaimers. 110 | 111 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 112 | # surrendered, licensed or otherwise affected by this document. 113 | 114 | # b. Affirmer offers the Work as-is and makes no representations or warranties 115 | # of any kind concerning the Work, express, implied, statutory or otherwise, 116 | # including without limitation warranties of title, merchantability, fitness 117 | # for a particular purpose, non infringement, or the absence of latent or 118 | # other defects, accuracy, or the present or absence of errors, whether or not 119 | # discoverable, all to the greatest extent permissible under applicable law. 120 | 121 | # c. Affirmer disclaims responsibility for clearing rights of other persons 122 | # that may apply to the Work or any use thereof, including without limitation 123 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 124 | # disclaims responsibility for obtaining any necessary consents, permissions 125 | # or other rights required for any use of the Work. 126 | 127 | # d. Affirmer understands and acknowledges that Creative Commons is not a 128 | # party to this document and has no duty or obligation with respect to this 129 | # CC0 or use of the Work. 130 | 131 | # For more information, please see 132 | # 133 | -------------------------------------------------------------------------------- /dqm/batch/run_108.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_108.pkl -------------------------------------------------------------------------------- /dqm/batch/run_108.py: -------------------------------------------------------------------------------- 1 | from pandas import pandas as pd 2 | from pandas import DataFrame 3 | 4 | run_108 =[ 5 | ['108', 'el503', 'el5_3', 'el5.3'] 6 | ] 7 | 8 | df = DataFrame(run_108, columns=['series', 'cb', 'measure_id', 'id']) 9 | df['measure_id'] = df['measure_id'].str.upper() 10 | 11 | 12 | with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 2000): 13 | print (df) 14 | 15 | df.to_pickle('./run_108.pkl') 16 | 17 | # CC0 1.0 Universal 18 | 19 | # Statement of Purpose 20 | 21 | # The laws of most jurisdictions throughout the world automatically confer 22 | # exclusive Copyright and Related Rights (defined below) upon the creator and 23 | # subsequent owner(s) (each and all, an "owner") of an original work of 24 | # authorship and/or a database (each, a "Work"). 25 | 26 | # Certain owners wish to permanently relinquish those rights to a Work for the 27 | # purpose of contributing to a commons of creative, cultural and scientific 28 | # works ("Commons") that the public can reliably and without fear of later 29 | # claims of infringement build upon, modify, incorporate in other works, reuse 30 | # and redistribute as freely as possible in any form whatsoever and for any 31 | # purposes, including without limitation commercial purposes. These owners may 32 | # contribute to the Commons to promote the ideal of a free culture and the 33 | # further production of creative, cultural and scientific works, or to gain 34 | # reputation or greater distribution for their Work in part through the use and 35 | # efforts of others. 36 | 37 | # For these and/or other purposes and motivations, and without any expectation 38 | # of additional consideration or compensation, the person associating CC0 with a 39 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 40 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 41 | # and publicly distribute the Work under its terms, with knowledge of his or her 42 | # Copyright and Related Rights in the Work and the meaning and intended legal 43 | # effect of CC0 on those rights. 44 | 45 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 46 | # protected by copyright and related or neighboring rights ("Copyright and 47 | # Related Rights"). Copyright and Related Rights include, but are not limited 48 | # to, the following: 49 | 50 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 51 | # and translate a Work; 52 | 53 | # ii. moral rights retained by the original author(s) and/or performer(s); 54 | 55 | # iii. publicity and privacy rights pertaining to a person's image or likeness 56 | # depicted in a Work; 57 | 58 | # iv. rights protecting against unfair competition in regards to a Work, 59 | # subject to the limitations in paragraph 4(a), below; 60 | 61 | # v. rights protecting the extraction, dissemination, use and reuse of data in 62 | # a Work; 63 | 64 | # vi. database rights (such as those arising under Directive 96/9/EC of the 65 | # European Parliament and of the Council of 11 March 1996 on the legal 66 | # protection of databases, and under any national implementation thereof, 67 | # including any amended or successor version of such directive); and 68 | 69 | # vii. other similar, equivalent or corresponding rights throughout the world 70 | # based on applicable law or treaty, and any national implementations thereof. 71 | 72 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 73 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 74 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 75 | # and Related Rights and associated claims and causes of action, whether now 76 | # known or unknown (including existing as well as future claims and causes of 77 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 78 | # duration provided by applicable law or treaty (including future time 79 | # extensions), (iii) in any current or future medium and for any number of 80 | # copies, and (iv) for any purpose whatsoever, including without limitation 81 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 82 | # the Waiver for the benefit of each member of the public at large and to the 83 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 84 | # shall not be subject to revocation, rescission, cancellation, termination, or 85 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 86 | # by the public as contemplated by Affirmer's express Statement of Purpose. 87 | 88 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 89 | # judged legally invalid or ineffective under applicable law, then the Waiver 90 | # shall be preserved to the maximum extent permitted taking into account 91 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 92 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 93 | # non transferable, non sublicensable, non exclusive, irrevocable and 94 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 95 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 96 | # provided by applicable law or treaty (including future time extensions), (iii) 97 | # in any current or future medium and for any number of copies, and (iv) for any 98 | # purpose whatsoever, including without limitation commercial, advertising or 99 | # promotional purposes (the "License"). The License shall be deemed effective as 100 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 101 | # License for any reason be judged legally invalid or ineffective under 102 | # applicable law, such partial invalidity or ineffectiveness shall not 103 | # invalidate the remainder of the License, and in such case Affirmer hereby 104 | # affirms that he or she will not (i) exercise any of his or her remaining 105 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 106 | # and causes of action with respect to the Work, in either case contrary to 107 | # Affirmer's express Statement of Purpose. 108 | 109 | # 4. Limitations and Disclaimers. 110 | 111 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 112 | # surrendered, licensed or otherwise affected by this document. 113 | 114 | # b. Affirmer offers the Work as-is and makes no representations or warranties 115 | # of any kind concerning the Work, express, implied, statutory or otherwise, 116 | # including without limitation warranties of title, merchantability, fitness 117 | # for a particular purpose, non infringement, or the absence of latent or 118 | # other defects, accuracy, or the present or absence of errors, whether or not 119 | # discoverable, all to the greatest extent permissible under applicable law. 120 | 121 | # c. Affirmer disclaims responsibility for clearing rights of other persons 122 | # that may apply to the Work or any use thereof, including without limitation 123 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 124 | # disclaims responsibility for obtaining any necessary consents, permissions 125 | # or other rights required for any use of the Work. 126 | 127 | # d. Affirmer understands and acknowledges that Creative Commons is not a 128 | # party to this document and has no duty or obligation with respect to this 129 | # CC0 or use of the Work. 130 | 131 | # For more information, please see 132 | # 133 | -------------------------------------------------------------------------------- /dqm/batch/run_109.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_109.pkl -------------------------------------------------------------------------------- /dqm/batch/run_109.py: -------------------------------------------------------------------------------- 1 | from pandas import pandas as pd 2 | from pandas import DataFrame 3 | 4 | run_109 =[ 5 | ['109', 'el_13_1', 'EL13_1', 'EL13.1'] 6 | 7 | ] 8 | df = DataFrame(run_109, columns=['series', 'cb', 'measure_id', 'id']) 9 | df['measure_id'] = df['measure_id'].str.upper() 10 | 11 | 12 | with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 2000): 13 | print (df) 14 | 15 | df.to_pickle('./run_109.pkl') 16 | 17 | # CC0 1.0 Universal 18 | 19 | # Statement of Purpose 20 | 21 | # The laws of most jurisdictions throughout the world automatically confer 22 | # exclusive Copyright and Related Rights (defined below) upon the creator and 23 | # subsequent owner(s) (each and all, an "owner") of an original work of 24 | # authorship and/or a database (each, a "Work"). 25 | 26 | # Certain owners wish to permanently relinquish those rights to a Work for the 27 | # purpose of contributing to a commons of creative, cultural and scientific 28 | # works ("Commons") that the public can reliably and without fear of later 29 | # claims of infringement build upon, modify, incorporate in other works, reuse 30 | # and redistribute as freely as possible in any form whatsoever and for any 31 | # purposes, including without limitation commercial purposes. These owners may 32 | # contribute to the Commons to promote the ideal of a free culture and the 33 | # further production of creative, cultural and scientific works, or to gain 34 | # reputation or greater distribution for their Work in part through the use and 35 | # efforts of others. 36 | 37 | # For these and/or other purposes and motivations, and without any expectation 38 | # of additional consideration or compensation, the person associating CC0 with a 39 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 40 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 41 | # and publicly distribute the Work under its terms, with knowledge of his or her 42 | # Copyright and Related Rights in the Work and the meaning and intended legal 43 | # effect of CC0 on those rights. 44 | 45 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 46 | # protected by copyright and related or neighboring rights ("Copyright and 47 | # Related Rights"). Copyright and Related Rights include, but are not limited 48 | # to, the following: 49 | 50 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 51 | # and translate a Work; 52 | 53 | # ii. moral rights retained by the original author(s) and/or performer(s); 54 | 55 | # iii. publicity and privacy rights pertaining to a person's image or likeness 56 | # depicted in a Work; 57 | 58 | # iv. rights protecting against unfair competition in regards to a Work, 59 | # subject to the limitations in paragraph 4(a), below; 60 | 61 | # v. rights protecting the extraction, dissemination, use and reuse of data in 62 | # a Work; 63 | 64 | # vi. database rights (such as those arising under Directive 96/9/EC of the 65 | # European Parliament and of the Council of 11 March 1996 on the legal 66 | # protection of databases, and under any national implementation thereof, 67 | # including any amended or successor version of such directive); and 68 | 69 | # vii. other similar, equivalent or corresponding rights throughout the world 70 | # based on applicable law or treaty, and any national implementations thereof. 71 | 72 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 73 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 74 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 75 | # and Related Rights and associated claims and causes of action, whether now 76 | # known or unknown (including existing as well as future claims and causes of 77 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 78 | # duration provided by applicable law or treaty (including future time 79 | # extensions), (iii) in any current or future medium and for any number of 80 | # copies, and (iv) for any purpose whatsoever, including without limitation 81 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 82 | # the Waiver for the benefit of each member of the public at large and to the 83 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 84 | # shall not be subject to revocation, rescission, cancellation, termination, or 85 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 86 | # by the public as contemplated by Affirmer's express Statement of Purpose. 87 | 88 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 89 | # judged legally invalid or ineffective under applicable law, then the Waiver 90 | # shall be preserved to the maximum extent permitted taking into account 91 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 92 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 93 | # non transferable, non sublicensable, non exclusive, irrevocable and 94 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 95 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 96 | # provided by applicable law or treaty (including future time extensions), (iii) 97 | # in any current or future medium and for any number of copies, and (iv) for any 98 | # purpose whatsoever, including without limitation commercial, advertising or 99 | # promotional purposes (the "License"). The License shall be deemed effective as 100 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 101 | # License for any reason be judged legally invalid or ineffective under 102 | # applicable law, such partial invalidity or ineffectiveness shall not 103 | # invalidate the remainder of the License, and in such case Affirmer hereby 104 | # affirms that he or she will not (i) exercise any of his or her remaining 105 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 106 | # and causes of action with respect to the Work, in either case contrary to 107 | # Affirmer's express Statement of Purpose. 108 | 109 | # 4. Limitations and Disclaimers. 110 | 111 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 112 | # surrendered, licensed or otherwise affected by this document. 113 | 114 | # b. Affirmer offers the Work as-is and makes no representations or warranties 115 | # of any kind concerning the Work, express, implied, statutory or otherwise, 116 | # including without limitation warranties of title, merchantability, fitness 117 | # for a particular purpose, non infringement, or the absence of latent or 118 | # other defects, accuracy, or the present or absence of errors, whether or not 119 | # discoverable, all to the greatest extent permissible under applicable law. 120 | 121 | # c. Affirmer disclaims responsibility for clearing rights of other persons 122 | # that may apply to the Work or any use thereof, including without limitation 123 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 124 | # disclaims responsibility for obtaining any necessary consents, permissions 125 | # or other rights required for any use of the Work. 126 | 127 | # d. Affirmer understands and acknowledges that Creative Commons is not a 128 | # party to this document and has no duty or obligation with respect to this 129 | # CC0 or use of the Work. 130 | 131 | # For more information, please see 132 | # 133 | -------------------------------------------------------------------------------- /dqm/batch/run_110.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_110.pkl -------------------------------------------------------------------------------- /dqm/batch/run_110.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | 3 | run_110 = [ 4 | 5 | ['110', 'el1_29', 'el1_29'], 6 | ['110', 'el19_1', 'el19_1'] 7 | ] 8 | 9 | df = DataFrame(run_110, columns=['series', 'cb', 'measure_id']) 10 | df['measure_id'] = df['measure_id'].str.upper() 11 | 12 | df = df.sort_values(by=['series', 'cb', 'measure_id']) 13 | 14 | print(df.head(10)) 15 | 16 | df.to_pickle('./run_110.pkl') 17 | 18 | # CC0 1.0 Universal 19 | 20 | # Statement of Purpose 21 | 22 | # The laws of most jurisdictions throughout the world automatically confer 23 | # exclusive Copyright and Related Rights (defined below) upon the creator and 24 | # subsequent owner(s) (each and all, an "owner") of an original work of 25 | # authorship and/or a database (each, a "Work"). 26 | 27 | # Certain owners wish to permanently relinquish those rights to a Work for the 28 | # purpose of contributing to a commons of creative, cultural and scientific 29 | # works ("Commons") that the public can reliably and without fear of later 30 | # claims of infringement build upon, modify, incorporate in other works, reuse 31 | # and redistribute as freely as possible in any form whatsoever and for any 32 | # purposes, including without limitation commercial purposes. These owners may 33 | # contribute to the Commons to promote the ideal of a free culture and the 34 | # further production of creative, cultural and scientific works, or to gain 35 | # reputation or greater distribution for their Work in part through the use and 36 | # efforts of others. 37 | 38 | # For these and/or other purposes and motivations, and without any expectation 39 | # of additional consideration or compensation, the person associating CC0 with a 40 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 41 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 42 | # and publicly distribute the Work under its terms, with knowledge of his or her 43 | # Copyright and Related Rights in the Work and the meaning and intended legal 44 | # effect of CC0 on those rights. 45 | 46 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 47 | # protected by copyright and related or neighboring rights ("Copyright and 48 | # Related Rights"). Copyright and Related Rights include, but are not limited 49 | # to, the following: 50 | 51 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 52 | # and translate a Work; 53 | 54 | # ii. moral rights retained by the original author(s) and/or performer(s); 55 | 56 | # iii. publicity and privacy rights pertaining to a person's image or likeness 57 | # depicted in a Work; 58 | 59 | # iv. rights protecting against unfair competition in regards to a Work, 60 | # subject to the limitations in paragraph 4(a), below; 61 | 62 | # v. rights protecting the extraction, dissemination, use and reuse of data in 63 | # a Work; 64 | 65 | # vi. database rights (such as those arising under Directive 96/9/EC of the 66 | # European Parliament and of the Council of 11 March 1996 on the legal 67 | # protection of databases, and under any national implementation thereof, 68 | # including any amended or successor version of such directive); and 69 | 70 | # vii. other similar, equivalent or corresponding rights throughout the world 71 | # based on applicable law or treaty, and any national implementations thereof. 72 | 73 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 74 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 75 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 76 | # and Related Rights and associated claims and causes of action, whether now 77 | # known or unknown (including existing as well as future claims and causes of 78 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 79 | # duration provided by applicable law or treaty (including future time 80 | # extensions), (iii) in any current or future medium and for any number of 81 | # copies, and (iv) for any purpose whatsoever, including without limitation 82 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 83 | # the Waiver for the benefit of each member of the public at large and to the 84 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 85 | # shall not be subject to revocation, rescission, cancellation, termination, or 86 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 87 | # by the public as contemplated by Affirmer's express Statement of Purpose. 88 | 89 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 90 | # judged legally invalid or ineffective under applicable law, then the Waiver 91 | # shall be preserved to the maximum extent permitted taking into account 92 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 93 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 94 | # non transferable, non sublicensable, non exclusive, irrevocable and 95 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 96 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 97 | # provided by applicable law or treaty (including future time extensions), (iii) 98 | # in any current or future medium and for any number of copies, and (iv) for any 99 | # purpose whatsoever, including without limitation commercial, advertising or 100 | # promotional purposes (the "License"). The License shall be deemed effective as 101 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 102 | # License for any reason be judged legally invalid or ineffective under 103 | # applicable law, such partial invalidity or ineffectiveness shall not 104 | # invalidate the remainder of the License, and in such case Affirmer hereby 105 | # affirms that he or she will not (i) exercise any of his or her remaining 106 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 107 | # and causes of action with respect to the Work, in either case contrary to 108 | # Affirmer's express Statement of Purpose. 109 | 110 | # 4. Limitations and Disclaimers. 111 | 112 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 113 | # surrendered, licensed or otherwise affected by this document. 114 | 115 | # b. Affirmer offers the Work as-is and makes no representations or warranties 116 | # of any kind concerning the Work, express, implied, statutory or otherwise, 117 | # including without limitation warranties of title, merchantability, fitness 118 | # for a particular purpose, non infringement, or the absence of latent or 119 | # other defects, accuracy, or the present or absence of errors, whether or not 120 | # discoverable, all to the greatest extent permissible under applicable law. 121 | 122 | # c. Affirmer disclaims responsibility for clearing rights of other persons 123 | # that may apply to the Work or any use thereof, including without limitation 124 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 125 | # disclaims responsibility for obtaining any necessary consents, permissions 126 | # or other rights required for any use of the Work. 127 | 128 | # d. Affirmer understands and acknowledges that Creative Commons is not a 129 | # party to this document and has no duty or obligation with respect to this 130 | # CC0 or use of the Work. 131 | 132 | # For more information, please see 133 | # 134 | -------------------------------------------------------------------------------- /dqm/batch/run_201.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_201.pkl -------------------------------------------------------------------------------- /dqm/batch/run_202.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_202.pkl -------------------------------------------------------------------------------- /dqm/batch/run_204.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_204.pkl -------------------------------------------------------------------------------- /dqm/batch/run_205.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_205.pkl -------------------------------------------------------------------------------- /dqm/batch/run_206.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_206.pkl -------------------------------------------------------------------------------- /dqm/batch/run_501.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_501.pkl -------------------------------------------------------------------------------- /dqm/batch/run_502.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_502.pkl -------------------------------------------------------------------------------- /dqm/batch/run_502.py: -------------------------------------------------------------------------------- 1 | from pandas import pandas as pd 2 | from pandas import DataFrame 3 | 4 | run_502 =[ 5 | 6 | ['502', 'sumprv', 'sumprv_1', '01'], 7 | ['502', 'sumprv', 'sumprv_2', '03'], 8 | ['502', 'sumprv', 'sumprv_3', '02'] 9 | ] 10 | 11 | df = DataFrame(run_502, columns=['series', 'cb', 'measure_id', 'fac_grp_indvdl_cd']) 12 | df['measure_id'] = df['measure_id'].str.upper() 13 | 14 | df = df.sort_values(by=['series', 'cb', 'measure_id', 'fac_grp_indvdl_cd']) 15 | 16 | print(df.head(10)) 17 | 18 | df.to_pickle('./run_502.pkl') 19 | 20 | # CC0 1.0 Universal 21 | 22 | # Statement of Purpose 23 | 24 | # The laws of most jurisdictions throughout the world automatically confer 25 | # exclusive Copyright and Related Rights (defined below) upon the creator and 26 | # subsequent owner(s) (each and all, an "owner") of an original work of 27 | # authorship and/or a database (each, a "Work"). 28 | 29 | # Certain owners wish to permanently relinquish those rights to a Work for the 30 | # purpose of contributing to a commons of creative, cultural and scientific 31 | # works ("Commons") that the public can reliably and without fear of later 32 | # claims of infringement build upon, modify, incorporate in other works, reuse 33 | # and redistribute as freely as possible in any form whatsoever and for any 34 | # purposes, including without limitation commercial purposes. These owners may 35 | # contribute to the Commons to promote the ideal of a free culture and the 36 | # further production of creative, cultural and scientific works, or to gain 37 | # reputation or greater distribution for their Work in part through the use and 38 | # efforts of others. 39 | 40 | # For these and/or other purposes and motivations, and without any expectation 41 | # of additional consideration or compensation, the person associating CC0 with a 42 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 43 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 44 | # and publicly distribute the Work under its terms, with knowledge of his or her 45 | # Copyright and Related Rights in the Work and the meaning and intended legal 46 | # effect of CC0 on those rights. 47 | 48 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 49 | # protected by copyright and related or neighboring rights ("Copyright and 50 | # Related Rights"). Copyright and Related Rights include, but are not limited 51 | # to, the following: 52 | 53 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 54 | # and translate a Work; 55 | 56 | # ii. moral rights retained by the original author(s) and/or performer(s); 57 | 58 | # iii. publicity and privacy rights pertaining to a person's image or likeness 59 | # depicted in a Work; 60 | 61 | # iv. rights protecting against unfair competition in regards to a Work, 62 | # subject to the limitations in paragraph 4(a), below; 63 | 64 | # v. rights protecting the extraction, dissemination, use and reuse of data in 65 | # a Work; 66 | 67 | # vi. database rights (such as those arising under Directive 96/9/EC of the 68 | # European Parliament and of the Council of 11 March 1996 on the legal 69 | # protection of databases, and under any national implementation thereof, 70 | # including any amended or successor version of such directive); and 71 | 72 | # vii. other similar, equivalent or corresponding rights throughout the world 73 | # based on applicable law or treaty, and any national implementations thereof. 74 | 75 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 76 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 77 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 78 | # and Related Rights and associated claims and causes of action, whether now 79 | # known or unknown (including existing as well as future claims and causes of 80 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 81 | # duration provided by applicable law or treaty (including future time 82 | # extensions), (iii) in any current or future medium and for any number of 83 | # copies, and (iv) for any purpose whatsoever, including without limitation 84 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 85 | # the Waiver for the benefit of each member of the public at large and to the 86 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 87 | # shall not be subject to revocation, rescission, cancellation, termination, or 88 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 89 | # by the public as contemplated by Affirmer's express Statement of Purpose. 90 | 91 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 92 | # judged legally invalid or ineffective under applicable law, then the Waiver 93 | # shall be preserved to the maximum extent permitted taking into account 94 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 95 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 96 | # non transferable, non sublicensable, non exclusive, irrevocable and 97 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 98 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 99 | # provided by applicable law or treaty (including future time extensions), (iii) 100 | # in any current or future medium and for any number of copies, and (iv) for any 101 | # purpose whatsoever, including without limitation commercial, advertising or 102 | # promotional purposes (the "License"). The License shall be deemed effective as 103 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 104 | # License for any reason be judged legally invalid or ineffective under 105 | # applicable law, such partial invalidity or ineffectiveness shall not 106 | # invalidate the remainder of the License, and in such case Affirmer hereby 107 | # affirms that he or she will not (i) exercise any of his or her remaining 108 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 109 | # and causes of action with respect to the Work, in either case contrary to 110 | # Affirmer's express Statement of Purpose. 111 | 112 | # 4. Limitations and Disclaimers. 113 | 114 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 115 | # surrendered, licensed or otherwise affected by this document. 116 | 117 | # b. Affirmer offers the Work as-is and makes no representations or warranties 118 | # of any kind concerning the Work, express, implied, statutory or otherwise, 119 | # including without limitation warranties of title, merchantability, fitness 120 | # for a particular purpose, non infringement, or the absence of latent or 121 | # other defects, accuracy, or the present or absence of errors, whether or not 122 | # discoverable, all to the greatest extent permissible under applicable law. 123 | 124 | # c. Affirmer disclaims responsibility for clearing rights of other persons 125 | # that may apply to the Work or any use thereof, including without limitation 126 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 127 | # disclaims responsibility for obtaining any necessary consents, permissions 128 | # or other rights required for any use of the Work. 129 | 130 | # d. Affirmer understands and acknowledges that Creative Commons is not a 131 | # party to this document and has no duty or obligation with respect to this 132 | # CC0 or use of the Work. 133 | 134 | # For more information, please see 135 | # 136 | -------------------------------------------------------------------------------- /dqm/batch/run_503.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_503.pkl -------------------------------------------------------------------------------- /dqm/batch/run_503.py: -------------------------------------------------------------------------------- 1 | from pandas import pandas as pd 2 | from pandas import DataFrame 3 | 4 | run_503 =[ 5 | 6 | ['503', 'prvdr_freq', 'prv4_1'] 7 | ] 8 | 9 | df = DataFrame(run_503, columns=['series', 'cb', 'measure_id']) 10 | df['measure_id'] = df['measure_id'].str.upper() 11 | 12 | df = df.sort_values(by=['series', 'cb', 'measure_id']) 13 | 14 | print(df.head(10)) 15 | 16 | df.to_pickle('./run_503.pkl') 17 | 18 | # CC0 1.0 Universal 19 | 20 | # Statement of Purpose 21 | 22 | # The laws of most jurisdictions throughout the world automatically confer 23 | # exclusive Copyright and Related Rights (defined below) upon the creator and 24 | # subsequent owner(s) (each and all, an "owner") of an original work of 25 | # authorship and/or a database (each, a "Work"). 26 | 27 | # Certain owners wish to permanently relinquish those rights to a Work for the 28 | # purpose of contributing to a commons of creative, cultural and scientific 29 | # works ("Commons") that the public can reliably and without fear of later 30 | # claims of infringement build upon, modify, incorporate in other works, reuse 31 | # and redistribute as freely as possible in any form whatsoever and for any 32 | # purposes, including without limitation commercial purposes. These owners may 33 | # contribute to the Commons to promote the ideal of a free culture and the 34 | # further production of creative, cultural and scientific works, or to gain 35 | # reputation or greater distribution for their Work in part through the use and 36 | # efforts of others. 37 | 38 | # For these and/or other purposes and motivations, and without any expectation 39 | # of additional consideration or compensation, the person associating CC0 with a 40 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 41 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 42 | # and publicly distribute the Work under its terms, with knowledge of his or her 43 | # Copyright and Related Rights in the Work and the meaning and intended legal 44 | # effect of CC0 on those rights. 45 | 46 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 47 | # protected by copyright and related or neighboring rights ("Copyright and 48 | # Related Rights"). Copyright and Related Rights include, but are not limited 49 | # to, the following: 50 | 51 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 52 | # and translate a Work; 53 | 54 | # ii. moral rights retained by the original author(s) and/or performer(s); 55 | 56 | # iii. publicity and privacy rights pertaining to a person's image or likeness 57 | # depicted in a Work; 58 | 59 | # iv. rights protecting against unfair competition in regards to a Work, 60 | # subject to the limitations in paragraph 4(a), below; 61 | 62 | # v. rights protecting the extraction, dissemination, use and reuse of data in 63 | # a Work; 64 | 65 | # vi. database rights (such as those arising under Directive 96/9/EC of the 66 | # European Parliament and of the Council of 11 March 1996 on the legal 67 | # protection of databases, and under any national implementation thereof, 68 | # including any amended or successor version of such directive); and 69 | 70 | # vii. other similar, equivalent or corresponding rights throughout the world 71 | # based on applicable law or treaty, and any national implementations thereof. 72 | 73 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 74 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 75 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 76 | # and Related Rights and associated claims and causes of action, whether now 77 | # known or unknown (including existing as well as future claims and causes of 78 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 79 | # duration provided by applicable law or treaty (including future time 80 | # extensions), (iii) in any current or future medium and for any number of 81 | # copies, and (iv) for any purpose whatsoever, including without limitation 82 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 83 | # the Waiver for the benefit of each member of the public at large and to the 84 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 85 | # shall not be subject to revocation, rescission, cancellation, termination, or 86 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 87 | # by the public as contemplated by Affirmer's express Statement of Purpose. 88 | 89 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 90 | # judged legally invalid or ineffective under applicable law, then the Waiver 91 | # shall be preserved to the maximum extent permitted taking into account 92 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 93 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 94 | # non transferable, non sublicensable, non exclusive, irrevocable and 95 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 96 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 97 | # provided by applicable law or treaty (including future time extensions), (iii) 98 | # in any current or future medium and for any number of copies, and (iv) for any 99 | # purpose whatsoever, including without limitation commercial, advertising or 100 | # promotional purposes (the "License"). The License shall be deemed effective as 101 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 102 | # License for any reason be judged legally invalid or ineffective under 103 | # applicable law, such partial invalidity or ineffectiveness shall not 104 | # invalidate the remainder of the License, and in such case Affirmer hereby 105 | # affirms that he or she will not (i) exercise any of his or her remaining 106 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 107 | # and causes of action with respect to the Work, in either case contrary to 108 | # Affirmer's express Statement of Purpose. 109 | 110 | # 4. Limitations and Disclaimers. 111 | 112 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 113 | # surrendered, licensed or otherwise affected by this document. 114 | 115 | # b. Affirmer offers the Work as-is and makes no representations or warranties 116 | # of any kind concerning the Work, express, implied, statutory or otherwise, 117 | # including without limitation warranties of title, merchantability, fitness 118 | # for a particular purpose, non infringement, or the absence of latent or 119 | # other defects, accuracy, or the present or absence of errors, whether or not 120 | # discoverable, all to the greatest extent permissible under applicable law. 121 | 122 | # c. Affirmer disclaims responsibility for clearing rights of other persons 123 | # that may apply to the Work or any use thereof, including without limitation 124 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 125 | # disclaims responsibility for obtaining any necessary consents, permissions 126 | # or other rights required for any use of the Work. 127 | 128 | # d. Affirmer understands and acknowledges that Creative Commons is not a 129 | # party to this document and has no duty or obligation with respect to this 130 | # CC0 or use of the Work. 131 | 132 | # For more information, please see 133 | # 134 | -------------------------------------------------------------------------------- /dqm/batch/run_504.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_504.pkl -------------------------------------------------------------------------------- /dqm/batch/run_601.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_601.pkl -------------------------------------------------------------------------------- /dqm/batch/run_602.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_602.pkl -------------------------------------------------------------------------------- /dqm/batch/run_603.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_603.pkl -------------------------------------------------------------------------------- /dqm/batch/run_603.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | 3 | run_603 =[ 4 | 5 | ['603', 'tpl_prsn_mn_prsn', 'tpl1_1'], 6 | ['603', 'tpl_prsn_mn_cvrg', 'tpl1_4'], 7 | 8 | ['603', 'tpl_prsn_hi_cvrg', 'tpl1_2'], 9 | ['603', 'tpl_prsn_hi_insrnc', 'tpl1_3'], 10 | 11 | ['603', 'ever_tpl_elig_prsn_mn_sql', 'tpl1_5'] 12 | 13 | ] 14 | 15 | df = DataFrame(run_603, columns=['series', 'cb', 'measure']) 16 | df['measure_id'] = df['measure'].str.upper() 17 | 18 | df = df.sort_values(by=['series', 'cb', 'measure_id']) 19 | print(df.head(5)) 20 | 21 | df.to_pickle('./run_603.pkl') 22 | 23 | # CC0 1.0 Universal 24 | 25 | # Statement of Purpose 26 | 27 | # The laws of most jurisdictions throughout the world automatically confer 28 | # exclusive Copyright and Related Rights (defined below) upon the creator and 29 | # subsequent owner(s) (each and all, an "owner") of an original work of 30 | # authorship and/or a database (each, a "Work"). 31 | 32 | # Certain owners wish to permanently relinquish those rights to a Work for the 33 | # purpose of contributing to a commons of creative, cultural and scientific 34 | # works ("Commons") that the public can reliably and without fear of later 35 | # claims of infringement build upon, modify, incorporate in other works, reuse 36 | # and redistribute as freely as possible in any form whatsoever and for any 37 | # purposes, including without limitation commercial purposes. These owners may 38 | # contribute to the Commons to promote the ideal of a free culture and the 39 | # further production of creative, cultural and scientific works, or to gain 40 | # reputation or greater distribution for their Work in part through the use and 41 | # efforts of others. 42 | 43 | # For these and/or other purposes and motivations, and without any expectation 44 | # of additional consideration or compensation, the person associating CC0 with a 45 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 46 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 47 | # and publicly distribute the Work under its terms, with knowledge of his or her 48 | # Copyright and Related Rights in the Work and the meaning and intended legal 49 | # effect of CC0 on those rights. 50 | 51 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 52 | # protected by copyright and related or neighboring rights ("Copyright and 53 | # Related Rights"). Copyright and Related Rights include, but are not limited 54 | # to, the following: 55 | 56 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 57 | # and translate a Work; 58 | 59 | # ii. moral rights retained by the original author(s) and/or performer(s); 60 | 61 | # iii. publicity and privacy rights pertaining to a person's image or likeness 62 | # depicted in a Work; 63 | 64 | # iv. rights protecting against unfair competition in regards to a Work, 65 | # subject to the limitations in paragraph 4(a), below; 66 | 67 | # v. rights protecting the extraction, dissemination, use and reuse of data in 68 | # a Work; 69 | 70 | # vi. database rights (such as those arising under Directive 96/9/EC of the 71 | # European Parliament and of the Council of 11 March 1996 on the legal 72 | # protection of databases, and under any national implementation thereof, 73 | # including any amended or successor version of such directive); and 74 | 75 | # vii. other similar, equivalent or corresponding rights throughout the world 76 | # based on applicable law or treaty, and any national implementations thereof. 77 | 78 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 79 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 80 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 81 | # and Related Rights and associated claims and causes of action, whether now 82 | # known or unknown (including existing as well as future claims and causes of 83 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 84 | # duration provided by applicable law or treaty (including future time 85 | # extensions), (iii) in any current or future medium and for any number of 86 | # copies, and (iv) for any purpose whatsoever, including without limitation 87 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 88 | # the Waiver for the benefit of each member of the public at large and to the 89 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 90 | # shall not be subject to revocation, rescission, cancellation, termination, or 91 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 92 | # by the public as contemplated by Affirmer's express Statement of Purpose. 93 | 94 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 95 | # judged legally invalid or ineffective under applicable law, then the Waiver 96 | # shall be preserved to the maximum extent permitted taking into account 97 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 98 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 99 | # non transferable, non sublicensable, non exclusive, irrevocable and 100 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 101 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 102 | # provided by applicable law or treaty (including future time extensions), (iii) 103 | # in any current or future medium and for any number of copies, and (iv) for any 104 | # purpose whatsoever, including without limitation commercial, advertising or 105 | # promotional purposes (the "License"). The License shall be deemed effective as 106 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 107 | # License for any reason be judged legally invalid or ineffective under 108 | # applicable law, such partial invalidity or ineffectiveness shall not 109 | # invalidate the remainder of the License, and in such case Affirmer hereby 110 | # affirms that he or she will not (i) exercise any of his or her remaining 111 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 112 | # and causes of action with respect to the Work, in either case contrary to 113 | # Affirmer's express Statement of Purpose. 114 | 115 | # 4. Limitations and Disclaimers. 116 | 117 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 118 | # surrendered, licensed or otherwise affected by this document. 119 | 120 | # b. Affirmer offers the Work as-is and makes no representations or warranties 121 | # of any kind concerning the Work, express, implied, statutory or otherwise, 122 | # including without limitation warranties of title, merchantability, fitness 123 | # for a particular purpose, non infringement, or the absence of latent or 124 | # other defects, accuracy, or the present or absence of errors, whether or not 125 | # discoverable, all to the greatest extent permissible under applicable law. 126 | 127 | # c. Affirmer disclaims responsibility for clearing rights of other persons 128 | # that may apply to the Work or any use thereof, including without limitation 129 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 130 | # disclaims responsibility for obtaining any necessary consents, permissions 131 | # or other rights required for any use of the Work. 132 | 133 | # d. Affirmer understands and acknowledges that Creative Commons is not a 134 | # party to this document and has no duty or obligation with respect to this 135 | # CC0 or use of the Work. 136 | 137 | # For more information, please see 138 | # 139 | -------------------------------------------------------------------------------- /dqm/batch/run_701.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_701.pkl -------------------------------------------------------------------------------- /dqm/batch/run_701.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | 3 | run_701 =[ 4 | ['701', 'all2_1', 'all2_1'] 5 | ] 6 | 7 | df = DataFrame(run_701, columns=['series', 'cb', 'measure_id']) 8 | df['measure_id'] = df['measure_id'].str.upper() 9 | df = df.sort_values(by=['series', 'cb', 'measure_id']) 10 | 11 | print(df.head()) 12 | df.to_pickle('./run_701.pkl') 13 | 14 | # CC0 1.0 Universal 15 | 16 | # Statement of Purpose 17 | 18 | # The laws of most jurisdictions throughout the world automatically confer 19 | # exclusive Copyright and Related Rights (defined below) upon the creator and 20 | # subsequent owner(s) (each and all, an "owner") of an original work of 21 | # authorship and/or a database (each, a "Work"). 22 | 23 | # Certain owners wish to permanently relinquish those rights to a Work for the 24 | # purpose of contributing to a commons of creative, cultural and scientific 25 | # works ("Commons") that the public can reliably and without fear of later 26 | # claims of infringement build upon, modify, incorporate in other works, reuse 27 | # and redistribute as freely as possible in any form whatsoever and for any 28 | # purposes, including without limitation commercial purposes. These owners may 29 | # contribute to the Commons to promote the ideal of a free culture and the 30 | # further production of creative, cultural and scientific works, or to gain 31 | # reputation or greater distribution for their Work in part through the use and 32 | # efforts of others. 33 | 34 | # For these and/or other purposes and motivations, and without any expectation 35 | # of additional consideration or compensation, the person associating CC0 with a 36 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 37 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 38 | # and publicly distribute the Work under its terms, with knowledge of his or her 39 | # Copyright and Related Rights in the Work and the meaning and intended legal 40 | # effect of CC0 on those rights. 41 | 42 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 43 | # protected by copyright and related or neighboring rights ("Copyright and 44 | # Related Rights"). Copyright and Related Rights include, but are not limited 45 | # to, the following: 46 | 47 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 48 | # and translate a Work; 49 | 50 | # ii. moral rights retained by the original author(s) and/or performer(s); 51 | 52 | # iii. publicity and privacy rights pertaining to a person's image or likeness 53 | # depicted in a Work; 54 | 55 | # iv. rights protecting against unfair competition in regards to a Work, 56 | # subject to the limitations in paragraph 4(a), below; 57 | 58 | # v. rights protecting the extraction, dissemination, use and reuse of data in 59 | # a Work; 60 | 61 | # vi. database rights (such as those arising under Directive 96/9/EC of the 62 | # European Parliament and of the Council of 11 March 1996 on the legal 63 | # protection of databases, and under any national implementation thereof, 64 | # including any amended or successor version of such directive); and 65 | 66 | # vii. other similar, equivalent or corresponding rights throughout the world 67 | # based on applicable law or treaty, and any national implementations thereof. 68 | 69 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 70 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 71 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 72 | # and Related Rights and associated claims and causes of action, whether now 73 | # known or unknown (including existing as well as future claims and causes of 74 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 75 | # duration provided by applicable law or treaty (including future time 76 | # extensions), (iii) in any current or future medium and for any number of 77 | # copies, and (iv) for any purpose whatsoever, including without limitation 78 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 79 | # the Waiver for the benefit of each member of the public at large and to the 80 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 81 | # shall not be subject to revocation, rescission, cancellation, termination, or 82 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 83 | # by the public as contemplated by Affirmer's express Statement of Purpose. 84 | 85 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 86 | # judged legally invalid or ineffective under applicable law, then the Waiver 87 | # shall be preserved to the maximum extent permitted taking into account 88 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 89 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 90 | # non transferable, non sublicensable, non exclusive, irrevocable and 91 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 92 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 93 | # provided by applicable law or treaty (including future time extensions), (iii) 94 | # in any current or future medium and for any number of copies, and (iv) for any 95 | # purpose whatsoever, including without limitation commercial, advertising or 96 | # promotional purposes (the "License"). The License shall be deemed effective as 97 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 98 | # License for any reason be judged legally invalid or ineffective under 99 | # applicable law, such partial invalidity or ineffectiveness shall not 100 | # invalidate the remainder of the License, and in such case Affirmer hereby 101 | # affirms that he or she will not (i) exercise any of his or her remaining 102 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 103 | # and causes of action with respect to the Work, in either case contrary to 104 | # Affirmer's express Statement of Purpose. 105 | 106 | # 4. Limitations and Disclaimers. 107 | 108 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 109 | # surrendered, licensed or otherwise affected by this document. 110 | 111 | # b. Affirmer offers the Work as-is and makes no representations or warranties 112 | # of any kind concerning the Work, express, implied, statutory or otherwise, 113 | # including without limitation warranties of title, merchantability, fitness 114 | # for a particular purpose, non infringement, or the absence of latent or 115 | # other defects, accuracy, or the present or absence of errors, whether or not 116 | # discoverable, all to the greatest extent permissible under applicable law. 117 | 118 | # c. Affirmer disclaims responsibility for clearing rights of other persons 119 | # that may apply to the Work or any use thereof, including without limitation 120 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 121 | # disclaims responsibility for obtaining any necessary consents, permissions 122 | # or other rights required for any use of the Work. 123 | 124 | # d. Affirmer understands and acknowledges that Creative Commons is not a 125 | # party to this document and has no duty or obligation with respect to this 126 | # CC0 or use of the Work. 127 | 128 | # For more information, please see 129 | # 130 | -------------------------------------------------------------------------------- /dqm/batch/run_702.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_702.pkl -------------------------------------------------------------------------------- /dqm/batch/run_703.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_703.pkl -------------------------------------------------------------------------------- /dqm/batch/run_704.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_704.pkl -------------------------------------------------------------------------------- /dqm/batch/run_705.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_705.pkl -------------------------------------------------------------------------------- /dqm/batch/run_706.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_706.pkl -------------------------------------------------------------------------------- /dqm/batch/run_707.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_707.pkl -------------------------------------------------------------------------------- /dqm/batch/run_708.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_708.pkl -------------------------------------------------------------------------------- /dqm/batch/run_709.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_709.pkl -------------------------------------------------------------------------------- /dqm/batch/run_709.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | 3 | run_709 =[ 4 | 5 | ['709', 'utl_tab_ah_sql', 'all20_1', 'ip'], 6 | ['709', 'utl_tab_ah_sql', 'all20_2', 'lt'], 7 | ['709', 'utl_tab_ah_sql', 'all20_3', 'ot'], 8 | ['709', 'utl_tab_ah_sql', 'all20_4', 'rx'] 9 | 10 | ] 11 | 12 | df = DataFrame(run_709, columns=['series', 'cb', 'measure_id', 'claim_type']) 13 | df['measure_id'] = df['measure_id'].str.upper() 14 | 15 | df = df.sort_values(by=['series', 'cb', 'measure_id']) 16 | 17 | print(df.head(4)) 18 | 19 | df.to_pickle('./run_709.pkl') 20 | 21 | # CC0 1.0 Universal 22 | 23 | # Statement of Purpose 24 | 25 | # The laws of most jurisdictions throughout the world automatically confer 26 | # exclusive Copyright and Related Rights (defined below) upon the creator and 27 | # subsequent owner(s) (each and all, an "owner") of an original work of 28 | # authorship and/or a database (each, a "Work"). 29 | 30 | # Certain owners wish to permanently relinquish those rights to a Work for the 31 | # purpose of contributing to a commons of creative, cultural and scientific 32 | # works ("Commons") that the public can reliably and without fear of later 33 | # claims of infringement build upon, modify, incorporate in other works, reuse 34 | # and redistribute as freely as possible in any form whatsoever and for any 35 | # purposes, including without limitation commercial purposes. These owners may 36 | # contribute to the Commons to promote the ideal of a free culture and the 37 | # further production of creative, cultural and scientific works, or to gain 38 | # reputation or greater distribution for their Work in part through the use and 39 | # efforts of others. 40 | 41 | # For these and/or other purposes and motivations, and without any expectation 42 | # of additional consideration or compensation, the person associating CC0 with a 43 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 44 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 45 | # and publicly distribute the Work under its terms, with knowledge of his or her 46 | # Copyright and Related Rights in the Work and the meaning and intended legal 47 | # effect of CC0 on those rights. 48 | 49 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 50 | # protected by copyright and related or neighboring rights ("Copyright and 51 | # Related Rights"). Copyright and Related Rights include, but are not limited 52 | # to, the following: 53 | 54 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 55 | # and translate a Work; 56 | 57 | # ii. moral rights retained by the original author(s) and/or performer(s); 58 | 59 | # iii. publicity and privacy rights pertaining to a person's image or likeness 60 | # depicted in a Work; 61 | 62 | # iv. rights protecting against unfair competition in regards to a Work, 63 | # subject to the limitations in paragraph 4(a), below; 64 | 65 | # v. rights protecting the extraction, dissemination, use and reuse of data in 66 | # a Work; 67 | 68 | # vi. database rights (such as those arising under Directive 96/9/EC of the 69 | # European Parliament and of the Council of 11 March 1996 on the legal 70 | # protection of databases, and under any national implementation thereof, 71 | # including any amended or successor version of such directive); and 72 | 73 | # vii. other similar, equivalent or corresponding rights throughout the world 74 | # based on applicable law or treaty, and any national implementations thereof. 75 | 76 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 77 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 78 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 79 | # and Related Rights and associated claims and causes of action, whether now 80 | # known or unknown (including existing as well as future claims and causes of 81 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 82 | # duration provided by applicable law or treaty (including future time 83 | # extensions), (iii) in any current or future medium and for any number of 84 | # copies, and (iv) for any purpose whatsoever, including without limitation 85 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 86 | # the Waiver for the benefit of each member of the public at large and to the 87 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 88 | # shall not be subject to revocation, rescission, cancellation, termination, or 89 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 90 | # by the public as contemplated by Affirmer's express Statement of Purpose. 91 | 92 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 93 | # judged legally invalid or ineffective under applicable law, then the Waiver 94 | # shall be preserved to the maximum extent permitted taking into account 95 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 96 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 97 | # non transferable, non sublicensable, non exclusive, irrevocable and 98 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 99 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 100 | # provided by applicable law or treaty (including future time extensions), (iii) 101 | # in any current or future medium and for any number of copies, and (iv) for any 102 | # purpose whatsoever, including without limitation commercial, advertising or 103 | # promotional purposes (the "License"). The License shall be deemed effective as 104 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 105 | # License for any reason be judged legally invalid or ineffective under 106 | # applicable law, such partial invalidity or ineffectiveness shall not 107 | # invalidate the remainder of the License, and in such case Affirmer hereby 108 | # affirms that he or she will not (i) exercise any of his or her remaining 109 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 110 | # and causes of action with respect to the Work, in either case contrary to 111 | # Affirmer's express Statement of Purpose. 112 | 113 | # 4. Limitations and Disclaimers. 114 | 115 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 116 | # surrendered, licensed or otherwise affected by this document. 117 | 118 | # b. Affirmer offers the Work as-is and makes no representations or warranties 119 | # of any kind concerning the Work, express, implied, statutory or otherwise, 120 | # including without limitation warranties of title, merchantability, fitness 121 | # for a particular purpose, non infringement, or the absence of latent or 122 | # other defects, accuracy, or the present or absence of errors, whether or not 123 | # discoverable, all to the greatest extent permissible under applicable law. 124 | 125 | # c. Affirmer disclaims responsibility for clearing rights of other persons 126 | # that may apply to the Work or any use thereof, including without limitation 127 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 128 | # disclaims responsibility for obtaining any necessary consents, permissions 129 | # or other rights required for any use of the Work. 130 | 131 | # d. Affirmer understands and acknowledges that Creative Commons is not a 132 | # party to this document and has no duty or obligation with respect to this 133 | # CC0 or use of the Work. 134 | 135 | # For more information, please see 136 | # 137 | -------------------------------------------------------------------------------- /dqm/batch/run_710.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_710.pkl -------------------------------------------------------------------------------- /dqm/batch/run_711.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_711.pkl -------------------------------------------------------------------------------- /dqm/batch/run_712.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_712.pkl -------------------------------------------------------------------------------- /dqm/batch/run_712.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | 3 | run_712 =[ 4 | 5 | ['712', 'frq', 'ALL28_1', 'IP', 'CLL', 'stc_cd', 'AJ'], 6 | ['712', 'frq', 'ALL29_1', 'LT', 'CLL', 'stc_cd', 'AJ'], 7 | ['712', 'frq', 'ALL30_1', 'OT', 'CLL', 'stc_cd', 'AJ'], 8 | ['712', 'frq', 'ALL31_1', 'RX', 'CLL', 'stc_cd', 'AJ'] 9 | 10 | ] 11 | 12 | df = DataFrame(run_712, columns=['series', 'cb', 'measure_id', 'claim_type', 'level', 'var', 'constraint']) 13 | df['measure_id'] = df['measure_id'].str.upper() 14 | 15 | df = df.sort_values(by=['series', 'cb', 'measure_id']) 16 | 17 | print(df.head(4)) 18 | 19 | df.to_pickle('./run_712.pkl') 20 | 21 | # CC0 1.0 Universal 22 | 23 | # Statement of Purpose 24 | 25 | # The laws of most jurisdictions throughout the world automatically confer 26 | # exclusive Copyright and Related Rights (defined below) upon the creator and 27 | # subsequent owner(s) (each and all, an "owner") of an original work of 28 | # authorship and/or a database (each, a "Work"). 29 | 30 | # Certain owners wish to permanently relinquish those rights to a Work for the 31 | # purpose of contributing to a commons of creative, cultural and scientific 32 | # works ("Commons") that the public can reliably and without fear of later 33 | # claims of infringement build upon, modify, incorporate in other works, reuse 34 | # and redistribute as freely as possible in any form whatsoever and for any 35 | # purposes, including without limitation commercial purposes. These owners may 36 | # contribute to the Commons to promote the ideal of a free culture and the 37 | # further production of creative, cultural and scientific works, or to gain 38 | # reputation or greater distribution for their Work in part through the use and 39 | # efforts of others. 40 | 41 | # For these and/or other purposes and motivations, and without any expectation 42 | # of additional consideration or compensation, the person associating CC0 with a 43 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 44 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 45 | # and publicly distribute the Work under its terms, with knowledge of his or her 46 | # Copyright and Related Rights in the Work and the meaning and intended legal 47 | # effect of CC0 on those rights. 48 | 49 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 50 | # protected by copyright and related or neighboring rights ("Copyright and 51 | # Related Rights"). Copyright and Related Rights include, but are not limited 52 | # to, the following: 53 | 54 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 55 | # and translate a Work; 56 | 57 | # ii. moral rights retained by the original author(s) and/or performer(s); 58 | 59 | # iii. publicity and privacy rights pertaining to a person's image or likeness 60 | # depicted in a Work; 61 | 62 | # iv. rights protecting against unfair competition in regards to a Work, 63 | # subject to the limitations in paragraph 4(a), below; 64 | 65 | # v. rights protecting the extraction, dissemination, use and reuse of data in 66 | # a Work; 67 | 68 | # vi. database rights (such as those arising under Directive 96/9/EC of the 69 | # European Parliament and of the Council of 11 March 1996 on the legal 70 | # protection of databases, and under any national implementation thereof, 71 | # including any amended or successor version of such directive); and 72 | 73 | # vii. other similar, equivalent or corresponding rights throughout the world 74 | # based on applicable law or treaty, and any national implementations thereof. 75 | 76 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 77 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 78 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 79 | # and Related Rights and associated claims and causes of action, whether now 80 | # known or unknown (including existing as well as future claims and causes of 81 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 82 | # duration provided by applicable law or treaty (including future time 83 | # extensions), (iii) in any current or future medium and for any number of 84 | # copies, and (iv) for any purpose whatsoever, including without limitation 85 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 86 | # the Waiver for the benefit of each member of the public at large and to the 87 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 88 | # shall not be subject to revocation, rescission, cancellation, termination, or 89 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 90 | # by the public as contemplated by Affirmer's express Statement of Purpose. 91 | 92 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 93 | # judged legally invalid or ineffective under applicable law, then the Waiver 94 | # shall be preserved to the maximum extent permitted taking into account 95 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 96 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 97 | # non transferable, non sublicensable, non exclusive, irrevocable and 98 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 99 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 100 | # provided by applicable law or treaty (including future time extensions), (iii) 101 | # in any current or future medium and for any number of copies, and (iv) for any 102 | # purpose whatsoever, including without limitation commercial, advertising or 103 | # promotional purposes (the "License"). The License shall be deemed effective as 104 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 105 | # License for any reason be judged legally invalid or ineffective under 106 | # applicable law, such partial invalidity or ineffectiveness shall not 107 | # invalidate the remainder of the License, and in such case Affirmer hereby 108 | # affirms that he or she will not (i) exercise any of his or her remaining 109 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 110 | # and causes of action with respect to the Work, in either case contrary to 111 | # Affirmer's express Statement of Purpose. 112 | 113 | # 4. Limitations and Disclaimers. 114 | 115 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 116 | # surrendered, licensed or otherwise affected by this document. 117 | 118 | # b. Affirmer offers the Work as-is and makes no representations or warranties 119 | # of any kind concerning the Work, express, implied, statutory or otherwise, 120 | # including without limitation warranties of title, merchantability, fitness 121 | # for a particular purpose, non infringement, or the absence of latent or 122 | # other defects, accuracy, or the present or absence of errors, whether or not 123 | # discoverable, all to the greatest extent permissible under applicable law. 124 | 125 | # c. Affirmer disclaims responsibility for clearing rights of other persons 126 | # that may apply to the Work or any use thereof, including without limitation 127 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 128 | # disclaims responsibility for obtaining any necessary consents, permissions 129 | # or other rights required for any use of the Work. 130 | 131 | # d. Affirmer understands and acknowledges that Creative Commons is not a 132 | # party to this document and has no duty or obligation with respect to this 133 | # CC0 or use of the Work. 134 | 135 | # For more information, please see 136 | # 137 | -------------------------------------------------------------------------------- /dqm/batch/run_713.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_713.pkl -------------------------------------------------------------------------------- /dqm/batch/run_714.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_714.pkl -------------------------------------------------------------------------------- /dqm/batch/run_714.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | 3 | run_714 = [ 4 | 5 | ['714', 'all36_1', 'all36_1', ''], 6 | 7 | ['714', 'all34_1', 'all34_1', ''], 8 | ['714', 'all34_2', 'all34_2', ''], 9 | 10 | ['714', 'all35_1_or_2', 'all35_1', 'AX'], 11 | ['714', 'all35_1_or_2', 'all35_2', 'AY'], 12 | ['714', 'all35_3_or_4', 'all35_3', 'AX'], 13 | ['714', 'all35_3_or_4', 'all35_4', 'AY'] 14 | ] 15 | 16 | df = DataFrame(run_714, columns=['series', 'cb', 'measure_id', 'claim_cat']) 17 | df['measure_id'] = df['measure_id'].str.upper() 18 | 19 | df = df.sort_values(by=['series', 'cb', 'measure_id']) 20 | 21 | print(df.head(10)) 22 | 23 | df.to_pickle('./run_714.pkl') 24 | 25 | # CC0 1.0 Universal 26 | 27 | # Statement of Purpose 28 | 29 | # The laws of most jurisdictions throughout the world automatically confer 30 | # exclusive Copyright and Related Rights (defined below) upon the creator and 31 | # subsequent owner(s) (each and all, an "owner") of an original work of 32 | # authorship and/or a database (each, a "Work"). 33 | 34 | # Certain owners wish to permanently relinquish those rights to a Work for the 35 | # purpose of contributing to a commons of creative, cultural and scientific 36 | # works ("Commons") that the public can reliably and without fear of later 37 | # claims of infringement build upon, modify, incorporate in other works, reuse 38 | # and redistribute as freely as possible in any form whatsoever and for any 39 | # purposes, including without limitation commercial purposes. These owners may 40 | # contribute to the Commons to promote the ideal of a free culture and the 41 | # further production of creative, cultural and scientific works, or to gain 42 | # reputation or greater distribution for their Work in part through the use and 43 | # efforts of others. 44 | 45 | # For these and/or other purposes and motivations, and without any expectation 46 | # of additional consideration or compensation, the person associating CC0 with a 47 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 48 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 49 | # and publicly distribute the Work under its terms, with knowledge of his or her 50 | # Copyright and Related Rights in the Work and the meaning and intended legal 51 | # effect of CC0 on those rights. 52 | 53 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 54 | # protected by copyright and related or neighboring rights ("Copyright and 55 | # Related Rights"). Copyright and Related Rights include, but are not limited 56 | # to, the following: 57 | 58 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 59 | # and translate a Work; 60 | 61 | # ii. moral rights retained by the original author(s) and/or performer(s); 62 | 63 | # iii. publicity and privacy rights pertaining to a person's image or likeness 64 | # depicted in a Work; 65 | 66 | # iv. rights protecting against unfair competition in regards to a Work, 67 | # subject to the limitations in paragraph 4(a), below; 68 | 69 | # v. rights protecting the extraction, dissemination, use and reuse of data in 70 | # a Work; 71 | 72 | # vi. database rights (such as those arising under Directive 96/9/EC of the 73 | # European Parliament and of the Council of 11 March 1996 on the legal 74 | # protection of databases, and under any national implementation thereof, 75 | # including any amended or successor version of such directive); and 76 | 77 | # vii. other similar, equivalent or corresponding rights throughout the world 78 | # based on applicable law or treaty, and any national implementations thereof. 79 | 80 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 81 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 82 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 83 | # and Related Rights and associated claims and causes of action, whether now 84 | # known or unknown (including existing as well as future claims and causes of 85 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 86 | # duration provided by applicable law or treaty (including future time 87 | # extensions), (iii) in any current or future medium and for any number of 88 | # copies, and (iv) for any purpose whatsoever, including without limitation 89 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 90 | # the Waiver for the benefit of each member of the public at large and to the 91 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 92 | # shall not be subject to revocation, rescission, cancellation, termination, or 93 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 94 | # by the public as contemplated by Affirmer's express Statement of Purpose. 95 | 96 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 97 | # judged legally invalid or ineffective under applicable law, then the Waiver 98 | # shall be preserved to the maximum extent permitted taking into account 99 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 100 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 101 | # non transferable, non sublicensable, non exclusive, irrevocable and 102 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 103 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 104 | # provided by applicable law or treaty (including future time extensions), (iii) 105 | # in any current or future medium and for any number of copies, and (iv) for any 106 | # purpose whatsoever, including without limitation commercial, advertising or 107 | # promotional purposes (the "License"). The License shall be deemed effective as 108 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 109 | # License for any reason be judged legally invalid or ineffective under 110 | # applicable law, such partial invalidity or ineffectiveness shall not 111 | # invalidate the remainder of the License, and in such case Affirmer hereby 112 | # affirms that he or she will not (i) exercise any of his or her remaining 113 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 114 | # and causes of action with respect to the Work, in either case contrary to 115 | # Affirmer's express Statement of Purpose. 116 | 117 | # 4. Limitations and Disclaimers. 118 | 119 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 120 | # surrendered, licensed or otherwise affected by this document. 121 | 122 | # b. Affirmer offers the Work as-is and makes no representations or warranties 123 | # of any kind concerning the Work, express, implied, statutory or otherwise, 124 | # including without limitation warranties of title, merchantability, fitness 125 | # for a particular purpose, non infringement, or the absence of latent or 126 | # other defects, accuracy, or the present or absence of errors, whether or not 127 | # discoverable, all to the greatest extent permissible under applicable law. 128 | 129 | # c. Affirmer disclaims responsibility for clearing rights of other persons 130 | # that may apply to the Work or any use thereof, including without limitation 131 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 132 | # disclaims responsibility for obtaining any necessary consents, permissions 133 | # or other rights required for any use of the Work. 134 | 135 | # d. Affirmer understands and acknowledges that Creative Commons is not a 136 | # party to this document and has no duty or obligation with respect to this 137 | # CC0 or use of the Work. 138 | 139 | # For more information, please see 140 | # 141 | -------------------------------------------------------------------------------- /dqm/batch/run_801.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_801.pkl -------------------------------------------------------------------------------- /dqm/batch/run_801_miss_claims.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_801_miss_claims.pkl -------------------------------------------------------------------------------- /dqm/batch/run_801_miss_non_claims.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_801_miss_non_claims.pkl -------------------------------------------------------------------------------- /dqm/batch/run_801_missvar.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_801_missvar.pkl -------------------------------------------------------------------------------- /dqm/batch/run_802.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_802.pkl -------------------------------------------------------------------------------- /dqm/batch/run_803.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_803.pkl -------------------------------------------------------------------------------- /dqm/batch/run_901.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_901.pkl -------------------------------------------------------------------------------- /dqm/batch/run_902.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_902.pkl -------------------------------------------------------------------------------- /dqm/batch/run_903.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_903.pkl -------------------------------------------------------------------------------- /dqm/batch/run_904.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_904.pkl -------------------------------------------------------------------------------- /dqm/batch/run_905.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_905.pkl -------------------------------------------------------------------------------- /dqm/batch/run_906.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_906.pkl -------------------------------------------------------------------------------- /dqm/batch/run_906.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | 3 | run_906_ffs_clms_ad = [ 4 | ['906', 'tab_clm_ad', 'FFS43_1', 'ip', '43'], 5 | ['906', 'tab_clm_ad', 'FFS44_1', 'lt', '44'], 6 | ['906', 'tab_clm_ad', 'FFS45_1', 'ot', '45'], 7 | ['906', 'tab_clm_ad', 'FFS46_1', 'rx', '46'] 8 | ] 9 | 10 | df = DataFrame(run_906_ffs_clms_ad, columns=['series', 'cb', 'measure_id', 'filetype', 'msrnum']) 11 | df['measure_id'] = df['measure_id'].str.upper() 12 | # df = df.set_index("measure_id", drop = False) 13 | print(df.head()) 14 | df.to_pickle('./run_906.pkl') 15 | 16 | # CC0 1.0 Universal 17 | 18 | # Statement of Purpose 19 | 20 | # The laws of most jurisdictions throughout the world automatically confer 21 | # exclusive Copyright and Related Rights (defined below) upon the creator and 22 | # subsequent owner(s) (each and all, an "owner") of an original work of 23 | # authorship and/or a database (each, a "Work"). 24 | 25 | # Certain owners wish to permanently relinquish those rights to a Work for the 26 | # purpose of contributing to a commons of creative, cultural and scientific 27 | # works ("Commons") that the public can reliably and without fear of later 28 | # claims of infringement build upon, modify, incorporate in other works, reuse 29 | # and redistribute as freely as possible in any form whatsoever and for any 30 | # purposes, including without limitation commercial purposes. These owners may 31 | # contribute to the Commons to promote the ideal of a free culture and the 32 | # further production of creative, cultural and scientific works, or to gain 33 | # reputation or greater distribution for their Work in part through the use and 34 | # efforts of others. 35 | 36 | # For these and/or other purposes and motivations, and without any expectation 37 | # of additional consideration or compensation, the person associating CC0 with a 38 | # Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 39 | # and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 40 | # and publicly distribute the Work under its terms, with knowledge of his or her 41 | # Copyright and Related Rights in the Work and the meaning and intended legal 42 | # effect of CC0 on those rights. 43 | 44 | # 1. Copyright and Related Rights. A Work made available under CC0 may be 45 | # protected by copyright and related or neighboring rights ("Copyright and 46 | # Related Rights"). Copyright and Related Rights include, but are not limited 47 | # to, the following: 48 | 49 | # i. the right to reproduce, adapt, distribute, perform, display, communicate, 50 | # and translate a Work; 51 | 52 | # ii. moral rights retained by the original author(s) and/or performer(s); 53 | 54 | # iii. publicity and privacy rights pertaining to a person's image or likeness 55 | # depicted in a Work; 56 | 57 | # iv. rights protecting against unfair competition in regards to a Work, 58 | # subject to the limitations in paragraph 4(a), below; 59 | 60 | # v. rights protecting the extraction, dissemination, use and reuse of data in 61 | # a Work; 62 | 63 | # vi. database rights (such as those arising under Directive 96/9/EC of the 64 | # European Parliament and of the Council of 11 March 1996 on the legal 65 | # protection of databases, and under any national implementation thereof, 66 | # including any amended or successor version of such directive); and 67 | 68 | # vii. other similar, equivalent or corresponding rights throughout the world 69 | # based on applicable law or treaty, and any national implementations thereof. 70 | 71 | # 2. Waiver. To the greatest extent permitted by, but not in contravention of, 72 | # applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 73 | # unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 74 | # and Related Rights and associated claims and causes of action, whether now 75 | # known or unknown (including existing as well as future claims and causes of 76 | # action), in the Work (i) in all territories worldwide, (ii) for the maximum 77 | # duration provided by applicable law or treaty (including future time 78 | # extensions), (iii) in any current or future medium and for any number of 79 | # copies, and (iv) for any purpose whatsoever, including without limitation 80 | # commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 81 | # the Waiver for the benefit of each member of the public at large and to the 82 | # detriment of Affirmer's heirs and successors, fully intending that such Waiver 83 | # shall not be subject to revocation, rescission, cancellation, termination, or 84 | # any other legal or equitable action to disrupt the quiet enjoyment of the Work 85 | # by the public as contemplated by Affirmer's express Statement of Purpose. 86 | 87 | # 3. Public License Fallback. Should any part of the Waiver for any reason be 88 | # judged legally invalid or ineffective under applicable law, then the Waiver 89 | # shall be preserved to the maximum extent permitted taking into account 90 | # Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 91 | # is so judged Affirmer hereby grants to each affected person a royalty-free, 92 | # non transferable, non sublicensable, non exclusive, irrevocable and 93 | # unconditional license to exercise Affirmer's Copyright and Related Rights in 94 | # the Work (i) in all territories worldwide, (ii) for the maximum duration 95 | # provided by applicable law or treaty (including future time extensions), (iii) 96 | # in any current or future medium and for any number of copies, and (iv) for any 97 | # purpose whatsoever, including without limitation commercial, advertising or 98 | # promotional purposes (the "License"). The License shall be deemed effective as 99 | # of the date CC0 was applied by Affirmer to the Work. Should any part of the 100 | # License for any reason be judged legally invalid or ineffective under 101 | # applicable law, such partial invalidity or ineffectiveness shall not 102 | # invalidate the remainder of the License, and in such case Affirmer hereby 103 | # affirms that he or she will not (i) exercise any of his or her remaining 104 | # Copyright and Related Rights in the Work or (ii) assert any associated claims 105 | # and causes of action with respect to the Work, in either case contrary to 106 | # Affirmer's express Statement of Purpose. 107 | 108 | # 4. Limitations and Disclaimers. 109 | 110 | # a. No trademark or patent rights held by Affirmer are waived, abandoned, 111 | # surrendered, licensed or otherwise affected by this document. 112 | 113 | # b. Affirmer offers the Work as-is and makes no representations or warranties 114 | # of any kind concerning the Work, express, implied, statutory or otherwise, 115 | # including without limitation warranties of title, merchantability, fitness 116 | # for a particular purpose, non infringement, or the absence of latent or 117 | # other defects, accuracy, or the present or absence of errors, whether or not 118 | # discoverable, all to the greatest extent permissible under applicable law. 119 | 120 | # c. Affirmer disclaims responsibility for clearing rights of other persons 121 | # that may apply to the Work or any use thereof, including without limitation 122 | # any person's Copyright and Related Rights in the Work. Further, Affirmer 123 | # disclaims responsibility for obtaining any necessary consents, permissions 124 | # or other rights required for any use of the Work. 125 | 126 | # d. Affirmer understands and acknowledges that Creative Commons is not a 127 | # party to this document and has no duty or obligation with respect to this 128 | # CC0 or use of the Work. 129 | 130 | # For more information, please see 131 | # 132 | -------------------------------------------------------------------------------- /dqm/batch/run_907.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_907.pkl -------------------------------------------------------------------------------- /dqm/batch/run_909.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_909.pkl -------------------------------------------------------------------------------- /dqm/batch/run_910.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_910.pkl -------------------------------------------------------------------------------- /dqm/batch/run_911.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_911.pkl -------------------------------------------------------------------------------- /dqm/batch/run_912.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_912.pkl -------------------------------------------------------------------------------- /dqm/batch/run_913.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_913.pkl -------------------------------------------------------------------------------- /dqm/batch/run_914.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_914.pkl -------------------------------------------------------------------------------- /dqm/batch/run_915.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_915.pkl -------------------------------------------------------------------------------- /dqm/batch/run_916.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_916.pkl -------------------------------------------------------------------------------- /dqm/batch/run_917.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_917.pkl -------------------------------------------------------------------------------- /dqm/batch/run_918.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_918.pkl -------------------------------------------------------------------------------- /dqm/batch/run_919.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_919.pkl -------------------------------------------------------------------------------- /dqm/batch/run_920.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/batch/run_920.pkl -------------------------------------------------------------------------------- /dqm/cfg/abd.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/abd.pkl -------------------------------------------------------------------------------- /dqm/cfg/apdxc.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/apdxc.pkl -------------------------------------------------------------------------------- /dqm/cfg/atypical_provider_table.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/atypical_provider_table.pkl -------------------------------------------------------------------------------- /dqm/cfg/countystate_lookup.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/countystate_lookup.pkl -------------------------------------------------------------------------------- /dqm/cfg/fmg.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/fmg.pkl -------------------------------------------------------------------------------- /dqm/cfg/missVar.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/missVar.pkl -------------------------------------------------------------------------------- /dqm/cfg/prgncy.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/prgncy.pkl -------------------------------------------------------------------------------- /dqm/cfg/provider_classification_lookup.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/provider_classification_lookup.pkl -------------------------------------------------------------------------------- /dqm/cfg/prvtxnmy.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/prvtxnmy.pkl -------------------------------------------------------------------------------- /dqm/cfg/sauths.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/sauths.pkl -------------------------------------------------------------------------------- /dqm/cfg/schip.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/schip.pkl -------------------------------------------------------------------------------- /dqm/cfg/splans.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/splans.pkl -------------------------------------------------------------------------------- /dqm/cfg/st2_name.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/st2_name.pkl -------------------------------------------------------------------------------- /dqm/cfg/st_fips.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/st_fips.pkl -------------------------------------------------------------------------------- /dqm/cfg/st_name.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/st_name.pkl -------------------------------------------------------------------------------- /dqm/cfg/st_usps.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/st_usps.pkl -------------------------------------------------------------------------------- /dqm/cfg/stabr.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/stabr.pkl -------------------------------------------------------------------------------- /dqm/cfg/stc_cd.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/stc_cd.pkl -------------------------------------------------------------------------------- /dqm/cfg/thresh.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/thresh.pkl -------------------------------------------------------------------------------- /dqm/cfg/thresh_measures.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/thresh_measures.pkl -------------------------------------------------------------------------------- /dqm/cfg/thresholds.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/thresholds.pkl -------------------------------------------------------------------------------- /dqm/cfg/zcc.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/zcc.pkl -------------------------------------------------------------------------------- /dqm/cfg/zipstate_lookup.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/cfg/zipstate_lookup.pkl -------------------------------------------------------------------------------- /dqm/submodules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/dqm/submodules/__init__.py -------------------------------------------------------------------------------- /dqm/thresholds.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import glob 4 | 5 | thresholds = glob.glob('./dqm/cfg/Thresholds*.xlsx') 6 | print(thresholds) 7 | assert len(thresholds) == 1, 'Either no thresholds file or multiple files found at ./dqm/cfg/Thresholds*.xlsx; expecting one' 8 | 9 | df = pd.read_excel(thresholds[0], sheet_name='Measures', header=None, dtype=str, engine='openpyxl') 10 | df.columns = ['_c' + str(col) for col in df.columns] 11 | 12 | col_names = {'_c0': 'claim_file', 13 | '_c3': 'display_order', 14 | '_c4': 'measure', 15 | '_c5': 'measure_id_w_display_order', 16 | '_c6': 'measure_name', 17 | '_c7': 'Measure_Type', 18 | '_c9': 'claim_category', 19 | '_c13': 'Active_Ind', 20 | '_c27': 'decimal_places', 21 | '_c33': 'Display_Type' 22 | } 23 | 24 | df = df[col_names.keys()] 25 | df = df.iloc[1:,:] 26 | df = df.rename(columns=col_names) 27 | 28 | df = df[df['Active_Ind'] == 'Y'] 29 | df = df[df['measure'].notnull()] 30 | 31 | df['measure_id'] = df['measure'].str.replace('.', '_', regex=False).str.upper() 32 | df['display_order'] = df['display_order'].str.replace('.', '_', regex=False) 33 | df['claim_category'] = df['claim_category'].fillna('N/A') 34 | 35 | # assign measure parts 36 | def createMeasureCols(measure_id): 37 | import re 38 | 39 | try: 40 | measure_cat = re.search( 41 | r"[a-zA-Z]*", measure_id, re.IGNORECASE).group() 42 | except AttributeError: 43 | measure_cat = re.search(r"[a-zA-Z]*", measure_id, re.IGNORECASE) 44 | 45 | if (measure_cat.startswith('SUM')): 46 | measure_major = measure_id[len(measure_cat):].split('_', 1)[1] 47 | measure_minor = '' 48 | else: 49 | measure_major = measure_id[len(measure_cat):].split('_', 1)[0] 50 | measure_minor = measure_id[len(measure_cat):].split('_', 1)[1] 51 | 52 | return {'cat': measure_cat, 'major': measure_major, 'minor': measure_minor} 53 | 54 | df[['measure_cat', 'measure_major', 'measure_minor']] = df.apply( 55 | lambda x: createMeasureCols(x['measure_id']), axis=1).apply(pd.Series) 56 | df[['z_display_order', 'display_suffix']] = df['display_order'].str.split('_', expand=True) 57 | 58 | df['z_display_order'] = df['z_display_order'].str.zfill(3) 59 | 60 | df['z_display_order_suffix'] = df.apply(lambda x: x['z_display_order'] + '_' + x['display_suffix'] 61 | if pd.notnull(x['display_suffix']) else x['z_display_order'], axis=1) 62 | 63 | df['decimal_places_int'] = df['decimal_places'].mask(df['decimal_places'] == 'N/A') 64 | df['decimal_places_int'] = df['decimal_places_int'].fillna(0) 65 | df['decimal_places_int'] = df['decimal_places_int'].astype(np.int64) 66 | df['decimal_places'] = df['decimal_places_int'] 67 | 68 | df.to_csv('./dqm/cfg/thresholds.csv', index=False) 69 | df.to_pickle('./dqm/cfg/thresholds.pkl') 70 | -------------------------------------------------------------------------------- /notebooks/val/Shared/DQ-Measures/DQ Measures (submodule - prepare).py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | from dqm import DQMeasures as s 3 | import logging 4 | 5 | # COMMAND ---------- 6 | 7 | # MAGIC %md 8 | # MAGIC ##### 1) Job Parameters 9 | 10 | # COMMAND ---------- 11 | 12 | _reportMonth = dbutils.widgets.get("reportMonth") 13 | _stateCode = dbutils.widgets.get("stateCode") 14 | _entity = dbutils.widgets.get("entity") 15 | 16 | # COMMAND ---------- 17 | 18 | # MAGIC %md 19 | # MAGIC ###### 2) Instantiate DQ Measures 20 | 21 | # COMMAND ---------- 22 | 23 | dqm = s.DQMeasures(_reportMonth, _stateCode, _entity) 24 | 25 | # COMMAND ---------- 26 | 27 | # MAGIC %md 28 | # MAGIC ###### 3) Logging Threshold 29 | 30 | # COMMAND ---------- 31 | 32 | dqm.logger.setLevel(logging.INFO) 33 | 34 | # COMMAND ---------- 35 | 36 | # MAGIC %md 37 | # MAGIC ###### 4) Initialize Run 38 | 39 | # COMMAND ---------- 40 | 41 | dqm.init() 42 | 43 | # COMMAND ---------- 44 | 45 | # MAGIC %md 46 | # MAGIC 47 | # MAGIC ###### 5) Run submodules 100, 200, 500, 600, 700, 800, and 900 in Parallel 48 | 49 | # COMMAND ---------- 50 | 51 | -------------------------------------------------------------------------------- /notebooks/val/Shared/DQ-Measures/DQ Measures (submodule).py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | from dqm import DQMeasures as s 3 | import logging 4 | 5 | # COMMAND ---------- 6 | 7 | # MAGIC %md 8 | # MAGIC ##### 1) Job Parameters 9 | 10 | # COMMAND ---------- 11 | 12 | _reportMonth = dbutils.widgets.get("reportMonth") 13 | _stateCode = dbutils.widgets.get("stateCode") 14 | _entity = dbutils.widgets.get("entity") 15 | _module = dbutils.widgets.get("module") 16 | 17 | # COMMAND ---------- 18 | 19 | # MAGIC %md 20 | # MAGIC ###### 2) Instantiate DQ Measures 21 | 22 | # COMMAND ---------- 23 | 24 | dqm = s.DQMeasures(_reportMonth, _stateCode, _entity) 25 | 26 | # COMMAND ---------- 27 | 28 | # MAGIC %md 29 | # MAGIC ###### 3) Logging Threshold 30 | 31 | # COMMAND ---------- 32 | 33 | dqm.logger.setLevel(logging.INFO) 34 | 35 | # COMMAND ---------- 36 | 37 | # MAGIC %md 38 | # MAGIC ###### 4) Initialize Run 39 | 40 | # COMMAND ---------- 41 | 42 | dqm.init() 43 | 44 | # COMMAND ---------- 45 | 46 | # MAGIC %md 47 | # MAGIC ###### 5) Run Measures 48 | 49 | # COMMAND ---------- 50 | 51 | # redirect s3 bucket folder path 52 | # conditionally handle seperate entities for medicaid and chip 53 | if (_entity == '0'): 54 | dqm.setS3Bucket('macbis-dw-dqm-val') 55 | dqm.s3folder = 'sas-dqm/uat/3.2/' + dqm.rpt_state + '/' + dqm.rpt_fldr + '/' + dqm.z_run_id 56 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 57 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 58 | dqm.s3path 59 | elif (_entity == '1'): 60 | dqm.setS3Bucket('macbis-dw-dqm-val') 61 | dqm.s3folder = 'sas-dqm/uat/3.2/' + dqm.rpt_state + '-M/' + dqm.rpt_fldr + '/' + dqm.z_run_id 62 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 63 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 64 | dqm.s3path 65 | elif (_entity == '2'): 66 | dqm.setS3Bucket('macbis-dw-dqm-val') 67 | dqm.s3folder = 'sas-dqm/uat/3.2/' + dqm.rpt_state + '-C/' + dqm.rpt_fldr + '/' + dqm.z_run_id 68 | dqm.s3xlsx = dqm.s3proto + dqm.s3bucket + '/' + dqm.s3folder 69 | dqm.s3path = dqm.s3xlsx + '/' + dqm.pgmstart 70 | dqm.s3path 71 | 72 | # COMMAND ---------- 73 | 74 | # configure spark with the task prefix for this run 75 | spark.conf.set('dqm.taskprefix', dqm.taskprefix) 76 | 77 | # COMMAND ---------- 78 | 79 | # display useful information about this run 80 | dqm.print() 81 | 82 | # COMMAND ---------- 83 | 84 | if _module == "100": 85 | dqm.run( 86 | spark, 87 | dqm.where(series="101") 88 | + dqm.where(series="102") 89 | + dqm.where(series="103") 90 | + dqm.where(series="104") 91 | + dqm.where(series="105") 92 | + dqm.where(series="106") 93 | + dqm.where(series="107") 94 | + dqm.where(series="108") 95 | + dqm.where(series="109") 96 | + dqm.where(series="110"), 97 | ) 98 | elif _module == "200": 99 | dqm.run( 100 | spark, 101 | dqm.where(series="201") 102 | + dqm.where(series="202") 103 | + dqm.where(series="204") 104 | + dqm.where(series="205") 105 | + dqm.where(series="206"), 106 | ) 107 | elif _module == "500": 108 | dqm.run( 109 | spark, 110 | dqm.where(series="501") 111 | + dqm.where(series="502") 112 | + dqm.where(series="503") 113 | + dqm.where(series="504"), 114 | ) 115 | elif _module == "600": 116 | dqm.run( 117 | spark, 118 | dqm.where(series="601") + dqm.where(series="602") + dqm.where(series="603"), 119 | ) 120 | elif _module == "700": 121 | dqm.run( 122 | spark, 123 | dqm.where(series="701") 124 | + dqm.where(series="702") 125 | + dqm.where(series="703") 126 | + dqm.where(series="704") 127 | + dqm.where(series="705") 128 | + dqm.where(series="706") 129 | + dqm.where(series="707") 130 | + dqm.where(series="708") 131 | + dqm.where(series="709") 132 | + dqm.where(series="710") 133 | + dqm.where(series="711") 134 | + dqm.where(series="712") 135 | + dqm.where(series="713") 136 | + dqm.where(series="714") 137 | + dqm.where(series="715") 138 | + dqm.where(series="716"), 139 | ) 140 | elif _module == "800": 141 | dqm.run(spark, dqm.where(series="802") + dqm.where(series="803")) 142 | elif _module == "900": 143 | dqm.run( 144 | spark, 145 | dqm.where(series="901") 146 | + dqm.where(series="902") 147 | + dqm.where(series="903") 148 | + dqm.where(series="904") 149 | + dqm.where(series="905") 150 | + dqm.where(series="906") 151 | + dqm.where(series="907") 152 | + dqm.where(series="909") 153 | + dqm.where(series="910") 154 | + dqm.where(series="911") 155 | + dqm.where(series="912") 156 | + dqm.where(series="913") 157 | + dqm.where(series="914") 158 | + dqm.where(series="915") 159 | + dqm.where(series="916") 160 | + dqm.where(series="917") 161 | + dqm.where(series="918") 162 | + dqm.where(series="919"), 163 | ) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | autoflake 2 | awscli 3 | bandit 4 | black 5 | databricks-cli 6 | flake8 7 | isort 8 | numpy==1.21.5 9 | pandas==1.4.4 10 | pyspark==3.4.0 11 | pytest 12 | setuptools==63.4.1 13 | wheel==0.37.1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open(".github/README.md", "r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="dqm", 8 | version="3.13.0", 9 | author="Shinu Verghese", 10 | author_email="shinu.verghese@cms.hhs.gov", 11 | description="A package to calculate data quality measures on T-MSIS data using Databricks", 12 | # long_description=long_description, 13 | # long_description_content_type="text/markdown", 14 | url="https://git-codecommit.us-east-2.amazonaws.com/v1/repos/DQ-Measures", 15 | classifiers=[ 16 | "Programming Language :: Python :: 3.9.13 :: Only", 17 | "Operating System :: OS Independent", 18 | ], 19 | packages=setuptools.find_packages(), 20 | package_data={ 21 | "dqm": ["cfg/*.pkl", "batch/*.pkl", "testing/*.pkl"], 22 | }, 23 | project_urls={ 24 | 'Documentation': 'https://tmsis2.atlassian.net/wiki/spaces/DQM/pages/2758180868/Data+Quality+Measures+Python+Library', 25 | 'Tracker': 'https://tmsis2.atlassian.net/browse/DQM', 26 | }, 27 | python_requires=">=3.8", 28 | ) 29 | -------------------------------------------------------------------------------- /static/csv/abd.csv: -------------------------------------------------------------------------------- 1 | State,Medicaid Determination 2 | Alabama,1634 3 | Alaska,SSI 4 | Amer. Samoa,N/A 5 | Arizona,1634 6 | Arkansas,1634 7 | California,1634 8 | Colorado,1634 9 | Connecticut,209b 10 | Delaware,1634 11 | Dist. Of Col.,1634 12 | Florida,1634 13 | Georgia,1634 14 | Guam,N/A 15 | Hawaii,209b 16 | Idaho,SSI 17 | Illinois,209b 18 | Indiana,1634 19 | Iowa,1634 20 | Kansas,SSI 21 | Kentucky,1634 22 | Louisiana,1634 23 | Maine,1634 24 | Maryland,1634 25 | Massachusetts,1634 26 | Michigan,1634 27 | Minnesota,209b 28 | Mississippi,1634 29 | Missouri,209b 30 | Montana,1634 31 | N. Mariana Islands,SSI 32 | Nebraska,SSI 33 | Nevada,SSI 34 | New Hampshire,209b 35 | New Jersey,1634 36 | New Mexico,1634 37 | New York,1634 38 | North Carolina,1634 39 | North Dakota,209b 40 | Ohio,1634 41 | Oklahoma,209b 42 | Oregon,SSI 43 | Pennsylvania,1634 44 | Puerto Rico,N/A 45 | Rhode Island,1634 46 | South Carolina,1634 47 | South Dakota,1634 48 | Tennessee,1634 49 | Texas,1634 50 | Utah,SSI 51 | Vermont,1634 52 | Virgin Islands,N/A 53 | Virginia,209b 54 | Washington,1634 55 | West Virginia,1634 56 | Wisconsin,1634 57 | Wyoming,1634 58 | -------------------------------------------------------------------------------- /static/csv/fmg.csv: -------------------------------------------------------------------------------- 1 | State,EL Group 72,"EL Group 73, 74, or 75",Notes 2 | Alabama,NO,NO, 3 | Alaska,YES,YES,"Alaska expanded its Medicaid program by adopting the VIII Group on September 1, 2015." 4 | Amer. Samoa,NO,NO, 5 | Arizona,YES,YES, 6 | Arkansas,YES,YES, 7 | California,YES,YES, 8 | Colorado,YES,YES, 9 | Connecticut,YES,YES, 10 | Delaware,YES,YES, 11 | Dist. Of Col.,YES,YES, 12 | Florida,NO,NO, 13 | Georgia,NO,NO, 14 | Guam,NO,YES, 15 | Hawaii,YES,YES, 16 | Idaho,YES,NO, 17 | Illinois,YES,YES, 18 | Indiana,YES,NO, 19 | Iowa,YES,YES, 20 | Kansas,NO,NO, 21 | Kentucky,YES,NO, 22 | Louisiana,YES,NO, 23 | Maine,YES,YES,Updated 2/15/23 based on March 2022 MBES data on Medicaid.gov 24 | Maryland,YES,NO, 25 | Massachusetts,NO,YES,Updated 2/15/23 based on March 2022 MBES data on Medicaid.gov 26 | Michigan,YES,YES, 27 | Minnesota,YES,NO, 28 | Mississippi,NO,NO, 29 | Missouri,YES,NO, 30 | Montana,YES,NO, 31 | N. Mariana Islands,NO,NO,https://www.medicaid.gov/medicaid/national-medicaid-chip-program-information/medicaid-chip-enrollment-data/medicaid-enrollment-data-collected-through-mbes/index.html 32 | Nebraska,YES,YES, 33 | Nevada,YES,NO, 34 | New Hampshire,YES,YES, 35 | New Jersey,YES,NO, 36 | New Mexico,YES,NO, 37 | New York,YES,YES, 38 | North Carolina,NO,NO, 39 | North Dakota,YES,YES, 40 | Ohio,YES,YES, 41 | Oklahoma,YES,NO, 42 | Oregon,YES,YES, 43 | Pennsylvania,YES,YES, 44 | Puerto Rico,NO,YES, 45 | Rhode Island,YES,NO, 46 | South Carolina,NO,NO, 47 | South Dakota,YES,NO,"South Dakota expanded its Medicaid program by adopting the VIII Group on July 1, 2023." 48 | Tennessee,NO,NO, 49 | Texas,NO,NO, 50 | Utah,YES,YES, 51 | Vermont,NO,YES, 52 | Virgin Islands,NO,YES, 53 | Virginia,YES,NO,Updated 2/15/23 based on March 2022 MBES data on Medicaid.gov 54 | Washington,YES,YES, 55 | West Virginia,YES,NO, 56 | Wisconsin,NO,NO, 57 | Wyoming,NO,NO, 58 | -------------------------------------------------------------------------------- /static/csv/provider_classification_lookup.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMSgov/T-MSIS-Data-Quality-Measures-Generation-Code/c774d61e027f9b29e4ac66b313393b9aa95c692f/static/csv/provider_classification_lookup.csv -------------------------------------------------------------------------------- /static/csv/sauths.csv: -------------------------------------------------------------------------------- 1 | State_Name,State_Abbreviation,FIPS_State_Numeric_Code,_01,_11,_09,_04,_06,_13,_05,_12,_02,_03,_07,_14,_10,_15,_08 2 | Alabama,AL,1,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No,Yes 3 | Alaska,AK,2,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 4 | Arizona,AZ,4,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 5 | Arkansas,AR,5,Yes,No,No,No,No,No,No,No,Yes,No,No,No,No,No,Yes 6 | California,CA,6,Yes,No,No,Yes,Yes,No,No,No,Yes,No,No,No,No,No,Yes 7 | Colorado,CO,8,Yes,No,No,Yes,No,No,No,No,Yes,Yes,No,No,No,No,Yes 8 | Connecticut,CT,9,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 9 | "Washington, DC",DC,10,Yes,Yes,No,Yes,No,No,No,No,No,No,No,No,No,No,No 10 | Delaware,DE,11,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 11 | Florida,FL,12,Yes,No,No,No,Yes,No,No,No,Yes,No,No,No,No,No,Yes 12 | Georgia,GA,13,Yes,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No 13 | Hawaii,HI,15,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 14 | Idaho,ID,16,No,Yes,No,No,No,No,No,No,Yes,No,No,Yes,Yes,Yes,No 15 | Illinois,IL,17,No,No,No,Yes,No,No,No,No,No,Yes,No,No,No,No,No 16 | Indiana,IN,18,Yes,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No 17 | Iowa,IA,19,No,Yes,No,No,No,No,No,Yes,Yes,Yes,No,No,No,No,Yes 18 | Kansas,KS,20,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 19 | Kentucky,KY,21,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No,No 20 | Louisiana,LA,22,Yes,No,No,No,No,No,Yes,Yes,No,Yes,No,No,No,No,Yes 21 | Maine,ME,23,No,No,No,No,No,No,No,No,No,Yes,No,No,No,Yes,No 22 | Maryland,MD,24,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 23 | Massachusetts,MA,25,Yes,No,No,No,Yes,No,No,No,No,No,No,No,No,No,Yes 24 | Michigan,MI,26,No,No,No,No,No,No,Yes,No,Yes,No,No,No,No,No,Yes 25 | Minnesota,MN,27,Yes,No,No,Yes,Yes,No,Yes,No,No,No,No,No,No,No,No 26 | Mississippi,MS,28,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No 27 | Missouri,MO,29,No,Yes,No,No,No,No,No,No,Yes,No,No,No,No,No,Yes 28 | Montana,MT,30,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No,No 29 | Nebraska,NE,31,No,No,No,No,No,No,No,No,Yes,Yes,No,No,No,No,Yes 30 | Nevada,NV,32,No,Yes,No,No,No,No,No,No,No,Yes,No,No,No,No,No 31 | New Hampshire,NH,33,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No 32 | New Jersey,NJ,34,Yes,Yes,No,No,No,No,No,No,No,Yes,No,No,No,No,Yes 33 | New Mexico,NM,35,Yes,No,No,No,No,No,Yes,No,Yes,No,No,No,No,No,Yes 34 | New York,NY,36,Yes,No,No,Yes,No,No,No,No,No,No,No,No,No,No,Yes 35 | North Carolina,NC,37,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,Yes 36 | North Dakota,ND,38,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,Yes 37 | Ohio,OH,39,No,No,No,No,No,No,No,No,Yes,Yes,No,No,No,No,Yes 38 | Oklahoma,OK,40,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,Yes 39 | Oregon,OR,41,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 40 | Pennsylvania,PA,42,No,Yes,No,Yes,No,No,No,No,Yes,No,No,No,No,No,Yes 41 | Puerto Rico,PR,72,No,No,No,Yes,No,No,No,No,Yes,No,No,No,No,No,No 42 | Rhode Island,RI,44,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 43 | South Carolina,SC,45,No,Yes,No,Yes,No,No,No,No,No,No,No,No,No,No,Yes 44 | South Dakota,SD,46,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No 45 | Tennessee,TN,47,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 46 | Texas,TX,48,Yes,Yes,No,Yes,No,No,No,No,Yes,No,No,No,No,No,Yes 47 | Utah,UT,49,No,No,No,Yes,No,No,No,No,Yes,No,No,No,No,No,No 48 | Vermont,VT,50,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 49 | Virginia,VA,51,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No,Yes 50 | Washington,WA,53,Yes,Yes,No,No,No,No,No,No,Yes,Yes,No,No,No,Yes,Yes 51 | West Virginia,WV,54,No,No,No,No,No,No,No,No,Yes,No,No,No,Yes,No,No 52 | Wisconsin,WI,55,No,No,No,Yes,No,No,Yes,No,No,Yes,Yes,No,No,No,Yes 53 | Wyoming,WY,56,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 54 | -------------------------------------------------------------------------------- /static/csv/schip.csv: -------------------------------------------------------------------------------- 1 | STABBREV,STATE_NAME,FIPS_CD,HAS_SCHIP 2 | AL,Alabama,1,YES 3 | AK,Alaska,2,No 4 | AZ,Arizona,4,YES 5 | AR,Arkansas,5,YES 6 | CA,California,6,YES 7 | CO,Colorado,8,YES 8 | CT,Connecticut,9,YES 9 | DE,Delaware,10,YES 10 | DC,District of Columbia,11,No 11 | FL,Florida,12,YES 12 | GA,Georgia,13,YES 13 | HI,Hawaii,15,No 14 | ID,Idaho,16,YES 15 | IL,Illinois,17,YES 16 | IN,Indiana,18,YES 17 | IA,Iowa,"19, 96",YES 18 | KS,Kansas,20,YES 19 | KY,Kentucky,21,YES 20 | LA,Louisiana,22,YES 21 | ME,Maine,23,YES 22 | MD,Maryland,24,No 23 | MA,Massachusetts,25,YES 24 | MI,Michigan,26,YES 25 | MN,Minnesota,27,YES 26 | MS,Mississippi,28,YES 27 | MO,Missouri,29,YES 28 | MT,Montana,30,YES 29 | NE,Nebraska,31,YES 30 | NV,Nevada,32,YES 31 | NH,New Hampshire,33,No 32 | NJ,New Jersey,34,YES 33 | NM,New Mexico,35,No 34 | NY,New York,36,YES 35 | NC,North Carolina,37,YES 36 | ND,North Dakota,38,YES 37 | OH,Ohio,39,No 38 | OK,Oklahoma,40,YES 39 | OR,Oregon,41,YES 40 | PA,Pennsylvania,"42, 97",YES 41 | RI,Rhode Island,44,YES 42 | SC,South Carolina,45,No 43 | SD,South Dakota,46,YES 44 | TN,Tennessee,47,YES 45 | TX,Texas,48,YES 46 | UT,Utah,49,YES 47 | VT,Vermont,50,No 48 | VA,Virginia,51,YES 49 | WA,Washington,53,YES 50 | WV,West Virginia,54,YES 51 | WI,Wisconsin,55,YES 52 | WY,Wyoming,56,YES 53 | VI,Virgin Islands,78,YES 54 | PR,Puerto Rico,72,YES 55 | GU,Guam,66,No -------------------------------------------------------------------------------- /static/csv/splans.csv: -------------------------------------------------------------------------------- 1 | State_Name,State_Abbreviation,FIPS_State_Numeric_Code,_01,_04,_80,_02,_03,_70,_07,_08,_09,_10,_11,_12,_13,_14,_15,_05,_06,_16,_18,_17 2 | Alabama,AL,1,No,No,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,Yes,Yes,Yes,Yes,Yes 3 | Alaska,AK,2,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 4 | Arizona,AZ,4,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 5 | Arkansas,AR,5,No,No,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,Yes,No,No,No,No,Yes 6 | California,CA,6,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,Yes,No,Yes,Yes,Yes,Yes,Yes 7 | Colorado,CO,8,Yes,Yes,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,Yes,Yes,Yes,Yes,Yes 8 | Connecticut,CT,9,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 9 | "Washington, DC",DC,10,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No 10 | Delaware,DE,11,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 11 | Florida,FL,12,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes 12 | Georgia,GA,13,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 13 | Hawaii,HI,15,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 14 | Idaho,ID,16,No,No,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,No 15 | Illinois,IL,17,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 16 | Indiana,IN,18,Yes,Yes,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 17 | Iowa,IA,19,Yes,Yes,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,No,No,No,No,Yes 18 | Kansas,KS,20,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 19 | Kentucky,KY,21,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 20 | Louisiana,LA,22,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,Yes 21 | Maine,ME,23,No,No,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 22 | Maryland,MD,24,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 23 | Massachusetts,MA,25,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,Yes 24 | Michigan,MI,26,Yes,Yes,No,No,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,Yes 25 | Minnesota,MN,27,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 26 | Mississippi,MS,28,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 27 | Missouri,MO,29,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,Yes 28 | Montana,MT,30,No,No,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 29 | Nebraska,NE,31,Yes,Yes,No,No,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,Yes 30 | Nevada,NV,32,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,No 31 | New Hampshire,NH,33,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No 32 | New Jersey,NJ,34,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,Yes,No,No,No,No,Yes 33 | New Mexico,NM,35,Yes,Yes,Yes,No,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,Yes 34 | New York,NY,36,Yes,Yes,Yes,No,No,No,Yes,No,No,No,No,No,No,No,No,No,No,No,No,Yes 35 | North Carolina,NC,37,No,No,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,Yes 36 | North Dakota,ND,38,No,No,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,Yes 37 | Ohio,OH,39,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 38 | Oklahoma,OK,40,No,No,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,Yes,No,No,No,No,Yes 39 | Oregon,OR,41,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 40 | Pennsylvania,PA,42,Yes,Yes,Yes,No,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,No,No,No,No,Yes 41 | Puerto Rico,PR,72,Yes,Yes,No,No,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,No 42 | Rhode Island,RI,44,Yes,Yes,No,Yes,Yes,Yes,No,No,No,No,No,No,No,Yes,No,No,No,No,No,Yes 43 | South Carolina,SC,45,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,No,No,Yes,No,No,No,No,Yes 44 | South Dakota,SD,46,No,No,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 45 | Tennessee,TN,47,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 46 | Texas,TX,48,Yes,Yes,Yes,No,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes,Yes 47 | Utah,UT,49,Yes,Yes,No,No,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,No,No,No,No,No 48 | Vermont,VT,50,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 49 | Virginia,VA,51,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 50 | Washington,WA,53,Yes,Yes,No,Yes,Yes,Yes,No,Yes,Yes,Yes,Yes,Yes,Yes,No,Yes,No,No,No,No,Yes 51 | West Virginia,WV,54,Yes,Yes,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No 52 | Wisconsin,WI,55,Yes,Yes,Yes,No,No,No,Yes,Yes,Yes,Yes,Yes,Yes,Yes,No,No,No,No,No,No,Yes 53 | Wyoming,WY,56,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,Yes 54 | ,,,,,,,,,,,,,,,,,,,,,, 55 | -------------------------------------------------------------------------------- /static/csv/st2_name.csv: -------------------------------------------------------------------------------- 1 | STABBREV,STNAME 2 | AL,Alabama 3 | AK,Alaska 4 | AZ,Arizona 5 | AR,Arkansas 6 | AS,American Samoa 7 | CA,California 8 | CO,Colorado 9 | CT,Connecticut 10 | DE,Delaware 11 | DC,District of Columbia 12 | FL,Florida 13 | GA,Georgia 14 | GU,Guam 15 | HI,Hawaii 16 | ID,Idaho 17 | IL,Illinois 18 | IN,Indiana 19 | IA,Iowa 20 | KS,Kansas 21 | KY,Kentucky 22 | LA,Louisiana 23 | ME,Maine 24 | MD,Maryland 25 | MA,Massachusetts 26 | MI,Michigan 27 | MN,Minnesota 28 | MS,Mississippi 29 | MO,Missouri 30 | MP,Northern Mariana Islands 31 | MT,Montana 32 | NE,Nebraska 33 | NV,Nevada 34 | NH,New Hampshire 35 | NJ,New Jersey 36 | NM,New Mexico 37 | NY,New York 38 | NC,North Carolina 39 | ND,North Dakota 40 | OH,Ohio 41 | OK,Oklahoma 42 | OR,Oregon 43 | PA,Pennsylvania 44 | PR,Puerto Rico 45 | RI,Rhode Island 46 | SC,South Carolina 47 | SD,South Dakota 48 | TN,Tennessee 49 | TX,Texas 50 | UT,Utah 51 | VI,Virgin Islands 52 | VT,Vermont 53 | VA,Virginia 54 | WA,Washington 55 | WV,West Virginia 56 | WI,Wisconsin 57 | WY,Wyoming 58 | -------------------------------------------------------------------------------- /static/csv/st_fips.csv: -------------------------------------------------------------------------------- 1 | STABBREV,FIPS 2 | AL,01 3 | AK,02 4 | AZ,04 5 | AR,05 6 | AS,60 7 | CA,06 8 | CO,08 9 | CT,09 10 | DE,10 11 | DC,11 12 | FL,12 13 | GA,13 14 | GU,66 15 | HI,15 16 | ID,16 17 | IL,17 18 | IN,18 19 | IA,"19,96" 20 | KS,20 21 | KY,21 22 | LA,22 23 | ME,23 24 | MD,24 25 | MA,25 26 | MI,26 27 | MN,27 28 | MS,28 29 | MO,29 30 | MP,69 31 | MT,"30,94" 32 | NE,31 33 | NV,32 34 | NH,33 35 | NJ,34 36 | NM,35 37 | NY,36 38 | NC,37 39 | ND,38 40 | OH,39 41 | OK,40 42 | OR,41 43 | PA,"42,97" 44 | PR,72 45 | RI,44 46 | SC,45 47 | SD,46 48 | TN,47 49 | TX,48 50 | UT,49 51 | VT,50 52 | VA,51 53 | VI,78 54 | WA,53 55 | WV,54 56 | WI,55 57 | WY,"56,93" 58 | -------------------------------------------------------------------------------- /static/csv/st_name.csv: -------------------------------------------------------------------------------- 1 | USPS,STNAME 2 | 01,Alabama 3 | 02,Alaska 4 | 04,Arizona 5 | 05,Arkansas 6 | 06,California 7 | 08,Colorado 8 | 09,Connecticut 9 | 10,Delaware 10 | 11,District of Columbia 11 | 12,Florida 12 | 13,Georgia 13 | 15,Hawaii 14 | 16,Idaho 15 | 17,Illinois 16 | 18,Indiana 17 | 19,Iowa 18 | 20,Kansas 19 | 21,Kentucky 20 | 22,Louisiana 21 | 23,Maine 22 | 24,Maryland 23 | 25,Massachusetts 24 | 26,Michigan 25 | 27,Minnesota 26 | 28,Mississippi 27 | 29,Missouri 28 | 30,Montana 29 | 31,Nebraska 30 | 32,Nevada 31 | 33,New Hampshire 32 | 34,New Jersey 33 | 35,New Mexico 34 | 36,New York 35 | 37,North Carolina 36 | 38,North Dakota 37 | 39,Ohio 38 | 40,Oklahoma 39 | 41,Oregon 40 | 42,Pennsylvania 41 | 44,Rhode Island 42 | 45,South Carolina 43 | 46,South Dakota 44 | 47,Tennessee 45 | 48,Texas 46 | 49,Utah 47 | 50,Vermont 48 | 51,Virginia 49 | 53,Washington 50 | 54,West Virginia 51 | 55,Wisconsin 52 | 56,Wyoming 53 | 72,Puerto Rico 54 | 78,Virgin Islands 55 | 93,Wyoming 56 | 94,Montana 57 | 96,Iowa 58 | 97,Pennsylvania 59 | 60,American Samoa 60 | 66,Guam 61 | 69,Northern Mariana Islands 62 | -------------------------------------------------------------------------------- /static/csv/st_usps.csv: -------------------------------------------------------------------------------- 1 | USPS,STABBREV 2 | 01,AL 3 | 02,AK 4 | 04,AZ 5 | 05,AR 6 | 06,CA 7 | 08,CO 8 | 09,CT 9 | 10,DE 10 | 11,DC 11 | 12,FL 12 | 13,GA 13 | 15,HI 14 | 16,ID 15 | 17,IL 16 | 18,IN 17 | 19,IA 18 | 20,KS 19 | 21,KY 20 | 22,LA 21 | 23,ME 22 | 24,MD 23 | 25,MA 24 | 26,MI 25 | 27,MN 26 | 28,MS 27 | 29,MO 28 | 30,MT 29 | 31,NE 30 | 32,NV 31 | 33,NH 32 | 34,NJ 33 | 35,NM 34 | 36,NY 35 | 37,NC 36 | 38,ND 37 | 39,OH 38 | 40,OK 39 | 41,OR 40 | 42,PA 41 | 44,RI 42 | 45,SC 43 | 46,SD 44 | 47,TN 45 | 48,TX 46 | 49,UT 47 | 50,VT 48 | 51,VA 49 | 53,WA 50 | 54,WV 51 | 55,WI 52 | 56,WY 53 | 72,PR 54 | 78,VI 55 | 93,WY 56 | 94,MT 57 | 96,IA 58 | 97,PA 59 | 60,AS 60 | 66,GU 61 | 69,MP 62 | -------------------------------------------------------------------------------- /static/csv/stabr.csv: -------------------------------------------------------------------------------- 1 | STNAME,STABBREV 2 | Alabama,AL 3 | Alaska,AK 4 | Amer. Samoa,AS 5 | Arizona,AZ 6 | Arkansas,AR 7 | California,CA 8 | Colorado,CO 9 | Connecticut,CT 10 | Delaware,DE 11 | Dist. Of Col.,DC 12 | Florida,FL 13 | Georgia,GA 14 | Hawaii,HI 15 | Idaho,ID 16 | Illinois,IL 17 | Indiana,IN 18 | Iowa,IA 19 | Kansas,KS 20 | Kentucky,KY 21 | Louisiana,LA 22 | Maine,ME 23 | Maryland,MD 24 | Massachusetts,MA 25 | Michigan,MI 26 | Minnesota,MN 27 | Mississippi,MS 28 | Missouri,MO 29 | Montana,MT 30 | Nebraska,NE 31 | Nevada,NV 32 | New Hampshire,NH 33 | New Jersey,NJ 34 | New Mexico,NM 35 | New York,NY 36 | North Carolina,NC 37 | North Dakota,ND 38 | Ohio,OH 39 | Oklahoma,OK 40 | Oregon,OR 41 | Pennsylvania,PA 42 | Puerto Rico,PR 43 | Rhode Island,RI 44 | South Carolina,SC 45 | South Dakota,SD 46 | Tennessee,TN 47 | Texas,TX 48 | Utah,UT 49 | Vermont,VT 50 | Virginia,VA 51 | Washington,WA 52 | West Virginia,WV 53 | Wisconsin,WI 54 | Wyoming,WY 55 | N. Mariana Islands,MP 56 | Virgin Islands,VI 57 | Guam,GU 58 | -------------------------------------------------------------------------------- /static/csv2pkl.py: -------------------------------------------------------------------------------- 1 | 2 | import pandas as pd 3 | import os 4 | 5 | 6 | def csv2pkl(pkl_name: str, csv_name: str = None, dtype_dict = 'str', na_filter_val = True, encode = 'utf-8'): 7 | 8 | if csv_name is None: 9 | csv_name = pkl_name 10 | 11 | if os.path.isfile(f'./csv/{csv_name}.csv'): 12 | print('Reading file ' + csv_name) 13 | pdf = pd.read_csv(f'./csv/{csv_name}.csv', dtype = dtype_dict, na_filter = na_filter_val, encoding = encode) 14 | pdf.to_pickle(f'../dqm/cfg/{pkl_name}.pkl') 15 | 16 | 17 | csv2pkl('abd', na_filter_val = False) 18 | csv2pkl('apdxc') 19 | csv2pkl('countystate_lookup') 20 | csv2pkl('fmg') 21 | csv2pkl('prgncy') 22 | csv2pkl('provider_classification_lookup', encode = 'cp1252') 23 | csv2pkl('atypical_provider_table') 24 | csv2pkl('prvtxnmy') 25 | csv2pkl('sauths') 26 | csv2pkl('schip') 27 | csv2pkl('splans') 28 | csv2pkl('st_fips') 29 | csv2pkl('st_name') 30 | csv2pkl('st_usps') 31 | csv2pkl('st2_name') 32 | csv2pkl('stabr') 33 | csv2pkl('stc_cd', csv_name = 'type_of_service', dtype_dict = {'TypeOfService': int, 'Description': str}) 34 | csv2pkl('zcc', csv_name = 'zip_county_crosswalk', dtype_dict = {'ZipCode': str, 'Sequence': int, 'State': str, 'City': str, 'County': str, 'StateFIPS': str, 'CountyFIPS': str, 'Percent': int}) 35 | csv2pkl('zipstate_lookup') 36 | -------------------------------------------------------------------------------- /static/pkl2csv.py: -------------------------------------------------------------------------------- 1 | 2 | import pandas as pd 3 | import os 4 | 5 | 6 | def pkl2csv(pkl_name: str, csv_name: str = None): 7 | 8 | if csv_name is None: 9 | csv_name = pkl_name 10 | 11 | if os.path.isfile(f'../dqm/cfg/{pkl_name}.pkl'): 12 | print('Reading file ' + pkl_name) 13 | pdf = pd.read_pickle(f'../dqm/cfg/{pkl_name}.pkl') 14 | pdf.to_csv(f'./csv/{csv_name}.csv', index=False) 15 | 16 | 17 | pkl2csv('abd') 18 | pkl2csv('apdxc') 19 | pkl2csv('countystate_lookup') 20 | pkl2csv('fmg') 21 | pkl2csv('prgncy') 22 | pkl2csv('provider_classification_lookup') 23 | pkl2csv('atypical_provider_table') 24 | pkl2csv('prvtxnmy') 25 | pkl2csv('sauths') 26 | pkl2csv('schip') 27 | pkl2csv('splans') 28 | pkl2csv('st_fips') 29 | pkl2csv('st_name') 30 | pkl2csv('st_usps') 31 | pkl2csv('st2_name') 32 | pkl2csv('stabr') 33 | pkl2csv('stc_cd', 'type_of_service') 34 | pkl2csv('zcc', 'zip_county_crosswalk') 35 | pkl2csv('zipstate_lookup') 36 | --------------------------------------------------------------------------------