├── .env ├── .gitignore ├── .python-version ├── CITATION.cff ├── LICENSE ├── README.md ├── docker-compose.yml ├── packages ├── irr_metrics │ ├── README.md │ ├── pyproject.toml │ └── src │ │ └── irr_metrics │ │ ├── __init__.py │ │ ├── binarize.py │ │ ├── constants.py │ │ ├── interrater.py │ │ ├── melt_wide_to_long.py │ │ ├── metric_bunch.py │ │ ├── py.typed │ │ ├── sens_spec_ppv_npv.py │ │ └── type_utils.py ├── llm_judge │ ├── README.md │ ├── pyproject.toml │ └── src │ │ └── llm_judge │ │ ├── __init__.py │ │ ├── judge.py │ │ ├── judge_cohort.py │ │ ├── judge_subject.py │ │ ├── prompts.py │ │ ├── py.typed │ │ ├── reference_context.py │ │ ├── schema.py │ │ └── score_report.py ├── llm_writer │ ├── README.md │ ├── pyproject.toml │ └── src │ │ └── llm_writer │ │ ├── __init__.py │ │ ├── hospital_course.py │ │ ├── prompts.py │ │ └── py.typed ├── proposition_validity │ ├── README.md │ ├── pyproject.toml │ └── src │ │ └── proposition_validity │ │ ├── __init__.py │ │ ├── judge.py │ │ ├── prompts.py │ │ └── py.typed ├── pydantic_utils │ ├── README.md │ ├── pyproject.toml │ └── src │ │ └── pydantic_utils │ │ ├── __init__.py │ │ ├── base.py │ │ ├── claim.py │ │ ├── llm_base_model.py │ │ ├── prompts.py │ │ └── py.typed ├── rag │ ├── README.md │ ├── pyproject.toml │ └── src │ │ └── rag │ │ ├── __init__.py │ │ ├── components │ │ ├── __init__.py │ │ ├── models.py │ │ ├── node_parsers.py │ │ └── stores.py │ │ ├── embedding │ │ ├── __init__.py │ │ ├── base.py │ │ ├── instrumentation.py │ │ ├── m3_embedding.py │ │ └── utils.py │ │ ├── fusion │ │ ├── __init__.py │ │ └── fusion.py │ │ ├── llms │ │ ├── __init__.py │ │ └── openai_like.py │ │ ├── node_parser │ │ ├── __init__.py │ │ ├── atomic_claims │ │ │ ├── __init__.py │ │ │ ├── atomic_claims.py │ │ │ └── prompts.py │ │ ├── node_utils.py │ │ ├── semantic_parser │ │ │ ├── __init__.py │ │ │ ├── semantic_parser.py │ │ │ └── semantic_utils.py │ │ └── single_sentence_parser.py │ │ ├── postprocessor │ │ ├── __init__.py │ │ └── m3_reranker.py │ │ ├── py.typed │ │ ├── readers │ │ ├── __init__.py │ │ └── note_reader.py │ │ ├── retry.py │ │ ├── schema │ │ ├── __init__.py │ │ ├── base.py │ │ ├── base_node.py │ │ ├── node_schema.py │ │ └── node_utils.py │ │ └── vector_stores │ │ ├── __init__.py │ │ ├── payload_utils.py │ │ ├── proposition.py │ │ ├── qdrant.py │ │ └── query_utils.py ├── rq_utils │ ├── README.md │ ├── pyproject.toml │ └── src │ │ └── rq_utils │ │ ├── __init__.py │ │ ├── py.typed │ │ ├── rq.py │ │ └── rq_demo │ │ ├── rq_demo.py │ │ └── rq_shutdown_workers.py └── utils │ ├── README.md │ ├── pyproject.toml │ └── src │ └── utils │ ├── __init__.py │ ├── async_utils.py │ ├── env.py │ ├── file_utils.py │ ├── func_utils.py │ ├── json_utils.py │ ├── list_utils.py │ ├── log_utils.py │ ├── parallel_process.py │ ├── py.typed │ ├── retry.py │ ├── tiktoken.py │ ├── time_utils.py │ ├── type_utils.py │ └── uuid.py ├── pyproject.toml ├── scripts ├── README.md ├── analysis_dataset │ ├── 1_proposition_validity.py │ └── README.md ├── analysis_interrater │ ├── 1_compute_interrater_agreement.py │ ├── 2_interrater_agreement_tables.py │ └── README.md ├── analysis_verifact │ ├── 0_check_completeness.py │ ├── 1_verifact_BHC_dataset_statistics.py │ ├── 2_compute_verifact_agreement.py │ ├── 3_sensitivity_analysis_plots.py │ ├── 4_top_n_vs_reference_context_length.py │ ├── 5_best_model_sens_spec_ppv_npv.py │ ├── 6_metric_result_tables.py │ ├── 7_error_analysis.py │ ├── 8_score_sheet_figure.py │ └── README.md ├── dataset │ ├── README.md │ ├── decompose_text.py │ ├── sample_mimic_dataset.py │ └── write_llm_hospital_course.py └── evaluate │ ├── README.md │ ├── run_verifact.py │ ├── run_verifact2.py │ └── score_report.pkl ├── services ├── README.md ├── embed │ ├── Dockerfile │ ├── README.md │ ├── infinity_patch │ │ ├── fastapi_schemas │ │ │ └── pymodels.py │ │ └── transformer │ │ │ └── embedder │ │ │ └── sentence_transformer.py │ └── tests │ │ ├── embedding_speed_test.py │ │ ├── embedding_test.py │ │ └── lorem_ipsum.txt ├── rerank │ ├── Dockerfile │ ├── README.md │ ├── infinity_patch │ │ └── transformer │ │ │ └── crossencoder │ │ │ └── torch.py │ └── tests │ │ ├── lorem_ipsum.txt │ │ ├── rerank_speed_test.py │ │ └── rerank_test.py └── vllm │ ├── monitoring │ ├── README.md │ ├── grafana.json │ └── prometheus.yml │ └── tests │ ├── aux_llm_context_window_test.py │ ├── llm_context_window_test.py │ └── lorem_ipsum.txt └── uv.lock /.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/.env -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/.gitignore -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.11.4 2 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/CITATION.cff -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/README.md -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /packages/irr_metrics/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/README.md -------------------------------------------------------------------------------- /packages/irr_metrics/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/pyproject.toml -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/__init__.py -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/binarize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/binarize.py -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/constants.py -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/interrater.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/interrater.py -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/melt_wide_to_long.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/melt_wide_to_long.py -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/metric_bunch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/metric_bunch.py -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/sens_spec_ppv_npv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/sens_spec_ppv_npv.py -------------------------------------------------------------------------------- /packages/irr_metrics/src/irr_metrics/type_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/irr_metrics/src/irr_metrics/type_utils.py -------------------------------------------------------------------------------- /packages/llm_judge/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/README.md -------------------------------------------------------------------------------- /packages/llm_judge/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/pyproject.toml -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/__init__.py -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/judge.py -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/judge_cohort.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/judge_cohort.py -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/judge_subject.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/judge_subject.py -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/prompts.py -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/reference_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/reference_context.py -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/schema.py -------------------------------------------------------------------------------- /packages/llm_judge/src/llm_judge/score_report.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_judge/src/llm_judge/score_report.py -------------------------------------------------------------------------------- /packages/llm_writer/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_writer/README.md -------------------------------------------------------------------------------- /packages/llm_writer/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_writer/pyproject.toml -------------------------------------------------------------------------------- /packages/llm_writer/src/llm_writer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_writer/src/llm_writer/__init__.py -------------------------------------------------------------------------------- /packages/llm_writer/src/llm_writer/hospital_course.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_writer/src/llm_writer/hospital_course.py -------------------------------------------------------------------------------- /packages/llm_writer/src/llm_writer/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/llm_writer/src/llm_writer/prompts.py -------------------------------------------------------------------------------- /packages/llm_writer/src/llm_writer/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/proposition_validity/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/proposition_validity/README.md -------------------------------------------------------------------------------- /packages/proposition_validity/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/proposition_validity/pyproject.toml -------------------------------------------------------------------------------- /packages/proposition_validity/src/proposition_validity/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/proposition_validity/src/proposition_validity/__init__.py -------------------------------------------------------------------------------- /packages/proposition_validity/src/proposition_validity/judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/proposition_validity/src/proposition_validity/judge.py -------------------------------------------------------------------------------- /packages/proposition_validity/src/proposition_validity/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/proposition_validity/src/proposition_validity/prompts.py -------------------------------------------------------------------------------- /packages/proposition_validity/src/proposition_validity/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/pydantic_utils/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/pydantic_utils/README.md -------------------------------------------------------------------------------- /packages/pydantic_utils/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/pydantic_utils/pyproject.toml -------------------------------------------------------------------------------- /packages/pydantic_utils/src/pydantic_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/pydantic_utils/src/pydantic_utils/__init__.py -------------------------------------------------------------------------------- /packages/pydantic_utils/src/pydantic_utils/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/pydantic_utils/src/pydantic_utils/base.py -------------------------------------------------------------------------------- /packages/pydantic_utils/src/pydantic_utils/claim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/pydantic_utils/src/pydantic_utils/claim.py -------------------------------------------------------------------------------- /packages/pydantic_utils/src/pydantic_utils/llm_base_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/pydantic_utils/src/pydantic_utils/llm_base_model.py -------------------------------------------------------------------------------- /packages/pydantic_utils/src/pydantic_utils/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/pydantic_utils/src/pydantic_utils/prompts.py -------------------------------------------------------------------------------- /packages/pydantic_utils/src/pydantic_utils/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/rag/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/README.md -------------------------------------------------------------------------------- /packages/rag/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/pyproject.toml -------------------------------------------------------------------------------- /packages/rag/src/rag/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/components/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/components/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/components/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/components/models.py -------------------------------------------------------------------------------- /packages/rag/src/rag/components/node_parsers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/components/node_parsers.py -------------------------------------------------------------------------------- /packages/rag/src/rag/components/stores.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/components/stores.py -------------------------------------------------------------------------------- /packages/rag/src/rag/embedding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/embedding/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/embedding/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/embedding/base.py -------------------------------------------------------------------------------- /packages/rag/src/rag/embedding/instrumentation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/embedding/instrumentation.py -------------------------------------------------------------------------------- /packages/rag/src/rag/embedding/m3_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/embedding/m3_embedding.py -------------------------------------------------------------------------------- /packages/rag/src/rag/embedding/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/embedding/utils.py -------------------------------------------------------------------------------- /packages/rag/src/rag/fusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/fusion/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/fusion/fusion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/fusion/fusion.py -------------------------------------------------------------------------------- /packages/rag/src/rag/llms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/llms/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/llms/openai_like.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/llms/openai_like.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/atomic_claims/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/atomic_claims/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/atomic_claims/atomic_claims.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/atomic_claims/atomic_claims.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/atomic_claims/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/atomic_claims/prompts.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/node_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/node_utils.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/semantic_parser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/semantic_parser/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/semantic_parser/semantic_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/semantic_parser/semantic_parser.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/semantic_parser/semantic_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/semantic_parser/semantic_utils.py -------------------------------------------------------------------------------- /packages/rag/src/rag/node_parser/single_sentence_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/node_parser/single_sentence_parser.py -------------------------------------------------------------------------------- /packages/rag/src/rag/postprocessor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/postprocessor/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/postprocessor/m3_reranker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/postprocessor/m3_reranker.py -------------------------------------------------------------------------------- /packages/rag/src/rag/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/rag/src/rag/readers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/readers/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/readers/note_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/readers/note_reader.py -------------------------------------------------------------------------------- /packages/rag/src/rag/retry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/retry.py -------------------------------------------------------------------------------- /packages/rag/src/rag/schema/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/schema/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/schema/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/schema/base.py -------------------------------------------------------------------------------- /packages/rag/src/rag/schema/base_node.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/schema/base_node.py -------------------------------------------------------------------------------- /packages/rag/src/rag/schema/node_schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/schema/node_schema.py -------------------------------------------------------------------------------- /packages/rag/src/rag/schema/node_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/schema/node_utils.py -------------------------------------------------------------------------------- /packages/rag/src/rag/vector_stores/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/vector_stores/__init__.py -------------------------------------------------------------------------------- /packages/rag/src/rag/vector_stores/payload_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/vector_stores/payload_utils.py -------------------------------------------------------------------------------- /packages/rag/src/rag/vector_stores/proposition.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/vector_stores/proposition.py -------------------------------------------------------------------------------- /packages/rag/src/rag/vector_stores/qdrant.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/vector_stores/qdrant.py -------------------------------------------------------------------------------- /packages/rag/src/rag/vector_stores/query_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rag/src/rag/vector_stores/query_utils.py -------------------------------------------------------------------------------- /packages/rq_utils/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rq_utils/README.md -------------------------------------------------------------------------------- /packages/rq_utils/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rq_utils/pyproject.toml -------------------------------------------------------------------------------- /packages/rq_utils/src/rq_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rq_utils/src/rq_utils/__init__.py -------------------------------------------------------------------------------- /packages/rq_utils/src/rq_utils/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/rq_utils/src/rq_utils/rq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rq_utils/src/rq_utils/rq.py -------------------------------------------------------------------------------- /packages/rq_utils/src/rq_utils/rq_demo/rq_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rq_utils/src/rq_utils/rq_demo/rq_demo.py -------------------------------------------------------------------------------- /packages/rq_utils/src/rq_utils/rq_demo/rq_shutdown_workers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/rq_utils/src/rq_utils/rq_demo/rq_shutdown_workers.py -------------------------------------------------------------------------------- /packages/utils/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/README.md -------------------------------------------------------------------------------- /packages/utils/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/pyproject.toml -------------------------------------------------------------------------------- /packages/utils/src/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/__init__.py -------------------------------------------------------------------------------- /packages/utils/src/utils/async_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/async_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/env.py -------------------------------------------------------------------------------- /packages/utils/src/utils/file_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/file_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/func_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/func_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/json_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/json_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/list_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/list_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/log_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/log_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/parallel_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/parallel_process.py -------------------------------------------------------------------------------- /packages/utils/src/utils/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/utils/src/utils/retry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/retry.py -------------------------------------------------------------------------------- /packages/utils/src/utils/tiktoken.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/tiktoken.py -------------------------------------------------------------------------------- /packages/utils/src/utils/time_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/time_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/type_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/type_utils.py -------------------------------------------------------------------------------- /packages/utils/src/utils/uuid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/packages/utils/src/utils/uuid.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/README.md -------------------------------------------------------------------------------- /scripts/analysis_dataset/1_proposition_validity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_dataset/1_proposition_validity.py -------------------------------------------------------------------------------- /scripts/analysis_dataset/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_dataset/README.md -------------------------------------------------------------------------------- /scripts/analysis_interrater/1_compute_interrater_agreement.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_interrater/1_compute_interrater_agreement.py -------------------------------------------------------------------------------- /scripts/analysis_interrater/2_interrater_agreement_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_interrater/2_interrater_agreement_tables.py -------------------------------------------------------------------------------- /scripts/analysis_interrater/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_interrater/README.md -------------------------------------------------------------------------------- /scripts/analysis_verifact/0_check_completeness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/0_check_completeness.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/1_verifact_BHC_dataset_statistics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/1_verifact_BHC_dataset_statistics.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/2_compute_verifact_agreement.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/2_compute_verifact_agreement.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/3_sensitivity_analysis_plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/3_sensitivity_analysis_plots.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/4_top_n_vs_reference_context_length.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/4_top_n_vs_reference_context_length.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/5_best_model_sens_spec_ppv_npv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/5_best_model_sens_spec_ppv_npv.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/6_metric_result_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/6_metric_result_tables.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/7_error_analysis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/7_error_analysis.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/8_score_sheet_figure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/8_score_sheet_figure.py -------------------------------------------------------------------------------- /scripts/analysis_verifact/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/analysis_verifact/README.md -------------------------------------------------------------------------------- /scripts/dataset/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/dataset/README.md -------------------------------------------------------------------------------- /scripts/dataset/decompose_text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/dataset/decompose_text.py -------------------------------------------------------------------------------- /scripts/dataset/sample_mimic_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/dataset/sample_mimic_dataset.py -------------------------------------------------------------------------------- /scripts/dataset/write_llm_hospital_course.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/dataset/write_llm_hospital_course.py -------------------------------------------------------------------------------- /scripts/evaluate/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/evaluate/README.md -------------------------------------------------------------------------------- /scripts/evaluate/run_verifact.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/evaluate/run_verifact.py -------------------------------------------------------------------------------- /scripts/evaluate/run_verifact2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/evaluate/run_verifact2.py -------------------------------------------------------------------------------- /scripts/evaluate/score_report.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/scripts/evaluate/score_report.pkl -------------------------------------------------------------------------------- /services/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/README.md -------------------------------------------------------------------------------- /services/embed/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/embed/Dockerfile -------------------------------------------------------------------------------- /services/embed/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/embed/README.md -------------------------------------------------------------------------------- /services/embed/infinity_patch/fastapi_schemas/pymodels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/embed/infinity_patch/fastapi_schemas/pymodels.py -------------------------------------------------------------------------------- /services/embed/infinity_patch/transformer/embedder/sentence_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/embed/infinity_patch/transformer/embedder/sentence_transformer.py -------------------------------------------------------------------------------- /services/embed/tests/embedding_speed_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/embed/tests/embedding_speed_test.py -------------------------------------------------------------------------------- /services/embed/tests/embedding_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/embed/tests/embedding_test.py -------------------------------------------------------------------------------- /services/embed/tests/lorem_ipsum.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/embed/tests/lorem_ipsum.txt -------------------------------------------------------------------------------- /services/rerank/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/rerank/Dockerfile -------------------------------------------------------------------------------- /services/rerank/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/rerank/README.md -------------------------------------------------------------------------------- /services/rerank/infinity_patch/transformer/crossencoder/torch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/rerank/infinity_patch/transformer/crossencoder/torch.py -------------------------------------------------------------------------------- /services/rerank/tests/lorem_ipsum.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/rerank/tests/lorem_ipsum.txt -------------------------------------------------------------------------------- /services/rerank/tests/rerank_speed_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/rerank/tests/rerank_speed_test.py -------------------------------------------------------------------------------- /services/rerank/tests/rerank_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/rerank/tests/rerank_test.py -------------------------------------------------------------------------------- /services/vllm/monitoring/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/vllm/monitoring/README.md -------------------------------------------------------------------------------- /services/vllm/monitoring/grafana.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/vllm/monitoring/grafana.json -------------------------------------------------------------------------------- /services/vllm/monitoring/prometheus.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/vllm/monitoring/prometheus.yml -------------------------------------------------------------------------------- /services/vllm/tests/aux_llm_context_window_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/vllm/tests/aux_llm_context_window_test.py -------------------------------------------------------------------------------- /services/vllm/tests/llm_context_window_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/vllm/tests/llm_context_window_test.py -------------------------------------------------------------------------------- /services/vllm/tests/lorem_ipsum.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/services/vllm/tests/lorem_ipsum.txt -------------------------------------------------------------------------------- /uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/philipchung/verifact/HEAD/uv.lock --------------------------------------------------------------------------------