├── .coveragerc ├── .flake8 ├── .git-blame-ignore-revs ├── .gitattributes ├── .github ├── .stale.yml ├── CODEOWNERS ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── feature_request.md │ ├── giskard_bug_template.yaml │ └── question.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── release-drafter.yml └── workflows │ ├── build-python.yml │ ├── clear-cache.yml │ ├── create-release.yml │ ├── lock-deps.yml │ ├── nightly-test.yml │ ├── pre-commit-checks.yml │ └── retry-workflow.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── .vscode └── launch.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUES.md ├── LICENSE ├── README.md ├── SECURITY.md ├── conftest.py ├── docs ├── Makefile ├── _static │ ├── catalog_example.png │ ├── catalog_slice.png │ ├── catalog_transfo.png │ ├── css │ │ └── custom.css │ ├── debug.png │ ├── giskard_logo.png │ ├── hey.png │ ├── js │ │ ├── githubStargazers.js │ │ ├── reodev.js │ │ └── sidebarKeepScroll.js │ ├── logo_black.png │ ├── logo_white.png │ ├── ragas_metrics.png │ ├── raget.png │ ├── scan_example.png │ └── test_suite_example.png ├── _templates │ ├── base.html │ ├── page.html │ └── sidebar │ │ └── brand.html ├── assets │ ├── Architecture.jpg │ ├── RAG.png │ ├── catalog.png │ ├── comparison.png │ ├── create_project.gif │ ├── credit_scoring_comment.png │ ├── discussion.png │ ├── feedback.png │ ├── gh_discussion.png │ ├── gh_readme.png │ ├── image_(1)_(1)_(2).png │ ├── image_(3).png │ ├── integrations │ │ ├── avid │ │ │ └── avid_taxonomy.gif │ │ ├── dagshub │ │ │ └── dagshub.png │ │ ├── hfs │ │ │ ├── copy_token.png │ │ │ ├── create_from_template.png │ │ │ ├── create_new_space.png │ │ │ ├── duplicate_this_space.png │ │ │ ├── eval_copy_id.png │ │ │ ├── eval_input_hf_access_token.png │ │ │ ├── eval_input_model_and_dataset_id.png │ │ │ ├── eval_job_id.png │ │ │ ├── eval_label_matched.png │ │ │ ├── eval_label_matching.png │ │ │ ├── eval_label_unmatched.png │ │ │ ├── eval_logs.png │ │ │ ├── eval_model_and_dataset_checking.png │ │ │ ├── eval_scan_conf.png │ │ │ ├── free_tier.png │ │ │ ├── generate_token.png │ │ │ ├── giskard_client.png │ │ │ ├── hfs.svg │ │ │ ├── input_hf_access_token.png │ │ │ ├── mlworker.png │ │ │ ├── paid_tier.png │ │ │ └── where_to_create_access_token.png │ │ ├── mlflow │ │ │ ├── MLflow-logo-final-white-TM.png │ │ │ ├── llms │ │ │ │ ├── artifact_comparison.png │ │ │ │ ├── scan-summary.png │ │ │ │ ├── table_view.png │ │ │ │ ├── text-davinci-001-metrics.png │ │ │ │ ├── text-davinci-001-scanresults.png │ │ │ │ └── text-davinci-001-scanresults2.png │ │ │ └── tabular │ │ │ │ ├── metrics.png │ │ │ │ ├── scan-summary.png │ │ │ │ ├── scanresults.png │ │ │ │ └── table_view.png │ │ └── wandb │ │ │ ├── categorical-chart.png │ │ │ ├── dataset.png │ │ │ ├── global-chart.png │ │ │ ├── numerical-chart.png │ │ │ ├── scanning-result.png │ │ │ ├── test-suite-result.png │ │ │ ├── wandb-categorical-chart.png │ │ │ ├── wandb-dataset.png │ │ │ ├── wandb-global-chart.png │ │ │ ├── wandb-logo-yellow-dots-black-wb.png │ │ │ ├── wandb-numerical-chart.png │ │ │ ├── wandb-scanning-result.png │ │ │ └── wandb-test-suite-result.png │ ├── intro │ │ ├── Giskard_Turtle_Computer.png │ │ ├── Giskard_Turtle_Kungfu.png │ │ ├── Giskard_Turtle_Lab.png │ │ ├── Giskard_Turtle_SK8.png │ │ ├── Giskard_Turtle_new_set_FINAL_05.png │ │ ├── Giskard_Turtle_new_set_FINAL_06.png │ │ ├── Giskard_Turtle_waving.png │ │ ├── coffee.png │ │ ├── hey.png │ │ ├── integrations.png │ │ ├── ninja.png │ │ ├── slice.jpg │ │ └── test_turtle.png │ ├── llm_debug.png │ ├── llm_monitoring_dashboard.gif │ ├── model_insights_titanic.png │ ├── ngrok_aut.png │ ├── ngrok_aut2.png │ ├── output_filter.png │ ├── push.png │ ├── scan_llm.png │ ├── scan_nlp.png │ ├── scan_results.png │ ├── scan_tabular.png │ ├── scan_vision.png │ ├── scan_widget.html │ ├── slice.png │ ├── test_suite_scan_llm.png │ └── test_suite_tabular.png ├── community │ ├── contribution_guidelines │ │ ├── configuration.md │ │ ├── dev-environment.md │ │ ├── giskard-architecture.md │ │ └── index.rst │ ├── discord │ │ └── index.md │ ├── github │ │ └── index.md │ └── index.md ├── conf.py ├── favicon.ico ├── getting_started │ ├── index.md │ └── quickstart │ │ ├── index.md │ │ ├── quickstart_llm.ipynb │ │ ├── quickstart_nlp.ipynb │ │ ├── quickstart_tabular.ipynb │ │ └── quickstart_vision.ipynb ├── index.md ├── integrations │ ├── avid │ │ ├── avid-integration-llm.ipynb │ │ └── index.md │ ├── cicd │ │ ├── index.md │ │ └── pipeline.ipynb │ ├── dagshub │ │ └── index.md │ ├── huggingface │ │ ├── evaluator.md │ │ └── index.md │ ├── index.md │ ├── mlflow │ │ ├── index.md │ │ ├── mlflow-evaluation.png │ │ ├── mlflow-harmfulness.png │ │ ├── mlflow-llm-example.ipynb │ │ ├── mlflow-metrics.png │ │ ├── mlflow-prompt-injection.png │ │ ├── mlflow-runs.png │ │ ├── mlflow-scan.png │ │ ├── mlflow-tabular-example.ipynb │ │ └── mlflow_overview.png │ ├── nemoguardrails │ │ ├── index.md │ │ └── nemoguardrails-integration.ipynb │ ├── pytest │ │ ├── full_example.ipynb │ │ └── index.md │ └── wandb │ │ ├── index.md │ │ ├── wandb-llm-example.ipynb │ │ └── wandb-tabular-example.ipynb ├── knowledge │ ├── catalogs │ │ ├── index.md │ │ ├── slicing-function-catalog │ │ │ └── index.rst │ │ ├── test-catalog │ │ │ ├── classification │ │ │ │ └── index.rst │ │ │ ├── index.rst │ │ │ ├── regression │ │ │ │ └── index.rst │ │ │ └── text_generation │ │ │ │ └── index.rst │ │ └── transformation-function-catalog │ │ │ └── index.rst │ ├── index.md │ ├── key_vulnerabilities │ │ ├── data_leakage │ │ │ └── index.md │ │ ├── ethics │ │ │ └── index.md │ │ ├── index.rst │ │ ├── overconfidence │ │ │ └── index.md │ │ ├── performance_bias │ │ │ └── index.md │ │ ├── robustness │ │ │ └── index.md │ │ ├── spurious │ │ │ └── index.md │ │ ├── stochasticity │ │ │ └── index.md │ │ └── underconfidence │ │ │ └── index.md │ └── llm_vulnerabilities │ │ ├── disclosure │ │ └── index.md │ │ ├── formatting │ │ └── index.md │ │ ├── hallucination │ │ └── index.md │ │ ├── harmfulness │ │ └── index.md │ │ ├── index.rst │ │ ├── injection │ │ └── index.md │ │ ├── robustness │ │ └── index.md │ │ └── stereotypes │ │ └── index.md ├── make.bat ├── open_source │ ├── ai_quality_copilot │ │ └── index.md │ ├── customize_tests │ │ ├── data_slices │ │ │ └── index.md │ │ ├── data_transformations │ │ │ └── index.md │ │ ├── index.md │ │ └── test_model │ │ │ └── index.md │ ├── index.md │ ├── installation_library │ │ └── index.md │ ├── integrate_tests │ │ └── index.md │ ├── scan │ │ ├── advanced_scan │ │ │ └── index.rst │ │ ├── index.md │ │ ├── scan_llm │ │ │ ├── html.rst │ │ │ ├── index.md │ │ │ └── scan_result_iframe.html │ │ ├── scan_nlp │ │ │ └── index.md │ │ ├── scan_tabular │ │ │ └── index.md │ │ └── scan_vision │ │ │ └── index.md │ ├── setting_up │ │ └── index.md │ └── testset_generation │ │ ├── index.md │ │ ├── rag_evaluation │ │ └── index.md │ │ └── testset_generation │ │ └── index.md ├── reference │ ├── datasets │ │ └── index.rst │ ├── index.rst │ ├── models │ │ ├── index.rst │ │ └── integrations │ │ │ ├── base_classes.rst │ │ │ ├── catboost.rst │ │ │ ├── function.rst │ │ │ ├── huggingface.rst │ │ │ ├── langchain.rst │ │ │ ├── pytorch.rst │ │ │ ├── sklearn.rst │ │ │ └── tensorflow.rst │ ├── notebooks │ │ ├── LLM_Description_Product.ipynb │ │ ├── LLM_Newspaper_Comment_Generation.ipynb │ │ ├── LLM_QA_Documentation.ipynb │ │ ├── LLM_QA_Google.ipynb │ │ ├── LLM_QA_IPCC.ipynb │ │ ├── LLM_QA_Winter_Olympics.ipynb │ │ ├── RAGET_Banking_Supervision.ipynb │ │ ├── RAGET_IPCC.ipynb │ │ ├── airline_tweets_sentiment_analysis.ipynb │ │ ├── amazon_review_classification_sklearn.ipynb │ │ ├── cancer_detection_xgboost.ipynb │ │ ├── churn_prediction_lgbm.ipynb │ │ ├── credit_scoring.ipynb │ │ ├── drug_classification_sklearn.ipynb │ │ ├── enron_email_classification_sklearn.ipynb │ │ ├── fake_real_news_classification.ipynb │ │ ├── hotel_text_regression.ipynb │ │ ├── ieee_fraud_detection_adversarial_validation.ipynb │ │ ├── insurance_prediction_lgbm.ipynb │ │ ├── m5_sales_prediction_lgbm.ipynb │ │ ├── medical_transcript_classification_sklearn.ipynb │ │ ├── movie_review_sentiment_classification_pytorch_sklearn.ipynb │ │ ├── newspaper_classification_pytorch.ipynb │ │ ├── tripadvisor_sentiment_classification.ipynb │ │ ├── twitter_sentiment_analysis_roberta.ipynb │ │ ├── vision_landmark_detection.ipynb │ │ ├── vision_object_detection.ipynb │ │ └── wage_classification.ipynb │ ├── rag-toolset │ │ ├── evaluation.rst │ │ ├── index.rst │ │ ├── knowledge_base.rst │ │ ├── metrics.rst │ │ ├── question_generation.rst │ │ └── testset_generation.rst │ ├── scan │ │ ├── detectors.rst │ │ ├── index.rst │ │ ├── llm_detectors.rst │ │ └── report.rst │ ├── slicing-functions │ │ └── index.rst │ ├── suite │ │ └── index.rst │ ├── tests │ │ ├── data.rst │ │ ├── drift.rst │ │ ├── index.rst │ │ ├── llm.rst │ │ ├── metamorphic.rst │ │ ├── performance.rst │ │ ├── stability.rst │ │ └── statistic.rst │ └── transformation-functions │ │ └── index.rst ├── scrapper.py └── tutorials │ ├── index.md │ ├── llm_tutorials │ └── index.md │ ├── nlp_tutorials │ └── index.md │ ├── rag_tutorials │ └── index.md │ ├── tabular_tutorials │ └── index.md │ └── vision_tutorials │ └── index.md ├── giskard ├── __init__.py ├── client │ ├── dtos.py │ ├── io_utils.py │ └── python_utils.py ├── core │ ├── __init__.py │ ├── core.py │ ├── dataset_validation.py │ ├── errors.py │ ├── kwargs_utils.py │ ├── model_validation.py │ ├── savable.py │ ├── suite.py │ ├── test_result.py │ └── validation.py ├── datasets │ ├── __init__.py │ ├── base │ │ └── __init__.py │ └── metadata │ │ ├── __init__.py │ │ ├── indexing.py │ │ ├── registry.py │ │ └── text_metadata_provider.py ├── demo │ ├── __init__.py │ ├── linear_regression.py │ ├── llm_ipcc_climate_change.py │ ├── titanic.csv │ └── titanic_classification.py ├── exceptions │ ├── IllegalArgumentError.py │ ├── __init__.py │ └── giskard_exception.py ├── functions │ ├── __init__.py │ ├── slicing.py │ └── transformation.py ├── integrations │ ├── __init__.py │ ├── avid.py │ ├── mlflow │ │ ├── __init__.py │ │ ├── evaluation_artifacts.py │ │ ├── giskard_evaluator.py │ │ └── giskard_evaluator_utils.py │ ├── nemoguardrails.py │ └── wandb │ │ ├── __init__.py │ │ └── wandb_utils.py ├── llm │ ├── __init__.py │ ├── client │ │ ├── __init__.py │ │ ├── base.py │ │ ├── bedrock.py │ │ ├── copilot.py │ │ ├── gemini.py │ │ ├── litellm.py │ │ ├── logger.py │ │ ├── mistral.py │ │ └── openai.py │ ├── config.py │ ├── embeddings │ │ ├── __init__.py │ │ ├── base.py │ │ ├── bedrock.py │ │ ├── fastembed.py │ │ ├── litellm.py │ │ └── openai.py │ ├── errors.py │ ├── evaluators │ │ ├── __init__.py │ │ ├── base.py │ │ ├── coherency.py │ │ ├── correctness.py │ │ ├── plausibility.py │ │ ├── requirements.py │ │ ├── string_matcher.py │ │ └── utils.py │ ├── generators │ │ ├── __init__.py │ │ ├── adversarial.py │ │ ├── base.py │ │ ├── implausible.py │ │ ├── simple.py │ │ └── sycophancy.py │ ├── loaders │ │ ├── __init__.py │ │ ├── local │ │ │ ├── giskard_meta_data.csv │ │ │ └── prompt_injections.csv │ │ └── prompt_injections.py │ ├── talk │ │ ├── config.py │ │ ├── tools │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── metric.py │ │ │ ├── predict.py │ │ │ ├── scan.py │ │ │ └── shap.py │ │ └── utils │ │ │ └── shap.py │ ├── testcase.py │ └── utils.py ├── models │ ├── __init__.py │ ├── _precooked.py │ ├── automodel.py │ ├── base │ │ ├── __init__.py │ │ ├── model.py │ │ ├── model_prediction.py │ │ ├── serialization.py │ │ └── wrapper.py │ ├── cache │ │ ├── __init__.py │ │ └── cache.py │ ├── catboost.py │ ├── function.py │ ├── huggingface.py │ ├── langchain.py │ ├── model_explanation.py │ ├── pytorch.py │ ├── shap_result.py │ ├── sklearn.py │ ├── talk_result.py │ ├── tensorflow.py │ └── utils.py ├── path_utils.py ├── rag │ ├── __init__.py │ ├── base.py │ ├── evaluate.py │ ├── knowledge_base.py │ ├── knowledge_base_plots.py │ ├── metrics │ │ ├── __init__.py │ │ ├── base.py │ │ ├── correctness.py │ │ └── ragas_metrics.py │ ├── question_generators │ │ ├── __init__.py │ │ ├── base.py │ │ ├── complex_questions.py │ │ ├── conversational_questions.py │ │ ├── distracting_questions.py │ │ ├── double_questions.py │ │ ├── oos_questions.py │ │ ├── prompt.py │ │ ├── question_types.py │ │ ├── simple_questions.py │ │ ├── situational_questions.py │ │ └── utils.py │ ├── recommendation.py │ ├── report.py │ ├── testset.py │ └── testset_generation.py ├── registry │ ├── __init__.py │ ├── decorators.py │ ├── decorators_utils.py │ ├── giskard_test.py │ ├── registry.py │ ├── slicing_function.py │ ├── transformation_function.py │ └── utils.py ├── scanner │ ├── __init__.py │ ├── calibration │ │ ├── __init__.py │ │ ├── overconfidence_detector.py │ │ └── underconfidence_detector.py │ ├── common │ │ ├── examples.py │ │ ├── loss_based_detector.py │ │ └── utils.py │ ├── correlation │ │ ├── __init__.py │ │ └── spurious_correlation_detector.py │ ├── data_leakage │ │ ├── __init__.py │ │ └── data_leakage_detector.py │ ├── decorators.py │ ├── issues.py │ ├── llm │ │ ├── __init__.py │ │ ├── base.py │ │ ├── llm_basic_sycophancy_detector.py │ │ ├── llm_chars_injection_detector.py │ │ ├── llm_faithfulness_detector.py │ │ ├── llm_harmful_content_detector.py │ │ ├── llm_implausible_output_detector.py │ │ ├── llm_information_disclosure_detector.py │ │ ├── llm_output_formatting_detector.py │ │ ├── llm_prompt_injection_detector.py │ │ └── llm_stereotypes_detector.py │ ├── logger.py │ ├── performance │ │ ├── __init__.py │ │ ├── metrics.py │ │ └── performance_bias_detector.py │ ├── registry.py │ ├── report.py │ ├── robustness │ │ ├── __init__.py │ │ ├── base_detector.py │ │ ├── base_perturbation_function.py │ │ ├── entity_swap.py │ │ ├── ethical_bias_detector.py │ │ ├── nationalities.json │ │ ├── numerical_perturbation_detector.py │ │ ├── numerical_transformations.py │ │ ├── text_perturbation_detector.py │ │ └── text_transformations.py │ ├── scanner.py │ ├── stochasticity │ │ ├── __init__.py │ │ └── stochasticity_detector.py │ └── templates │ │ └── static │ │ ├── external.js │ │ ├── internal.js │ │ └── style.css ├── settings.py ├── slicing │ ├── __init__.py │ ├── base.py │ ├── bruteforce_slicer.py │ ├── category_slicer.py │ ├── multiscale_slicer.py │ ├── opt_slicer.py │ ├── slice.py │ ├── slice_finder.py │ ├── stop_words.py │ ├── text_slicer.py │ ├── tree_slicer.py │ └── utils.py ├── testing │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── calibration.py │ │ ├── data_quality.py │ │ ├── debug_slicing_functions.py │ │ ├── drift.py │ │ ├── llm │ │ │ ├── __init__.py │ │ │ ├── correctness.py │ │ │ ├── ground_truth.py │ │ │ ├── hallucination.py │ │ │ ├── injections.py │ │ │ └── output_requirements.py │ │ ├── metamorphic.py │ │ ├── performance.py │ │ ├── stability.py │ │ └── statistic.py │ └── utils │ │ ├── stat_utils.py │ │ └── utils.py ├── utils │ ├── __init__.py │ ├── analytics_collector.py │ ├── artifacts.py │ ├── display.py │ ├── environment_detector.py │ ├── file_utils.py │ ├── iterables.py │ ├── language_detection.py │ ├── logging_utils.py │ └── versions.py └── visualization │ ├── __init__.py │ ├── custom_jinja.py │ ├── templates │ ├── rag_report │ │ ├── rag_report.html │ │ └── static │ │ │ ├── internal.js │ │ │ └── style.css │ ├── scan_report │ │ ├── html │ │ │ ├── _code_snippet.html │ │ │ ├── _issue.html │ │ │ ├── _issues_table.html │ │ │ ├── _main_content.html │ │ │ ├── _tab_header.html │ │ │ ├── base.html │ │ │ ├── full.html │ │ │ └── static │ │ │ │ ├── external.js │ │ │ │ ├── internal.js │ │ │ │ └── style.css │ │ └── markdown │ │ │ ├── github.md │ │ │ ├── huggingface.md │ │ │ └── summary.md │ └── suite_results │ │ ├── _suite_results_cards.html │ │ ├── _suite_results_header.html │ │ └── suite_results.html │ └── widget.py ├── gulpfile.js ├── package-lock.json ├── package.json ├── pdm.lock ├── postcss.config.js ├── pyproject.toml ├── readme ├── Discord.png ├── Give_feedback.jpeg ├── Logo_full_darkgreen.png ├── RAGET_updated.gif ├── architechture_giskard.png ├── catalog_example.png ├── catalog_slice.png ├── catalog_transfo.png ├── demo.png ├── design_partner.png ├── feedback.png ├── feedback1.png ├── giskard_logo.png ├── giskard_logo_green.png ├── inspect.png ├── perturbation.png ├── pipeline.png ├── scan_example.gif ├── scan_example.png ├── scan_results.png ├── scan_updates.gif ├── suite_example.png ├── test.png ├── test1.png ├── test_suite_example.png ├── tools.png ├── tools_updated.png ├── upload.png └── workflow.png ├── sample_data ├── classification │ ├── credit │ │ ├── german_credit.csv │ │ └── german_credit_prepared.csv │ └── titanic │ │ ├── titanic-train-big-duplicate.csv │ │ └── titanic-train.csv ├── other │ └── addresses.csv └── regression │ └── house-prices │ ├── house_price_updated.csv │ ├── test.csv │ └── train.csv ├── scripts ├── install-giskard-client-dev.sh └── make-release.sh ├── sonar-project.properties ├── src └── scan-widget │ ├── _highlight_theme.css │ ├── _highlightjs_copy.css │ ├── external-js │ └── iframeResizer.min.js │ ├── internal-js │ ├── highlight.min.js │ ├── highlightjs-copy.min.js │ ├── iframeResizer.contentWindow.min.js │ └── scan.js │ └── style.css ├── tailwind.config.js └── tests ├── __init__.py ├── conftest.py ├── core ├── __init__.py ├── test_core.py ├── test_test_result.py └── test_validation.py ├── datasets ├── test_base_dataset.py ├── test_dataset_languages_extraction.py ├── test_dataset_row_hashes.py ├── test_dataset_serialization.py ├── test_metadata.py └── test_metadata_loading.py ├── debug ├── test_calibration.py ├── test_drift.py ├── test_metamorphic.py ├── test_performance.py └── test_statistic.py ├── fixtures ├── __init__.py ├── amazon_review__binary_classification.py ├── diabetes__regression.py ├── drug_classification__multiclass_classification.py ├── enron_multilabel_classification.py ├── fraud_detection__binary_classification.py ├── german_credit_scoring.py ├── hotel_text__regression.py ├── imdb.py ├── medical_transcript_multiclass_classification.py ├── pytorch_sst2.py ├── titanic.py ├── tripadvisor_text_classification_torch.py ├── utils.py └── xboost_classification.py ├── functions └── test_transformation.py ├── google_drive_utils.py ├── integrations ├── test_avid.py ├── test_mlflow.py ├── test_nemoguardrails.py └── test_wandb.py ├── llm ├── evaluators │ ├── test_coherency_evaluator.py │ ├── test_correctness_evaluator.py │ ├── test_llm_based_evaluators.py │ ├── test_requirements_evaluator.py │ └── utils.py ├── generators │ ├── test_adversarial_generator.py │ ├── test_base_llm_generators.py │ └── test_sycophancy_generator.py ├── test_embedding_client.py └── test_llm_client.py ├── models ├── automodel │ ├── test_automodel.py │ ├── test_infer_giskard_cls.py │ └── test_loading.py ├── fixtures │ ├── func │ │ ├── 3.10 │ │ │ ├── giskard-model-meta.yaml │ │ │ ├── giskard-model-wrapper-meta.yaml │ │ │ └── model.pkl │ │ ├── 3.11 │ │ │ ├── giskard-model-meta.yaml │ │ │ ├── giskard-model-wrapper-meta.yaml │ │ │ └── model.pkl │ │ └── 3.9 │ │ │ ├── giskard-model-meta.yaml │ │ │ ├── giskard-model-wrapper-meta.yaml │ │ │ └── model.pkl │ └── ipcc │ │ ├── 3.10 │ │ ├── ModelClass.pkl │ │ ├── giskard-model-meta.yaml │ │ ├── giskard-model-wrapper-meta.yaml │ │ └── model.json │ │ └── 3.11 │ │ ├── ModelClass.pkl │ │ ├── giskard-model-meta.yaml │ │ ├── giskard-model-wrapper-meta.yaml │ │ └── model.json ├── huggingface │ ├── test_email_classification_bert.py │ ├── test_email_classification_bert_custom_model.py │ ├── test_sequence_classification_pytorch.py │ ├── test_sequence_classification_pytorch_pipeline.py │ └── test_sequence_classification_tensorflow.py ├── langchain │ ├── state_of_the_union.txt │ ├── test_llm_chain.py │ └── test_qa_retreiver.py ├── pytorch │ ├── test_dtypes.py │ ├── test_error.py │ ├── test_linear_regression_pytorch_dataframe.py │ ├── test_newspaper_classification_pytorch_custom_model.py │ ├── test_newspaper_classification_pytorch_dataset.py │ └── test_sst2_pytorch.py ├── talk │ ├── talk_test_resources.py │ └── test_model_talk.py ├── tensorflow │ ├── test_mnist.py │ ├── test_tabular_titanic_binary_classification.py │ ├── test_text_classification.py │ ├── test_text_classification_tfhub.csv │ ├── test_text_classification_tfhub.py │ └── test_tf_auto_model_as_embedding_layer.py ├── test_base_model.py ├── test_catboost_wrapper.py ├── test_function_model.py ├── test_model.py ├── test_model_auto_inference.py ├── test_model_cache.py ├── test_model_explanation.py ├── test_model_postprocess.py ├── test_model_serialization.py ├── test_precooked_model.py ├── test_sklearn_wrapper.py ├── test_wrap_model.py └── test_wrapper_model.py ├── rag ├── test_base_question_generator.py ├── test_document_creation.py ├── test_evaluate.py ├── test_knowledge_base.py ├── test_prompt.py ├── test_qa_testset.py ├── test_question_generators.py ├── test_ragas_metrics.py ├── test_report.py ├── test_testset_generator.py └── test_testset_suite_conversion.py ├── registry ├── module_utils.py └── test_giskard_test.py ├── scan ├── llm │ ├── test_basic_sycophancy_detector.py │ ├── test_chars_injection_detector.py │ ├── test_implausible_output_detector.py │ ├── test_prompt_injection_detector.py │ └── test_requirement_based_detectors.py ├── test_data_leakage_detector.py ├── test_dataset_subsampling.py ├── test_detector_registry.py ├── test_example_extractor.py ├── test_numerical_perturbation_detector.py ├── test_overconfidence_detector.py ├── test_performance_bias_detector.py ├── test_performance_metrics.py ├── test_scan_report.py ├── test_scanner.py ├── test_slicers.py ├── test_spurious_correlation_detector.py ├── test_stochasticity_detector.py ├── test_suite_generation.py ├── test_text_perturbation_detector.py ├── test_text_transformations.py └── test_underconfidence_detector.py ├── slicing ├── test_query_slicer.py └── test_slicing_function.py ├── test_data ├── enron_data.csv └── german_credit_prepared.csv ├── test_data_drift.py ├── test_data_processing_pipeline.py ├── test_dataset.py ├── test_import_giskard.py ├── test_metamorphic_direction.py ├── test_metamorphic_invariance.py ├── test_performance.py ├── test_programmatic.py ├── test_project_uploads.py ├── test_settings.py ├── test_statistical.py ├── test_suite.py ├── test_utils.py ├── testing ├── test_calibration_tests.py ├── test_data_quality.py ├── test_llm_chars_injector.py ├── test_llm_ground_truth.py ├── test_llm_output_requirement.py ├── test_side_effects.py ├── test_stability.py └── test_test_metadata.py ├── url_utils.py ├── utils.py └── utils └── test_logging_utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | relative_files = True -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | extend-ignore = 3 | DAR003 4 | DAR102 5 | DAR201 6 | E501 7 | DAR201 8 | DAR101 9 | DAR401 10 | E203 11 | exclude = 12 | .git 13 | __pycache__ 14 | giskard/models/vectorstore/__init__.py 15 | giskard/__init__.py 16 | 17 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Black 2 | 7e9ea13e0d06f6d0a1cf1821ee39a08511d02f03 3 | c75f43099c1fa1ee3863aa5d38eeaafa2f78d745 4 | # iSort 5 | f7bbe212aec81c3c4bd8aadff08080ab82d31cee -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior, in case people don't have core.autocrlf set. 2 | * text=auto 3 | docs/** linguist-documentation -------------------------------------------------------------------------------- /.github/.stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | # Label to use when marking an issue as stale 10 | staleLabel: wontfix 11 | # Comment to post when marking an issue as stale. Set to `false` to disable 12 | markComment: > 13 | This issue has been automatically marked as stale because it has not had 14 | recent activity. It will be closed if no further activity occurs. Thank you 15 | for your contributions. 16 | # Comment to post when closing a stale issue. Set to `false` to disable 17 | closeComment: false 18 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | ### Limit any modification to .github folder to have an owner review 2 | .github @Giskard-AI/giskard -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: Giskard-AI 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository 2 | 3 | blank_issues_enabled: true 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🚀 Feature request 3 | about: Suggest an idea for this project 🏖 4 | title: '' 5 | labels: enhancement 6 | assignees: 7 | --- 8 | 9 | ## 🚀 Feature Request 10 | 11 | 12 | 13 | ## 🔈 Motivation 14 | 15 | 16 | 17 | ## 🛰 Alternatives 18 | 19 | 20 | 21 | ## 📎 Additional context 22 | 23 | 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: ❓ Question 3 | about: Ask a question about this project 🎓 4 | title: '' 5 | labels: question 6 | assignees: 7 | --- 8 | 9 | ## Checklist 10 | 11 | 12 | 13 | - [ ] I've searched the project's [`issues`](https://github.com/Giskard-AI/giskard/issues?q=is%3Aissue). 14 | 15 | ## ❓ Question 16 | 17 | 18 | 19 | How can I [...]? 20 | 21 | Is it possible to [...]? 22 | 23 | ## 📎 Additional context 24 | 25 | 26 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 4 | 5 | ## Related Issue 6 | 7 | 8 | 9 | ## Type of Change 10 | 11 | 12 | 13 | - [ ] 📚 Examples / docs / tutorials / dependencies update 14 | - [ ] 🔧 Bug fix (non-breaking change which fixes an issue) 15 | - [ ] 🥂 Improvement (non-breaking change which improves an existing feature) 16 | - [ ] 🚀 New feature (non-breaking change which adds functionality) 17 | - [ ] 💥 Breaking change (fix or feature that would cause existing functionality to change) 18 | - [ ] 🔐 Security fix 19 | 20 | ## Checklist 21 | 22 | 23 | 24 | - [ ] I've read the [`CODE_OF_CONDUCT.md`](https://github.com/Giskard-AI/ai-inspector/blob/master/CODE_OF_CONDUCT.md) document. 25 | - [ ] I've read the [`CONTRIBUTING.md`](https://github.com/Giskard-AI/ai-inspector/blob/master/CONTRIBUTING.md) guide. 26 | - [ ] I've written tests for all new methods and classes that I created. 27 | - [ ] I've written the docstring in Google format for all the methods and classes that I used. 28 | - [ ] I've updated the `pdm.lock` running `pdm update-lock` (only applicable when `pyproject.toml` has been 29 | modified) 30 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "npm" 9 | directory: "/" 10 | schedule: 11 | interval: "monthly" 12 | 13 | - package-ecosystem: "pip" 14 | directory: "/" 15 | schedule: 16 | interval: "monthly" 17 | 18 | - package-ecosystem: "github-actions" 19 | directory: "/" 20 | schedule: 21 | interval: "monthly" 22 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | # Release drafter configuration https://github.com/release-drafter/release-drafter#configuration 2 | # Emojis were chosen to match the https://gitmoji.carloscuesta.me/ 3 | 4 | name-template: "v$NEXT_PATCH_VERSION" 5 | tag-template: "v$NEXT_PATCH_VERSION" 6 | 7 | categories: 8 | - title: ":rocket: Features" 9 | labels: [enhancement, feature] 10 | - title: ":wrench: Fixes & Refactoring" 11 | labels: [bug, refactoring, bugfix, fix] 12 | - title: ":package: Build System & CI/CD" 13 | labels: [build, ci, testing] 14 | - title: ":boom: Breaking Changes" 15 | labels: [breaking] 16 | - title: ":pencil: Documentation" 17 | labels: [documentation] 18 | - title: ":arrow_up: Dependencies updates" 19 | labels: [dependencies] 20 | 21 | template: | 22 | ## What’s Changed 23 | 24 | $CHANGES 25 | 26 | ## :busts_in_silhouette: List of contributors 27 | 28 | $CONTRIBUTORS 29 | -------------------------------------------------------------------------------- /.github/workflows/clear-cache.yml: -------------------------------------------------------------------------------- 1 | name: Clear all caches 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: '0 0 * * 0' # Every week 6 | 7 | permissions: 8 | actions: write 9 | 10 | jobs: 11 | clear: 12 | name: Clear all cache 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout code 16 | uses: actions/checkout@v4 17 | with: 18 | fetch-depth: 1 19 | - name: Clear cache 20 | run: gh cache delete --all 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | -------------------------------------------------------------------------------- /.github/workflows/nightly-test.yml: -------------------------------------------------------------------------------- 1 | name: Nightly test 2 | on: 3 | schedule: 4 | - cron: '0 1 * * *' # Always run at 1:00 a.m. 5 | workflow_dispatch: 6 | 7 | jobs: 8 | call-full-ci: 9 | uses: ./.github/workflows/build-python.yml 10 | with: 11 | run-integration-tests: true 12 | use-cache: false -------------------------------------------------------------------------------- /.github/workflows/pre-commit-checks.yml: -------------------------------------------------------------------------------- 1 | name: Pre-commit checks 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | env: 10 | GSK_DISABLE_ANALYTICS: true 11 | SENTRY_ENABLED: false 12 | GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} 13 | defaults: 14 | run: 15 | shell: bash 16 | jobs: 17 | pre-commit: 18 | name: Pre-commit checks 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | - uses: actions/setup-python@v5 23 | - uses: pre-commit/action@v3.0.1 24 | env: 25 | SKIP: ggshield 26 | -------------------------------------------------------------------------------- /.github/workflows/retry-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Retry workflow 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | run_id: 7 | required: true 8 | description: Id of the failed workflow to retry 9 | jobs: 10 | rerun: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: rerun ${{ inputs.run_id }} 14 | env: 15 | GH_REPO: ${{ github.repository }} 16 | GH_TOKEN: ${{ github.token }} 17 | run: | 18 | gh run watch ${{ inputs.run_id }} > /dev/null 2>&1 19 | gh run rerun ${{ inputs.run_id }} --failed 20 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pycqa/isort 3 | rev: 5.13.2 4 | hooks: 5 | - id: isort 6 | files: '^.*\.py$' 7 | args: 8 | - "--settings-path" 9 | - "pyproject.toml" 10 | 11 | - repo: https://github.com/ambv/black 12 | rev: 23.12.1 13 | hooks: 14 | - id: black 15 | files: '^.*\.py$' 16 | args: 17 | - "--config" 18 | - "pyproject.toml" 19 | stages: [pre-commit] 20 | - repo: https://github.com/astral-sh/ruff-pre-commit 21 | rev: v0.1.6 22 | hooks: 23 | - id: ruff 24 | files: '^.*\.py$' 25 | args: 26 | - "--config" 27 | - "pyproject.toml" 28 | - "--fix" 29 | - "--exit-non-zero-on-fix" 30 | 31 | - repo: https://github.com/gitguardian/ggshield 32 | rev: v1.23.0 33 | hooks: 34 | - id: ggshield 35 | language_version: python3 36 | stages: [pre-commit] -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.10" 7 | jobs: 8 | post_create_environment: 9 | - pip install pdm 10 | - pdm export --without-hashes -G doc,llm -o requirements-sphinx.txt 11 | 12 | sphinx: 13 | configuration: docs/conf.py 14 | 15 | python: 16 | install: 17 | - requirements: requirements-sphinx.txt 18 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Remote Attach", 9 | "type": "python", 10 | "request": "attach", 11 | "connect": { 12 | "host": "localhost", 13 | "port": 5678 14 | }, 15 | "pathMappings": [ 16 | { 17 | "localRoot": "${workspaceFolder}", 18 | "remoteRoot": "." 19 | } 20 | ], 21 | "justMyCode": true 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /ISSUES.md: -------------------------------------------------------------------------------- 1 | If you open a GitHub Issue, here is our policy: 2 | 3 | 1. It must be a bug/performance issue or a feature request or a build issue or 4 | a documentation issue (for small doc fixes please send a PR instead). 5 | 2. Make sure the Issue Template is filled out. 6 | 3. The issue should be related to the repo it is created in. 7 | 8 | **Here's why we have this policy:** We want to focus on the work that benefits 9 | the whole community, e.g., fixing bugs and adding features. Individual support 10 | should be sought on our [Discord](https://discord.com/invite/ABvfpbu69R) community. It helps us to 11 | address bugs and feature requests in a timely manner. -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | If you believe you have found a security vulnerability in Giskard, you can report it to us in two ways: 4 | - by privately submitting a report from this repository's [security advisories](https://github.com/Giskard-AI/giskard/security/advisories) page 5 | - by sending an email to security@giskard.ai 6 | 7 | Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests. 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/catalog_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/catalog_example.png -------------------------------------------------------------------------------- /docs/_static/catalog_slice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/catalog_slice.png -------------------------------------------------------------------------------- /docs/_static/catalog_transfo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/catalog_transfo.png -------------------------------------------------------------------------------- /docs/_static/debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/debug.png -------------------------------------------------------------------------------- /docs/_static/giskard_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/giskard_logo.png -------------------------------------------------------------------------------- /docs/_static/hey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/hey.png -------------------------------------------------------------------------------- /docs/_static/js/reodev.js: -------------------------------------------------------------------------------- 1 | !function () { 2 | var e, t, n; 3 | e = "7f26c9419b1f16e", t = function () { 4 | Reo.init({clientID: "7f26c9419b1f16e"}) 5 | }, (n = document.createElement("script")).src = "https://static.reo.dev/" + e + "/reo.js", n.async = !0, n.onload = t, document.head.appendChild(n) 6 | }(); -------------------------------------------------------------------------------- /docs/_static/js/sidebarKeepScroll.js: -------------------------------------------------------------------------------- 1 | document.addEventListener("DOMContentLoaded", () => { 2 | let sidebar = document.querySelector(".sidebar-scroll"); 3 | let top = localStorage.getItem("sidebar-scroll"); 4 | if (sidebar && top !== null) { 5 | sidebar.scrollTo({ 6 | top: parseInt(top), 7 | behavior: "instant", 8 | }); 9 | } 10 | window.addEventListener("beforeunload", () => { 11 | localStorage.setItem("sidebar-scroll", sidebar.scrollTop); 12 | }); 13 | }); -------------------------------------------------------------------------------- /docs/_static/logo_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/logo_black.png -------------------------------------------------------------------------------- /docs/_static/logo_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/logo_white.png -------------------------------------------------------------------------------- /docs/_static/ragas_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/ragas_metrics.png -------------------------------------------------------------------------------- /docs/_static/raget.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/raget.png -------------------------------------------------------------------------------- /docs/_static/scan_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/scan_example.png -------------------------------------------------------------------------------- /docs/_static/test_suite_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/_static/test_suite_example.png -------------------------------------------------------------------------------- /docs/_templates/sidebar/brand.html: -------------------------------------------------------------------------------- 1 | {#- 2 | 3 | Hi there! 4 | 5 | You might be interested in https://pradyunsg.me/furo/customisation/sidebar/ 6 | 7 | Although if you're reading this, chances are that you're either familiar 8 | enough with Sphinx that you know what you're doing, or landed here from that 9 | documentation page. 10 | 11 | Hope your day's going well. :) 12 | 13 | -#} 14 | 29 |
30 |
31 |
33 |
-------------------------------------------------------------------------------- /docs/assets/Architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/Architecture.jpg -------------------------------------------------------------------------------- /docs/assets/RAG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/RAG.png -------------------------------------------------------------------------------- /docs/assets/catalog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/catalog.png -------------------------------------------------------------------------------- /docs/assets/comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/comparison.png -------------------------------------------------------------------------------- /docs/assets/create_project.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/create_project.gif -------------------------------------------------------------------------------- /docs/assets/credit_scoring_comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/credit_scoring_comment.png -------------------------------------------------------------------------------- /docs/assets/discussion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/discussion.png -------------------------------------------------------------------------------- /docs/assets/feedback.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/feedback.png -------------------------------------------------------------------------------- /docs/assets/gh_discussion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/gh_discussion.png -------------------------------------------------------------------------------- /docs/assets/gh_readme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/gh_readme.png -------------------------------------------------------------------------------- /docs/assets/image_(1)_(1)_(2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/image_(1)_(1)_(2).png -------------------------------------------------------------------------------- /docs/assets/image_(3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/image_(3).png -------------------------------------------------------------------------------- /docs/assets/integrations/avid/avid_taxonomy.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/avid/avid_taxonomy.gif -------------------------------------------------------------------------------- /docs/assets/integrations/dagshub/dagshub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/dagshub/dagshub.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/copy_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/copy_token.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/create_from_template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/create_from_template.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/create_new_space.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/create_new_space.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/duplicate_this_space.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/duplicate_this_space.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_copy_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_copy_id.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_input_hf_access_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_input_hf_access_token.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_input_model_and_dataset_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_input_model_and_dataset_id.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_job_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_job_id.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_label_matched.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_label_matched.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_label_matching.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_label_matching.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_label_unmatched.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_label_unmatched.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_logs.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_model_and_dataset_checking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_model_and_dataset_checking.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/eval_scan_conf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/eval_scan_conf.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/free_tier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/free_tier.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/generate_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/generate_token.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/giskard_client.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/giskard_client.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/hfs.svg: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/input_hf_access_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/input_hf_access_token.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/mlworker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/mlworker.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/paid_tier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/paid_tier.png -------------------------------------------------------------------------------- /docs/assets/integrations/hfs/where_to_create_access_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/hfs/where_to_create_access_token.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/MLflow-logo-final-white-TM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/MLflow-logo-final-white-TM.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/llms/artifact_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/llms/artifact_comparison.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/llms/scan-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/llms/scan-summary.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/llms/table_view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/llms/table_view.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/llms/text-davinci-001-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/llms/text-davinci-001-metrics.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/llms/text-davinci-001-scanresults.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/llms/text-davinci-001-scanresults.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/llms/text-davinci-001-scanresults2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/llms/text-davinci-001-scanresults2.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/tabular/metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/tabular/metrics.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/tabular/scan-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/tabular/scan-summary.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/tabular/scanresults.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/tabular/scanresults.png -------------------------------------------------------------------------------- /docs/assets/integrations/mlflow/tabular/table_view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/mlflow/tabular/table_view.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/categorical-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/categorical-chart.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/dataset.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/global-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/global-chart.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/numerical-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/numerical-chart.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/scanning-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/scanning-result.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/test-suite-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/test-suite-result.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/wandb-categorical-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/wandb-categorical-chart.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/wandb-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/wandb-dataset.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/wandb-global-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/wandb-global-chart.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/wandb-logo-yellow-dots-black-wb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/wandb-logo-yellow-dots-black-wb.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/wandb-numerical-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/wandb-numerical-chart.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/wandb-scanning-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/wandb-scanning-result.png -------------------------------------------------------------------------------- /docs/assets/integrations/wandb/wandb-test-suite-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/integrations/wandb/wandb-test-suite-result.png -------------------------------------------------------------------------------- /docs/assets/intro/Giskard_Turtle_Computer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/Giskard_Turtle_Computer.png -------------------------------------------------------------------------------- /docs/assets/intro/Giskard_Turtle_Kungfu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/Giskard_Turtle_Kungfu.png -------------------------------------------------------------------------------- /docs/assets/intro/Giskard_Turtle_Lab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/Giskard_Turtle_Lab.png -------------------------------------------------------------------------------- /docs/assets/intro/Giskard_Turtle_SK8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/Giskard_Turtle_SK8.png -------------------------------------------------------------------------------- /docs/assets/intro/Giskard_Turtle_new_set_FINAL_05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/Giskard_Turtle_new_set_FINAL_05.png -------------------------------------------------------------------------------- /docs/assets/intro/Giskard_Turtle_new_set_FINAL_06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/Giskard_Turtle_new_set_FINAL_06.png -------------------------------------------------------------------------------- /docs/assets/intro/Giskard_Turtle_waving.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/Giskard_Turtle_waving.png -------------------------------------------------------------------------------- /docs/assets/intro/coffee.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/coffee.png -------------------------------------------------------------------------------- /docs/assets/intro/hey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/hey.png -------------------------------------------------------------------------------- /docs/assets/intro/integrations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/integrations.png -------------------------------------------------------------------------------- /docs/assets/intro/ninja.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/ninja.png -------------------------------------------------------------------------------- /docs/assets/intro/slice.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/slice.jpg -------------------------------------------------------------------------------- /docs/assets/intro/test_turtle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/intro/test_turtle.png -------------------------------------------------------------------------------- /docs/assets/llm_debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/llm_debug.png -------------------------------------------------------------------------------- /docs/assets/llm_monitoring_dashboard.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/llm_monitoring_dashboard.gif -------------------------------------------------------------------------------- /docs/assets/model_insights_titanic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/model_insights_titanic.png -------------------------------------------------------------------------------- /docs/assets/ngrok_aut.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/ngrok_aut.png -------------------------------------------------------------------------------- /docs/assets/ngrok_aut2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/ngrok_aut2.png -------------------------------------------------------------------------------- /docs/assets/output_filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/output_filter.png -------------------------------------------------------------------------------- /docs/assets/push.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/push.png -------------------------------------------------------------------------------- /docs/assets/scan_llm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/scan_llm.png -------------------------------------------------------------------------------- /docs/assets/scan_nlp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/scan_nlp.png -------------------------------------------------------------------------------- /docs/assets/scan_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/scan_results.png -------------------------------------------------------------------------------- /docs/assets/scan_tabular.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/scan_tabular.png -------------------------------------------------------------------------------- /docs/assets/scan_vision.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/scan_vision.png -------------------------------------------------------------------------------- /docs/assets/slice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/slice.png -------------------------------------------------------------------------------- /docs/assets/test_suite_scan_llm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/test_suite_scan_llm.png -------------------------------------------------------------------------------- /docs/assets/test_suite_tabular.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/assets/test_suite_tabular.png -------------------------------------------------------------------------------- /docs/community/contribution_guidelines/giskard-architecture.md: -------------------------------------------------------------------------------- 1 | --- 2 | description: Giskard process architecture 3 | --- 4 | 5 | # Giskard architecture 6 | ![](<../../assets/Architecture.jpg>) 7 | -------------------------------------------------------------------------------- /docs/community/discord/index.md: -------------------------------------------------------------------------------- 1 | # Discord community 2 | 3 | 🤖 Our discord community is an inclusive place where anyone interested in ML Quality is welcome! Leverage best practices from the community, contribute new tests, build the future of AI safety standards. 4 | 5 | 🤝 Use the following link to join the conversation: [Discord community link](https://discord.com/invite/ABvfpbu69R) 6 | -------------------------------------------------------------------------------- /docs/community/github/index.md: -------------------------------------------------------------------------------- 1 | # GitHub community 2 | 3 | 💬 [Open a discussion](https://github.com/orgs/Giskard-AI/discussions) to interact with our contributing members and users! 4 | 5 | 🌟 You can also drop by to [leave us a star](https://github.com/Giskard-AI/giskard), it helps the project to get discovered by others and keeps us motivated to build awesome open-source tools! 🌟 6 | 7 | ❤️ Feel free to [sponsor us](https://github.com/sponsors/Giskard-AI) on GitHub. With a monthly sponsor subscription, you can get a sponsor badge and get your bug reports prioritized. We also offer one-time sponsoring if you want us to get involved in a consulting project, run a workshop, or give a talk at your company. 8 | -------------------------------------------------------------------------------- /docs/community/index.md: -------------------------------------------------------------------------------- 1 | # Community 2 | ```{toctree} 3 | :caption: Community 4 | :maxdepth: 1 5 | :hidden: 6 | 7 | discord/index 8 | github/index 9 | contribution_guidelines/index 10 | ``` 11 | 12 | :::::{grid} 1 1 2 2 13 | 14 | 15 | ::::{grid-item-card}

Discord community

16 | :text-align: center 17 | :link: discord/index.html 18 | :::: 19 | 20 | ::::{grid-item-card}

Github community

21 | :text-align: center 22 | :link: github/index.html 23 | :::: 24 | 25 | ::::{grid-item-card}

Contribute to Giskard

26 | :text-align: center 27 | :link: contribution_guidelines/index.html 28 | :::: 29 | 30 | ::::: 31 | -------------------------------------------------------------------------------- /docs/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/favicon.ico -------------------------------------------------------------------------------- /docs/getting_started/quickstart/index.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | ```{toctree} 4 | :caption: Table of Contents 5 | :maxdepth: 2 6 | :hidden: 7 | 8 | quickstart_llm 9 | quickstart_tabular 10 | quickstart_nlp 11 | quickstart_vision 12 | ``` 13 | 14 | :::::{grid} 1 1 2 2 15 | 16 | ::::{grid-item-card}

📚 LLM Quickstart

17 | :text-align: center 18 | :link: quickstart_llm.ipynb 19 | :::: 20 | 21 | ::::{grid-item-card}

📊 Tabular Quickstart

22 | :text-align: center 23 | :link: quickstart_tabular.ipynb 24 | :::: 25 | 26 | ::::{grid-item-card}

🗣️ NLP Quickstart

27 | :text-align: center 28 | :link: quickstart_nlp.ipynb 29 | :::: 30 | 31 | ::::{grid-item-card}

📸 Vision Quickstart

32 | :text-align: center 33 | :link: quickstart_vision.ipynb 34 | :::: 35 | 36 | ::::: 37 | -------------------------------------------------------------------------------- /docs/integrations/huggingface/index.md: -------------------------------------------------------------------------------- 1 | # 🤗 Hugging Face 2 | 3 | **Leverage Hugging Face (HF) Spaces to easily scan, test & debug your own ML models.** 4 | 5 | ```{toctree} 6 | :caption: Table of Contents 7 | :maxdepth: 1 8 | :hidden: 9 | 10 | ./evaluator.md 11 | 12 | ``` 13 | 14 | ::::::{grid} 1 1 1 1 15 | 16 | ::::{grid-item-card}

🔍 Giskard Evaluator

17 | :text-align: center 18 | :link: ./evaluator.md 19 | :::: 20 | -------------------------------------------------------------------------------- /docs/integrations/index.md: -------------------------------------------------------------------------------- 1 | # Integrations 2 | ```{toctree} 3 | :caption: Integrations 4 | :maxdepth: 1 5 | :hidden: 6 | 7 | cicd/index 8 | mlflow/index 9 | nemoguardrails/index 10 | wandb/index 11 | dagshub/index 12 | huggingface/index 13 | avid/index 14 | pytest/index 15 | ``` 16 | 17 | ::::{grid} 1 1 2 2 18 | 19 | 20 | :::{grid-item-card}

🐙️ GitHub

21 | :text-align: center 22 | :link: cicd/index.html 23 | ::: 24 | 25 | :::{grid-item-card}

🤗 HuggingFace

26 | :text-align: center 27 | :link: huggingface/index.html 28 | ::: 29 | 30 | :::{grid-item-card}

🏃 MLflow

31 | :text-align: center 32 | :link: mlflow/index.html 33 | ::: 34 | 35 | :::{grid-item-card}

🟩 NeMo Guardrails

36 | :text-align: center 37 | :link: nemoguardrails/index.html 38 | ::: 39 | 40 | :::{grid-item-card}

🐝 Weights & Biases

41 | :text-align: center 42 | :link: wandb/index.html 43 | ::: 44 | 45 | :::{grid-item-card}

🐶 DagsHub

46 | :text-align: center 47 | :link: dagshub/index.html 48 | ::: 49 | 50 | :::{grid-item-card}

📒 AVID

51 | :text-align: center 52 | :link: avid/index.html 53 | ::: 54 | 55 | :::{grid-item-card}

🧪 Pytest

56 | :text-align: center 57 | :link: pytest/index.html 58 | ::: 59 | -------------------------------------------------------------------------------- /docs/integrations/mlflow/mlflow-evaluation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/integrations/mlflow/mlflow-evaluation.png -------------------------------------------------------------------------------- /docs/integrations/mlflow/mlflow-harmfulness.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/integrations/mlflow/mlflow-harmfulness.png -------------------------------------------------------------------------------- /docs/integrations/mlflow/mlflow-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/integrations/mlflow/mlflow-metrics.png -------------------------------------------------------------------------------- /docs/integrations/mlflow/mlflow-prompt-injection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/integrations/mlflow/mlflow-prompt-injection.png -------------------------------------------------------------------------------- /docs/integrations/mlflow/mlflow-runs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/integrations/mlflow/mlflow-runs.png -------------------------------------------------------------------------------- /docs/integrations/mlflow/mlflow-scan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/integrations/mlflow/mlflow-scan.png -------------------------------------------------------------------------------- /docs/integrations/mlflow/mlflow_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/docs/integrations/mlflow/mlflow_overview.png -------------------------------------------------------------------------------- /docs/knowledge/catalogs/index.md: -------------------------------------------------------------------------------- 1 | # Catalogs 2 | 3 | ```{toctree} 4 | :caption: Table of Contents 5 | :maxdepth: 2 6 | :hidden: 7 | 8 | test-catalog/index 9 | slicing-function-catalog/index 10 | transformation-function-catalog/index 11 | ``` 12 | 13 | ::::::{grid} 1 1 2 2 14 | 15 | 16 | ::::{grid-item-card}

Tests

17 | :text-align: center 18 | :link: test-catalog/index.html 19 | :::: 20 | 21 | ::::{grid-item-card}

Slicing functions

22 | :text-align: center 23 | :link: slicing-function-catalog/index.html 24 | :::: 25 | 26 | ::::{grid-item-card}

Transformation functions

27 | :text-align: center 28 | :link: transformation-function-catalog/index.html 29 | :::: 30 | :::::: -------------------------------------------------------------------------------- /docs/knowledge/catalogs/slicing-function-catalog/index.rst: -------------------------------------------------------------------------------- 1 | Slicing functions 2 | ================= 3 | 4 | - **Textual slicing function** 5 | 6 | - :func:`~.giskard.functions.slicing.short_comment_slicing_fn` 7 | - :func:`~.giskard.functions.slicing.keyword_lookup_slicing_fn` 8 | - :func:`~.giskard.functions.slicing.positive_sentiment_analysis` 9 | - :func:`~.giskard.functions.slicing.offensive_sentiment_analysis` 10 | - :func:`~.giskard.functions.slicing.irony_sentiment_analysis` 11 | - :func:`~.giskard.functions.slicing.hate_sentiment_analysis` 12 | - :func:`~.giskard.functions.slicing.emotion_sentiment_analysis` 13 | 14 | - **Numerical slicing function** 15 | - :func:`~.giskard.functions.slicing.outlier_filter` 16 | -------------------------------------------------------------------------------- /docs/knowledge/catalogs/test-catalog/index.rst: -------------------------------------------------------------------------------- 1 | Tests 2 | ============ 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | classification/index 8 | regression/index 9 | text_generation/index 10 | -------------------------------------------------------------------------------- /docs/knowledge/catalogs/test-catalog/regression/index.rst: -------------------------------------------------------------------------------- 1 | Regression tests 2 | ================= 3 | 4 | - **Metamorphic tests** 5 | 6 | - :func:`~.giskard.testing.test_metamorphic_invariance` 7 | - :func:`~.giskard.testing.test_metamorphic_increasing` 8 | - :func:`~.giskard.testing.test_metamorphic_decreasing` 9 | 10 | - **Statistical tests** 11 | 12 | - :func:`~.giskard.testing.test_output_in_range` 13 | 14 | - **Performance tests** 15 | 16 | - :func:`~.giskard.testing.test_mae` 17 | - :func:`~.giskard.testing.test_rmse` 18 | - :func:`~.giskard.testing.test_diff_rmse` 19 | - :func:`~.giskard.testing.test_r2` 20 | 21 | - **Drift tests** 22 | 23 | - :func:`~.giskard.testing.test_drift_prediction_psi` 24 | - :func:`~.giskard.testing.test_drift_prediction_chi_square` 25 | - :func:`~.giskard.testing.test_drift_prediction_ks` 26 | - :func:`~.giskard.testing.test_drift_prediction_earth_movers_distance` 27 | 28 | - **Stability tests** 29 | - :func:`~.giskard.testing.test_smoothness` 30 | - :func:`~.giskard.testing.test_monotonicity` -------------------------------------------------------------------------------- /docs/knowledge/catalogs/test-catalog/text_generation/index.rst: -------------------------------------------------------------------------------- 1 | Text generation tests 2 | ===================== 3 | 4 | - **Injections** 5 | 6 | - :func:`~.giskard.testing.tests.llm.test_llm_char_injection` 7 | - :func:`~.giskard.testing.tests.llm.test_llm_single_output_against_strings` 8 | - :func:`~.giskard.testing.tests.llm.test_llm_output_against_strings` 9 | 10 | - **LLM-as-ajudge** 11 | 12 | - :func:`~.giskard.testing.tests.llm.test_llm_output_coherency` 13 | - :func:`~.giskard.testing.tests.llm.test_llm_output_plausibility` 14 | - :func:`~.giskard.testing.tests.llm.test_llm_output_against_requirement_per_row` 15 | - :func:`~.giskard.testing.tests.llm.test_llm_single_output_against_requirement` 16 | - :func:`~.giskard.testing.tests.llm.test_llm_output_against_requirement` 17 | 18 | - **Ground Truth** 19 | 20 | - :func:`~.giskard.testing.tests.llm.test_llm_ground_truth` 21 | - :func:`~.giskard.testing.tests.llm.test_llm_ground_truth_similarity` 22 | - :func:`~.giskard.testing.tests.llm.test_llm_as_a_judge_ground_truth_similarity` 23 | 24 | -------------------------------------------------------------------------------- /docs/knowledge/catalogs/transformation-function-catalog/index.rst: -------------------------------------------------------------------------------- 1 | Transformation functions 2 | ======================== 3 | 4 | - **Text transformation function** 5 | 6 | - :func:`~.giskard.functions.transformation.keyboard_typo_transformation` 7 | - :func:`~.giskard.functions.transformation.text_uppercase` 8 | - :func:`~.giskard.functions.transformation.text_lowercase` 9 | - :func:`~.giskard.functions.transformation.text_title_case` 10 | - :func:`~.giskard.functions.transformation.text_typo` 11 | - :func:`~.giskard.functions.transformation.text_typo_from_ocr` 12 | - :func:`~.giskard.functions.transformation.text_punctuation_removal` 13 | - :func:`~.giskard.functions.transformation.text_accent_removal` 14 | - :func:`~.giskard.functions.transformation.text_gender_switch` 15 | - :func:`~.giskard.functions.transformation.text_number_to_word` 16 | - :func:`~.giskard.functions.transformation.text_religion_switch` 17 | - :func:`~.giskard.functions.transformation.text_nationality_switch` 18 | - :func:`~.giskard.functions.transformation.text_typo_from_speech` 19 | -------------------------------------------------------------------------------- /docs/knowledge/index.md: -------------------------------------------------------------------------------- 1 | # Knowledge 2 | ```{toctree} 3 | :caption: Knowledge 4 | :maxdepth: 1 5 | :hidden: 6 | 7 | llm_vulnerabilities/index 8 | key_vulnerabilities/index 9 | catalogs/index 10 | ``` 11 | 12 | ::::::{grid} 1 1 2 2 13 | 14 | 15 | ::::{grid-item-card}

📚 Catalogs

16 | :text-align: center 17 | :link: catalogs/index.html 18 | :::: 19 | 20 | ::::{grid-item-card}

📊 ML models vulnerabilities

21 | :text-align: center 22 | :link: key_vulnerabilities/index.html 23 | :::: 24 | 25 | ::::{grid-item-card}

🗣️ LLM vulnerabilities

26 | :text-align: center 27 | :link: llm_vulnerabilities/index.html 28 | :::: 29 | -------------------------------------------------------------------------------- /docs/knowledge/key_vulnerabilities/index.rst: -------------------------------------------------------------------------------- 1 | ML Model Vulnerabilities 2 | ======================== 3 | 4 | Before testing Machine Learning models, it's crucial to have a comprehensive understanding of the critical vulnerabilities that can impact your model. Giskard provides an `automatic scan functionality <../../open_source/scan/index.md>`_ that is designed to automatically detect a variety of risks associated with your ML model. You can learn more about the different vulnerabilities it can detect here: 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | performance_bias/index 10 | robustness/index 11 | overconfidence/index 12 | underconfidence/index 13 | ethics/index 14 | data_leakage/index 15 | stochasticity/index 16 | spurious/index 17 | 18 | By conducting a `Giskard Scan <../../open_source/scan/index.md>`_, you can proactively identify and address these vulnerabilities to ensure the reliability, fairness, and robustness of your Machine Learning models. 19 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/open_source/customize_tests/index.md: -------------------------------------------------------------------------------- 1 | # 🧪 Customize your tests 2 | 3 | ```{toctree} 4 | :caption: Table of Contents 5 | :maxdepth: 2 6 | :hidden: 7 | 8 | test_model/index 9 | data_slices/index 10 | data_transformations/index 11 | ``` 12 | 13 | :::::{grid} 1 1 2 2 14 | 15 | 16 | ::::{grid-item-card}

👨‍🔬 Create tests

17 | :text-align: center 18 | :link: test_model/index.html 19 | :::: 20 | 21 | ::::{grid-item-card}

🔪 Create data slices

22 | :text-align: center 23 | :link: data_slices/index.html 24 | :::: 25 | 26 | ::::{grid-item-card}

🔄 Create data transformations

27 | :text-align: center 28 | :link: data_transformations/index.html 29 | :::: 30 | 31 | ::::: 32 | -------------------------------------------------------------------------------- /docs/open_source/index.md: -------------------------------------------------------------------------------- 1 | # Open Source 2 | ```{toctree} 3 | :caption: Open Source 4 | :maxdepth: 1 5 | :hidden: 6 | 7 | installation_library/index 8 | setting_up/index 9 | scan/index 10 | testset_generation/index 11 | customize_tests/index 12 | integrate_tests/index 13 | ``` 14 | 15 | ::::::{grid} 1 1 2 2 16 | 17 | 18 | ::::{grid-item-card}

📥 Install the Giskard Python Library

19 | :text-align: center 20 | :link: installation_library/index.html 21 | :::: 22 | 23 | ::::{grid-item-card}

🤖 Setting up the LLM Client

24 | :text-align: center 25 | :link: setting_up/index.html 26 | :::: 27 | 28 | ::::{grid-item-card}

🔍 Scan a model

29 | :text-align: center 30 | :link: scan/index.html 31 | :::: 32 | 33 | ::::{grid-item-card}

🧰 RAG Evaluation Toolkit

34 | :text-align: center 35 | :link: testset_generation/index.html 36 | :::: 37 | 38 | ::::{grid-item-card}

🤖 AI Quality Copilot

39 | :text-align: center 40 | :link: ai_quality_copilot/index.html 41 | :::: 42 | 43 | ::::{grid-item-card}

🧪 Customize your tests

44 | :text-align: center 45 | :link: customize_tests/index.html 46 | :::: 47 | 48 | ::::{grid-item-card}

🔁 Integrate your tests

49 | :text-align: center 50 | :link: integrate_tests/index.html 51 | :::: -------------------------------------------------------------------------------- /docs/open_source/installation_library/index.md: -------------------------------------------------------------------------------- 1 | # 📥 Install the Giskard Python Library 2 | 3 | Giskard supports Python `3.9`, `3.10` and `3.11`. 4 | 5 | ## Install 6 | 7 | ::::{tab-set} 8 | :::{tab-item} LLMs 9 | 10 | ```sh 11 | pip install giskard[llm] -U 12 | ``` 13 | 14 | ::: 15 | 16 | :::{tab-item} Tabular and NLP 17 | 18 | ```sh 19 | pip install giskard -U 20 | ``` 21 | 22 | ::: 23 | :::: 24 | 25 | ## Upgrade 26 | 27 | ```sh 28 | pip uninstall giskard 29 | pip install giskard -U 30 | ``` 31 | 32 | ## Dependency issues 33 | 34 | If you run into conflicts with Giskard dependencies, please uninstall the conflicting libraries and re-install Giskard. 35 | 36 | For instance, if you have dependency conflicts with `pandas`, you can run: 37 | 38 | ```sh 39 | pip uninstall giskard 40 | pip uninstall pandas 41 | pip install giskard -U 42 | ``` 43 | -------------------------------------------------------------------------------- /docs/open_source/integrate_tests/index.md: -------------------------------------------------------------------------------- 1 | # 🔁 Integrate your tests 2 | 3 | Once you've created a test suite for a model, integrating it into existing workflows allow you to automate the testing and logging of your model quality. You will find a couple options on this page to do so. 4 | 5 | ```{toctree} 6 | :caption: Table of Contents 7 | :maxdepth: 2 8 | :hidden: 9 | 10 | ../../integrations/cicd/pipeline.ipynb 11 | ../../integrations/mlflow/index 12 | ../../integrations/wandb/index 13 | ../../integrations/pytest/index 14 | 15 | ``` 16 | 17 | :::::{grid} 1 1 2 2 18 | 19 | 20 | ::::{grid-item-card}

🚀 Execute your test suite in your CI/CD pipeline

21 | :text-align: center 22 | :link: ../../integrations/cicd/pipeline.ipynb 23 | :::: 24 | 25 | ::::{grid-item-card}

🏃 Evaluate & log with MLflow

26 | :text-align: center 27 | :link: ../../integrations/mlflow/index.md 28 | :::: 29 | 30 | ::::{grid-item-card}

🐝 Evaluate & log with Weights & Biases

31 | :text-align: center 32 | :link: ../../integrations/wandb/index.md 33 | :::: 34 | 35 | ::::{grid-item-card}

🧪 Execute your test with pytest

36 | :text-align: center 37 | :link: ../../integrations/pytest/index.md 38 | :::: 39 | 40 | ::::: 41 | -------------------------------------------------------------------------------- /docs/open_source/scan/index.md: -------------------------------------------------------------------------------- 1 | # 🔍 Scan a model 2 | 3 | ```{toctree} 4 | :caption: Table of Contents 5 | :maxdepth: 2 6 | :hidden: 7 | 8 | scan_llm/index 9 | scan_tabular/index 10 | scan_nlp/index 11 | scan_vision/index 12 | advanced_scan/index 13 | ``` 14 | 15 | ::::::{grid} 1 1 2 2 16 | 17 | ::::{grid-item-card}

📚 LLM scan

18 | :text-align: center 19 | :link: scan_llm/index.html 20 | :::: 21 | 22 | ::::{grid-item-card}

📊 Tabular model scan

23 | :text-align: center 24 | :link: scan_tabular/index.html 25 | :::: 26 | 27 | ::::{grid-item-card}

🗣️ NLP model scan

28 | :text-align: center 29 | :link: scan_nlp/index.html 30 | :::: 31 | 32 | ::::{grid-item-card}

📸 Vision model scan

33 | :text-align: center 34 | :link: scan_vision/index.html 35 | :::: 36 | 37 | ::::{grid-item-card}

Advanced scan usage

38 | :text-align: center 39 | :link: advanced_scan/index.rst 40 | :::: 41 | -------------------------------------------------------------------------------- /docs/open_source/scan/scan_llm/html.rst: -------------------------------------------------------------------------------- 1 | .. raw:: html 2 | :file: scan_result_iframe.html 3 | 4 | -------------------------------------------------------------------------------- /docs/open_source/testset_generation/index.md: -------------------------------------------------------------------------------- 1 | # 🧰 RAG Evaluation Toolkit 2 | 3 | ```{toctree} 4 | :caption: Table of Contents 5 | :maxdepth: 2 6 | :hidden: 7 | 8 | testset_generation/index 9 | rag_evaluation/index 10 | ``` 11 | 12 | ::::::{grid} 1 1 2 2 13 | 14 | 15 | ::::{grid-item-card}

🎯 Testset Generation

16 | :text-align: center 17 | :link: testset_generation/index.html 18 | :::: 19 | 20 | ::::{grid-item-card}

🥇 Evaluate your RAG

21 | :text-align: center 22 | :link: rag_evaluation/index.html 23 | :::: 24 | -------------------------------------------------------------------------------- /docs/reference/datasets/index.rst: -------------------------------------------------------------------------------- 1 | Dataset 2 | ============ 3 | 4 | .. autoclass:: giskard.Dataset 5 | 6 | .. automethod:: __init__ 7 | .. automethod:: _infer_column_types 8 | .. automethod:: add_slicing_function 9 | .. automethod:: add_transformation_function 10 | .. automethod:: slice 11 | .. automethod:: transform 12 | .. automethod:: process -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | models/index 8 | datasets/index 9 | scan/index 10 | tests/index 11 | slicing-functions/index 12 | transformation-functions/index 13 | suite/index 14 | rag-toolset/index 15 | -------------------------------------------------------------------------------- /docs/reference/models/index.rst: -------------------------------------------------------------------------------- 1 | Models 2 | ====== 3 | 4 | .. automodule:: giskard.models 5 | 6 | Integrations 7 | ------------ 8 | 9 | .. toctree:: 10 | :glob: 11 | :maxdepth: 1 12 | 13 | integrations/* 14 | 15 | 16 | The :class:`giskard.Model` class 17 | -------------------------------- 18 | 19 | .. autoclass:: giskard.Model 20 | 21 | .. automethod:: __new__ 22 | .. automethod:: is_classification 23 | .. automethod:: is_binary_classification 24 | .. automethod:: is_regression 25 | .. automethod:: is_text_generation 26 | .. automethod:: model_predict 27 | .. automethod:: predict 28 | .. automethod:: save_model 29 | .. automethod:: load_model 30 | 31 | 32 | 33 | 34 | 35 | Model Prediction 36 | ---------------- 37 | .. autoclass:: giskard.models.base.ModelPredictionResults 38 | 39 | 40 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/base_classes.rst: -------------------------------------------------------------------------------- 1 | Base model classes 2 | ================== 3 | 4 | .. automodule:: giskard.models.base 5 | :members: BaseModel, WrapperModel, MLFlowSerializableModel, CloudpickleSerializableModel 6 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/catboost.rst: -------------------------------------------------------------------------------- 1 | Catboost models 2 | =============== 3 | 4 | .. automodule:: giskard.models.catboost 5 | :members: CatboostModel 6 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/function.rst: -------------------------------------------------------------------------------- 1 | Prediction function 2 | =================== 3 | 4 | .. automodule:: giskard.models.function 5 | :members: PredictionFunctionModel 6 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/huggingface.rst: -------------------------------------------------------------------------------- 1 | HuggingFace models 2 | ================== 3 | 4 | .. automodule:: giskard.models.huggingface 5 | :members: HuggingFaceModel 6 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/langchain.rst: -------------------------------------------------------------------------------- 1 | Langchain models 2 | ================ 3 | 4 | .. automodule:: giskard.models.langchain 5 | :members: LangchainModel 6 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/pytorch.rst: -------------------------------------------------------------------------------- 1 | Pytorch models 2 | ============== 3 | 4 | .. automodule:: giskard.models.pytorch 5 | :members: PyTorchModel 6 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/sklearn.rst: -------------------------------------------------------------------------------- 1 | Sklearn models 2 | ============== 3 | 4 | .. automodule:: giskard.models.sklearn 5 | :members: SKLearnModel 6 | -------------------------------------------------------------------------------- /docs/reference/models/integrations/tensorflow.rst: -------------------------------------------------------------------------------- 1 | Tensorflow models 2 | ================= 3 | 4 | .. automodule:: giskard.models.tensorflow 5 | :members: TensorFlowModel 6 | -------------------------------------------------------------------------------- /docs/reference/rag-toolset/evaluation.rst: -------------------------------------------------------------------------------- 1 | Evaluation 2 | ========== 3 | 4 | .. autofunction:: giskard.rag.evaluate 5 | 6 | .. autoclass:: giskard.llm.evaluators.CorrectnessEvaluator 7 | :members: 8 | -------------------------------------------------------------------------------- /docs/reference/rag-toolset/index.rst: -------------------------------------------------------------------------------- 1 | RAG Evaluation Toolkit 2 | ====================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | testset_generation 8 | knowledge_base 9 | evaluation 10 | question_generation 11 | metrics 12 | 13 | -------------------------------------------------------------------------------- /docs/reference/rag-toolset/knowledge_base.rst: -------------------------------------------------------------------------------- 1 | Knowledge Base 2 | ============== 3 | 4 | .. autoclass:: giskard.rag.knowledge_base.KnowledgeBase 5 | :members: 6 | 7 | .. autoclass:: giskard.rag.knowledge_base.Document 8 | :members: 9 | -------------------------------------------------------------------------------- /docs/reference/rag-toolset/metrics.rst: -------------------------------------------------------------------------------- 1 | 2 | Available Metric functions 3 | ========================== 4 | 5 | Correctness 6 | ----------- 7 | Using LLM as a judge strategy, the correctness metrics check if an answer is correct compared to the reference answer. 8 | 9 | .. autofunction:: giskard.rag.metrics.correctness.correctness_metric 10 | 11 | RAGAS Metrics 12 | ------------- 13 | We provide wrappers for some RAGAS metrics. You can implement other RAGAS metrics using the `RAGASMetric` class. 14 | .. autofunction:: giskard.rag.metrics.ragas_metrics.ragas_context_precision 15 | 16 | .. autofunction:: giskard.rag.metrics.ragas_metrics.ragas_faithfulness 17 | 18 | .. autofunction:: giskard.rag.metrics.ragas_metrics.ragas_answer_relevancy 19 | 20 | .. autofunction:: giskard.rag.metrics.ragas_metrics.ragas_context_recall 21 | 22 | Base Metric 23 | ----------- 24 | .. autoclass:: giskard.rag.metrics.Metric 25 | :members: 26 | :special-members: __call__ 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/reference/rag-toolset/question_generation.rst: -------------------------------------------------------------------------------- 1 | Question Generation 2 | =================== 3 | 4 | .. autoclass:: giskard.rag.question_generators.SimpleQuestionsGenerator 5 | :members: 6 | 7 | .. autoclass:: giskard.rag.question_generators.ComplexQuestionsGenerator 8 | :members: 9 | 10 | .. autoclass:: giskard.rag.question_generators.DistractingQuestionsGenerator 11 | :members: 12 | 13 | .. autoclass:: giskard.rag.question_generators.SituationalQuestionsGenerator 14 | :members: 15 | 16 | .. autoclass:: giskard.rag.question_generators.DoubleQuestionsGenerator 17 | :members: 18 | 19 | .. autoclass:: giskard.rag.question_generators.ConversationalQuestionsGenerator 20 | :members: 21 | 22 | .. autoclass:: giskard.rag.question_generators.OutOfScopeGenerator 23 | :members: 24 | -------------------------------------------------------------------------------- /docs/reference/rag-toolset/testset_generation.rst: -------------------------------------------------------------------------------- 1 | Testset Generation 2 | ================== 3 | 4 | .. autofunction:: giskard.rag.generate_testset 5 | 6 | .. autoclass:: giskard.rag.QATestset 7 | :members: -------------------------------------------------------------------------------- /docs/reference/scan/detectors.rst: -------------------------------------------------------------------------------- 1 | Tabular & NLP Detectors 2 | ======================= 3 | 4 | Performance 5 | ----------- 6 | .. automodule:: giskard.scanner.performance 7 | :members: 8 | :show-inheritance: 9 | 10 | Robustness 11 | ---------- 12 | .. automodule:: giskard.scanner.robustness 13 | :members: 14 | :show-inheritance: 15 | 16 | Calibration 17 | ----------- 18 | .. automodule:: giskard.scanner.calibration 19 | :members: 20 | :show-inheritance: 21 | 22 | Data Leakage 23 | ------------ 24 | .. automodule:: giskard.scanner.data_leakage 25 | :members: 26 | :show-inheritance: 27 | 28 | Stochasticity 29 | ------------- 30 | .. automodule:: giskard.scanner.stochasticity 31 | :members: 32 | :show-inheritance: 33 | 34 | -------------------------------------------------------------------------------- /docs/reference/scan/index.rst: -------------------------------------------------------------------------------- 1 | Model Scanner 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | report 8 | detectors 9 | llm_detectors 10 | 11 | .. automodule:: giskard.scanner 12 | :members: -------------------------------------------------------------------------------- /docs/reference/scan/llm_detectors.rst: -------------------------------------------------------------------------------- 1 | Detectors for LLM models 2 | ======================== 3 | 4 | 5 | Injection attacks 6 | ----------------- 7 | 8 | .. autoclass :: giskard.scanner.llm.LLMCharsInjectionDetector 9 | :members: 10 | :show-inheritance: 11 | 12 | .. autoclass :: giskard.scanner.llm.LLMPromptInjectionDetector 13 | :members: 14 | :show-inheritance: 15 | 16 | Hallucination & misinformation 17 | ------------------------------ 18 | 19 | .. autoclass :: giskard.scanner.llm.LLMBasicSycophancyDetector 20 | :members: 21 | :show-inheritance: 22 | 23 | .. autoclass :: giskard.scanner.llm.LLMImplausibleOutputDetector 24 | :members: 25 | :show-inheritance: 26 | 27 | 28 | Harmful content generation 29 | -------------------------- 30 | 31 | .. autoclass :: giskard.scanner.llm.LLMHarmfulContentDetector 32 | :members: 33 | :show-inheritance: 34 | 35 | Stereotypes 36 | ----------- 37 | 38 | .. autoclass :: giskard.scanner.llm.LLMStereotypesDetector 39 | :members: 40 | :show-inheritance: 41 | 42 | 43 | Information disclosure 44 | ---------------------- 45 | 46 | .. autoclass :: giskard.scanner.llm.LLMInformationDisclosureDetector 47 | :members: 48 | :show-inheritance: 49 | 50 | Output formatting 51 | ----------------- 52 | 53 | .. autoclass :: giskard.scanner.llm.LLMOutputFormattingDetector 54 | :members: 55 | :show-inheritance: 56 | -------------------------------------------------------------------------------- /docs/reference/scan/report.rst: -------------------------------------------------------------------------------- 1 | Scan Report 2 | =========== 3 | 4 | .. autoclass:: giskard.scanner.report.ScanReport 5 | :members: 6 | :show-inheritance: 7 | 8 | -------------------------------------------------------------------------------- /docs/reference/slicing-functions/index.rst: -------------------------------------------------------------------------------- 1 | Slicing functions 2 | ================= 3 | 4 | .. autofunction:: giskard.slicing_function 5 | 6 | .. autoclass:: giskard.registry.slicing_function.SlicingFunction 7 | 8 | .. automethod:: execute 9 | 10 | Textual slicing 11 | --------------- 12 | 13 | .. autofunction:: giskard.functions.slicing.short_comment_slicing_fn 14 | .. autofunction:: giskard.functions.slicing.keyword_lookup_slicing_fn 15 | .. autofunction:: giskard.functions.slicing.positive_sentiment_analysis 16 | .. autofunction:: giskard.functions.slicing.offensive_sentiment_analysis 17 | .. autofunction:: giskard.functions.slicing.irony_sentiment_analysis 18 | .. autofunction:: giskard.functions.slicing.hate_sentiment_analysis 19 | .. autofunction:: giskard.functions.slicing.emotion_sentiment_analysis 20 | 21 | Numerical slicing functions 22 | --------------------------- 23 | 24 | .. autofunction:: giskard.functions.slicing.outlier_filter 25 | -------------------------------------------------------------------------------- /docs/reference/suite/index.rst: -------------------------------------------------------------------------------- 1 | Test suite 2 | ============== 3 | 4 | .. autoclass:: giskard.Suite 5 | 6 | .. automethod:: __init__ 7 | .. automethod:: run 8 | .. automethod:: add_test 9 | .. automethod:: add_test 10 | .. automethod:: remove_test 11 | .. automethod:: upgrade_test 12 | .. automethod:: update_test_params 13 | 14 | .. autoclass:: giskard.core.suite.SuiteInput 15 | 16 | .. autoclass:: giskard.core.suite.DatasetInput 17 | 18 | .. autoclass:: giskard.core.suite.ModelInput 19 | 20 | .. autoclass:: giskard.core.suite.TestSuiteResult 21 | 22 | .. autoclass:: giskard.core.test_result.TestResult 23 | -------------------------------------------------------------------------------- /docs/reference/tests/data.rst: -------------------------------------------------------------------------------- 1 | Data quality tests 2 | ^^^^^^^^^^^^^^^^^^^^^ 3 | .. autofunction:: giskard.testing.test_data_uniqueness 4 | .. autofunction:: giskard.testing.test_data_completeness 5 | .. autofunction:: giskard.testing.test_valid_range 6 | .. autofunction:: giskard.testing.test_valid_values 7 | .. autofunction:: giskard.testing.test_data_correlation 8 | .. autofunction:: giskard.testing.test_outlier_value 9 | .. autofunction:: giskard.testing.test_foreign_constraint 10 | .. autofunction:: giskard.testing.test_label_consistency 11 | .. autofunction:: giskard.testing.test_mislabeling 12 | .. autofunction:: giskard.testing.test_feature_importance 13 | .. autofunction:: giskard.testing.test_class_imbalance 14 | -------------------------------------------------------------------------------- /docs/reference/tests/drift.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Drift tests 4 | ^^^^^^^^^^^^^^^^^^^^^ 5 | .. autofunction:: giskard.testing.test_drift_psi 6 | .. autofunction:: giskard.testing.test_drift_chi_square 7 | .. autofunction:: giskard.testing.test_drift_ks 8 | .. autofunction:: giskard.testing.test_drift_earth_movers_distance 9 | .. autofunction:: giskard.testing.test_drift_prediction_psi 10 | .. autofunction:: giskard.testing.test_drift_prediction_chi_square 11 | .. autofunction:: giskard.testing.test_drift_prediction_ks 12 | .. autofunction:: giskard.testing.test_drift_prediction_earth_movers_distance -------------------------------------------------------------------------------- /docs/reference/tests/index.rst: -------------------------------------------------------------------------------- 1 | Tests 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | metamorphic 8 | statistic 9 | performance 10 | drift 11 | llm 12 | data 13 | stability 14 | -------------------------------------------------------------------------------- /docs/reference/tests/llm.rst: -------------------------------------------------------------------------------- 1 | LLM tests 2 | ^^^^^^^^^^^^^^^^^^^^^ 3 | 4 | Injections 5 | ---------- 6 | .. autofunction:: giskard.testing.tests.llm.test_llm_char_injection 7 | .. autofunction:: giskard.testing.tests.llm.test_llm_single_output_against_strings 8 | .. autofunction:: giskard.testing.tests.llm.test_llm_output_against_strings 9 | 10 | LLM-as-a-judge 11 | -------------- 12 | .. autofunction:: giskard.testing.tests.llm.test_llm_output_coherency 13 | .. autofunction:: giskard.testing.tests.llm.test_llm_output_plausibility 14 | .. autofunction:: giskard.testing.tests.llm.test_llm_output_against_requirement_per_row 15 | .. autofunction:: giskard.testing.tests.llm.test_llm_single_output_against_requirement 16 | .. autofunction:: giskard.testing.tests.llm.test_llm_output_against_requirement 17 | .. autofunction:: giskard.testing.tests.llm.test_llm_correctness 18 | 19 | Ground Truth 20 | -------------- 21 | .. autofunction:: giskard.testing.tests.llm.test_llm_ground_truth 22 | .. autofunction:: giskard.testing.tests.llm.test_llm_ground_truth_similarity 23 | .. autofunction:: giskard.testing.tests.llm.test_llm_as_a_judge_ground_truth_similarity 24 | -------------------------------------------------------------------------------- /docs/reference/tests/metamorphic.rst: -------------------------------------------------------------------------------- 1 | Metamorphic tests 2 | ^^^^^^^^^^^^^^^^^^^^^ 3 | .. autofunction:: giskard.testing.test_metamorphic_invariance 4 | .. autofunction:: giskard.testing.test_metamorphic_increasing 5 | .. autofunction:: giskard.testing.test_metamorphic_decreasing 6 | .. autofunction:: giskard.testing.test_metamorphic_decreasing_t_test 7 | .. autofunction:: giskard.testing.test_metamorphic_increasing_t_test 8 | .. autofunction:: giskard.testing.test_metamorphic_invariance_t_test 9 | .. autofunction:: giskard.testing.test_metamorphic_decreasing_wilcoxon 10 | .. autofunction:: giskard.testing.test_metamorphic_increasing_wilcoxon 11 | .. autofunction:: giskard.testing.test_metamorphic_invariance_wilcoxon 12 | -------------------------------------------------------------------------------- /docs/reference/tests/performance.rst: -------------------------------------------------------------------------------- 1 | Performance tests 2 | ^^^^^^^^^^^^^^^^^^^^^ 3 | .. autofunction:: giskard.testing.test_mae 4 | .. autofunction:: giskard.testing.test_rmse 5 | .. autofunction:: giskard.testing.test_recall 6 | .. autofunction:: giskard.testing.test_auc 7 | .. autofunction:: giskard.testing.test_accuracy 8 | .. autofunction:: giskard.testing.test_precision 9 | .. autofunction:: giskard.testing.test_f1 10 | .. autofunction:: giskard.testing.test_r2 11 | .. autofunction:: giskard.testing.test_diff_recall 12 | .. autofunction:: giskard.testing.test_diff_accuracy 13 | .. autofunction:: giskard.testing.test_diff_precision 14 | .. autofunction:: giskard.testing.test_diff_rmse 15 | .. autofunction:: giskard.testing.test_diff_f1 16 | .. autofunction:: giskard.testing.test_brier 17 | -------------------------------------------------------------------------------- /docs/reference/tests/stability.rst: -------------------------------------------------------------------------------- 1 | Stability tests 2 | ^^^^^^^^^^^^^^^^^^^^^ 3 | .. autofunction:: giskard.testing.test_smoothness 4 | .. autofunction:: giskard.testing.test_monotonicity 5 | -------------------------------------------------------------------------------- /docs/reference/tests/statistic.rst: -------------------------------------------------------------------------------- 1 | Statistical tests 2 | ^^^^^^^^^^^^^^^^^^^^^ 3 | .. autofunction:: giskard.testing.test_right_label 4 | .. autofunction:: giskard.testing.test_output_in_range 5 | .. autofunction:: giskard.testing.test_disparate_impact 6 | .. autofunction:: giskard.testing.test_nominal_association 7 | .. autofunction:: giskard.testing.test_cramer_v 8 | .. autofunction:: giskard.testing.test_mutual_information 9 | .. autofunction:: giskard.testing.test_theil_u 10 | -------------------------------------------------------------------------------- /docs/reference/transformation-functions/index.rst: -------------------------------------------------------------------------------- 1 | .. _transformation_functions: 2 | 3 | Transformation functions 4 | ======================== 5 | 6 | .. autofunction:: giskard.transformation_function 7 | 8 | .. autoclass:: giskard.registry.transformation_function.TransformationFunction 9 | 10 | .. automethod:: execute 11 | 12 | Textual transformation functions 13 | -------------------------------- 14 | 15 | .. autofunction:: giskard.functions.transformation.keyboard_typo_transformation 16 | .. autofunction:: giskard.functions.transformation.text_uppercase 17 | .. autofunction:: giskard.functions.transformation.text_lowercase 18 | .. autofunction:: giskard.functions.transformation.text_title_case 19 | .. autofunction:: giskard.functions.transformation.text_typo 20 | .. autofunction:: giskard.functions.transformation.text_typo_from_ocr 21 | .. autofunction:: giskard.functions.transformation.text_punctuation_removal 22 | .. autofunction:: giskard.functions.transformation.text_accent_removal 23 | .. autofunction:: giskard.functions.transformation.text_gender_switch 24 | .. autofunction:: giskard.functions.transformation.text_number_to_word 25 | .. autofunction:: giskard.functions.transformation.text_religion_switch 26 | .. autofunction:: giskard.functions.transformation.text_nationality_switch 27 | .. autofunction:: giskard.functions.transformation.text_typo_from_speech 28 | 29 | 30 | Special transformations used by the scan 31 | ---------------------------------------- 32 | 33 | .. automodule:: giskard.scanner.robustness.text_transformations 34 | :members: 35 | :show-inheritance: 36 | -------------------------------------------------------------------------------- /docs/tutorials/index.md: -------------------------------------------------------------------------------- 1 | # Tutorials 2 | ```{toctree} 3 | :caption: Tutorials 4 | :maxdepth: 1 5 | :hidden: 6 | 7 | llm_tutorials/index 8 | rag_tutorials/index 9 | tabular_tutorials/index 10 | nlp_tutorials/index 11 | vision_tutorials/index 12 | ``` 13 | 14 | ::::::{grid} 1 1 2 2 15 | 16 | 17 | ::::{grid-item-card}

📚 LLM tutorials

18 | :text-align: center 19 | :link: llm_tutorials/index.html 20 | :::: 21 | 22 | ::::{grid-item-card}

🗄️ RAG tutorials

23 | :text-align: center 24 | :link: rag_tutorials/index.html 25 | :::: 26 | 27 | ::::{grid-item-card}

📊 Tabular tutorials

28 | :text-align: center 29 | :link: tabular_tutorials/index.html 30 | :::: 31 | 32 | ::::{grid-item-card}

🗣️ NLP tutorials

33 | :text-align: center 34 | :link: nlp_tutorials/index.html 35 | :::: 36 | 37 | ::::{grid-item-card}

📸 Vision tutorials

38 | :text-align: center 39 | :link: vision_tutorials/index.html 40 | :::: -------------------------------------------------------------------------------- /docs/tutorials/rag_tutorials/index.md: -------------------------------------------------------------------------------- 1 | # RAG Tutorials 2 | 3 | ```{toctree} 4 | :caption: Table of Contents 5 | :maxdepth: 1 6 | :hidden: 7 | 8 | ../../reference/notebooks/RAGET_IPCC.ipynb 9 | ../../reference/notebooks/RAGET_Banking_Supervision.ipynb 10 | 11 | 12 | 13 | ``` 14 | 15 | :::::{grid} 1 1 2 2 16 | 17 | ::::{grid-item-card}

IPCC Climate Change Report

RAGET Demo with LlamaIndex RAG 18 | :text-align: center 19 | :link: ../../reference/notebooks/RAGET_IPCC.ipynb 20 | :::: 21 | 22 | ::::{grid-item-card}

ECB Banking Supervision Report

RAGET Demo with LlamaIndex RAG 23 | :text-align: center 24 | :link: ../../reference/notebooks/RAGET_Banking_Supervision.ipynb 25 | :::: 26 | 27 | ::::: 28 | -------------------------------------------------------------------------------- /docs/tutorials/vision_tutorials/index.md: -------------------------------------------------------------------------------- 1 | # Vision Tutorials 2 | ```{toctree} 3 | :caption: Table of Contents 4 | :maxdepth: 1 5 | :hidden: 6 | 7 | ../../reference/notebooks/vision_landmark_detection.ipynb 8 | ../../reference/notebooks/vision_object_detection.ipynb 9 | ``` 10 | 11 | :::::{grid} 1 1 2 2 12 | 13 | ::::{grid-item-card}

Face landmark detection on the FFHQ dataset

Landmark detection (Open CV) 14 | :text-align: center 15 | :link: ../../reference/notebooks/vision_landmark_detection.ipynb 16 | :::: 17 | 18 | ::::{grid-item-card}

Detection of faces on the 300W dataset

Object detection (Hugging Face) 19 | :text-align: center 20 | :link: ../../reference/notebooks/vision_object_detection.ipynb 21 | :::: 22 | 23 | ::::: 24 | -------------------------------------------------------------------------------- /giskard/client/dtos.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Optional 2 | 3 | from giskard.core.validation import ConfiguredBaseModel 4 | 5 | 6 | class ServerInfo(ConfiguredBaseModel): 7 | instanceId: Optional[str] = None 8 | serverVersion: Optional[str] = None 9 | instanceLicenseId: Optional[str] = None 10 | user: Optional[str] = None 11 | 12 | 13 | class ModelMetaInfo(ConfiguredBaseModel): 14 | id: str 15 | name: str 16 | modelType: str 17 | featureNames: List[str] 18 | threshold: Optional[float] = None 19 | description: Optional[str] = None 20 | classificationLabels: Optional[List[str]] = None 21 | classificationLabelsDtype: Optional[str] = None 22 | languageVersion: str 23 | language: str 24 | createdDate: str 25 | size: int 26 | projectId: int 27 | 28 | 29 | class DatasetMetaInfo(ConfiguredBaseModel): 30 | target: Optional[str] = None 31 | columnTypes: Dict[str, str] 32 | columnDtypes: Dict[str, str] 33 | numberOfRows: int 34 | categoryFeatures: Dict[str, List[str]] 35 | name: Optional[str] = None 36 | originalSizeBytes: int 37 | compressedSizeBytes: int 38 | createdDate: str 39 | id: str 40 | -------------------------------------------------------------------------------- /giskard/client/python_utils.py: -------------------------------------------------------------------------------- 1 | """Various utility functions to manage Python environments""" 2 | 3 | import os 4 | import sys 5 | import warnings 6 | from platform import python_version 7 | 8 | import importlib_metadata 9 | 10 | # Libs to be excluded when create a kernel automatically from the current env 11 | EXCLUDED_PYLIBS = [ 12 | "setuptools", 13 | "pip", 14 | ] 15 | 16 | 17 | def get_python_requirements() -> str: 18 | pip_requirements = os.popen(f"{sys.executable} -m pip list --format freeze").read() 19 | if pip_requirements: 20 | return pip_requirements 21 | else: 22 | raise RuntimeError( 23 | "Python requirements could not be resolved. " 24 | + "Please use one of the following Python package managers: " 25 | + "Poetry, Pipenv or Pip." 26 | ) 27 | 28 | 29 | def get_python_version() -> str: 30 | return python_version() 31 | 32 | 33 | def warning(content: str): 34 | warnings.warn(content, stacklevel=2) 35 | 36 | 37 | def format_pylib_extras(name): 38 | extras = importlib_metadata.metadata(name).get_all("Provides-Extra") 39 | if not extras or len(extras) == 0: 40 | return "" 41 | else: 42 | return f"[{', '.join(extras)}]" 43 | -------------------------------------------------------------------------------- /giskard/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/core/__init__.py -------------------------------------------------------------------------------- /giskard/core/errors.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | 4 | class GiskardInstallationError(ImportError): 5 | functionality: Optional[str] 6 | flavor: Optional[str] 7 | 8 | def __init__(self, flavor: str = None, functionality: str = None, msg: str = None) -> None: 9 | self.msg = msg 10 | if not self.msg: 11 | self.flavor = flavor or self.flavor 12 | self.functionality = functionality or self.functionality 13 | assert self.flavor, "Either giskard package flavor or custom error message should be provided" 14 | 15 | self.msg = ( 16 | f"It seems that you are using Giskard {self.functionality or self.flavor} functionality " 17 | "but you are missing some required package. Please install Giskard " 18 | f"with {self.functionality or self.flavor} support with `pip install giskard[{self.flavor}]`." 19 | ) 20 | 21 | 22 | class GiskardImportError(ImportError): 23 | def __init__(self, missing_package: str) -> None: 24 | self.msg = f"The '{missing_package}' Python package is not installed; please execute 'pip install {missing_package}' to obtain it." 25 | -------------------------------------------------------------------------------- /giskard/core/kwargs_utils.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Optional, Set, Tuple, Union 2 | 3 | import inspect 4 | from collections import defaultdict 5 | 6 | 7 | def get_imports_code(obj): 8 | imports = _get_imports(obj) 9 | return "\n".join([f'from {module} import {", ".join(classes)}' for module, classes in imports.items()]) 10 | 11 | 12 | def _get_imports(obj, imports: Optional[defaultdict] = None): 13 | imports = imports or defaultdict(set) 14 | module = inspect.getmodule(obj) 15 | 16 | if isinstance(obj, Dict): 17 | for i in obj.values(): 18 | imports = _get_imports(i, imports) 19 | elif isinstance(obj, Union[List, Set, Tuple]): 20 | for i in obj: 21 | print(i) 22 | imports = _get_imports(i, imports) 23 | elif hasattr(obj, "__dict__"): 24 | imports = _get_imports(obj.__dict__.values(), imports) 25 | 26 | if module is None: 27 | return imports 28 | 29 | imports[module.__name__].add(obj.__class__.__name__) 30 | return imports 31 | -------------------------------------------------------------------------------- /giskard/datasets/metadata/__init__.py: -------------------------------------------------------------------------------- 1 | from .indexing import ColumnMetadataMixin 2 | from .registry import MetadataProvider, MetadataProviderRegistry 3 | 4 | __all__ = [ 5 | "MetadataProviderRegistry", 6 | "MetadataProvider", 7 | "ColumnMetadataMixin", 8 | ] 9 | -------------------------------------------------------------------------------- /giskard/datasets/metadata/registry.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | from abc import ABC, abstractmethod 4 | 5 | import pandas as pd 6 | 7 | from ...core.core import ColumnType 8 | 9 | 10 | class MetadataProvider(ABC): 11 | name: str 12 | 13 | def __init__(self, name): 14 | self.name = name 15 | 16 | @abstractmethod 17 | def generate_metadata(self, values: pd.Series) -> pd.DataFrame: 18 | ... 19 | 20 | @abstractmethod 21 | def supported_types(self) -> Sequence[ColumnType]: 22 | ... 23 | 24 | 25 | class MetadataProviderRegistry: 26 | _provider = dict() 27 | 28 | @classmethod 29 | def register(cls, provider): 30 | cls._provider[provider.name] = provider 31 | 32 | @classmethod 33 | def get_provider(cls, name) -> MetadataProvider: 34 | return cls._provider[name] 35 | 36 | @classmethod 37 | def get_available_providers(cls) -> Sequence[str]: 38 | return list(cls._provider.keys()) 39 | -------------------------------------------------------------------------------- /giskard/datasets/metadata/text_metadata_provider.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from langdetect import DetectorFactory 3 | 4 | from ...utils.language_detection import detect_lang 5 | from .registry import MetadataProvider 6 | 7 | DetectorFactory.seed = 0 8 | 9 | 10 | class TextMetadataProvider(MetadataProvider): 11 | def __init__(self, name="text"): 12 | self.name = name 13 | 14 | def generate_metadata(self, values: pd.Series): 15 | # Ensure this is text encoded as a string 16 | values = values.astype(str) 17 | 18 | return pd.DataFrame( 19 | { 20 | "text_length": values.map(len), 21 | "charset": pd.Categorical(values.map(_detect_charset)), 22 | "language": values.map(detect_lang), 23 | }, 24 | index=values.index, 25 | ) 26 | 27 | def supported_types(self): 28 | return ["text"] 29 | 30 | 31 | def _detect_charset(text: str): 32 | import chardet 33 | 34 | charset = chardet.detect(text.encode("utf-8", errors="ignore"))["encoding"] 35 | return charset or "undefined" 36 | -------------------------------------------------------------------------------- /giskard/demo/__init__.py: -------------------------------------------------------------------------------- 1 | from . import linear_regression, titanic_classification 2 | 3 | 4 | def titanic_df(): 5 | return titanic_classification.get_test_df() 6 | 7 | 8 | def titanic(model="LogisticRegression", max_iter=100): 9 | return titanic_classification.get_model_and_df(model=model, max_iter=max_iter) 10 | 11 | 12 | def titanic_pipeline(): 13 | return titanic_classification.get_pipeline() 14 | 15 | 16 | def linear_df(): 17 | return linear_regression.get_df() 18 | 19 | 20 | def linear(): 21 | return linear_regression.get_model_and_df() 22 | 23 | 24 | def linear_pipeline(): 25 | return linear_regression.get_pipeline() 26 | 27 | 28 | __all__ = [ 29 | "titanic_df", 30 | "titanic", 31 | "titanic_pipeline", 32 | "linear_df", 33 | "linear", 34 | "linear_pipeline", 35 | ] 36 | -------------------------------------------------------------------------------- /giskard/demo/linear_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | 5 | def get_df(): 6 | df = pd.DataFrame({"x": np.arange(100), "y": np.arange(100)}) 7 | return df 8 | 9 | 10 | def get_model_and_df(): 11 | from sklearn.linear_model import LinearRegression 12 | 13 | df = get_df() 14 | 15 | reg = LinearRegression() 16 | reg.fit(df["x"].to_numpy().reshape(100, 1), df["y"].to_numpy().reshape(100, 1)) 17 | return reg, df 18 | 19 | 20 | def get_pipeline(): 21 | reg, _ = get_model_and_df() 22 | 23 | def preprocessor(df): 24 | return df["x"].to_numpy().reshape(len(df["x"]), 1) 25 | 26 | return preprocessor, reg 27 | -------------------------------------------------------------------------------- /giskard/exceptions/IllegalArgumentError.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class StatusCode(str, Enum): 5 | NO_ERROR = "NO_ERROR" 6 | INTERNAL = "INTERNAL" 7 | INVALID_ARGUMENT = "INVALID_ARGUMENT" 8 | 9 | 10 | class CodedError(Exception): 11 | code: StatusCode = StatusCode.INTERNAL 12 | 13 | def __init__(self, code: StatusCode, *args: object) -> None: 14 | super().__init__(*args) 15 | self.code = code 16 | 17 | 18 | class IllegalArgumentError(CodedError): 19 | def __init__(self, *args: object) -> None: 20 | super().__init__(StatusCode.INVALID_ARGUMENT, *args) 21 | -------------------------------------------------------------------------------- /giskard/exceptions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/exceptions/__init__.py -------------------------------------------------------------------------------- /giskard/functions/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["slicing", "transformation"] 2 | -------------------------------------------------------------------------------- /giskard/integrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/integrations/__init__.py -------------------------------------------------------------------------------- /giskard/integrations/mlflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/integrations/mlflow/__init__.py -------------------------------------------------------------------------------- /giskard/integrations/mlflow/evaluation_artifacts.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from mlflow.models import EvaluationArtifact 3 | from zstandard import ZstdDecompressor 4 | 5 | from giskard.client.io_utils import save_df 6 | 7 | 8 | class GiskardDatasetEvaluationArtifact(EvaluationArtifact): 9 | def _save(self, output_artifact_path): 10 | with open(output_artifact_path, "wb") as fw: 11 | uncompressed_bytes = save_df(self._content) 12 | fw.write(uncompressed_bytes) 13 | 14 | def _load_content_from_file(self, local_artifact_path): 15 | with open(local_artifact_path, "rb") as ds_stream: 16 | return pd.read_csv( 17 | ZstdDecompressor().stream_reader(ds_stream), 18 | keep_default_na=False, 19 | na_values=["_GSK_NA_"], 20 | ) 21 | 22 | 23 | class GiskardScanResultEvaluationArtifact(EvaluationArtifact): 24 | def _save(self, output_artifact_path): 25 | self._content.to_html(output_artifact_path) 26 | 27 | def _load_content_from_file(self, local_artifact_path): 28 | with open(local_artifact_path, "rb") as f: 29 | return f.read() 30 | 31 | 32 | class GiskardScanSummaryEvaluationArtifact(GiskardDatasetEvaluationArtifact): 33 | def _load_content_from_file(self, local_artifact_path): 34 | with open(local_artifact_path, "rb") as ds_stream: 35 | return pd.read_csv( 36 | ZstdDecompressor().stream_reader(ds_stream), 37 | keep_default_na=False, 38 | ) 39 | -------------------------------------------------------------------------------- /giskard/integrations/wandb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/integrations/wandb/__init__.py -------------------------------------------------------------------------------- /giskard/llm/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import get_default_client, set_default_client, set_llm_api, set_llm_model 2 | from .embeddings import set_default_embedding, set_embedding_model 3 | from .errors import LLMImportError 4 | 5 | __all__ = [ 6 | "LLMImportError", 7 | "get_default_client", 8 | "set_llm_model", 9 | "set_default_client", 10 | "set_llm_api", 11 | "set_default_embedding", 12 | "set_embedding_model", 13 | ] 14 | -------------------------------------------------------------------------------- /giskard/llm/client/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Sequence 2 | 3 | from abc import ABC, abstractmethod 4 | from dataclasses import dataclass 5 | 6 | from .logger import LLMLogger 7 | 8 | 9 | @dataclass 10 | class ChatMessage: 11 | role: str 12 | content: Optional[str] = None 13 | 14 | 15 | _logger = LLMLogger() 16 | 17 | 18 | class LLMClient(ABC): 19 | @property 20 | def logger(self) -> LLMLogger: 21 | return _logger 22 | 23 | @abstractmethod 24 | def complete( 25 | self, 26 | messages: Sequence[ChatMessage], 27 | temperature: float = 1, 28 | max_tokens: Optional[int] = None, 29 | caller_id: Optional[str] = None, 30 | seed: Optional[int] = None, 31 | format=None, 32 | ) -> ChatMessage: 33 | ... 34 | -------------------------------------------------------------------------------- /giskard/llm/client/logger.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | 4 | class LLMLogger: 5 | def __init__(self): 6 | self.calls = [] 7 | self.errors = [] 8 | 9 | def log_call( 10 | self, 11 | prompt_tokens: int, 12 | sampled_tokens: int, 13 | model: Optional[str] = None, 14 | client_class: Optional[str] = None, 15 | caller_id: Optional[str] = None, 16 | ): 17 | self.calls.append( 18 | { 19 | "prompt_tokens": prompt_tokens, 20 | "sampled_tokens": sampled_tokens, 21 | "model": model, 22 | "client_class": client_class, 23 | "caller_id": caller_id, 24 | } 25 | ) 26 | 27 | def log_error(self, error): 28 | self.errors.append(error) 29 | 30 | def reset(self): 31 | self.calls = [] 32 | 33 | def get_num_calls(self): 34 | return len(self.calls) 35 | 36 | def get_num_prompt_tokens(self): 37 | return sum(c["prompt_tokens"] for c in self.calls) 38 | 39 | def get_num_sampled_tokens(self): 40 | return sum(c["sampled_tokens"] for c in self.calls) 41 | -------------------------------------------------------------------------------- /giskard/llm/config.py: -------------------------------------------------------------------------------- 1 | class LLMConfigurationError(ValueError): 2 | """Raised when the LLM client is not configured properly.""" 3 | -------------------------------------------------------------------------------- /giskard/llm/embeddings/base.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | from abc import ABC, abstractmethod 4 | 5 | import numpy as np 6 | 7 | 8 | class BaseEmbedding(ABC): 9 | @abstractmethod 10 | def embed(self, texts: Sequence[str]) -> np.ndarray: 11 | ... 12 | -------------------------------------------------------------------------------- /giskard/llm/embeddings/bedrock.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | from typing_extensions import deprecated 3 | 4 | import json 5 | 6 | import numpy as np 7 | 8 | from .base import BaseEmbedding 9 | 10 | 11 | @deprecated("BedrockEmbedding is deprecated: https://docs.giskard.ai/en/latest/open_source/setting_up/index.html") 12 | class BedrockEmbedding(BaseEmbedding): 13 | def __init__(self, client, model: str): 14 | """ 15 | Parameters 16 | ---------- 17 | client : Bedrock 18 | boto3 based Bedrock runtime client instance. 19 | model : str 20 | Model name. 21 | """ 22 | self.model = model 23 | self.client = client 24 | 25 | def embed(self, texts: Sequence[str]) -> np.ndarray: 26 | if "titan" not in self.model: 27 | raise ValueError(f"Only titan embedding models are supported currently, got {self.model} instead") 28 | 29 | if isinstance(texts, str): 30 | texts = [texts] 31 | 32 | accept = "application/json" 33 | contentType = "application/json" 34 | embeddings = [] 35 | for text in texts: 36 | body = json.dumps({"inputText": text}) 37 | response = self.client.invoke_model(body=body, modelId=self.model, accept=accept, contentType=contentType) 38 | response_body = json.loads(response.get("body").read()) 39 | embedding = response_body.get("embedding") 40 | embeddings.append(embedding) 41 | 42 | return np.array(embeddings) 43 | -------------------------------------------------------------------------------- /giskard/llm/embeddings/fastembed.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Optional, Sequence 4 | 5 | import numpy as np 6 | 7 | from .base import BaseEmbedding 8 | 9 | if TYPE_CHECKING: 10 | from fastembed import TextEmbedding 11 | 12 | 13 | class FastEmbedEmbedding(BaseEmbedding): 14 | def __init__(self, text_embedding: TextEmbedding): 15 | """ 16 | Parameters 17 | ---------- 18 | text_embedding : TextEmbedding 19 | FastEmbed TextEmbedding model. 20 | """ 21 | self.text_embedding = text_embedding 22 | 23 | def embed(self, texts: Sequence[str]) -> np.ndarray: 24 | if isinstance(texts, str): 25 | texts = [texts] 26 | 27 | return np.array(list(self.text_embedding.embed(texts))) 28 | 29 | 30 | def try_get_fastembed_embeddings() -> Optional[FastEmbedEmbedding]: 31 | try: 32 | from fastembed import TextEmbedding 33 | 34 | from .fastembed import FastEmbedEmbedding 35 | 36 | return FastEmbedEmbedding( 37 | text_embedding=TextEmbedding("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2") 38 | ) 39 | except ImportError: 40 | return None 41 | -------------------------------------------------------------------------------- /giskard/llm/embeddings/litellm.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional, Sequence 2 | 3 | import numpy as np 4 | 5 | from ...utils.iterables import batched 6 | from .. import LLMImportError 7 | from .base import BaseEmbedding 8 | 9 | try: 10 | import litellm 11 | except ImportError as err: 12 | raise LLMImportError(flavor="litellm") from err 13 | 14 | 15 | class LiteLLMEmbedding(BaseEmbedding): 16 | def __init__(self, model: str, embedding_params: Optional[Dict[str, Any]] = None, batch_size=40): 17 | """Initialize a LiteLLM embedding client 18 | 19 | Parameters 20 | ---------- 21 | model : str 22 | Model name. 23 | embedding_params : dict, optional) 24 | A dictionary containing params for the completion. 25 | batch_size : int, optional 26 | Batch size for embeddings, by default 40. 27 | """ 28 | self.model = model 29 | self.embedding_params = embedding_params or dict() 30 | self.batch_size = batch_size 31 | 32 | def embed(self, texts: Sequence[str]) -> np.ndarray: 33 | if isinstance(texts, str): 34 | texts = [texts] 35 | 36 | embeddings = [] 37 | for batch in batched(texts, self.batch_size): 38 | response = litellm.embedding(model=self.model, input=batch, **self.embedding_params) 39 | embeddings.extend([item["embedding"] for item in response.data]) 40 | 41 | return np.array(embeddings) 42 | -------------------------------------------------------------------------------- /giskard/llm/errors.py: -------------------------------------------------------------------------------- 1 | from ..core.errors import GiskardInstallationError 2 | 3 | 4 | class LLMImportError(GiskardInstallationError): 5 | flavor = "llm" 6 | functionality = "LLM" 7 | 8 | 9 | class LLMGenerationError(RuntimeError): 10 | """Indicates a failure in LLM-based generation""" 11 | -------------------------------------------------------------------------------- /giskard/llm/evaluators/__init__.py: -------------------------------------------------------------------------------- 1 | from .coherency import CoherencyEvaluator 2 | from .correctness import CorrectnessEvaluator 3 | from .plausibility import PlausibilityEvaluator 4 | from .requirements import RequirementEvaluator 5 | 6 | __all__ = [ 7 | "CoherencyEvaluator", 8 | "RequirementEvaluator", 9 | "PlausibilityEvaluator", 10 | "CorrectnessEvaluator", 11 | ] 12 | -------------------------------------------------------------------------------- /giskard/llm/evaluators/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Sequence 2 | 3 | 4 | def format_conversation(conversation: Sequence[Dict]): 5 | return "\n\n".join([f"<{msg['role'].lower()}>{msg['content']}" for msg in conversation]) 6 | -------------------------------------------------------------------------------- /giskard/llm/generators/__init__.py: -------------------------------------------------------------------------------- 1 | from .adversarial import AdversarialDataGenerator 2 | from .base import LLMBasedDataGenerator 3 | from .implausible import ImplausibleDataGenerator 4 | from .sycophancy import SycophancyDataGenerator 5 | 6 | __all__ = [ 7 | "LLMBasedDataGenerator", 8 | "SycophancyDataGenerator", 9 | "ImplausibleDataGenerator", 10 | "AdversarialDataGenerator", 11 | ] 12 | -------------------------------------------------------------------------------- /giskard/llm/loaders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/llm/loaders/__init__.py -------------------------------------------------------------------------------- /giskard/llm/talk/tools/__init__.py: -------------------------------------------------------------------------------- 1 | from giskard.llm.talk.tools.base import BaseTool 2 | from giskard.llm.talk.tools.metric import MetricTool 3 | from giskard.llm.talk.tools.predict import PredictTool 4 | from giskard.llm.talk.tools.scan import IssuesScannerTool 5 | from giskard.llm.talk.tools.shap import SHAPExplanationTool 6 | 7 | __all__ = ["BaseTool", "PredictTool", "MetricTool", "IssuesScannerTool", "SHAPExplanationTool"] 8 | -------------------------------------------------------------------------------- /giskard/models/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import BaseModel 2 | from .model_prediction import ModelPredictionResults 3 | from .serialization import CloudpickleSerializableModel, MLFlowSerializableModel 4 | from .wrapper import WrapperModel 5 | 6 | __all__ = [ 7 | "BaseModel", 8 | "ModelPredictionResults", 9 | "WrapperModel", 10 | "MLFlowSerializableModel", 11 | "CloudpickleSerializableModel", 12 | ] 13 | -------------------------------------------------------------------------------- /giskard/models/cache/__init__.py: -------------------------------------------------------------------------------- 1 | from contextlib import AbstractContextManager 2 | 3 | from .cache import ModelCache 4 | 5 | # This is a global switch that controls model prediction cache. 6 | _cache_enabled = True 7 | 8 | 9 | def set_cache_enabled(value: bool): 10 | global _cache_enabled 11 | _cache_enabled = value 12 | 13 | 14 | def get_cache_enabled(): 15 | global _cache_enabled 16 | return _cache_enabled 17 | 18 | 19 | def enable_cache(): 20 | set_cache_enabled(True) 21 | 22 | 23 | def disable_cache(): 24 | set_cache_enabled(False) 25 | 26 | 27 | class no_cache(AbstractContextManager): 28 | def __init__(self): 29 | self.prev = get_cache_enabled() 30 | 31 | def __enter__(self): 32 | set_cache_enabled(False) 33 | 34 | def __exit__(self, exc_type, exc_val, exc_tb): 35 | set_cache_enabled(self.prev) 36 | 37 | 38 | __all__ = ["set_cache_enabled", "get_cache_enabled", "enable_cache", "disable_cache", "no_cache", "ModelCache"] 39 | -------------------------------------------------------------------------------- /giskard/models/catboost.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | import mlflow 4 | 5 | from .sklearn import SKLearnModel 6 | 7 | 8 | class CatboostModel(SKLearnModel): 9 | """Automatically wraps ``catboost`` models for use with Giskard.""" 10 | 11 | _feature_names_attr = "feature_names_" 12 | 13 | def save_model(self, local_path, mlflow_meta: mlflow.models.Model, *args, **kwargs): 14 | mlflow.catboost.save_model(self.model, path=local_path, mlflow_model=mlflow_meta) 15 | 16 | @classmethod 17 | def load_model(cls, local_dir, model_py_ver: Optional[Tuple[str, str, str]] = None, *args, **kwargs): 18 | return mlflow.catboost.load_model(local_dir) 19 | 20 | def to_mlflow(self, artifact_path: str = "catboost-model-from-giskard", **kwargs): 21 | return mlflow.catboost.log_model(self.model, artifact_path, **kwargs) 22 | -------------------------------------------------------------------------------- /giskard/models/function.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Iterable, Optional 2 | 3 | import numpy 4 | import pandas as pd 5 | 6 | from ..core.core import ModelType 7 | from ..core.validation import configured_validate_arguments 8 | from .base.serialization import CloudpickleSerializableModel 9 | 10 | 11 | class PredictionFunctionModel(CloudpickleSerializableModel): 12 | @configured_validate_arguments 13 | def __init__( 14 | self, 15 | model: Callable, 16 | model_type: ModelType, 17 | data_preprocessing_function: Optional[Callable[[pd.DataFrame], Any]] = None, 18 | model_postprocessing_function: Optional[Callable[[Any], Any]] = None, 19 | name: Optional[str] = None, 20 | feature_names: Optional[Iterable] = None, 21 | classification_threshold: Optional[float] = 0.5, 22 | classification_labels: Optional[Iterable] = None, 23 | id: Optional[str] = None, 24 | **kwargs, 25 | ) -> None: 26 | super().__init__( 27 | model, 28 | model_type, 29 | data_preprocessing_function, 30 | model_postprocessing_function, 31 | name, 32 | feature_names, 33 | classification_threshold, 34 | classification_labels, 35 | id=id, 36 | **kwargs, 37 | ) 38 | 39 | def model_predict(self, df: pd.DataFrame) -> numpy.ndarray: 40 | return self.model(df) 41 | -------------------------------------------------------------------------------- /giskard/models/talk_result.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from giskard.llm.client import ChatMessage 4 | 5 | 6 | @dataclass 7 | class TalkResult: 8 | """The dataclass containing the result of the 'talk' call. 9 | 10 | Attributes 11 | ---------- 12 | response : ChatMessage 13 | The response to the user's query. 14 | summary : ChatMessage 15 | The summary of the conversation between the user and the LLM agent. 16 | tool_errors : list[Exception] 17 | The list of errors raised during tools execution. 18 | """ 19 | 20 | response: ChatMessage 21 | summary: ChatMessage 22 | tool_errors: list[Exception] 23 | 24 | def __repr__(self) -> str: 25 | """Return the 'talk' result. 26 | 27 | Returns 28 | ------- 29 | str 30 | The 'Talk' result, containing an answer and a conversation summary. 31 | """ 32 | return ( 33 | f"LLM Response:\n" 34 | f"-------------\n" 35 | f"{self.response.content}\n\n" 36 | f"Full Conversation Summary:\n" 37 | f"--------------------------\n" 38 | f"{self.summary.content}\n\n" 39 | f"Tool Errors Raised: {len(self.tool_errors)}" 40 | ) 41 | -------------------------------------------------------------------------------- /giskard/models/utils.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import itertools 4 | import random 5 | from collections.abc import Iterator 6 | from functools import lru_cache 7 | 8 | import numpy as np 9 | 10 | 11 | def map_to_tuples(data: Iterator): 12 | item = next(data) 13 | data = itertools.chain([item], data) 14 | 15 | if isinstance(item, tuple): # nothing to do 16 | return data 17 | 18 | return map(lambda x: (x,), data) 19 | 20 | 21 | def np_types_to_native(some_list: List): 22 | if some_list is not None: 23 | return [np_type_to_native(i) for i in some_list] 24 | return some_list 25 | 26 | 27 | def np_type_to_native(i): 28 | return i.item() if isinstance(i, np.generic) else i 29 | 30 | 31 | def fix_seed(seed=1337): 32 | random.seed(seed) 33 | np.random.seed(seed) 34 | try: 35 | import torch 36 | 37 | torch.manual_seed(seed) 38 | torch.cuda.manual_seed_all(seed) 39 | except ImportError: 40 | pass 41 | 42 | 43 | @lru_cache(None) 44 | def warn_once(logger, msg: str): 45 | logger.warning(msg) 46 | -------------------------------------------------------------------------------- /giskard/path_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from giskard.settings import settings 4 | 5 | run_dir = settings.home_dir / "run" 6 | artifacts_dir = settings.home_dir / "artifacts" 7 | 8 | 9 | def get_size(path: str): 10 | size = 0 11 | 12 | for file_ in Path(path).rglob("*"): 13 | size += file_.stat().st_size 14 | 15 | return size 16 | -------------------------------------------------------------------------------- /giskard/rag/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import AgentAnswer 2 | from .evaluate import evaluate 3 | from .knowledge_base import KnowledgeBase 4 | from .report import RAGReport 5 | from .testset import QATestset, QuestionSample 6 | from .testset_generation import generate_testset 7 | 8 | __all__ = [ 9 | "QATestset", 10 | "QuestionSample", 11 | "generate_testset", 12 | "KnowledgeBase", 13 | "evaluate", 14 | "RAGReport", 15 | "AgentAnswer", 16 | ] 17 | -------------------------------------------------------------------------------- /giskard/rag/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Sequence 2 | 3 | from dataclasses import dataclass 4 | 5 | 6 | @dataclass 7 | class AgentAnswer: 8 | message: str 9 | documents: Optional[Sequence[str]] = None 10 | -------------------------------------------------------------------------------- /giskard/rag/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Metric 2 | from .correctness import CorrectnessMetric, correctness_metric 3 | 4 | __all__ = ["Metric", "correctness_metric", "CorrectnessMetric"] 5 | -------------------------------------------------------------------------------- /giskard/rag/metrics/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | from ...llm.client.base import LLMClient 4 | from ..base import AgentAnswer 5 | 6 | 7 | class Metric(ABC): 8 | """ 9 | Metric base class. All metrics should inherit from this class and implement the __call__ method. 10 | The instances of this class can be passed to the evaluate method. 11 | """ 12 | 13 | def __init__(self, name: str, llm_client: LLMClient = None) -> None: 14 | self.name = name 15 | self._llm_client = llm_client 16 | 17 | @abstractmethod 18 | def __call__(self, question_sample: dict, answer: AgentAnswer): 19 | """ 20 | Compute the metric on a single question and its associated answer. 21 | 22 | Parameters 23 | ---------- 24 | question_sample : dict 25 | A question sample from a QATestset. 26 | answer : AgentAnswer 27 | The agent answer on that question. 28 | 29 | Returns 30 | ------- 31 | dict 32 | The result of the metric computation. The keys should be the names of the metrics computed. 33 | """ 34 | pass 35 | -------------------------------------------------------------------------------- /giskard/rag/question_generators/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import QuestionGenerator 2 | from .complex_questions import ComplexQuestionsGenerator, complex_questions 3 | from .conversational_questions import ConversationalQuestionsGenerator, conversational_questions 4 | from .distracting_questions import DistractingQuestionsGenerator, distracting_questions 5 | from .double_questions import DoubleQuestionsGenerator, double_questions 6 | from .oos_questions import OutOfScopeGenerator, oos_questions 7 | from .question_types import COMPONENT_DESCRIPTIONS, QUESTION_ATTRIBUTION, RAGComponents 8 | from .simple_questions import SimpleQuestionsGenerator, simple_questions 9 | from .situational_questions import SituationalQuestionsGenerator, situational_questions 10 | 11 | __all__ = [ 12 | "RAGComponents", 13 | "QuestionGenerator", 14 | "QUESTION_ATTRIBUTION", 15 | "COMPONENT_DESCRIPTIONS", 16 | "SimpleQuestionsGenerator", 17 | "ComplexQuestionsGenerator", 18 | "ConversationalQuestionsGenerator", 19 | "DistractingQuestionsGenerator", 20 | "SituationalQuestionsGenerator", 21 | "DoubleQuestionsGenerator", 22 | "OutOfScopeGenerator", 23 | "simple_questions", 24 | "complex_questions", 25 | "conversational_questions", 26 | "distracting_questions", 27 | "situational_questions", 28 | "double_questions", 29 | "oos_questions", 30 | ] 31 | -------------------------------------------------------------------------------- /giskard/rag/question_generators/prompt.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from dataclasses import dataclass 4 | 5 | from ...llm.client import ChatMessage 6 | 7 | 8 | @dataclass 9 | class QAGenerationPrompt: 10 | system_prompt: str 11 | user_input_template: Optional[str] = None 12 | example_input: Optional[str] = None 13 | example_output: Optional[str] = None 14 | 15 | def _format_example_prompt(self, examples): 16 | if examples is not None: 17 | return examples 18 | 19 | examples = [] 20 | if self.example_input is not None: 21 | examples.append(ChatMessage(role="user", content=self.example_input)) 22 | if self.example_output is not None: 23 | examples.append(ChatMessage(role="assistant", content=self.example_output)) 24 | return examples 25 | 26 | def to_messages( 27 | self, 28 | system_prompt_input, 29 | user_input, 30 | add_examples=True, 31 | examples=None, 32 | ): 33 | messages = [ 34 | ChatMessage( 35 | role="system", 36 | content=self.system_prompt.format(**system_prompt_input), 37 | ) 38 | ] 39 | if add_examples: 40 | messages.extend(self._format_example_prompt(examples)) 41 | 42 | if self.user_input_template is None: 43 | messages.append(ChatMessage(role="user", content=user_input)) 44 | else: 45 | messages.append(ChatMessage(role="user", content=self.user_input_template.format(**user_input))) 46 | 47 | return messages 48 | -------------------------------------------------------------------------------- /giskard/rag/question_generators/question_types.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class RAGComponents(int, Enum): 5 | GENERATOR = 1 6 | RETRIEVER = 2 7 | REWRITER = 3 8 | ROUTING = 4 9 | KNOWLEDGE_BASE = 5 10 | 11 | 12 | QUESTION_ATTRIBUTION = { 13 | RAGComponents.GENERATOR: [ 14 | "simple", 15 | "complex", 16 | "distracting element", 17 | "situational", 18 | "double", 19 | ], 20 | RAGComponents.RETRIEVER: ["simple", "distracting element", "multi-context"], 21 | RAGComponents.REWRITER: [ 22 | "distracting element", 23 | "double", 24 | "conversational", 25 | "multi-context", 26 | ], 27 | RAGComponents.ROUTING: ["out of scope"], 28 | RAGComponents.KNOWLEDGE_BASE: ["out of scope"], 29 | } 30 | 31 | COMPONENT_DESCRIPTIONS = { 32 | "GENERATOR": "The Generator is the LLM inside the RAG to generate the answers.", 33 | "RETRIEVER": "The Retriever fetches relevant documents from the knowledge base according to a user query.", 34 | "REWRITER": "The Rewriter modifies the user query to match a predefined format or to include the context from the chat history.", 35 | "ROUTING": "The Router filters the query of the user based on his intentions (intentions detection).", 36 | "KNOWLEDGE_BASE": "The knowledge base is the set of documents given to the RAG to generate the answers. Its scores is computed differently from the other components: it is the difference between the maximum and minimum correctness score across all the topics of the knowledge base.", 37 | } 38 | -------------------------------------------------------------------------------- /giskard/registry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/registry/__init__.py -------------------------------------------------------------------------------- /giskard/scanner/calibration/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package provides detectors for potential problems related to model calibration. 3 | """ 4 | from .overconfidence_detector import OverconfidenceDetector 5 | from .underconfidence_detector import UnderconfidenceDetector 6 | 7 | __all__ = ["OverconfidenceDetector", "UnderconfidenceDetector"] 8 | -------------------------------------------------------------------------------- /giskard/scanner/common/utils.py: -------------------------------------------------------------------------------- 1 | from ...datasets.base import Dataset 2 | from ...models.base import BaseModel 3 | 4 | 5 | def get_dataset_subsample(dataset: Dataset, model: BaseModel, max_data_size: int): 6 | if max_data_size <= 0 or len(dataset) <= max_data_size: # just double checking 7 | return dataset 8 | if not model.is_classification: 9 | return dataset.slice(lambda df: df.sample(max_data_size, random_state=42), row_level=False) 10 | 11 | # Slice the dataset while keeping the class proportions 12 | keep_ratio = max_data_size / len(dataset) 13 | dataset = dataset.slice( 14 | lambda df: df.groupby(dataset.target, group_keys=False, sort=False).sample(frac=keep_ratio, random_state=42), 15 | row_level=False, 16 | ) 17 | return dataset 18 | -------------------------------------------------------------------------------- /giskard/scanner/correlation/__init__.py: -------------------------------------------------------------------------------- 1 | from .spurious_correlation_detector import SpuriousCorrelationDetector 2 | 3 | __all__ = ["SpuriousCorrelationDetector"] 4 | -------------------------------------------------------------------------------- /giskard/scanner/data_leakage/__init__.py: -------------------------------------------------------------------------------- 1 | from .data_leakage_detector import DataLeakageDetector 2 | 3 | __all__ = ["DataLeakageDetector"] 4 | -------------------------------------------------------------------------------- /giskard/scanner/decorators.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import re 3 | 4 | from .registry import DetectorRegistry 5 | 6 | 7 | def detector(name=None, tags=None): 8 | if inspect.isclass(name): 9 | # If the decorator is used without arguments, the first argument is the class 10 | cls = name 11 | DetectorRegistry.register(_to_snake_case(cls.__name__), cls) 12 | return cls 13 | 14 | def inner(cls): 15 | DetectorRegistry.register(name or _to_snake_case(cls.__name__), cls, tags) 16 | return cls 17 | 18 | return inner 19 | 20 | 21 | def _to_snake_case(string) -> str: 22 | s = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", string) 23 | return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s).lower() 24 | -------------------------------------------------------------------------------- /giskard/scanner/llm/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package provides LLM-specific detectors for the automatic model scan. 3 | """ 4 | from .llm_basic_sycophancy_detector import LLMBasicSycophancyDetector 5 | from .llm_chars_injection_detector import LLMCharsInjectionDetector 6 | from .llm_harmful_content_detector import LLMHarmfulContentDetector 7 | from .llm_implausible_output_detector import LLMImplausibleOutputDetector 8 | from .llm_information_disclosure_detector import LLMInformationDisclosureDetector 9 | from .llm_output_formatting_detector import LLMOutputFormattingDetector 10 | from .llm_prompt_injection_detector import LLMPromptInjectionDetector 11 | from .llm_stereotypes_detector import LLMStereotypesDetector 12 | 13 | __all__ = [ 14 | "LLMBasicSycophancyDetector", 15 | "LLMCharsInjectionDetector", 16 | "LLMHarmfulContentDetector", 17 | "LLMImplausibleOutputDetector", 18 | "LLMInformationDisclosureDetector", 19 | "LLMOutputFormattingDetector", 20 | "LLMPromptInjectionDetector", 21 | "LLMStereotypesDetector", 22 | ] 23 | -------------------------------------------------------------------------------- /giskard/scanner/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger(__name__) 4 | logger.setLevel(logging.INFO) 5 | -------------------------------------------------------------------------------- /giskard/scanner/performance/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains detectors that measure the performance of a model to spot potential performance problems. In 3 | particular, we aim at detecting performance problems affecting specific subpopulations of the data (data slices). 4 | 5 | For example, consider a census model that performs well on the overall population but performs poorly on a specific 6 | subpopulation (e.g. age < 30). This is a performance problem that we want to detect. 7 | """ 8 | from .performance_bias_detector import PerformanceBiasDetector 9 | 10 | __all__ = ["PerformanceBiasDetector"] 11 | -------------------------------------------------------------------------------- /giskard/scanner/registry.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional, Sequence 2 | 3 | from abc import ABC, abstractmethod 4 | from collections import defaultdict 5 | 6 | from ..datasets.base import Dataset 7 | from ..models.base import BaseModel 8 | from .issues import Issue 9 | 10 | 11 | class Detector(ABC): 12 | @abstractmethod 13 | def run(self, model: BaseModel, dataset: Dataset, features: Sequence[str]) -> Sequence[Issue]: 14 | ... 15 | 16 | 17 | class DetectorRegistry: 18 | _detectors: Dict[str, Detector] = dict() 19 | _tags = defaultdict(set) 20 | 21 | @classmethod 22 | def register(cls, name: str, detector: Detector, tags: Optional[Sequence[str]] = None): 23 | cls._detectors[name] = detector 24 | if tags is not None: 25 | cls._tags[name] = set(tags) 26 | 27 | @classmethod 28 | def get_detector_classes(cls, tags: Optional[Sequence[str]] = None) -> dict: 29 | if tags is None: 30 | return {n: d for n, d in cls._detectors.items()} 31 | 32 | return {n: d for n, d in cls._detectors.items() if cls._tags[n].intersection(tags)} 33 | -------------------------------------------------------------------------------- /giskard/scanner/robustness/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains detectors that measure the robustness of a model to spot potential robustness problems. For 3 | classification models, this means detecting a change in the predicted class as a result of a small change in the input 4 | data. For regression models, this means detecting a significant variation in the predicted value as a result of a small 5 | change in the input features. 6 | 7 | These detectors are generally based on some form of metamorphic invariance testing, e.g. by applying a transformation 8 | to the input data that is not supposed to affect the output significantly, and compare the output of the model before 9 | and after the transformation. 10 | """ 11 | from .base_detector import BaseTextPerturbationDetector 12 | from .ethical_bias_detector import EthicalBiasDetector 13 | from .text_perturbation_detector import TextPerturbationDetector 14 | 15 | __all__ = ["EthicalBiasDetector", "TextPerturbationDetector", "BaseTextPerturbationDetector"] 16 | -------------------------------------------------------------------------------- /giskard/scanner/robustness/base_perturbation_function.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import pandas as pd 4 | 5 | from ...core.core import DatasetProcessFunctionMeta 6 | from ...registry.registry import get_object_uuid 7 | from ...registry.transformation_function import TransformationFunction 8 | 9 | 10 | class PerturbationFunction(TransformationFunction): 11 | name: str 12 | 13 | def __init__(self, column: str, needs_dataset: bool = False) -> None: 14 | super().__init__(None, row_level=False, cell_level=False, needs_dataset=needs_dataset) 15 | self.column = column 16 | self.meta = DatasetProcessFunctionMeta(type="TRANSFORMATION") 17 | self.meta.uuid = get_object_uuid(self) 18 | self.meta.code = self.name 19 | self.meta.name = self.name 20 | self.meta.display_name = self.name 21 | self.meta.tags = ["pickle", "scan"] 22 | self.meta.doc = self.meta.default_doc("Automatically generated transformation function") 23 | 24 | def __str__(self) -> str: 25 | return self.name 26 | 27 | def make_perturbation(self, data_or_series: Any) -> Any: 28 | raise NotImplementedError() 29 | 30 | def execute(self, data: pd.DataFrame) -> pd.DataFrame: 31 | raise NotImplementedError() 32 | -------------------------------------------------------------------------------- /giskard/scanner/robustness/numerical_perturbation_detector.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | from ..decorators import detector 4 | from .base_detector import BaseNumericalPerturbationDetector 5 | from .numerical_transformations import NumericalTransformation 6 | 7 | 8 | class BoundClassWrapper: 9 | def __init__(self, cls, **bound_kwargs): 10 | self.cls = cls 11 | self.bound_kwargs = bound_kwargs 12 | 13 | def __call__(self, *args, **kwargs): 14 | return self.cls(*args, **self.bound_kwargs, **kwargs) 15 | 16 | def __getattr__(self, attr): 17 | # Forward attribute access to the wrapped class 18 | return getattr(self.cls, attr) 19 | 20 | 21 | @detector( 22 | name="numerical_perturbation", 23 | tags=[ 24 | "numerical_perturbation", 25 | "robustness", 26 | "classification", 27 | "regression", 28 | ], 29 | ) 30 | class NumericalPerturbationDetector(BaseNumericalPerturbationDetector): 31 | """Detects robustness problems in a model by applying numerical perturbations to the numerical features.""" 32 | 33 | def _get_default_transformations(self) -> Sequence[NumericalTransformation]: 34 | from .numerical_transformations import AddGaussianNoise, MultiplyByFactor 35 | 36 | return [ 37 | BoundClassWrapper(MultiplyByFactor, factor=1.01), 38 | BoundClassWrapper(MultiplyByFactor, factor=0.99), 39 | BoundClassWrapper(AddGaussianNoise, mean=0, std=0.01), 40 | ] 41 | -------------------------------------------------------------------------------- /giskard/scanner/stochasticity/__init__.py: -------------------------------------------------------------------------------- 1 | from .stochasticity_detector import StochasticityDetector 2 | 3 | __all__ = ["StochasticityDetector"] 4 | -------------------------------------------------------------------------------- /giskard/settings.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import os 4 | from pathlib import Path 5 | 6 | import pydantic 7 | from packaging import version 8 | from pydantic import BaseModel 9 | 10 | # See https://linear.app/giskard/issue/GSK-1745/upgrade-pydantic-to-20 11 | IS_PYDANTIC_V2 = version.parse(pydantic.version.VERSION) >= version.parse("2.0") 12 | 13 | if IS_PYDANTIC_V2: 14 | FIELD_ATTR = "model_fields" 15 | else: 16 | FIELD_ATTR = "__fields__" 17 | 18 | 19 | def expand_env_var(env_var: Optional[str]) -> Optional[str]: 20 | current = env_var 21 | previous = None 22 | while current != previous: 23 | previous = current 24 | current = os.path.expandvars(current) 25 | return current 26 | 27 | 28 | class Settings(BaseModel): 29 | home: str = "~/giskard-home" 30 | ws_port: int = 9000 31 | ws_path: str = "/websocket" 32 | host: str = "localhost" 33 | max_workers: int = 10 34 | loglevel: str = "INFO" 35 | cache_dir: str = "cache" 36 | disable_analytics: bool = False 37 | force_asyncio_event_loop: bool = False 38 | min_workers: int = 2 39 | use_pool: bool = True # For testing/debugging only, do not disable 40 | 41 | class Config: 42 | env_prefix = "GSK_" 43 | 44 | @property 45 | def home_dir(self) -> Path: 46 | return Path(expand_env_var(self.home)).expanduser().resolve() 47 | 48 | @classmethod 49 | def build_from_env(cls) -> "Settings": 50 | return Settings( 51 | **{k: os.getenv(cls.Config.env_prefix + k.upper(), v.default) for k, v in getattr(cls, FIELD_ATTR).items()} 52 | ) 53 | 54 | 55 | settings = Settings.build_from_env() 56 | -------------------------------------------------------------------------------- /giskard/slicing/__init__.py: -------------------------------------------------------------------------------- 1 | from .slice import QueryBasedSliceFunction 2 | 3 | __all__ = ["QueryBasedSliceFunction"] 4 | -------------------------------------------------------------------------------- /giskard/slicing/base.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, Sequence 2 | 3 | from abc import ABC, abstractmethod 4 | 5 | from ..datasets.base import Dataset 6 | from .slice import QueryBasedSliceFunction 7 | 8 | 9 | class BaseSlicer(ABC): 10 | def __init__( 11 | self, 12 | dataset: Dataset, 13 | features: Optional[Sequence[str]] = None, 14 | target: Optional[str] = None, 15 | min_deviation: float = 0.05, 16 | abs_deviation: bool = False, 17 | ): 18 | self.dataset = dataset 19 | self.features = features 20 | self.target = target 21 | self.min_deviation = min_deviation 22 | self.abs_deviation = abs_deviation 23 | 24 | @abstractmethod 25 | def find_slices(self, features, target=None) -> List[QueryBasedSliceFunction]: 26 | ... 27 | -------------------------------------------------------------------------------- /giskard/slicing/bruteforce_slicer.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from ..datasets import Dataset 4 | from .base import BaseSlicer 5 | from .slice import GreaterThan, LowerThan, Query, QueryBasedSliceFunction 6 | 7 | 8 | class BruteForceSlicer(BaseSlicer): 9 | def __init__(self, dataset: Dataset, features=None, target=None): 10 | self.dataset = dataset 11 | self.features = features 12 | self.target = target 13 | 14 | def find_slices(self, features, target=None): 15 | target = target or self.target 16 | data = self.dataset.df 17 | 18 | if len(features) > 1: 19 | raise NotImplementedError("Only single-feature slicing is implemented for now.") 20 | (feature,) = features 21 | 22 | # Quantile-based binning 23 | _, cut_bin = pd.qcut(data[feature], q=4, retbins=True, duplicates="drop", labels=False) 24 | 25 | result_df = [] 26 | for i in range(len(cut_bin) - 1): 27 | result_df.append([cut_bin[i], cut_bin[i + 1]]) 28 | 29 | clauses = [] 30 | for interval in result_df: 31 | clauses.append([GreaterThan(feature, interval[0], True), LowerThan(feature, interval[1], True)]) 32 | 33 | slice_candidates = [QueryBasedSliceFunction(Query(clause)) for clause in clauses] 34 | 35 | # Filter by relevance 36 | # filt = SignificanceFilter(target) 37 | # slices = filt.filter(slice_candidates) 38 | 39 | return slice_candidates 40 | -------------------------------------------------------------------------------- /giskard/slicing/category_slicer.py: -------------------------------------------------------------------------------- 1 | from .base import BaseSlicer 2 | from .slice import EqualTo, Query, QueryBasedSliceFunction 3 | 4 | 5 | class CategorySlicer(BaseSlicer): 6 | def find_slices(self, features, target=None): 7 | data = self.dataset.df 8 | target = target or self.target 9 | 10 | if len(features) > 1: 11 | raise NotImplementedError("Only single-feature slicing is implemented for now.") 12 | (feature,) = features 13 | 14 | # Make slices 15 | values = data[feature].dropna().unique().tolist() 16 | slice_candidates = [QueryBasedSliceFunction(Query([EqualTo(feature, val)])) for val in values] 17 | 18 | return slice_candidates 19 | -------------------------------------------------------------------------------- /giskard/slicing/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from .bruteforce_slicer import BruteForceSlicer 4 | from .multiscale_slicer import MultiscaleSlicer 5 | from .opt_slicer import OptSlicer 6 | from .slice import GreaterThan, LowerThan 7 | from .tree_slicer import DecisionTreeSlicer 8 | 9 | 10 | def get_slice_feature_interval(data_slice, feature, min_value=None, max_value=None): 11 | clauses = data_slice.query.clauses[feature] 12 | try: 13 | low = next(c for c in clauses if isinstance(c, GreaterThan)).threshold 14 | except StopIteration: 15 | low = min_value or -float("inf") 16 | 17 | try: 18 | high = next(c for c in clauses if isinstance(c, LowerThan)).threshold 19 | except StopIteration: 20 | high = max_value or float("inf") 21 | 22 | return low, high 23 | 24 | 25 | def get_slice_feature_intervals(slices, *args, **kwargs): 26 | return np.array([get_slice_feature_interval(s, *args, **kwargs) for s in slices]) 27 | 28 | 29 | def get_slicer(slicer_name, dataset, target): 30 | if slicer_name == "optimized": 31 | return OptSlicer(dataset, target=target) 32 | if slicer_name == "tree": 33 | return DecisionTreeSlicer(dataset, target=target) 34 | if slicer_name == "multiscale": 35 | return MultiscaleSlicer(dataset, target=target) 36 | if slicer_name == "bruteforce": 37 | return BruteForceSlicer(dataset, target=target) 38 | 39 | raise ValueError(f"Invalid slicer `{slicer_name}`.") 40 | -------------------------------------------------------------------------------- /giskard/testing/tests/__init__.py: -------------------------------------------------------------------------------- 1 | debug_prefix = "Debug: " 2 | debug_description_prefix = "This debugging session opens one by one all the examples " 3 | -------------------------------------------------------------------------------- /giskard/testing/tests/debug_slicing_functions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from giskard.registry.slicing_function import slicing_function 5 | 6 | 7 | # Performance tests: Classification 8 | @slicing_function(name="Incorrect rows", row_level=False) 9 | def incorrect_rows_slicing_fn(df: pd.DataFrame, target: str, prediction: np.ndarray) -> pd.DataFrame: 10 | """ 11 | Filter the rows where the ground truth (target) does not match the model predictions. 12 | """ 13 | return df[df[target] != prediction] 14 | 15 | 16 | # Performance tests: Regression 17 | @slicing_function(name="nlargest abs err rows", row_level=False) 18 | def nlargest_abs_err_rows_slicing_fn( 19 | df: pd.DataFrame, target: str, prediction: np.ndarray, debug_percent_rows: float = 0.3 20 | ) -> pd.DataFrame: 21 | """ 22 | Filter the largest debug_percent_rows of rows based on the absolute error between 23 | the ground truth (target) and the model predictions. 24 | """ 25 | df["metric"] = abs(df[target] - prediction) 26 | top_n = round(debug_percent_rows * len(df)) 27 | return df.nlargest(top_n, "metric").drop("metric", axis=1) 28 | 29 | 30 | # Miscellaneous: Monotonicity 31 | @slicing_function(name="Rows failing monotonicity", row_level=False) 32 | def row_failing_slicing_fn(df: pd.DataFrame, index_failure: pd.Index) -> pd.DataFrame: 33 | """ 34 | Filter the dataset by index 35 | """ 36 | return df.loc[index_failure] 37 | -------------------------------------------------------------------------------- /giskard/testing/tests/llm/__init__.py: -------------------------------------------------------------------------------- 1 | from .correctness import test_llm_correctness 2 | from .ground_truth import ( 3 | test_llm_as_a_judge_ground_truth_similarity, 4 | test_llm_ground_truth, 5 | test_llm_ground_truth_similarity, 6 | ) 7 | from .hallucination import test_llm_output_coherency, test_llm_output_plausibility 8 | from .injections import ( 9 | LLMCharInjector, 10 | test_llm_char_injection, 11 | test_llm_output_against_strings, 12 | test_llm_single_output_against_strings, 13 | ) 14 | from .output_requirements import ( 15 | test_llm_output_against_requirement, 16 | test_llm_output_against_requirement_per_row, 17 | test_llm_single_output_against_requirement, 18 | ) 19 | 20 | __all__ = [ 21 | "test_llm_char_injection", 22 | "LLMCharInjector", 23 | "test_llm_output_against_requirement", 24 | "test_llm_single_output_against_requirement", 25 | "test_llm_output_against_requirement_per_row", 26 | "test_llm_output_coherency", 27 | "test_llm_output_plausibility", 28 | "test_llm_single_output_against_strings", 29 | "test_llm_output_against_strings", 30 | "test_llm_ground_truth_similarity", 31 | "test_llm_ground_truth", 32 | "test_llm_correctness", 33 | "test_llm_as_a_judge_ground_truth_similarity", 34 | ] 35 | -------------------------------------------------------------------------------- /giskard/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from threading import Thread 3 | 4 | LOGGER = logging.getLogger(__name__) 5 | 6 | 7 | def threaded(fn): 8 | def wrapper(*args, **kwargs): 9 | thread = Thread(target=fn, daemon=True, args=args, kwargs=kwargs) 10 | thread.start() 11 | return thread 12 | 13 | return wrapper 14 | 15 | 16 | def fullname(o): 17 | klass = o.__class__ 18 | module = klass.__module__ 19 | if module == "__builtin__": 20 | return klass.__name__ 21 | return module + "." + klass.__name__ 22 | -------------------------------------------------------------------------------- /giskard/utils/display.py: -------------------------------------------------------------------------------- 1 | def format_number(value, n=3): 2 | if isinstance(value, float): 3 | if value > 1e6 or value < 1e-3: 4 | return f"{value:.{n}e}" 5 | else: 6 | return f"{value:.{3}f}" 7 | 8 | if isinstance(value, int) and abs(value) > 1e4: 9 | return f"{value:.{n}e}" 10 | 11 | return value 12 | 13 | 14 | def truncate(text, max_length=240, ellipsis="…"): 15 | """Truncates a text to the given length, adding an ellipsis at the end if needed.""" 16 | if len(text) > max_length: 17 | return text[: max_length - len(ellipsis)] + ellipsis 18 | return text 19 | -------------------------------------------------------------------------------- /giskard/utils/file_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from uuid import UUID 3 | 4 | from giskard.settings import settings 5 | 6 | 7 | def get_file_name(name: str, extension: str, sample: bool): 8 | return f"{name}.sample.{extension}" if sample else f"{name}.{extension}" 9 | 10 | 11 | def job_logs_path(job_id: UUID) -> Path: 12 | return settings.home_dir / "run" / "jobs" / (str(job_id) + ".log") 13 | -------------------------------------------------------------------------------- /giskard/utils/iterables.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | 4 | def batched(iterable, batch_size): 5 | """Batches an iterable into chunks of size batch_size.""" 6 | it = iter(iterable) 7 | while True: 8 | chunk = list(itertools.islice(it, batch_size)) 9 | if not chunk: 10 | break 11 | yield chunk 12 | -------------------------------------------------------------------------------- /giskard/utils/language_detection.py: -------------------------------------------------------------------------------- 1 | import langdetect 2 | import pandas as pd 3 | 4 | 5 | def detect_lang(text: str): 6 | if len(text.split()) <= 5: 7 | return pd.NA 8 | try: 9 | detected = langdetect.detect_langs(text) 10 | language = detected[0].lang 11 | except langdetect.lang_detect_exception.LangDetectException: 12 | language = "unknown" 13 | return language 14 | -------------------------------------------------------------------------------- /giskard/visualization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/giskard/visualization/__init__.py -------------------------------------------------------------------------------- /giskard/visualization/custom_jinja.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | import markdown 4 | from jinja2 import nodes 5 | from jinja2.ext import Extension 6 | from markupsafe import Markup 7 | 8 | from ..utils.display import format_number 9 | 10 | 11 | class IncludeRawExtension(Extension): 12 | tags = {"include_raw"} 13 | 14 | def parse(self, parser): 15 | lineno = next(parser.stream).lineno 16 | template = parser.parse_expression() 17 | result = self.call_method("_render", [template], lineno=lineno) 18 | return nodes.Output([result], lineno=lineno) 19 | 20 | def _render(self, filename): 21 | return Markup(self.environment.loader.get_source(self.environment, filename)[0]) 22 | 23 | 24 | def markdown_to_html(md_text): 25 | html = markdown.markdown(md_text) 26 | return Markup(html) 27 | 28 | 29 | def pluralize(count, singular="", plural="s"): 30 | if isinstance(count, Sequence): 31 | count = len(count) 32 | return singular if count == 1 else plural 33 | 34 | 35 | def format_metric(value, n=3): 36 | return format_number(value, n) 37 | -------------------------------------------------------------------------------- /giskard/visualization/templates/rag_report/static/internal.js: -------------------------------------------------------------------------------- 1 | function opentab(evt, name) { 2 | // Declare all variables 3 | let i, tabcontent, tablinks; 4 | 5 | // Get all elements with class="tabcontent" and hide them 6 | tabcontent = document.getElementsByClassName("tabcontent"); 7 | for (i = 0; i < tabcontent.length; i++) { 8 | tabcontent[i].style.display = "none"; 9 | } 10 | 11 | // Get all elements with class="tablinks" and remove the class "active" 12 | tablinks = document.getElementsByClassName("tablinks"); 13 | for (i = 0; i < tablinks.length; i++) { 14 | tablinks[i].className = tablinks[i].className.replace(" active", ""); 15 | } 16 | 17 | // Show the current tab, and add an "active" class to the button that opened the tab 18 | document.getElementById(name).style.display = "block"; 19 | evt.currentTarget.className += " active"; 20 | } 21 | -------------------------------------------------------------------------------- /giskard/visualization/templates/scan_report/html/_code_snippet.html: -------------------------------------------------------------------------------- 1 |
2 |

What's next?

3 |
4 |

5 | 1. Generate a test suite from your scan results 6 |

7 | 
{% raw %}test_suite = results.generate_test_suite("My first test suite"){% endraw %}
8 |
9 |

2. Run your test suite

10 | 
{% raw %}test_suite.run(){% endraw %}
11 |
12 |
13 | -------------------------------------------------------------------------------- /giskard/visualization/templates/scan_report/html/_issues_table.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | {% for issue in issues %} 4 | 5 | {% include "scan_report/html/_issue.html" %} 6 | 7 | {% endfor %} 8 |
9 |
-------------------------------------------------------------------------------- /giskard/visualization/templates/scan_report/html/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Giskard Scan Results 6 | 7 | 8 | {% block head %}{% endblock %} 9 | 10 | 11 | 12 | {% block content %}{% endblock %} 13 | {% block javascript %}{% endblock %} 14 | 15 | 16 | -------------------------------------------------------------------------------- /giskard/visualization/templates/scan_report/html/full.html: -------------------------------------------------------------------------------- 1 | {% extends "scan_report/html/base.html" %} 2 | 3 | {% block head %} 4 | 5 | {% endblock %} 6 | 7 | {% block content %} 8 |
9 |
10 | {% include "scan_report/html/_tab_header.html" %} 11 | {% include "scan_report/html/_main_content.html" %} 12 | {% if issues|length > 0 and not issues[0].model.is_text_generation and issues[0].display_footer_info %} 13 | {% include "scan_report/html/_code_snippet.html" %} 14 | {% endif %} 15 |
16 |
17 | {% endblock %} 18 | 19 | {% block javascript %} 20 | 23 | {% endblock %} 24 | -------------------------------------------------------------------------------- /giskard/visualization/templates/scan_report/markdown/github.md: -------------------------------------------------------------------------------- 1 | {% for view in groups -%} 2 |
3 | {{ view.group.name }} issues ({{ view.issues|length }}) 4 | 5 | | Vulnerability | Level | Data slice | Metric | Transformation | Deviation | Description | 6 | |---------------|-------|------------|--------|----------------|-----------|-------------| 7 | {% for issue in view.issues -%} 8 | | {{ view.group.name }} | {{ issue.level.value }} | {{ issue.slicing_fn if issue.slicing_fn else "—" }} | {% if "metric" in issue.meta %}{{ issue.meta.metric }} = {{ issue.meta.metric_value|format_metric }}{% else %} "—" {% endif %} | {{ issue.transformation_fn if issue.transformation_fn else "—" }} | {{ issue.meta["deviation"] if "deviation" in issue.meta else "—" }} | {{ issue.description }} | 9 | {% endfor %} 10 | 11 |
12 | {% endfor -%} -------------------------------------------------------------------------------- /giskard/visualization/templates/scan_report/markdown/summary.md: -------------------------------------------------------------------------------- 1 | | Vulnerability | Level | Data slice | Metric | Transformation | Deviation | Description | 2 | |---------------|-------|------------|--------|----------------|-----------|-------------| 3 | {%- for view in groups -%} 4 | {% for issue in view.issues %} 5 | | {{ view.group.name }} | {{ issue.level.value }} | {{ issue.slicing_fn if issue.slicing_fn else "—" }} | {% if "metric" in issue.meta %}{{ issue.meta.metric }} = {{ issue.meta.metric_value|format_metric }}{% else %} "—" {% endif %} | {{ issue.transformation_fn if issue.transformation_fn else "—" }} | {{ issue.meta["deviation"] if "deviation" in issue.meta else "—" }} | {{ issue.description }} | 6 | {%- endfor -%} 7 | {%- endfor -%} 8 | -------------------------------------------------------------------------------- /giskard/visualization/templates/suite_results/_suite_results_header.html: -------------------------------------------------------------------------------- 1 |
2 | {% if passed %} 3 | 4 | check 5 | 6 | 7 | Test suite passed. 8 | 9 | {% else %} 10 | 11 | close 12 | 15 | 16 | Test suite failed. 17 | {% endif %} 18 |
19 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "build": "tailwindcss -i ./src/scan-widget/style.css -o ./giskard/scanner/templates/static/style.css", 4 | "watch": "tailwindcss -i ./src/scan-widget/style.css -o ./giskard/scanner/templates/static/style.css --watch" 5 | }, 6 | "devDependencies": { 7 | "@babel/core": "^7.26.9", 8 | "@babel/preset-env": "^7.26.9", 9 | "autoprefixer": "^10.4.21", 10 | "cssnano": "^7.0.6", 11 | "gulp": "^5.0.0", 12 | "gulp-babel": "^8.0.0", 13 | "gulp-concat": "^2.6.1", 14 | "gulp-minify": "^3.1.0", 15 | "gulp-postcss": "^10.0.0", 16 | "postcss": "^8.5.3", 17 | "postcss-import": "^16.1.0", 18 | "tailwindcss": "^4.1.0", 19 | "uglify": "^0.1.5" 20 | }, 21 | "dependencies": { 22 | "alpinejs": "^3.14.9" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | "postcss-import": {}, 4 | tailwindcss: {}, 5 | autoprefixer: {}, 6 | cssnano: {}, 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /readme/Discord.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/Discord.png -------------------------------------------------------------------------------- /readme/Give_feedback.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/Give_feedback.jpeg -------------------------------------------------------------------------------- /readme/Logo_full_darkgreen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/Logo_full_darkgreen.png -------------------------------------------------------------------------------- /readme/RAGET_updated.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/RAGET_updated.gif -------------------------------------------------------------------------------- /readme/architechture_giskard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/architechture_giskard.png -------------------------------------------------------------------------------- /readme/catalog_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/catalog_example.png -------------------------------------------------------------------------------- /readme/catalog_slice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/catalog_slice.png -------------------------------------------------------------------------------- /readme/catalog_transfo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/catalog_transfo.png -------------------------------------------------------------------------------- /readme/demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/demo.png -------------------------------------------------------------------------------- /readme/design_partner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/design_partner.png -------------------------------------------------------------------------------- /readme/feedback.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/feedback.png -------------------------------------------------------------------------------- /readme/feedback1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/feedback1.png -------------------------------------------------------------------------------- /readme/giskard_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/giskard_logo.png -------------------------------------------------------------------------------- /readme/giskard_logo_green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/giskard_logo_green.png -------------------------------------------------------------------------------- /readme/inspect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/inspect.png -------------------------------------------------------------------------------- /readme/perturbation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/perturbation.png -------------------------------------------------------------------------------- /readme/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/pipeline.png -------------------------------------------------------------------------------- /readme/scan_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/scan_example.gif -------------------------------------------------------------------------------- /readme/scan_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/scan_example.png -------------------------------------------------------------------------------- /readme/scan_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/scan_results.png -------------------------------------------------------------------------------- /readme/scan_updates.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/scan_updates.gif -------------------------------------------------------------------------------- /readme/suite_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/suite_example.png -------------------------------------------------------------------------------- /readme/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/test.png -------------------------------------------------------------------------------- /readme/test1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/test1.png -------------------------------------------------------------------------------- /readme/test_suite_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/test_suite_example.png -------------------------------------------------------------------------------- /readme/tools.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/tools.png -------------------------------------------------------------------------------- /readme/tools_updated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/tools_updated.png -------------------------------------------------------------------------------- /readme/upload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/upload.png -------------------------------------------------------------------------------- /readme/workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/readme/workflow.png -------------------------------------------------------------------------------- /sample_data/other/addresses.csv: -------------------------------------------------------------------------------- 1 | Name,Last,Address,City,State,PostalCode 2 | John,Doe,120 jefferson st.,Riverside, NJ, 08075 3 | Jack,McGinnis,220 hobo Av.,Phila, PA,09119 4 | "John ""Da Man""",Repici,120 Jefferson St.,Riverside, NJ,08075 5 | Stephen,Tyler,"7452 Terrace ""At the Plaza"" road",SomeTown,SD, 91234 6 | ,Blankman,,SomeTown, SD, 00298 7 | "Joan ""the bone"", Anne",Jet,"9th, at Terrace plc",Desert City,CO,00123 8 | -------------------------------------------------------------------------------- /scripts/install-giskard-client-dev.sh: -------------------------------------------------------------------------------- 1 | pip="${1:-pip}" 2 | rm -f /tmp/giskard*whl* || true 3 | wget https://nightly.link/Giskard-AI/giskard/workflows/build/main/giskard-dev.whl.zip -P /tmp 4 | unzip -o /tmp/giskard-dev.whl.zip -d /tmp 5 | $pip install --upgrade "$(ls /tmp/giskard*whl)" 6 | rm -f /tmp/giskard*whl* -------------------------------------------------------------------------------- /scripts/make-release.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | DIR="$(dirname $(readlink -f $0))" 4 | 5 | die () { 6 | echo >&2 "$@" 7 | exit 1 8 | } 9 | 10 | [ "$#" -eq 1 ] || die "1 argument required, $# provided" 11 | echo $1 | grep -E -q '^[0-9]+\.[0-9]+\.[0-9]+([ab]\d+)?$' || 12 | die "Version should match '^[0-9]+\.[0-9]+\.[0-9]+([ab]\d+)?$', $1 provided" 13 | 14 | if [ -n "$1" ]; then 15 | git stash || true 16 | git pull 17 | pyprojpath="$DIR/../pyproject.toml" 18 | echo "Setting version to $1 in $pyprojpath" 19 | sed -i '' "s/^version = \".*$/version = \"$1\"/" "$pyprojpath" 20 | git add "$pyprojpath" 21 | git commit -m "Version $1" 22 | git tag -a "python-client-v$1" -m "v$1" 23 | git push 24 | git push origin "python-client-$1" 25 | else 26 | echo "New version is not specified" 27 | fi 28 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.sources=giskard 2 | sonar.tests=tests 3 | sonar.language=python 4 | sonar.python.version=3 5 | sonar.dynamicAnalysis=reuseReports 6 | sonar.core.codeCoveragePlugin=cobertura 7 | sonar.python.coverage.reportPaths=coverage.xml 8 | 9 | sonar.verbose=true 10 | sonar.projectKey=giskard 11 | sonar.organization=giskard 12 | 13 | sonar.host.url=https://sonarcloud.io 14 | sonar.sourceEncoding=UTF-8 15 | 16 | sonar.issue.ignore.multicriteria.S2245.resourceKey=**/* 17 | sonar.issue.ignore.multicriteria.S2245.ruleKey=python:S2245 -------------------------------------------------------------------------------- /src/scan-widget/internal-js/highlightjs-copy.min.js: -------------------------------------------------------------------------------- 1 | class CopyButtonPlugin{constructor(options={}){self.hook=options.hook;self.callback=options.callback}"after:highlightElement"({el,text}){let button=Object.assign(document.createElement("button"),{innerHTML:"Copy",className:"hljs-copy-button"});button.dataset.copied=false;el.parentElement.classList.add("hljs-copy-wrapper");el.parentElement.appendChild(button);el.parentElement.style.setProperty("--hljs-theme-background",window.getComputedStyle(el).backgroundColor);button.onclick=function(){if(!navigator.clipboard)return;let newText=text;if(hook&&typeof hook==="function"){newText=hook(text,el)||text}navigator.clipboard.writeText(newText).then(function(){button.innerHTML=" Copied! ";button.dataset.copied=true;let alert=Object.assign(document.createElement("div"),{role:"status",className:"hljs-copy-alert",innerHTML:"Copied to clipboard"});el.parentElement.appendChild(alert);setTimeout(()=>{button.innerHTML="Copy";button.dataset.copied=false;el.parentElement.removeChild(alert);alert=null},2e3)}).then(function(){if(typeof callback==="function")return callback(newText,el)})}}} -------------------------------------------------------------------------------- /src/scan-widget/internal-js/scan.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | // HighlightJS 3 | hljs.addPlugin(new CopyButtonPlugin()); 4 | hljs.highlightAll(); 5 | 6 | // Tabs 7 | let rows = document.querySelectorAll("#gsk-scan .gsk-issue") 8 | rows.forEach(rowEl => { 9 | rowEl.addEventListener("click", (event) => { 10 | { 11 | event.preventDefault() 12 | rowEl.classList.toggle("open") 13 | rowEl.classList.toggle("bg-zinc-700") 14 | } 15 | }) 16 | }); 17 | 18 | const tabs = document.querySelectorAll("#gsk-scan [role='tabpanel']") 19 | const tabHeaders = document.querySelectorAll("#gsk-scan [data-tab-target]") 20 | tabHeaders.forEach(tabHeader => { 21 | tabHeader.addEventListener("click", (event) => { 22 | event.preventDefault() 23 | const tabId = tabHeader.getAttribute("data-tab-target") 24 | 25 | tabs.forEach(tab => { 26 | tab.classList.add("hidden") 27 | }) 28 | tabHeaders.forEach(tabh => { 29 | tabh.classList.remove("active") 30 | }) 31 | 32 | tabHeader.classList.add("active") 33 | document.getElementById(tabId).classList.remove("hidden") 34 | 35 | }) 36 | }) 37 | })() -------------------------------------------------------------------------------- /src/scan-widget/style.css: -------------------------------------------------------------------------------- 1 | /* Imports */ 2 | @import "_highlight_theme.css"; 3 | @import "_highlightjs_copy.css"; 4 | 5 | /* Main styles */ 6 | @tailwind base; 7 | @tailwind components; 8 | @tailwind utilities; 9 | 10 | 11 | p a { 12 | text-decoration: underline; 13 | } 14 | 15 | /* Special styles for tab headers */ 16 | .tab-header { 17 | min-width: 3rem; 18 | } 19 | 20 | .active.tab-header { 21 | min-width: fit-content; 22 | } 23 | 24 | /* Pandas DataFrame table */ 25 | table.dataframe { 26 | width: 100%; 27 | max-width: 100%; 28 | overflow: auto; 29 | } 30 | 31 | .dataframe tr { 32 | border-bottom: 1px solid #555; 33 | vertical-align: top; 34 | } 35 | 36 | .dataframe td, 37 | .dataframe th { 38 | padding: 0.5rem; 39 | } 40 | 41 | .dataframe th { 42 | text-align: left !important; 43 | } 44 | 45 | .prose p { 46 | @apply my-1; 47 | } 48 | 49 | .prose ul { 50 | @apply list-disc ml-4 my-1; 51 | } -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | darkMode: "class", 4 | content: [ 5 | "./giskard/visualization/templates/**/*.{html,js}" 6 | ], 7 | theme: { 8 | extend: {}, 9 | }, 10 | plugins: [], 11 | } 12 | 13 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | def path(p): 5 | return Path(__file__).parent / p 6 | -------------------------------------------------------------------------------- /tests/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/tests/core/__init__.py -------------------------------------------------------------------------------- /tests/core/test_validation.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import pydantic 4 | import pytest 5 | from packaging import version 6 | from pydantic import ValidationError 7 | 8 | from giskard.core.validation import ConfiguredBaseModel 9 | 10 | IS_PYDANTIC_V2 = version.parse(pydantic.version.VERSION) >= version.parse("2.0") 11 | 12 | 13 | def test_model_forbid_extra(): 14 | class TestModel(ConfiguredBaseModel): 15 | a: str 16 | b: int 17 | 18 | TestModel(a="toot", b=5) 19 | with pytest.raises(ValidationError) as exc_info: 20 | TestModel(a="toto", b=5, c=True) 21 | 22 | with pytest.raises(ValidationError) as exc_info: 23 | TestModel.parse_obj({"a": "toto", "b": 5, "c": True}) 24 | if IS_PYDANTIC_V2: 25 | assert "Extra inputs are not permitted" in str(exc_info) 26 | else: 27 | assert "extra fields not permitted" in str(exc_info) 28 | 29 | 30 | @pytest.mark.skip(reason="GSK-2057, restore after proper handing of NaN in tests metrics") 31 | def test_model_forbid_inf_nan(): 32 | class TestModel(ConfiguredBaseModel): 33 | a: float 34 | 35 | TestModel(a=5.0) 36 | with pytest.raises(ValidationError) as exc_info: 37 | TestModel(a=math.inf) 38 | 39 | if IS_PYDANTIC_V2: 40 | assert "Input should be a finite number" in str(exc_info) 41 | else: 42 | assert "ensure this value is a finite number" in str(exc_info) 43 | -------------------------------------------------------------------------------- /tests/datasets/test_base_dataset.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import pandas as pd 4 | import pytest 5 | 6 | from giskard import Dataset 7 | 8 | 9 | @pytest.fixture 10 | def df() -> pd.DataFrame: 11 | return pd.DataFrame({"a": range(10), "b": ["x", "y"] * 5}) 12 | 13 | 14 | def test_named_and_IDed_dataset_str(df: pd.DataFrame): 15 | uid = uuid.uuid4() 16 | dataset = Dataset(name="foo", id=uid, df=df) 17 | assert str(dataset) == f"foo({uid})" 18 | 19 | 20 | def test_named_dataset_str(df: pd.DataFrame): 21 | dataset = Dataset(name="bar", df=df) 22 | assert str(dataset).split("(")[0] == "bar" 23 | 24 | 25 | def test_unnamed_dataset_str(df: pd.DataFrame): 26 | dataset = Dataset(df=df) 27 | assert hex(id(dataset)).lower()[2:] in str(dataset).lower() 28 | assert " pd.DataFrame: 16 | return df[TEXT_COLUMN_NAME].apply(lambda x: self.model.run({"query": x})) 17 | 18 | def save_model(self, path: str, *args, **kwargs): 19 | out_dest = Path(path) 20 | # Save the chain object 21 | self.model.save(out_dest.joinpath("model.json")) 22 | 23 | @classmethod 24 | def load_model(cls, path: str, **kwargs) -> Chain: 25 | src = Path(path) 26 | 27 | # Load the chain, passing the retriever 28 | chain = load_chain(src.joinpath("model.json")) 29 | return chain 30 | 31 | 32 | def test_load(): 33 | llm = FakeListLLM(responses=["foo", "bar", "baz"]) 34 | chain = LLMChain.from_string(llm=llm, template="{query}") 35 | 36 | model = MyModel( 37 | chain, 38 | "text_generation", 39 | name="Foo", 40 | description="Model that reply foo, bar, baz", 41 | feature_names=[TEXT_COLUMN_NAME], 42 | ) 43 | 44 | with tempfile.TemporaryDirectory() as f: 45 | model.save(f) 46 | giskard.Model.load(f) 47 | -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.10/giskard-model-meta.yaml: -------------------------------------------------------------------------------- 1 | classification_labels: 2 | - 0 3 | - 1 4 | description: No description 5 | feature_names: null 6 | id: 54a90cbf-3b44-43d4-8070-8926c8339791 7 | language: PYTHON 8 | language_version: 3.10.13 9 | loader_class: PredictionFunctionModel 10 | loader_module: giskard.models.function 11 | model_type: CLASSIFICATION 12 | name: PredictionFunctionModel 13 | size: 0 14 | threshold: 0.5 15 | -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.10/giskard-model-wrapper-meta.yaml: -------------------------------------------------------------------------------- 1 | batch_size: null 2 | -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.10/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/tests/models/fixtures/func/3.10/model.pkl -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.11/giskard-model-meta.yaml: -------------------------------------------------------------------------------- 1 | classification_labels: 2 | - 0 3 | - 1 4 | description: No description 5 | feature_names: null 6 | id: 01297928-e182-4b50-b9aa-57058dcbadbd 7 | language: PYTHON 8 | language_version: 3.11.5 9 | loader_class: PredictionFunctionModel 10 | loader_module: giskard.models.function 11 | model_type: CLASSIFICATION 12 | name: PredictionFunctionModel 13 | size: 0 14 | threshold: 0.5 15 | -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.11/giskard-model-wrapper-meta.yaml: -------------------------------------------------------------------------------- 1 | batch_size: null 2 | -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.11/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/tests/models/fixtures/func/3.11/model.pkl -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.9/giskard-model-meta.yaml: -------------------------------------------------------------------------------- 1 | classification_labels: 2 | - 0 3 | - 1 4 | feature_names: null 5 | id: 592e6f31-5d04-4915-9d26-798f21277a28 6 | language: PYTHON 7 | language_version: 3.9.6 8 | loader_class: PredictionFunctionModel 9 | loader_module: giskard.models.function 10 | model_type: CLASSIFICATION 11 | name: PredictionFunctionModel 12 | size: 0 13 | threshold: 0.5 14 | -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.9/giskard-model-wrapper-meta.yaml: -------------------------------------------------------------------------------- 1 | batch_size: null 2 | -------------------------------------------------------------------------------- /tests/models/fixtures/func/3.9/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/tests/models/fixtures/func/3.9/model.pkl -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.10/ModelClass.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/tests/models/fixtures/ipcc/3.10/ModelClass.pkl -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.10/giskard-model-meta.yaml: -------------------------------------------------------------------------------- 1 | classification_labels: null 2 | description: Model that reply foo, bar, baz 3 | feature_names: 4 | - query 5 | id: 172f1497-fb87-4484-8dcc-c01ca4197509 6 | language: PYTHON 7 | language_version: 3.10.9 8 | loader_class: MyModel 9 | loader_module: __main__ 10 | model_type: TEXT_GENERATION 11 | name: Foo 12 | size: 1608 13 | threshold: 0.5 14 | -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.10/giskard-model-wrapper-meta.yaml: -------------------------------------------------------------------------------- 1 | batch_size: null 2 | -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.10/model.json: -------------------------------------------------------------------------------- 1 | { 2 | "memory": null, 3 | "verbose": false, 4 | "tags": null, 5 | "metadata": null, 6 | "prompt": { 7 | "input_variables": [ 8 | "query" 9 | ], 10 | "output_parser": null, 11 | "partial_variables": {}, 12 | "template": "{query}", 13 | "template_format": "f-string", 14 | "validate_template": true, 15 | "_type": "prompt" 16 | }, 17 | "llm": { 18 | "responses": [ 19 | "foo", 20 | "bar", 21 | "baz" 22 | ], 23 | "_type": "fake-list" 24 | }, 25 | "output_key": "text", 26 | "output_parser": { 27 | "_type": "default" 28 | }, 29 | "return_final_only": true, 30 | "llm_kwargs": {}, 31 | "_type": "llm_chain" 32 | } -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.11/ModelClass.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Giskard-AI/giskard/6808a562cc1c565e6cf88aeb1fd1f77d28fe76ba/tests/models/fixtures/ipcc/3.11/ModelClass.pkl -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.11/giskard-model-meta.yaml: -------------------------------------------------------------------------------- 1 | classification_labels: null 2 | description: Model that reply foo, bar, baz 3 | feature_names: 4 | - query 5 | id: 9a6c36df-390c-4b13-81de-e7f310a909c0 6 | language: PYTHON 7 | language_version: 3.11.3 8 | loader_class: MyModel 9 | loader_module: __main__ 10 | model_type: TEXT_GENERATION 11 | name: Foo 12 | size: 2067 13 | threshold: 0.5 14 | -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.11/giskard-model-wrapper-meta.yaml: -------------------------------------------------------------------------------- 1 | batch_size: null 2 | -------------------------------------------------------------------------------- /tests/models/fixtures/ipcc/3.11/model.json: -------------------------------------------------------------------------------- 1 | { 2 | "memory": null, 3 | "verbose": false, 4 | "tags": null, 5 | "metadata": null, 6 | "prompt": { 7 | "input_variables": [ 8 | "query" 9 | ], 10 | "input_types": {}, 11 | "output_parser": null, 12 | "partial_variables": {}, 13 | "template": "{query}", 14 | "template_format": "f-string", 15 | "validate_template": false, 16 | "_type": "prompt" 17 | }, 18 | "llm": { 19 | "responses": [ 20 | "foo", 21 | "bar", 22 | "baz" 23 | ], 24 | "_type": "fake-list" 25 | }, 26 | "output_key": "text", 27 | "output_parser": { 28 | "_type": "default" 29 | }, 30 | "return_final_only": true, 31 | "llm_kwargs": {}, 32 | "_type": "llm_chain" 33 | } -------------------------------------------------------------------------------- /tests/models/langchain/test_llm_chain.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | 3 | import pandas as pd 4 | from langchain.chains import LLMChain 5 | from langchain.llms.fake import FakeListLLM 6 | from langchain.prompts import PromptTemplate 7 | 8 | from giskard import Dataset, Model 9 | from giskard.models.langchain import LangchainModel 10 | 11 | 12 | def test_llm_chain(): 13 | responses = ["\n\nHueFoots.", "\n\nEcoDrive Motors."] 14 | llm = FakeListLLM(responses=responses) 15 | prompt = PromptTemplate( 16 | input_variables=["product"], 17 | template="What is a good name for a company that makes {product}?", 18 | ) 19 | chain = LLMChain(llm=llm, prompt=prompt) 20 | 21 | wrapped_model = Model( 22 | chain, model_type="text_generation", name="test", description="fake", feature_names=["product"] 23 | ) 24 | 25 | df = pd.DataFrame(["colorful socks", "electric car"], columns=["product"]) 26 | 27 | wrapped_dataset = Dataset(df, cat_columns=[]) 28 | 29 | results = wrapped_model.predict(wrapped_dataset) 30 | 31 | assert list(results.raw) == responses[:2], f"{results.raw}" 32 | assert list(results.raw_prediction) == responses[:2] 33 | 34 | with tempfile.TemporaryDirectory() as tmpdirname: 35 | wrapped_model.save(tmpdirname) 36 | loaded_model = LangchainModel.load(tmpdirname) 37 | 38 | assert list(results.raw) == list(loaded_model.predict(wrapped_dataset).raw) 39 | -------------------------------------------------------------------------------- /tests/models/pytorch/test_dtypes.py: -------------------------------------------------------------------------------- 1 | from typing import get_args 2 | 3 | from giskard.models.pytorch import TorchDType, string_to_torch_dtype 4 | 5 | 6 | def test_dtypes(): 7 | for t in get_args(TorchDType): 8 | assert string_to_torch_dtype(t) 9 | -------------------------------------------------------------------------------- /tests/models/pytorch/test_error.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import pytest 4 | import torch 5 | import torch.nn as nn 6 | 7 | from giskard import Dataset 8 | from giskard.core.model_validation import validate_model 9 | from giskard.models.pytorch import PyTorchModel 10 | 11 | 12 | class ManualLinearRegression(nn.Module): 13 | def __init__(self): 14 | super().__init__() 15 | self.linear = nn.Linear(1, 1) 16 | 17 | def forward(self, x): 18 | return self.linear(x) 19 | 20 | 21 | def test_error(): 22 | device = "cuda" if torch.cuda.is_available() else "cpu" 23 | 24 | model = ManualLinearRegression().to(device) 25 | 26 | df = pd.DataFrame({"x": np.array([1]), "y": np.array([2])}) 27 | 28 | def preproc_func(df): 29 | return df.values.tolist() 30 | 31 | my_model = PyTorchModel( 32 | name="my_linear_model", 33 | model=model, 34 | feature_names=["x"], 35 | model_type="regression", 36 | data_preprocessing_function=preproc_func, 37 | ) 38 | 39 | my_test_dataset = Dataset(df.head(), name="test dataset", target="y") 40 | 41 | with pytest.raises(Exception) as e: 42 | validate_model(my_model, validate_ds=my_test_dataset) 43 | assert e.match( 44 | f"The output of data_preprocessing_function is of type={type(df.values.tolist())}.\n \ 45 | Make sure that your data_preprocessing_function outputs one of the following: \n \ 46 | - pandas.DataFrame \n \ 47 | - torch.Dataset \n \ 48 | - torch.DataLoader" 49 | ) 50 | -------------------------------------------------------------------------------- /tests/models/pytorch/test_sst2_pytorch.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from giskard.core.model_validation import validate_model 4 | 5 | 6 | @pytest.mark.memory_expensive 7 | @pytest.mark.skip(reason="Too memory expensive, to be replaced with smaller one") 8 | def test_sst2_pytorch_upload(sst2_model, sst2_data): 9 | validate_model(sst2_model, validate_ds=sst2_data) 10 | -------------------------------------------------------------------------------- /tests/models/tensorflow/test_tabular_titanic_binary_classification.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from giskard import Dataset 4 | from giskard.core.model_validation import validate_model 5 | from giskard.demo import titanic_classification 6 | from giskard.models.tensorflow import TensorFlowModel 7 | 8 | tf = pytest.importorskip("tensorflow") 9 | 10 | 11 | def test_tabular_titanic_binary_classification(): 12 | df = titanic_classification.get_test_df() 13 | preprocess, _ = titanic_classification.get_pipeline() 14 | 15 | input_shape = (preprocess(df).shape[-1],) 16 | 17 | model = tf.keras.models.Sequential( 18 | [ 19 | tf.keras.layers.Dense(128, input_shape=input_shape, activation="relu"), 20 | tf.keras.layers.Dropout(0.2), 21 | tf.keras.layers.Dense(2, activation="softmax"), 22 | ] 23 | ) 24 | 25 | my_model = TensorFlowModel( 26 | model, 27 | feature_names=df.columns.drop("Survived"), 28 | model_type="classification", 29 | classification_labels=["no", "yes"], 30 | data_preprocessing_function=preprocess, 31 | ) 32 | 33 | my_test_dataset = Dataset(df, target="Survived", name="test dataset") 34 | 35 | assert my_model.predict(my_test_dataset) 36 | assert my_model.predict(my_test_dataset).raw.shape == (len(df), 2) 37 | 38 | validate_model(my_model, validate_ds=my_test_dataset) 39 | -------------------------------------------------------------------------------- /tests/models/test_catboost_wrapper.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from catboost import CatBoostClassifier 4 | 5 | from giskard.models.catboost import CatboostModel 6 | 7 | 8 | def test_catboost_model_detects_feature_names(): 9 | df = pd.DataFrame({"feature_1": np.random.normal(size=100), "feature_2": ["cat1"] * 10 + ["cat2"] * 90}) 10 | classifier = CatBoostClassifier(cat_features=["feature_2"]) 11 | classifier.fit(df, ["target1"] * 90 + ["target2"] * 10) 12 | 13 | model = CatboostModel( 14 | model=classifier, 15 | model_type="classification", 16 | feature_names=None, # we are not passing feature names here 17 | classification_labels=None, # we are not passing labels here 18 | ) 19 | 20 | assert model.meta.feature_names == ["feature_1", "feature_2"] 21 | assert model.meta.classification_labels == ["target1", "target2"] 22 | -------------------------------------------------------------------------------- /tests/models/test_precooked_model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import pytest 4 | 5 | from giskard import slicing_function 6 | from giskard.models._precooked import PrecookedModel 7 | 8 | 9 | @slicing_function(name="random", row_level=False) 10 | def random_slice(df: pd.DataFrame) -> pd.DataFrame: 11 | return df.sample(100) 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "model,dataset", 16 | [("german_credit_model", "german_credit_data")], 17 | ) 18 | def test_precooked_model(model, dataset, request): 19 | model = request.getfixturevalue(model) 20 | dataset = request.getfixturevalue(dataset) 21 | precooked_model = PrecookedModel.from_model(model, dataset) 22 | data_slice = dataset.slice(random_slice()) 23 | # assert np.all(data_slice.df.columns == dataset.df.columns) 24 | # assert np.all(precooked_model.predict(dataset).raw == model.predict(dataset).raw) 25 | 26 | assert np.all(precooked_model.predict(data_slice).raw == model.predict(data_slice).raw) 27 | -------------------------------------------------------------------------------- /tests/models/test_sklearn_wrapper.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import pytest 4 | from sklearn.linear_model import LinearRegression 5 | 6 | from giskard.models.sklearn import SKLearnModel 7 | 8 | 9 | def test_must_provide_feature_names_if_has_preprocessing(): 10 | df = pd.DataFrame({"a": np.random.normal(size=100), "b": np.random.normal(size=100)}) 11 | reg = LinearRegression() 12 | reg.fit(df, np.ones(100)) 13 | 14 | with pytest.raises(ValueError): 15 | SKLearnModel(reg, model_type="regression", data_preprocessing_function=lambda df: df) 16 | 17 | assert SKLearnModel(reg, model_type="regression", feature_names=["test"], data_preprocessing_function=lambda df: df) 18 | 19 | 20 | def test_raises_save_error_if_unknown_model_type(): 21 | reg = LinearRegression() 22 | gsk_model = SKLearnModel(reg, model_type="regression") 23 | 24 | with pytest.raises(ValueError): 25 | gsk_model.meta.model_type = "unknown" 26 | gsk_model.save_model("test_path", None) 27 | -------------------------------------------------------------------------------- /tests/registry/module_utils.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | 3 | import importlib.util 4 | import os 5 | import sys 6 | import tempfile 7 | from dataclasses import dataclass 8 | from pathlib import Path 9 | 10 | 11 | @dataclass 12 | class PythonFile: 13 | relative_path: Path 14 | content: str 15 | 16 | 17 | @dataclass 18 | class PythonModule: 19 | module_name: str 20 | init_content: str 21 | files: List[PythonFile] 22 | 23 | 24 | def _write_file(dir: Path, file: Union[str, Path], content: str): 25 | os.makedirs(os.path.dirname(dir / file), exist_ok=True) 26 | with open(dir / file, "w", encoding="utf-8") as f: 27 | f.write(content) 28 | 29 | 30 | class TmpModule(object): 31 | def __init__(self, module_def: PythonModule): 32 | self.temp_dir = tempfile.TemporaryDirectory() 33 | self.module_def = module_def 34 | 35 | def __enter__(self): 36 | dir = Path(self.temp_dir.__enter__()) 37 | 38 | _write_file(dir, "__init__.py", self.module_def.init_content) 39 | for file_def in self.module_def.files: 40 | _write_file(dir, file_def.relative_path, file_def.content) 41 | 42 | spec = importlib.util.spec_from_file_location(self.module_def.module_name, dir / "__init__.py") 43 | loaded_module = importlib.util.module_from_spec(spec) 44 | sys.modules[self.module_def.module_name] = loaded_module 45 | spec.loader.exec_module(loaded_module) 46 | 47 | return loaded_module 48 | 49 | def __exit__(self, type, value, traceback): 50 | del sys.modules[self.module_def.module_name] 51 | self.temp_dir.__exit__(type, value, traceback) 52 | -------------------------------------------------------------------------------- /tests/scan/test_data_leakage_detector.py: -------------------------------------------------------------------------------- 1 | from giskard.scanner.data_leakage.data_leakage_detector import DataLeakageDetector 2 | from giskard.scanner.issues import DataLeakage 3 | 4 | 5 | def test_data_leakage_is_detected(enron_data, enron_model): 6 | detector = DataLeakageDetector() 7 | issues = detector.run(enron_model, enron_data) 8 | 9 | # should detect no issue 10 | assert len(issues) == 0 11 | 12 | # Now we add normalization problems 13 | ll = len(enron_data) 14 | 15 | def renormalize_df(df): 16 | numeric_cols = df.select_dtypes("number").columns 17 | df.loc[:, numeric_cols] /= ll / len(df) 18 | return df 19 | 20 | _prev_func = enron_model.data_preprocessing_function 21 | enron_model.data_preprocessing_function = renormalize_df 22 | issues = detector.run(enron_model, enron_data) 23 | 24 | assert len(issues) == 1 25 | assert issues[0].group == DataLeakage 26 | 27 | enron_model.data_preprocessing_function = _prev_func 28 | 29 | 30 | def test_data_leakage_works_on_small_dataset(enron_data, enron_model): 31 | test_data_leakage_is_detected(enron_data.slice(lambda df: df.sample(10), row_level=False), enron_model) 32 | -------------------------------------------------------------------------------- /tests/scan/test_dataset_subsampling.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from giskard import Dataset, Model 5 | from giskard.scanner.common.utils import get_dataset_subsample 6 | 7 | 8 | def test_get_dataset_subsample(): 9 | # Create dummy dataset with 20% of samples having label 1, 20% label 2, and the rest label 0 10 | data = {"feature": np.arange(1000), "target": np.zeros(1000)} 11 | data["target"][0:200] = 1 12 | data["target"][200:400] = 2 13 | np.random.shuffle(data["target"]) 14 | df = pd.DataFrame(data) 15 | dataset = Dataset(df, target="target") 16 | 17 | # Create dummy model 18 | model = Model(lambda df: np.ones(len(df)), model_type="classification", classification_labels=[0, 1, 2]) 19 | 20 | # Test new dataset size 21 | max_data_size = len(dataset) // 10 22 | result = get_dataset_subsample(dataset, model, max_data_size) 23 | assert len(result) == max_data_size 24 | 25 | # For each label, check that the proportion of samples is preserved 26 | label_counts = dataset.df.target.value_counts() 27 | for label in label_counts.index: 28 | assert result.df.target.tolist().count(label) == label_counts[label] // 10 29 | 30 | # Edge case: max size larger than original size (should return the original dataset) 31 | large_max_data_size = len(dataset) * 2 32 | result_large = get_dataset_subsample(dataset, model, large_max_data_size) 33 | assert len(result_large) == len(dataset) 34 | 35 | # Edge case: max size 0 (should return the original dataset) 36 | result_zero = get_dataset_subsample(dataset, model, 0) 37 | assert len(result_zero) == len(dataset) 38 | -------------------------------------------------------------------------------- /tests/scan/test_detector_registry.py: -------------------------------------------------------------------------------- 1 | from giskard.scanner.decorators import detector 2 | from giskard.scanner.registry import DetectorRegistry 3 | 4 | 5 | def test_detector_registry(): 6 | class MyTestDetector: 7 | def run(self, model, dataset, features=None): 8 | return [] 9 | 10 | DetectorRegistry.register("test_detector", MyTestDetector, tags=["tag_1", "tag_2", "classification"]) 11 | 12 | assert "test_detector" in DetectorRegistry.get_detector_classes().keys() 13 | assert DetectorRegistry.get_detector_classes()["test_detector"] == MyTestDetector 14 | assert DetectorRegistry.get_detector_classes(tags=["tag_1"])["test_detector"] == MyTestDetector 15 | assert "test_detector" not in DetectorRegistry.get_detector_classes(tags=["regression"]).keys() 16 | 17 | 18 | def test_detector_decorator(): 19 | @detector 20 | class MyDecoratedDetector: 21 | def run(self, model, dataset): 22 | return [] 23 | 24 | assert "my_decorated_detector" in DetectorRegistry.get_detector_classes().keys() 25 | 26 | @detector(name="other_detector", tags=["tag_1", "tag_2"]) 27 | class MyOtherDecoratedDetector: 28 | def run(self, model, dataset): 29 | return [] 30 | 31 | assert "other_detector" in DetectorRegistry.get_detector_classes().keys() 32 | assert "other_detector" in DetectorRegistry.get_detector_classes(tags=["tag_2"]).keys() 33 | assert "other_detector" not in DetectorRegistry.get_detector_classes(tags=["regression"]).keys() 34 | -------------------------------------------------------------------------------- /tests/scan/test_suite_generation.py: -------------------------------------------------------------------------------- 1 | from giskard import slicing_function 2 | from giskard.core.suite import Suite 3 | from giskard.scanner.issues import Issue, IssueLevel, Performance 4 | from giskard.scanner.performance.metrics import Accuracy, MetricResult 5 | from giskard.scanner.performance.performance_bias_detector import _generate_performance_tests 6 | from giskard.scanner.report import ScanReport 7 | 8 | 9 | def test_generate_test_suite_from_scan_result(german_credit_data, german_credit_model): 10 | @slicing_function 11 | def slice_fn(x): 12 | return True 13 | 14 | issues = [ 15 | Issue( 16 | model=german_credit_model, 17 | dataset=german_credit_data, 18 | group=Performance, 19 | level=IssueLevel.MAJOR, 20 | description="Description", 21 | meta={ 22 | "metric": "Accuracy", 23 | "metric_value": 0.2, 24 | "metric_reference_value": 1.0, 25 | "slice_metric": MetricResult(Accuracy(), 0.2, 100), 26 | "reference_metric": MetricResult(Accuracy(), 1, 100), 27 | "threshold": 0.95, 28 | "p_value": 1e-6, 29 | }, 30 | slicing_fn=slice_fn, 31 | tests=_generate_performance_tests, 32 | ) 33 | ] 34 | 35 | result = ScanReport(issues, model=german_credit_model, dataset=german_credit_data) 36 | test_suite = result.generate_test_suite("Custom name") 37 | 38 | assert isinstance(test_suite, Suite) 39 | assert test_suite.name == "Custom name" 40 | assert len(test_suite.tests) == 1 41 | 42 | test_suite.run() 43 | -------------------------------------------------------------------------------- /tests/slicing/test_slicing_function.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from giskard import slicing_function 5 | 6 | 7 | def test_slicing_function(german_credit_data): 8 | # Define a slicing function 9 | @slicing_function(row_level=False) 10 | def head_slice(df: pd.DataFrame) -> pd.DataFrame: 11 | return df.head(10) 12 | 13 | # Slice the dataset 14 | data_slice = german_credit_data.slice(head_slice) 15 | 16 | assert np.all(data_slice.df.values == german_credit_data.df.head(10).values) 17 | -------------------------------------------------------------------------------- /tests/test_import_giskard.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from time import time 3 | 4 | # As of 17.01.2024. The import of giskard takes about 2.5-3 seconds on a 2021 MacBook Pro. 5 | IMPORT_TIME_THRESHOLD_SECOND = 7 6 | 7 | 8 | def test_import_giskard(): 9 | start = time() 10 | 11 | # Start subprocess to ensure that giskard is not already imported 12 | # Average of 10 imports to avoid random issues 13 | for _ in range(10): 14 | subprocess.run(["python", "-c", "import giskard"]) 15 | 16 | end = time() 17 | assert ( 18 | end - start 19 | ) / 10 < IMPORT_TIME_THRESHOLD_SECOND, f"Import of Giskard took {(end - start) / 10} seconds on average (maximum threshold is set to {IMPORT_TIME_THRESHOLD_SECOND} second)" 20 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from giskard.settings import Settings 5 | 6 | 7 | def test_default_obj_equals_static_built(): 8 | assert Settings(disable_analytics=os.getenv("GSK_DISABLE_ANALYTICS", "false")) == Settings.build_from_env() 9 | 10 | 11 | def test_override_env(): 12 | os.environ["GSK_DISABLE_ANALYTICS"] = "true" 13 | os.environ["GSK_WS_PORT"] = "4444" 14 | os.environ["GSK_HOST"] = "toto" 15 | 16 | settings = Settings.build_from_env() 17 | assert settings.disable_analytics 18 | assert settings.ws_port == 4444 19 | assert settings.host == "toto" 20 | 21 | 22 | def test_expand_path(): 23 | os.environ["ENV1"] = "~/toto/../${ENV2}" 24 | os.environ["ENV2"] = "${ENV3}/${ENV4}" 25 | os.environ["ENV3"] = "tutu" 26 | os.environ["ENV4"] = "tata" 27 | 28 | settings = Settings(home="${ENV1}") 29 | print(settings.home_dir.as_posix()) 30 | assert settings.home_dir.as_posix() == (Path("~").expanduser().resolve() / "tutu/tata").as_posix() 31 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import assert_array_equal 2 | 3 | from giskard.utils.analytics_collector import anonymize 4 | 5 | 6 | def test_anonymize_simple(): 7 | assert anonymize("clear text") == "3ede875cc56517b8" 8 | assert anonymize(123) == "40bd001563085fc3" 9 | assert anonymize(1.23) == "1ee00eff570301a5" 10 | assert anonymize(None) is None 11 | 12 | 13 | def test_anonymize_complex(): 14 | assert_array_equal(anonymize(["hello", "world"]), ["aaf4c61ddcc5e8a2", "7c211433f0207159"]) 15 | -------------------------------------------------------------------------------- /tests/testing/test_side_effects.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from giskard import Dataset 5 | from giskard.testing import test_accuracy 6 | 7 | 8 | def test_dataset_index_is_preserved(german_credit_data, german_credit_model): 9 | dataset = german_credit_data 10 | model = german_credit_model 11 | 12 | # Permute the index to make it non banal, otherwise reset_index will produce, 13 | # by change, the same sequence. 14 | dataset.df.set_index(dataset.df.index.values[np.random.permutation(len(dataset.df))], inplace=True) 15 | original_idx = dataset.df.index.copy() 16 | 17 | _ = test_accuracy(dataset=dataset, model=model, threshold=0.8).execute() 18 | 19 | assert (dataset.df.index == original_idx).all() 20 | 21 | # Try with an index that is not a sequence of integers 22 | dataset = Dataset( 23 | dataset.df.set_index(pd.to_datetime(dataset.df.index)), column_types=dataset.column_types, target=dataset.target 24 | ) 25 | original_idx = dataset.df.index.copy() 26 | 27 | _ = test_accuracy(dataset=dataset, model=model, threshold=0.8).execute() 28 | 29 | assert (dataset.df.index == original_idx).all() 30 | -------------------------------------------------------------------------------- /tests/url_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from urllib.request import urlretrieve 3 | 4 | 5 | def fetch_test_data(url: str, file: Path) -> None: 6 | if not file.parent.exists(): 7 | file.parent.mkdir(parents=True, exist_ok=True) 8 | 9 | if not file.exists(): 10 | urlretrieve(url, file) 11 | -------------------------------------------------------------------------------- /tests/utils/test_logging_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import giskard 4 | 5 | 6 | def test_giskard_log_level(): 7 | assert ( 8 | logging.getLogger(giskard.__name__).level == logging.INFO 9 | ), "giskard log level should be set to INFO when importing giskard" 10 | 11 | 12 | def test_other_package_log_level_unset(): 13 | assert ( 14 | logging.getLogger(giskard.llm.client.__name__).level == logging.NOTSET 15 | ), "Non giskard package log level should't be touched by giskard (NOTSET)" 16 | 17 | 18 | def test_root_log_level_default_warning(): 19 | assert logging.getLogger().level == logging.WARNING, "Root package log level should be set to WARNING by default" 20 | --------------------------------------------------------------------------------