├── .gitignore ├── DESCRIPTION.rst ├── LICENSE ├── MANIFEST.in ├── README.md ├── docs ├── Makefile ├── imgs │ ├── AI靶场logo.png │ ├── 傲天.jpeg │ ├── 奇安信.jpeg │ ├── 昂楷科技.jpeg │ └── 组织.png ├── make.bat ├── source │ ├── README.md │ ├── api │ │ ├── index.rst │ │ ├── regex_generation │ │ │ └── index.rst │ │ └── sensitive_analyze │ │ │ └── index.rst │ ├── conf.py │ └── index.rst ├── 识别流程.png └── 问题描述.png ├── opendlp ├── __init__.py ├── regex_generation │ ├── __init__.py │ ├── bpe.py │ ├── config │ │ ├── __init__.py │ │ ├── conf.py │ │ └── evolve_param.py │ ├── dataset.py │ ├── evolution │ │ ├── __init__.py │ │ ├── evolve.py │ │ ├── selection.py │ │ └── variation.py │ ├── fitness │ │ ├── __init__.py │ │ ├── fitness.py │ │ └── objective.py │ ├── generations │ │ ├── __init__.py │ │ ├── dataset_population.py │ │ ├── population_init.py │ │ └── random_population.py │ ├── generator.py │ ├── node_factory.py │ ├── regex_tree │ │ ├── __init__.py │ │ ├── function_set │ │ │ ├── __init__.py │ │ │ ├── backreference.py │ │ │ ├── binary_operator.py │ │ │ ├── concatenator.py │ │ │ ├── group.py │ │ │ ├── list_match.py │ │ │ ├── list_not_match.py │ │ │ ├── lookahead_negative.py │ │ │ ├── lookahead_positive.py │ │ │ ├── lookaround.py │ │ │ ├── lookbehind_negative.py │ │ │ ├── lookbehind_positive.py │ │ │ ├── match_min_max.py │ │ │ ├── match_min_max_greedy.py │ │ │ ├── match_one_or_more.py │ │ │ ├── match_one_or_more_greedy.py │ │ │ ├── match_zero_or_more.py │ │ │ ├── match_zero_or_more_greedy.py │ │ │ ├── match_zero_or_one.py │ │ │ ├── match_zero_or_one_greedy.py │ │ │ ├── non_capturing_group.py │ │ │ ├── orr.py │ │ │ ├── quantifier.py │ │ │ ├── ternary_operator.py │ │ │ └── unary_operator.py │ │ ├── id_factory.py │ │ ├── node.py │ │ ├── regex_context.py │ │ └── terminal_set │ │ │ ├── __init__.py │ │ │ ├── anchor.py │ │ │ ├── constant.py │ │ │ └── regex_range.py │ └── utils.py └── sensitive_analyze │ ├── __init__.py │ ├── analyzer_engine.py │ ├── entity_classify │ ├── __init__.py │ ├── classifier.py │ ├── config.py │ ├── model.py │ ├── trained-model │ │ ├── label2id.pkl │ │ ├── model.pt │ │ └── vocab.pkl │ └── utils_infer.py │ ├── entity_recognize │ ├── __init__.py │ ├── conf │ │ ├── __init__.py │ │ └── config.py │ ├── entity_recognizer.py │ ├── pattern.py │ ├── pattern_recognizer.py │ ├── predefined_recognizer │ │ ├── __init__.py │ │ ├── nlp_recognizer │ │ │ ├── __init__.py │ │ │ └── nlp_recognizer.py │ │ └── rule_recognizer │ │ │ ├── __init__.py │ │ │ ├── bank_card_recognizer.py │ │ │ ├── date_recognizer.py │ │ │ ├── domain_recognizer.py │ │ │ ├── email_recognizer.py │ │ │ ├── idcard_recognizer.py │ │ │ ├── ipv4_recognizer.py │ │ │ ├── ipv6_recognizer.py │ │ │ ├── license_plate_recognizer.py │ │ │ ├── mac_recognizer.py │ │ │ ├── mobile_phone_recognizer.py │ │ │ ├── passport_recognizer.py │ │ │ ├── postcode_recognizer.py │ │ │ ├── resources │ │ │ ├── mobile-phone-code.json │ │ │ ├── postcode-province.json │ │ │ ├── social-credit-code.json │ │ │ └── telephone-code.json │ │ │ ├── social_credit_code_recognizer.py │ │ │ └── telephone_recognizer.py │ ├── recognizer_engine.py │ ├── recognizer_registry.py │ ├── recognizer_result.py │ ├── userdefined_recognizer │ │ ├── __init__.py │ │ └── udf_recognizer.py │ └── utils.py │ ├── exceptions.py │ ├── table_analyzer.py │ └── utils.py ├── requirements.txt ├── requirements_service.txt ├── service ├── README.md ├── __init__.py ├── examples │ ├── example.py │ └── example_regex_generate.py ├── grpc_module │ ├── __init__.py │ ├── readme.txt │ ├── sensitive_pb2.py │ └── sensitive_pb2_grpc.py ├── logging.ini ├── opendlp_server.py ├── proto │ └── sensitive.proto └── util.py ├── setup.py └── tests ├── data ├── data-empty.csv ├── data-udf.csv ├── data1.csv ├── data2.csv ├── data3.csv ├── data4.csv ├── dataset-test.csv ├── error-test │ ├── data-gbk.csv │ ├── empty.csv │ └── json-error.json ├── pattern.json └── regex_generation │ └── test-data │ ├── BANK_CARD.csv │ ├── DATE.csv │ ├── DOMAIN_NAME.csv │ ├── EMAIL.csv │ ├── ID_CARD.csv │ ├── IPV4.csv │ ├── IPV6.csv │ ├── LICENSE_PLATE.csv │ ├── MAC.csv │ ├── MOBILE_PHONE.csv │ ├── PASSPORT.csv │ ├── POSTCODE.csv │ ├── SOCIAL_CREDIT_CODE.csv │ └── TELEPHONE.csv ├── grpc_module ├── __init__.py ├── readme.txt ├── sensitive_pb2.py └── sensitive_pb2_grpc.py └── test_client.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/.gitignore -------------------------------------------------------------------------------- /DESCRIPTION.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/DESCRIPTION.rst -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/README.md -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/imgs/AI靶场logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/imgs/AI靶场logo.png -------------------------------------------------------------------------------- /docs/imgs/傲天.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/imgs/傲天.jpeg -------------------------------------------------------------------------------- /docs/imgs/奇安信.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/imgs/奇安信.jpeg -------------------------------------------------------------------------------- /docs/imgs/昂楷科技.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/imgs/昂楷科技.jpeg -------------------------------------------------------------------------------- /docs/imgs/组织.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/imgs/组织.png -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/source/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/source/README.md -------------------------------------------------------------------------------- /docs/source/api/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/source/api/index.rst -------------------------------------------------------------------------------- /docs/source/api/regex_generation/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/source/api/regex_generation/index.rst -------------------------------------------------------------------------------- /docs/source/api/sensitive_analyze/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/source/api/sensitive_analyze/index.rst -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /docs/识别流程.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/识别流程.png -------------------------------------------------------------------------------- /docs/问题描述.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/docs/问题描述.png -------------------------------------------------------------------------------- /opendlp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/__init__.py -------------------------------------------------------------------------------- /opendlp/regex_generation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/regex_generation/bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/bpe.py -------------------------------------------------------------------------------- /opendlp/regex_generation/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/regex_generation/config/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/config/conf.py -------------------------------------------------------------------------------- /opendlp/regex_generation/config/evolve_param.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/config/evolve_param.py -------------------------------------------------------------------------------- /opendlp/regex_generation/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/dataset.py -------------------------------------------------------------------------------- /opendlp/regex_generation/evolution/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/evolution/__init__.py -------------------------------------------------------------------------------- /opendlp/regex_generation/evolution/evolve.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/evolution/evolve.py -------------------------------------------------------------------------------- /opendlp/regex_generation/evolution/selection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/evolution/selection.py -------------------------------------------------------------------------------- /opendlp/regex_generation/evolution/variation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/evolution/variation.py -------------------------------------------------------------------------------- /opendlp/regex_generation/fitness/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/regex_generation/fitness/fitness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/fitness/fitness.py -------------------------------------------------------------------------------- /opendlp/regex_generation/fitness/objective.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/fitness/objective.py -------------------------------------------------------------------------------- /opendlp/regex_generation/generations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/generations/__init__.py -------------------------------------------------------------------------------- /opendlp/regex_generation/generations/dataset_population.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/generations/dataset_population.py -------------------------------------------------------------------------------- /opendlp/regex_generation/generations/population_init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/generations/population_init.py -------------------------------------------------------------------------------- /opendlp/regex_generation/generations/random_population.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/generations/random_population.py -------------------------------------------------------------------------------- /opendlp/regex_generation/generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/generator.py -------------------------------------------------------------------------------- /opendlp/regex_generation/node_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/node_factory.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/__init__.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/__init__.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/backreference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/backreference.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/binary_operator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/binary_operator.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/concatenator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/concatenator.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/group.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/group.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/list_match.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/list_match.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/list_not_match.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/list_not_match.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/lookahead_negative.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/lookahead_negative.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/lookahead_positive.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/lookahead_positive.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/lookaround.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/lookaround.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/lookbehind_negative.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/lookbehind_negative.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/lookbehind_positive.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/lookbehind_positive.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_min_max.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_min_max.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_min_max_greedy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_min_max_greedy.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_one_or_more.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_one_or_more.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_one_or_more_greedy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_one_or_more_greedy.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_zero_or_more.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_zero_or_more.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_zero_or_more_greedy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_zero_or_more_greedy.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_zero_or_one.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_zero_or_one.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/match_zero_or_one_greedy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/match_zero_or_one_greedy.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/non_capturing_group.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/non_capturing_group.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/orr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/orr.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/quantifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/quantifier.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/ternary_operator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/ternary_operator.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/function_set/unary_operator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/function_set/unary_operator.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/id_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/id_factory.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/node.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/node.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/regex_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/regex_context.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/terminal_set/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/terminal_set/__init__.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/terminal_set/anchor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/terminal_set/anchor.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/terminal_set/constant.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/terminal_set/constant.py -------------------------------------------------------------------------------- /opendlp/regex_generation/regex_tree/terminal_set/regex_range.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/regex_tree/terminal_set/regex_range.py -------------------------------------------------------------------------------- /opendlp/regex_generation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/regex_generation/utils.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/analyzer_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/analyzer_engine.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_classify/classifier.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_classify/config.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_classify/model.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/trained-model/label2id.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_classify/trained-model/label2id.pkl -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/trained-model/model.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_classify/trained-model/model.pt -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/trained-model/vocab.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_classify/trained-model/vocab.pkl -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_classify/utils_infer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_classify/utils_infer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/__init__.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/conf/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/conf/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/conf/config.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/entity_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/entity_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/pattern.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/pattern.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/pattern_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/pattern_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/__init__.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/nlp_recognizer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/nlp_recognizer/nlp_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/nlp_recognizer/nlp_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/bank_card_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/bank_card_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/date_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/date_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/domain_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/domain_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/email_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/email_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/idcard_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/idcard_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/ipv4_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/ipv4_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/ipv6_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/ipv6_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/license_plate_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/license_plate_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/mac_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/mac_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/mobile_phone_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/mobile_phone_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/passport_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/passport_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/postcode_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/postcode_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/mobile-phone-code.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/mobile-phone-code.json -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/postcode-province.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/postcode-province.json -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/social-credit-code.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/social-credit-code.json -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/telephone-code.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/resources/telephone-code.json -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/social_credit_code_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/social_credit_code_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/telephone_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/predefined_recognizer/rule_recognizer/telephone_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/recognizer_engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/recognizer_engine.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/recognizer_registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/recognizer_registry.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/recognizer_result.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/recognizer_result.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/userdefined_recognizer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/userdefined_recognizer/__init__.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/userdefined_recognizer/udf_recognizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/userdefined_recognizer/udf_recognizer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/entity_recognize/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/entity_recognize/utils.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/exceptions.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/table_analyzer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/table_analyzer.py -------------------------------------------------------------------------------- /opendlp/sensitive_analyze/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/opendlp/sensitive_analyze/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements_service.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/requirements_service.txt -------------------------------------------------------------------------------- /service/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/README.md -------------------------------------------------------------------------------- /service/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /service/examples/example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/examples/example.py -------------------------------------------------------------------------------- /service/examples/example_regex_generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/examples/example_regex_generate.py -------------------------------------------------------------------------------- /service/grpc_module/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /service/grpc_module/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/grpc_module/readme.txt -------------------------------------------------------------------------------- /service/grpc_module/sensitive_pb2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/grpc_module/sensitive_pb2.py -------------------------------------------------------------------------------- /service/grpc_module/sensitive_pb2_grpc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/grpc_module/sensitive_pb2_grpc.py -------------------------------------------------------------------------------- /service/logging.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/logging.ini -------------------------------------------------------------------------------- /service/opendlp_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/opendlp_server.py -------------------------------------------------------------------------------- /service/proto/sensitive.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/proto/sensitive.proto -------------------------------------------------------------------------------- /service/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/service/util.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/setup.py -------------------------------------------------------------------------------- /tests/data/data-empty.csv: -------------------------------------------------------------------------------- 1 | QQ,PASSWORD 2 | -------------------------------------------------------------------------------- /tests/data/data-udf.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/data-udf.csv -------------------------------------------------------------------------------- /tests/data/data1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/data1.csv -------------------------------------------------------------------------------- /tests/data/data2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/data2.csv -------------------------------------------------------------------------------- /tests/data/data3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/data3.csv -------------------------------------------------------------------------------- /tests/data/data4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/data4.csv -------------------------------------------------------------------------------- /tests/data/dataset-test.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/dataset-test.csv -------------------------------------------------------------------------------- /tests/data/error-test/data-gbk.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/error-test/data-gbk.csv -------------------------------------------------------------------------------- /tests/data/error-test/empty.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/data/error-test/json-error.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/error-test/json-error.json -------------------------------------------------------------------------------- /tests/data/pattern.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/pattern.json -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/BANK_CARD.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/BANK_CARD.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/DATE.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/DATE.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/DOMAIN_NAME.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/DOMAIN_NAME.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/EMAIL.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/EMAIL.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/ID_CARD.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/ID_CARD.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/IPV4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/IPV4.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/IPV6.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/IPV6.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/LICENSE_PLATE.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/LICENSE_PLATE.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/MAC.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/MAC.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/MOBILE_PHONE.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/MOBILE_PHONE.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/PASSPORT.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/PASSPORT.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/POSTCODE.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/POSTCODE.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/SOCIAL_CREDIT_CODE.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/SOCIAL_CREDIT_CODE.csv -------------------------------------------------------------------------------- /tests/data/regex_generation/test-data/TELEPHONE.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/data/regex_generation/test-data/TELEPHONE.csv -------------------------------------------------------------------------------- /tests/grpc_module/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/grpc_module/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/grpc_module/readme.txt -------------------------------------------------------------------------------- /tests/grpc_module/sensitive_pb2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/grpc_module/sensitive_pb2.py -------------------------------------------------------------------------------- /tests/grpc_module/sensitive_pb2_grpc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/grpc_module/sensitive_pb2_grpc.py -------------------------------------------------------------------------------- /tests/test_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hitsz-ids/openDLP/HEAD/tests/test_client.py --------------------------------------------------------------------------------