├── web └── play │ ├── node_modules │ └── go.mod │ ├── src │ ├── init_go.js │ ├── .prettierrc │ ├── Error.elm │ ├── Style.elm │ ├── Header.elm │ ├── Ports.elm │ ├── Play.elm │ ├── Examples.elm │ └── OutputPanel.elm │ ├── postcss.config.cjs │ ├── webpack.config.prod.js │ ├── tailwind.config.cjs │ ├── webpack.config.dev.js │ ├── webpack.config.wasm.prod.js │ ├── elm.json │ ├── webpack.config.wasm.js │ ├── package.json │ └── webpack.config.common.js ├── test ├── names_F.txt ├── names.txt ├── csvvalues.csv ├── names_M.txt ├── find_person.json ├── sample.txt ├── masking_template.yml ├── wrongMasking.yml ├── masking_agency.yml ├── masking_template_lower.yml ├── data.json ├── masking_template_noacc.yml ├── httpmock │ └── default.json ├── maskingTest.yml ├── masking_template_function.yml ├── suites │ ├── help.yml │ ├── without_masking_file.yml │ ├── jsonschema.yml │ ├── masking_rangemask.yml │ ├── masking_constant.yml │ ├── masking_command.yml │ ├── mask_chaining.yml │ ├── masking_remove.yml │ ├── mask_pipe.yml │ ├── multiple-selectors.yml │ ├── repeat.yml │ ├── serve.yml │ ├── play.yml │ ├── masking_incremental.yml │ ├── masking_multiple_time.yml │ ├── masking_replacement.yml │ ├── empty-input_test.yml │ ├── masking_hash.yml │ ├── masking_segment.yml │ ├── preserve-key-order.yml │ ├── other_masking_file.yml │ ├── one-liner-masking.yml │ ├── masking_luhn.yml │ ├── mask_add_transient.yml │ ├── new_field.yml │ ├── masking_partition.yml │ ├── masking_duration.yml │ ├── parsingXML.yml │ ├── masking_transcode.yml │ ├── masking_fluxuri.yml │ ├── masking_context.yml │ ├── masking_random.yml │ ├── masking_parserdate.yml │ ├── masking_markov.yml │ ├── mask_template_each.yml │ └── masking-transcode.yml ├── masking_template_range.yml ├── persons_dirty.csv ├── masking_account.yml ├── masking_attr.yml ├── persons.csv ├── persons_sep.csv ├── persons_exact_jaccard.csv ├── persons_same_name.csv ├── masking_pipe.yml ├── exemple_expected.xml ├── persons_sep_comment.csv ├── exemple.xml ├── masking_find_in_csv.yml ├── persons_sep_comment_trim.csv ├── masking.yml └── masking_findincsv_many.yml ├── doc └── img │ └── pimo.png ├── assets └── pimo-play.png ├── cmd └── pimo │ └── default.pgo ├── demo ├── demo5 │ ├── data.jsonl │ ├── masking.yml │ ├── demo.sh │ └── README.md ├── demo7 │ ├── data.jsonl │ ├── masking.yml │ ├── masking-decrypt.yml │ ├── masking-tweak.yml │ └── demo.sh ├── demo6 │ ├── data.jsonl │ ├── masking.yml │ ├── demo.sh │ └── README.md ├── demo8 │ ├── masking-org.yml │ ├── data-cache.jsonl │ ├── masking-root.yml │ ├── masking-pipe-1.yml │ ├── masking-wrong.yml │ ├── masking-alsowrong.yml │ ├── masking-cache.yml │ ├── demo.sh │ ├── masking-pipe-2.yml │ └── data.json ├── demo4 │ ├── masking.yml │ ├── demo.sh │ └── data.jsonl ├── demo3 │ ├── data.jsonl │ ├── demo.sh │ └── masking.yml ├── demo1 │ ├── data.jsonl │ ├── demo.sh │ └── masking.yml ├── demo2 │ ├── data.jsonl │ ├── demo.sh │ └── masking.yml ├── README.md └── demo9 │ └── masking.yml ├── pkg ├── model │ ├── common.go │ ├── iterable.go │ ├── logger.go │ ├── ordered_dict_test.go │ ├── sources_test.go │ ├── pack.go │ ├── wrapper.go │ ├── process_counter.go │ ├── sources.go │ ├── process_delete.go │ └── repeater.go ├── parquet │ ├── testdata │ │ ├── alltypes_plain.parquet │ │ └── alltypes_plain_out.parquet │ └── parquet_test.go ├── maskingdata │ ├── maskingdata.go │ └── surnameFR.go ├── markov │ └── markov_test.go ├── template │ └── template.go ├── remove │ ├── remove.go │ └── remove_test.go ├── constant │ ├── constant.go │ └── constant_test.go ├── replacement │ ├── replacement.go │ └── replacement_test.go ├── increment │ └── increment.go ├── add │ ├── add_test.go │ └── add.go ├── functions │ └── exec.go ├── fluxuri │ └── fluxuri.go ├── rangemask │ └── rangemask_test.go ├── fromjson │ └── fromjson.go ├── randomdecimal │ └── randomdecimal_test.go ├── templatemask │ └── templatemask.go ├── uri │ └── uri_test.go ├── randomint │ └── randomint_test.go ├── regex │ └── regex_test.go ├── transcode │ └── transcode_test.go ├── command │ └── command.go ├── axis │ ├── constraint.go │ └── generate_test.go ├── hash │ └── hash_test.go ├── logmask │ └── log.go ├── sequence │ └── sequence_test.go ├── weightedchoice │ └── weightedchoice_test.go ├── randomuri │ └── randomuri_test.go └── randdura │ └── randdura_test.go ├── .vscode └── settings.json ├── .env.example ├── .editorconfig ├── .goreleaser.template.yml ├── internal └── app │ └── pimo │ ├── client │ ├── readme.html │ └── .README.md │ ├── mock │ ├── consts.go │ └── routes.go │ └── xixo.go ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── .devcontainer ├── docker-compose.yml ├── Dockerfile ├── cgi_ca_root.crt ├── cgi_zscaler_ca_root.crt └── misc-sni-google-com.crt ├── githooks ├── pre-commit └── commit-msg ├── CONTRIBUTING.md ├── masking.yml ├── NOTICE.md └── Dockerfile /web/play/node_modules/go.mod: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/names_F.txt: -------------------------------------------------------------------------------- 1 | Marie 2 | Julie 3 | -------------------------------------------------------------------------------- /test/names.txt: -------------------------------------------------------------------------------- 1 | Mickael 2 | Marc 3 | Benjamin -------------------------------------------------------------------------------- /test/csvvalues.csv: -------------------------------------------------------------------------------- 1 | 1623 2 | 1512 3 | 905 4 | -------------------------------------------------------------------------------- /test/names_M.txt: -------------------------------------------------------------------------------- 1 | Mickael 2 | Marc 3 | Benjamin 4 | -------------------------------------------------------------------------------- /test/find_person.json: -------------------------------------------------------------------------------- 1 | {"nom":"Vidal","info_personne":""} 2 | -------------------------------------------------------------------------------- /test/sample.txt: -------------------------------------------------------------------------------- 1 | I want a cheese burger 2 | I need a cheese cake 3 | -------------------------------------------------------------------------------- /web/play/src/init_go.js: -------------------------------------------------------------------------------- 1 | (() => { 2 | this.go = new Go() 3 | })() 4 | -------------------------------------------------------------------------------- /doc/img/pimo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CGI-FR/PIMO/HEAD/doc/img/pimo.png -------------------------------------------------------------------------------- /assets/pimo-play.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CGI-FR/PIMO/HEAD/assets/pimo-play.png -------------------------------------------------------------------------------- /cmd/pimo/default.pgo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CGI-FR/PIMO/HEAD/cmd/pimo/default.pgo -------------------------------------------------------------------------------- /demo/demo5/data.jsonl: -------------------------------------------------------------------------------- 1 | {"birth": ""} 2 | {"birth": ""} 3 | {"birth": ""} 4 | {"birth": ""} 5 | {"birth": ""} -------------------------------------------------------------------------------- /demo/demo7/data.jsonl: -------------------------------------------------------------------------------- 1 | {"siret": "01234567891234"} 2 | {"siret": "12345678912340"} 3 | {"siret": "23456789123401"} -------------------------------------------------------------------------------- /web/play/src/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "tabWidth": 2, 3 | "useTabs": false, 4 | "printWidth": 300 5 | } 6 | -------------------------------------------------------------------------------- /pkg/model/common.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Resource interface { 4 | Open() error 5 | Close() error 6 | } 7 | -------------------------------------------------------------------------------- /pkg/parquet/testdata/alltypes_plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CGI-FR/PIMO/HEAD/pkg/parquet/testdata/alltypes_plain.parquet -------------------------------------------------------------------------------- /pkg/parquet/testdata/alltypes_plain_out.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CGI-FR/PIMO/HEAD/pkg/parquet/testdata/alltypes_plain_out.parquet -------------------------------------------------------------------------------- /web/play/postcss.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: [ 3 | require('autoprefixer'), 4 | require('cssnano') 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "yaml.schemas": { 3 | 4 | "*masking*": "file:///${WORKDIR}/schema/v1/pimo.schema.json", 5 | 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /demo/demo6/data.jsonl: -------------------------------------------------------------------------------- 1 | {} 2 | {"fieldToRemove":"Unimportant Value"} 3 | {"newField":"Some Value"} 4 | {"fieldToRemove":"Unimportant Value","newField":"Some Value"} -------------------------------------------------------------------------------- /demo/demo8/masking-org.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | masking: 3 | - selector: 4 | jsonpath: "email" 5 | mask: 6 | template: "{{.name}}.{{.surname}}@{{.org.domain}}" 7 | -------------------------------------------------------------------------------- /test/masking_template.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "mail" 6 | mask: 7 | template: "{{.surname}}.{{.name}}@gmail.com" -------------------------------------------------------------------------------- /test/wrongMasking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "name1" 6 | mask: 7 | constant: "Toto" 8 | command: "echo Dorothy" -------------------------------------------------------------------------------- /demo/demo8/data-cache.jsonl: -------------------------------------------------------------------------------- 1 | {"age": 10, "related": [{"age":30}]} 2 | {"age": 20, "related": [{"age":40}]} 3 | {"age": 30, "related": [{"age":10}]} 4 | {"age": 40, "related": [{"age":20}]} -------------------------------------------------------------------------------- /test/masking_agency.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | 4 | masking: 5 | - selector: 6 | jsonpath: "agency_number" 7 | mask: 8 | template: '{{MaskRegex "[0-9]{4}$"}}' 9 | -------------------------------------------------------------------------------- /test/masking_template_lower.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "mail" 6 | mask: 7 | template: "{{.surname | lower}}.{{.name | lower}}@gmail.com" -------------------------------------------------------------------------------- /demo/demo8/masking-root.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | masking: 3 | - selector: 4 | jsonpath: "organizations.persons" 5 | mask: 6 | pipe: 7 | injectParent: "org" 8 | file: "masking-org.yml" -------------------------------------------------------------------------------- /test/data.json: -------------------------------------------------------------------------------- 1 | {"age":35,"name":"Benjamin"} 2 | {"personne": {"name":"Benjamin","surname":"Toto"}} 3 | {"age":35,"name":"Benjamin", "surname":"Toto","address":{"town":"Nantes"},"mail":"Benjamin.Toto@hotmail.fr"} 4 | -------------------------------------------------------------------------------- /test/masking_template_noacc.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "mail" 6 | mask: 7 | template: "{{.surname | NoAccent | lower}}.{{.name | NoAccent | lower}}@gmail.com" -------------------------------------------------------------------------------- /web/play/webpack.config.prod.js: -------------------------------------------------------------------------------- 1 | import { merge } from 'webpack-merge'; 2 | 3 | import common from "./webpack.config.common.js"; 4 | 5 | export default merge(common(true), { 6 | mode: "production", 7 | 8 | }); 9 | -------------------------------------------------------------------------------- /test/httpmock/default.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "httpRequest": { 4 | "path": "/api/v1/stats" 5 | }, 6 | "httpResponse": { 7 | "statusCode": 200 8 | } 9 | } 10 | ] 11 | -------------------------------------------------------------------------------- /demo/demo6/masking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | masking: 3 | - selector: 4 | jsonpath: "newField" 5 | mask: 6 | add: "New Value" 7 | - selector: 8 | jsonpath: "fieldToRemove" 9 | mask: 10 | remove: true -------------------------------------------------------------------------------- /test/maskingTest.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "name" 6 | mask: 7 | constant: "Toto" 8 | - selector: 9 | jsonpath: "phone" 10 | mask: 11 | regex: "0[1-7]( ([0-9]){2}){4}" -------------------------------------------------------------------------------- /test/masking_template_function.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | functions: 3 | print: 4 | params: 5 | - name: "str" 6 | body: |- 7 | return str 8 | masking: 9 | - selector: 10 | jsonpath: "text" 11 | mask: 12 | template: '{{print "hello"}}' 13 | -------------------------------------------------------------------------------- /pkg/model/iterable.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "iter" 4 | 5 | // Iterable is a resource that can be iterated over to produce values of type T. It extends the Resource interface. 6 | type Iterable[T any] interface { 7 | Resource 8 | Values() iter.Seq2[T, error] 9 | } 10 | -------------------------------------------------------------------------------- /demo/demo4/masking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "fieldToReplace" 6 | mask: 7 | replacement: "replacement" 8 | - selector: 9 | jsonpath: "compositeField" 10 | mask: 11 | template: "{{.field1}}+{{.field2}}" -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # Renseigner les variables http_proxy, https_proxy et no_proxy (si besoin proxy CGI) 2 | # Attention à ne pas commiter ce fichier 3 | http_proxy=http://:@:/ 4 | https_proxy=http://:@:/ 5 | no_proxy=localhost 6 | -------------------------------------------------------------------------------- /demo/demo3/data.jsonl: -------------------------------------------------------------------------------- 1 | {"name":["Benjamin", "Jean", "Louis"], "surname": ["Martin", "Bernard"]} 2 | {"name":["Benjamin", "Jean", "Louis"], "surname": "Martin"} 3 | {"name":"Benjamin", "surname": ["Martin", "Bernard"]} 4 | {"name":"Benjamin", "surname": "Martin"} 5 | {"identity":{"name":["Jeanne", "Louise"]}} -------------------------------------------------------------------------------- /demo/demo5/masking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | masking: 3 | - selector: 4 | jsonpath: "birth" 5 | mask: 6 | randDate: 7 | dateMin: "2010-01-01T00:00:00Z" 8 | dateMax: "2020-01-01T00:00:00Z" 9 | - selector: 10 | jsonpath: "birth" 11 | mask: 12 | duration: "-P20Y" -------------------------------------------------------------------------------- /test/suites/help.yml: -------------------------------------------------------------------------------- 1 | name: help features 2 | testcases: 3 | - name: short version 4 | steps: 5 | - script: pimo -h 6 | - name: long version 7 | steps: 8 | - script: pimo --help 9 | - name: unknown flag 10 | steps: 11 | - script: pimo -unknown 12 | assertions: 13 | - result.code ShouldEqual 1 -------------------------------------------------------------------------------- /demo/demo8/masking-pipe-1.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "organizations.persons" 6 | mask: 7 | pipe: 8 | masking: 9 | - selector: 10 | jsonpath: "email" 11 | mask: 12 | template: "{{.name}}.{{.surname}}" -------------------------------------------------------------------------------- /demo/demo8/masking-wrong.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "organizations.persons.email" 6 | mask: 7 | # this go template syntax refer to a field that is not in a nested array 8 | template: "{{.organizations.persons.name}}.{{.organizations.persons.surname}}@{{.organizations.domain}}" -------------------------------------------------------------------------------- /web/play/tailwind.config.cjs: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | theme: { 4 | extend: {}, 5 | }, 6 | variants: [], 7 | plugins: [ 8 | require("@tailwindcss/typography"), 9 | require("@tailwindcss/forms"), 10 | require("@tailwindcss/aspect-ratio") 11 | ], 12 | } -------------------------------------------------------------------------------- /test/masking_template_range.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "PERMIS" 6 | mask: 7 | template: | 8 | {{- range $index, $rel := .REL_PERMIS -}} 9 | {{- if $index }},{{end -}} 10 | {"id": {{.ID_PERMIS}},"type":"PERMIS"} 11 | {{- end -}} -------------------------------------------------------------------------------- /test/persons_dirty.csv: -------------------------------------------------------------------------------- 1 | Benoît,Blanc 2 | Wilfried,Jean,wilfried.jean@yopmail.fr 3 | Patricia 4 | Jean-Claude,Leclercq,jean-claude.leclercq@yopmail.fr 5 | Gérard,Perez 6 | Anissa,anissa.mercier@yopmail.fr 7 | Aimée,Moreau,aimee.moreau@yopmail.fr 8 | Aurèle,Chevalier,aurele.chevalier@yopmail.fr, 9 | Luce 10 | Geoffroy,Dupuis,geoffroy.dupuis@yopmail.fr 11 | -------------------------------------------------------------------------------- /demo/demo1/data.jsonl: -------------------------------------------------------------------------------- 1 | {"age": 28, "name":"Dupont", "surname":"Martin", "town": "Nantes"} 2 | {"age": 28, "name":"Dupont", "surname":"Martin", "town": "Nantes"} 3 | {"age": 28, "name":"Dupont", "surname":"Martin", "town": "Nantes"} 4 | {"age": 28, "name":"Dupont", "surname":"Martin", "town": "Nantes"} 5 | {"age": 28, "name":"Dupont", "surname":"Martin", "town": "Nantes"} -------------------------------------------------------------------------------- /demo/demo2/data.jsonl: -------------------------------------------------------------------------------- 1 | {"town": "Nantes", "identity": {"age": 27, "name":"Jean", "surname": "Bertrand"}} 2 | {"town": "Nantes", "identity": {"age": 58, "name":"Benjamin", "surname": "Martin"}} 3 | {"town": "Rennes", "identity": {"age": 35, "name":"Benjamin", "surname": "Bertrand"}} 4 | {"town": "Rennes", "identity": {"age": 28, "name":"Jean", "surname": "Martin"}} 5 | -------------------------------------------------------------------------------- /test/masking_account.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | 4 | masking: 5 | - selector: 6 | jsonpath: "name" 7 | mask: 8 | randomChoiceInUri: "pimo://nameFR" 9 | - selector: 10 | jsonpath: "account_number" 11 | masks: 12 | - incremental: 13 | start: 1 14 | increment: 1 15 | - template: "{{.account_number}}" 16 | -------------------------------------------------------------------------------- /demo/demo1/demo.sh: -------------------------------------------------------------------------------- 1 | # this command apply the masks defined in masking.yml 2 | # to data contained in the file data.jsonl 3 | # and save the result in the file dataout.jsonl 4 | pimo < data.jsonl > dataout.jsonl 5 | 6 | # other alternative (same results) 7 | cat data.jsonl | pimo > dataout.jsonl 8 | 9 | # to also print the result on screen 10 | cat data.jsonl | pimo | tee dataout.jsonl 11 | -------------------------------------------------------------------------------- /demo/demo2/demo.sh: -------------------------------------------------------------------------------- 1 | # this command apply the masks defined in masking.yml 2 | # to data contained in the file data.jsonl 3 | # and save the result in the file dataout.jsonl 4 | pimo < data.jsonl > dataout.jsonl 5 | 6 | # other alternative (same results) 7 | cat data.jsonl | pimo > dataout.jsonl 8 | 9 | # to also print the result on screen 10 | cat data.jsonl | pimo | tee dataout.jsonl 11 | -------------------------------------------------------------------------------- /demo/demo3/demo.sh: -------------------------------------------------------------------------------- 1 | # this command apply the masks defined in masking.yml 2 | # to data contained in the file data.jsonl 3 | # and save the result in the file dataout.jsonl 4 | pimo < data.jsonl > dataout.jsonl 5 | 6 | # other alternative (same results) 7 | cat data.jsonl | pimo > dataout.jsonl 8 | 9 | # to also print the result on screen 10 | cat data.jsonl | pimo | tee dataout.jsonl 11 | -------------------------------------------------------------------------------- /demo/demo4/demo.sh: -------------------------------------------------------------------------------- 1 | # this command apply the masks defined in masking.yml 2 | # to data contained in the file data.jsonl 3 | # and save the result in the file dataout.jsonl 4 | pimo < data.jsonl > dataout.jsonl 5 | 6 | # other alternative (same results) 7 | cat data.jsonl | pimo > dataout.jsonl 8 | 9 | # to also print the result on screen 10 | cat data.jsonl | pimo | tee dataout.jsonl 11 | -------------------------------------------------------------------------------- /demo/demo5/demo.sh: -------------------------------------------------------------------------------- 1 | # this command apply the masks defined in masking.yml 2 | # to data contained in the file data.jsonl 3 | # and save the result in the file dataout.jsonl 4 | pimo < data.jsonl > dataout.jsonl 5 | 6 | # other alternative (same results) 7 | cat data.jsonl | pimo > dataout.jsonl 8 | 9 | # to also print the result on screen 10 | cat data.jsonl | pimo | tee dataout.jsonl 11 | -------------------------------------------------------------------------------- /demo/demo6/demo.sh: -------------------------------------------------------------------------------- 1 | # this command apply the masks defined in masking.yml 2 | # to data contained in the file data.jsonl 3 | # and save the result in the file dataout.jsonl 4 | pimo < data.jsonl > dataout.jsonl 5 | 6 | # other alternative (same results) 7 | cat data.jsonl | pimo > dataout.jsonl 8 | 9 | # to also print the result on screen 10 | cat data.jsonl | pimo | tee dataout.jsonl 11 | -------------------------------------------------------------------------------- /test/suites/without_masking_file.yml: -------------------------------------------------------------------------------- 1 | name: whitout file masking.yaml 2 | testcases: 3 | - name: without masking.yml file 4 | steps: 5 | - script: rm -f masking.yml 6 | - script: pimo 7 | assertions: 8 | #- result.systemerr ShouldContainSubstring masking.yml not working properly 9 | # open masking.yml: no such file or directory 10 | - result.code ShouldEqual 1 11 | 12 | -------------------------------------------------------------------------------- /demo/demo4/data.jsonl: -------------------------------------------------------------------------------- 1 | {"replacement": "Some better value", "fieldToReplace": "Some value"} 2 | {"field1": "First field data", "field2": "Second field data", "compositeField": "Unimportant value"} 3 | {"replacement": "Value1", "fieldToReplace": "Value0", "field1": "data1", "field2": "data2", "compositeField": "Unimportant value"} 4 | {"field1": "AàEéè", "field2": "äâGêë", "compositeField": "Unimportant value"} -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 4 7 | indent_style = space 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.go] 12 | indent_style = tab 13 | 14 | [{Makefile, *.mk}] 15 | indent_style = tab 16 | 17 | [*.{yaml,yml}] 18 | indent_size = 2 19 | 20 | [*.{md,txt}] 21 | indent_style = space 22 | indent_size = 4 23 | -------------------------------------------------------------------------------- /demo/demo3/masking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "name" 6 | mask: 7 | randomChoice: 8 | - Mickael 9 | - Jean 10 | - Benjamin 11 | - selector: 12 | jsonpath: "surname" 13 | mask: 14 | regex: "(Mar|Ber)t(rand|in)" 15 | - selector: 16 | jsonpath: "identity.name" 17 | mask: 18 | randomChoiceInUri: "pimo://nameFR" -------------------------------------------------------------------------------- /test/suites/jsonschema.yml: -------------------------------------------------------------------------------- 1 | name: jsonschema features 2 | testcases: 3 | - name: export jsonschema 4 | steps: 5 | - script: pimo jsonschema > pimo.schema.json 6 | assertions: 7 | - result.code ShouldEqual 0 8 | - script: diff pimo.schema.json ../../schema/v1/pimo.schema.json 9 | assertions: 10 | - result.code ShouldEqual 0 11 | - result.systemout ShouldBeEmpty 12 | -------------------------------------------------------------------------------- /demo/demo8/masking-alsowrong.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "organizations.persons.email" 6 | mask: 7 | # this go template syntax refer to a single value of index (0;0) 8 | # (and it's not very readable) 9 | template: "{{(index (index .organizations 0).persons 0).name}}.{{(index (index .organizations 0).persons 0).surname}}@{{(index .organizations 0).domain}}" 10 | -------------------------------------------------------------------------------- /.goreleaser.template.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - GO111MODULE=on 3 | before: 4 | hooks: 5 | - go mod download 6 | dist: bin/dist 7 | builds: 8 | {{range (getenv "BUILDPATHS" | jsonArray)}} 9 | - id: "{{.}}" 10 | binary: "{{ path.Base . }}" 11 | main: "./{{.}}" 12 | goos: 13 | - darwin 14 | - linux 15 | - windows 16 | ldflags: 17 | - -s -w -X main.name={{ path.Base . }} {{env.Getenv "LDFLAGS"}} 18 | {{end}} 19 | -------------------------------------------------------------------------------- /demo/demo7/masking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | masking: 3 | - selector: 4 | jsonpath: "siret" 5 | mask: 6 | # use of the FF1 mask 7 | ff1: 8 | # radix 10 specify that only the 10 digits are used in the output format 9 | radix: 10 10 | # name of the environment variable containing the base64-encoded secret key (note: key length must be 128, 192, or 256 bits) 11 | keyFromEnv: "FF1_ENCRYPTION_KEY" 12 | -------------------------------------------------------------------------------- /pkg/model/logger.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | type MsgLogger struct { 9 | file io.Writer 10 | } 11 | 12 | func NewMsgLogger(filename string) *MsgLogger { 13 | file, _ := os.Create(filename) 14 | return &MsgLogger{file} 15 | } 16 | 17 | func (logger *MsgLogger) Log(msg string) error { 18 | if _, err := logger.file.Write(append([]byte(msg), '\n')); err != nil { 19 | return err 20 | } 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /test/masking_attr.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "name" 6 | mask: 7 | randomChoiceInUri: "pimo://nameFR" 8 | - selector: 9 | jsonpath: "@name" 10 | mask: 11 | randomChoiceInUri: "pimo://nameFR" 12 | - selector: 13 | jsonpath: "name@age" 14 | masks: 15 | - randomInt: 16 | min: 18 17 | max: 95 18 | - template: "{{index . \"name@age\"}}" 19 | -------------------------------------------------------------------------------- /web/play/src/Error.elm: -------------------------------------------------------------------------------- 1 | module Error exposing (..) 2 | 3 | import Html.Styled exposing (Html, div, text) 4 | import Html.Styled.Attributes as Attr exposing (..) 5 | import Play exposing (Msg) 6 | import Tailwind.Utilities exposing (..) 7 | 8 | 9 | view : String -> Html Msg 10 | view error = 11 | div 12 | [ Attr.css [ flex_none, font_sans, text_lg, text_red_500 ] 13 | , Attr.id "result-error" 14 | ] 15 | [ text error ] 16 | -------------------------------------------------------------------------------- /internal/app/pimo/client/readme.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | PIMO Play client website 6 | 7 | 8 |
9 | If you see this page when using a PIMO binary file and the `pimo play` command, then it means the binary was misbuilt : please report an issue to the team! 10 |
11 | 12 | 13 | -------------------------------------------------------------------------------- /demo/demo7/masking-decrypt.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | masking: 3 | - selector: 4 | jsonpath: "siret" 5 | mask: 6 | # use of the FF1 mask 7 | ff1: 8 | # important: use the same radix parameter as for encryption (values will be decrypted incorrectly if 11 is used for example) 9 | radix: 10 10 | # use the same secret key as for encryption 11 | keyFromEnv: "FF1_ENCRYPTION_KEY" 12 | # activate decryption 13 | decrypt: true 14 | -------------------------------------------------------------------------------- /internal/app/pimo/client/.README.md: -------------------------------------------------------------------------------- 1 | # PIMO Play client website 2 | 3 | This folder should contains the PIMO Play website before compiling the `pimo` binary file. 4 | 5 | This can be achieved in 2 ways : 6 | 7 | - use the `neon build-web` command : this will build the static website and copy it into this directory 8 | - use the `neon release` command : this will trigger the `neon build-web` then compile binay for production release 9 | 10 | DO NOT COMMIT THE GENERATED STATIC WEBSITE 11 | -------------------------------------------------------------------------------- /test/persons.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,email 2 | Benoît,Blanc,benoit.blanc@yopmail.fr 3 | Wilfried,Jean,wilfried.jean@yopmail.fr 4 | Patricia,Garnier,patricia.garnier@yopmail.fr 5 | Jean-Claude,Leclercq,jean-claude.leclercq@yopmail.fr 6 | Gérard,Perez,gerard.perez@yopmail.fr 7 | Anissa,Mercier,anissa.mercier@yopmail.fr 8 | Aimée,Moreau,aimee.moreau@yopmail.fr 9 | Aurèle,Chevalier,aurele.chevalier@yopmail.fr 10 | Luce,Vidal,luce.vidal@yopmail.fr 11 | Geoffroy,Dupuis,geoffroy.dupuis@yopmail.fr 12 | -------------------------------------------------------------------------------- /test/persons_sep.csv: -------------------------------------------------------------------------------- 1 | first_name%last_name%email 2 | Benoît%Blanc%benoit.blanc@yopmail.fr 3 | Wilfried%Jean%wilfried.jean@yopmail.fr 4 | Patricia%Garnier%patricia.garnier@yopmail.fr 5 | Jean-Claude%Leclercq%jean-claude.leclercq@yopmail.fr 6 | Gérard%Perez%gerard.perez@yopmail.fr 7 | Anissa%Mercier%anissa.mercier@yopmail.fr 8 | Aimée%Moreau%aimee.moreau@yopmail.fr 9 | Aurèle%Chevalier%aurele.chevalier@yopmail.fr 10 | Luce%Vidal%luce.vidal@yopmail.fr 11 | Geoffroy%Dupuis%geoffroy.dupuis@yopmail.fr 12 | -------------------------------------------------------------------------------- /web/play/webpack.config.dev.js: -------------------------------------------------------------------------------- 1 | import { merge } from 'webpack-merge'; 2 | 3 | import common from "./webpack.config.common.js"; 4 | 5 | export default merge(common(false), { 6 | mode: "development", 7 | module: { 8 | rules: [ 9 | { 10 | test: /index\.es$/, 11 | loader: 'string-replace-loader', 12 | options: { 13 | search: '{{ version }}', 14 | replace: 'master', 15 | flags: 'g' 16 | } 17 | } 18 | ] 19 | } 20 | }); 21 | -------------------------------------------------------------------------------- /demo/demo8/masking-cache.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "age" 6 | cache: "age" 7 | mask: 8 | randomInt: 9 | min: 0 10 | max: 100 11 | - selector: 12 | jsonpath: "related" 13 | mask: 14 | pipe: 15 | masking: 16 | - selector: 17 | jsonpath: "age" 18 | cache: "age" 19 | mask: 20 | randomInt: 21 | min: 0 22 | max: 100 23 | caches: 24 | age : {} -------------------------------------------------------------------------------- /demo/README.md: -------------------------------------------------------------------------------- 1 | # A simple guide to learn to use PIMO 2 | 3 | This folder contains every file necessary for demos of PIMO program. Here are the available demos : 4 | 5 | * Demo 1 - Simple use of PIMO 6 | * Demo 2 - Masking nested data 7 | * Demo 3 - Masking Tables of data 8 | * Demo 4 - Using other data in mask 9 | * Demo 5 - Multiple successive masking 10 | * Demo 6 - Adding and removing fields 11 | * Demo 7 - Format preserving encryption 12 | * Demo 8 - Handle complex structures 13 | * Demo 9 - Logging 14 | 15 | More demos will come. 16 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /demo/demo1/masking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "name" 6 | mask: 7 | constant: "Benjamin" 8 | - selector: 9 | jsonpath: "surname" 10 | mask: 11 | randomChoice: 12 | - Dupont 13 | - Dupond 14 | - selector: 15 | jsonpath: "town" 16 | mask: 17 | hash: 18 | - "Emerald City" 19 | - "Ruby City" 20 | - "Sapphire City" 21 | - selector: 22 | jsonpath: "age" 23 | mask: 24 | randomInt: 25 | min: 18 26 | max: 90 -------------------------------------------------------------------------------- /demo/demo9/masking.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "name" 6 | mask: 7 | constant: "Benjamin" 8 | - selector: 9 | jsonpath: "surname" 10 | mask: 11 | randomChoice: 12 | - Dupont 13 | - Dupond 14 | - selector: 15 | jsonpath: "town" 16 | mask: 17 | hash: 18 | - "Emerald City" 19 | - "Ruby City" 20 | - "Sapphire City" 21 | - selector: 22 | jsonpath: "age" 23 | mask: 24 | randomInt: 25 | min: 18 26 | max: 90 -------------------------------------------------------------------------------- /test/suites/masking_rangemask.yml: -------------------------------------------------------------------------------- 1 | name: Rangemask features 2 | testcases: 3 | - name: Rangemask mask 4 | steps: 5 | - script: rm -f masking.yml 6 | - script: |- 7 | cat > masking.yml < masking.yml < masking.yml < Html.Styled.Attribute Msg 11 | onCustomClick msg = 12 | custom 13 | "click" 14 | (JD.succeed 15 | { message = msg 16 | , stopPropagation = True 17 | , preventDefault = True 18 | } 19 | ) 20 | 21 | 22 | h_x_px : Int -> Css.Style 23 | h_x_px height = 24 | Css.property "height" <| String.fromInt height ++ "px" 25 | -------------------------------------------------------------------------------- /test/masking_pipe.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "organizations.persons" 6 | mask: 7 | pipe: 8 | injectParent: "_" 9 | masking: 10 | - selector: 11 | jsonpath: "name" 12 | mask: 13 | template: "{{ title .name }}" 14 | - selector: 15 | jsonpath: "surname" 16 | mask: 17 | template: "{{ title .surname }}" 18 | - selector: 19 | jsonpath: "email" 20 | mask: 21 | template: "{{ lower .name }}.{{ lower .surname }}@{{ ._.domain }}" -------------------------------------------------------------------------------- /demo/demo8/masking-pipe-2.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 42 3 | masking: 4 | - selector: 5 | jsonpath: "organizations.persons" 6 | mask: 7 | pipe: 8 | # the parent of the person will be injected during the processing of the sub-pipeline, under the path ".org" 9 | # the name "org" is an example, any valid identifier can be chosen 10 | injectParent: "org" 11 | masking: 12 | - selector: 13 | jsonpath: "email" 14 | mask: 15 | # now the template can read the value of the organization domain 16 | template: "{{.name}}.{{.surname}}@{{.org.domain}}" -------------------------------------------------------------------------------- /test/suites/mask_chaining.yml: -------------------------------------------------------------------------------- 1 | name: Test of chaining of masks 2 | testcases: 3 | - name: the keyword masks instead of mask takes an array of mask 4 | steps: 5 | - script: |- 6 | cat > masking.yml < masking.yml < masking.yml < expected.txt < result.txt 20 | assertions: 21 | - result.code ShouldEqual 0 22 | - script: diff expected.txt result.txt 23 | assertions: 24 | - result.systemout ShouldBeEmpty 25 | - script: rm -f expected.txt 26 | - script: rm -f result.txt 27 | -------------------------------------------------------------------------------- /test/exemple_expected.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Nantes Agency 5 | 2308 6 | 7 | 8 | Rolande 9 | 1 10 | 50000 11 | 10000 12 | 13 | 14 | Matéo 15 | 2 16 | 60000 17 | 12000 18 | 19 | 20 | Rosalie 21 | 3 22 | 60000 23 | 12000 24 | 25 | 26 | -------------------------------------------------------------------------------- /pkg/model/ordered_dict_test.go: -------------------------------------------------------------------------------- 1 | package model_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/cgi-fr/pimo/pkg/model" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSimple(t *testing.T) { 11 | jsonl := `{"b":0,"a":1}` 12 | 13 | dict := model.NewDictionary() 14 | err := dict.UnmarshalJSON([]byte(jsonl)) 15 | assert.Nil(t, err) 16 | 17 | b, err := dict.MarshalJSON() 18 | assert.Nil(t, err) 19 | 20 | assert.Equal(t, jsonl, string(b)) 21 | } 22 | 23 | func TestComplex(t *testing.T) { 24 | jsonl := `{"c":"","b":"","a":{"3":null,"2":null,"1":null}}` 25 | 26 | dict := model.NewDictionary() 27 | err := dict.UnmarshalJSON([]byte(jsonl)) 28 | assert.Nil(t, err) 29 | 30 | b, err := dict.MarshalJSON() 31 | assert.Nil(t, err) 32 | 33 | assert.Equal(t, jsonl, string(b)) 34 | } 35 | -------------------------------------------------------------------------------- /test/persons_sep_comment.csv: -------------------------------------------------------------------------------- 1 | ############################################################# 2 | ## Ce fichier CSV est utilisé pour les tests de commentaire # 3 | ## Auteur : Adrien # 4 | ############################################################# 5 | first_name%last_name%email 6 | Benoît%Blanc%benoit.blanc@yopmail.fr 7 | Wilfried%Jean%wilfried.jean@yopmail.fr 8 | Patricia%Garnier%patricia.garnier@yopmail.fr 9 | Jean-Claude%Leclercq%jean-claude.leclercq@yopmail.fr 10 | Gérard%Perez%gerard.perez@yopmail.fr 11 | Anissa%Mercier%anissa.mercier@yopmail.fr 12 | Aimée%Moreau%aimee.moreau@yopmail.fr 13 | Aurèle%Chevalier%aurele.chevalier@yopmail.fr 14 | 15 | # Commentaire dans le fichier, et lignes vides 16 | 17 | Luce%Vidal%luce.vidal@yopmail.fr 18 | Geoffroy%Dupuis%geoffroy.dupuis@yopmail.fr 19 | -------------------------------------------------------------------------------- /test/suites/serve.yml: -------------------------------------------------------------------------------- 1 | name: random features 2 | testcases: 3 | - name: mask on HTTP 4 | steps: 5 | - script: rm -f masking.yml 6 | - script: |- 7 | cat > masking.yml < 13 | - result.systemout ShouldContainSubstring http server started on [::]:3010 14 | 15 | - name: pimo play should start a web server on port specified by flag 16 | steps: 17 | - type: exec 18 | script: | 19 | pimo play -p 3011 & 20 | sleep 2 21 | curl http://127.0.0.1:3011 22 | kill %1 23 | assertions: 24 | - result.systemout ShouldContainSubstring 25 | - result.systemout ShouldContainSubstring http server started on [::]:3011 26 | -------------------------------------------------------------------------------- /test/suites/masking_incremental.yml: -------------------------------------------------------------------------------- 1 | name: incremental features 2 | testcases: 3 | - name: incremental mask 4 | steps: 5 | - script: rm -f masking.yml 6 | - script: |- 7 | cat > masking.yml < 2 | 3 | 4 | Nantes Agency 5 | 0032 6 | 7 | 8 | Doe John 9 | 12345 10 | 50000 11 | 10000 12 | 13 | 14 | Smith Jane 15 | 67890 16 | 60000 17 | 12000 18 | 19 | 20 | Hello world 21 | 22 | 00000 23 | 60000 24 | 12000 25 | 26 | 27 | -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM adrienaury/go-devcontainer:v6.0 2 | 3 | USER root 4 | 5 | ADD --chmod=644 *.crt /usr/local/share/ca-certificates/ 6 | RUN update-ca-certificates 7 | 8 | RUN apk add --update --progress --no-cache make gomplate yarn 9 | 10 | ARG VERSION_GOLICENSE=0.2.0 11 | ARG VERSION_MILLER=6.13.0 12 | RUN wget -nv -O- https://github.com/mitchellh/golicense/releases/download/v${VERSION_GOLICENSE}/golicense_${VERSION_GOLICENSE}_linux_x86_64.tar.gz | tar xz -C /usr/bin golicense \ 13 | && wget -nv -O- https://github.com/johnkerl/miller/releases/download/v${VERSION_MILLER}/miller-${VERSION_MILLER}-linux-amd64.tar.gz | tar xz --strip-components 1 -C /usr/bin miller-${VERSION_MILLER}-linux-amd64/mlr \ 14 | && chmod +x /usr/bin/golicense /usr/bin/mlr 15 | 16 | ENV http_proxy ${http_proxy:-} 17 | ENV https_proxy ${https_proxy:-} 18 | ENV no_proxy ${no_proxy:-} 19 | 20 | USER vscode 21 | -------------------------------------------------------------------------------- /test/suites/masking_multiple_time.yml: -------------------------------------------------------------------------------- 1 | name: Multiple different masks 2 | testcases: 3 | - name: mask, use to replace, remove 4 | steps: 5 | - script: rm -f masking.yml 6 | - script: |- 7 | cat > masking.yml <} 4 | type PackProcess struct{} 5 | 6 | func NewPackProcess() *PackProcess { 7 | return &PackProcess{} 8 | } 9 | 10 | func (pp *PackProcess) Open() error { 11 | return nil 12 | } 13 | 14 | func (pp *PackProcess) ProcessDictionary(dictionary Dictionary, out Collector) error { 15 | if dictionary.IsPacked() { 16 | out.Collect(dictionary) 17 | } else { 18 | out.Collect(dictionary.Pack()) 19 | } 20 | return nil 21 | } 22 | 23 | // UnpackProcess will unpack the jsonline data from a wrapper {"": } 24 | type UnpackProcess struct{} 25 | 26 | func NewUnpackProcess() *UnpackProcess { 27 | return &UnpackProcess{} 28 | } 29 | 30 | func (up *UnpackProcess) Open() error { 31 | return nil 32 | } 33 | 34 | func (up *UnpackProcess) ProcessDictionary(dictionary Dictionary, out Collector) error { 35 | out.Collect(dictionary.Unpack()) 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (C) 2021 CGI France 3 | # 4 | # This file is part of PIMO. 5 | # 6 | # PIMO is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # PIMO is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with PIMO. If not, see . 18 | 19 | set -e 20 | if git status --porcelain | grep "^.M" > /dev/null 21 | then 22 | echo "🚽 Add or stash unstaged changes before commit" 23 | exit 1 24 | else 25 | echo "👌 Nothing to stash" 26 | fi 27 | -------------------------------------------------------------------------------- /web/play/elm.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "application", 3 | "source-directories": [ 4 | "src", 5 | "gen" 6 | ], 7 | "elm-version": "0.19.1", 8 | "dependencies": { 9 | "direct": { 10 | "elm/browser": "1.0.2", 11 | "elm/core": "1.0.5", 12 | "elm/html": "1.0.0", 13 | "elm/http": "2.0.0", 14 | "elm/json": "1.1.3", 15 | "elm/svg": "1.0.1", 16 | "jzxhuang/http-extras": "2.1.0", 17 | "rtfeldman/elm-css": "18.0.0" 18 | }, 19 | "indirect": { 20 | "elm/bytes": "1.0.8", 21 | "elm/file": "1.0.5", 22 | "elm/time": "1.0.0", 23 | "elm/url": "1.0.0", 24 | "elm/virtual-dom": "1.0.3", 25 | "robinheghan/murmur3": "1.0.0", 26 | "rtfeldman/elm-hex": "1.0.0" 27 | } 28 | }, 29 | "test-dependencies": { 30 | "direct": {}, 31 | "indirect": {} 32 | } 33 | } -------------------------------------------------------------------------------- /internal/app/pimo/xixo.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2022 CGI France 2 | // 3 | // This file is part of PIMO. 4 | // 5 | // PIMO is free software: you can redistribute it and/or modify 6 | // it under the terms of the GNU General Public License as published by 7 | // the Free Software Foundation, either version 3 of the License, or 8 | // (at your option) any later version. 9 | // 10 | // PIMO is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | // GNU General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU General Public License 16 | // along with PIMO. If not, see . 17 | 18 | package pimo 19 | 20 | import ( 21 | "io" 22 | 23 | "github.com/CGI-FR/xixo/pkg/xixo" 24 | ) 25 | 26 | func ParseXML(input io.Reader, output io.Writer) *xixo.XMLParser { 27 | return xixo.NewXMLParser(input, output).EnableXpath() 28 | } 29 | -------------------------------------------------------------------------------- /test/masking_find_in_csv.yml: -------------------------------------------------------------------------------- 1 | version: "1" 2 | seed: 1 3 | masking: 4 | - selector: 5 | jsonpath: "info_personne" 6 | mask: 7 | findInCSV: 8 | uri: "file://../persons.csv" 9 | exactMatch: 10 | csv: "{{index . \"1\"}}+123" 11 | entry: "{{.nom}}+123" 12 | expected: "only-one" 13 | header: false 14 | trim: true 15 | 16 | - selector: 17 | jsonpath: "info" 18 | mask: 19 | findInCSV: 20 | uri: "file://persons_same_name.csv" 21 | exactMatch: 22 | csv: "{{.last_name}}+123" 23 | entry: "{{.nom}}+123" 24 | expected: "many" 25 | header: true 26 | trim: true 27 | 28 | - selector: 29 | jsonpath: "info_empty" 30 | mask: 31 | findInCSV: 32 | uri: "file://persons_same_name.csv" 33 | exactMatch: 34 | csv: "{{.last_name}}+123" 35 | entry: "{{.nom}}+123" 36 | expected: "many" 37 | header: true 38 | trim: true 39 | -------------------------------------------------------------------------------- /test/persons_sep_comment_trim.csv: -------------------------------------------------------------------------------- 1 | ############################################################# 2 | ## Ce fichier CSV est utilisé pour les tests de trim # 3 | ## Il y a des espaces autour des valeurs # 4 | ## Auteur : Adrien # 5 | ############################################################# 6 | first_name %last_name%email 7 | Benoît % Blanc %benoit.blanc@yopmail.fr 8 | Wilfried % Jean %wilfried.jean@yopmail.fr 9 | Patricia % Garnier %patricia.garnier@yopmail.fr 10 | Jean-Claude % Leclercq %jean-claude.leclercq@yopmail.fr 11 | Gérard % Perez %gerard.perez@yopmail.fr 12 | Anissa % Mercier %anissa.mercier@yopmail.fr 13 | Aimée % Moreau %aimee.moreau@yopmail.fr 14 | Aurèle % Chevalier %aurele.chevalier@yopmail.fr 15 | 16 | # Test avec des tabulations 17 | 18 | Luce % Vidal % luce.vidal@yopmail.fr 19 | Geoffroy % Dupuis % geoffroy.dupuis@yopmail.fr 20 | -------------------------------------------------------------------------------- /pkg/maskingdata/maskingdata.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 CGI France 2 | // 3 | // This file is part of PIMO. 4 | // 5 | // PIMO is free software: you can redistribute it and/or modify 6 | // it under the terms of the GNU General Public License as published by 7 | // the Free Software Foundation, either version 3 of the License, or 8 | // (at your option) any later version. 9 | // 10 | // PIMO is distributed in the hope that it will be useful, 11 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | // GNU General Public License for more details. 14 | // 15 | // You should have received a copy of the GNU General Public License 16 | // along with PIMO. If not, see . 17 | 18 | package maskingdata 19 | 20 | var MapData = map[string][]string{ 21 | "nameFR": append(NameFRM, NameFRF...), 22 | "nameFRM": NameFRM, 23 | "nameFRF": NameFRF, 24 | "nameEN": append(NameENM, NameENF...), 25 | "nameENM": NameENM, 26 | "nameENF": NameENF, 27 | "surnameFR": SurnameFR, 28 | "townFR": TownFR, 29 | } 30 | -------------------------------------------------------------------------------- /pkg/markov/markov_test.go: -------------------------------------------------------------------------------- 1 | package markov 2 | 3 | import ( 4 | "math/rand" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/cgi-fr/pimo/pkg/model" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestFactoryShouldCreateAMask(t *testing.T) { 13 | maskingConfig := model.Masking{Mask: model.MaskType{Markov: model.MarkovType{MaxSize: 20, Sample: "pimo://nameFR"}}} 14 | factoryConfig := model.MaskFactoryConfiguration{Masking: maskingConfig, Seed: 0} 15 | _, present, err := Factory(factoryConfig) 16 | assert.True(t, present, "should be true") 17 | assert.Nil(t, err, "error should be nil") 18 | } 19 | 20 | func TestFactoryShouldCreateAChain(t *testing.T) { 21 | //nolint: gosec 22 | chain := NewChain(2, rand.New(rand.NewSource(42)), "") 23 | data := []string{"aaa", "bbb", "aba"} 24 | for _, el := range data { 25 | chain.Add(strings.Split(el, "")) 26 | } 27 | newState, err := chain.Generate([]string{"a", "b"}) 28 | assert.Nil(t, err, "error should be nil") 29 | assert.Equal(t, "a", newState) 30 | _, err = chain.Generate([]string{"a"}) 31 | assert.Error(t, err, "N-gram length does not match chain order") 32 | } 33 | -------------------------------------------------------------------------------- /test/suites/masking_replacement.yml: -------------------------------------------------------------------------------- 1 | name: replacement features 2 | testcases: 3 | - name: replacement mask 4 | steps: 5 | - script: rm -f masking.yml 6 | - script: |- 7 | cat > masking.yml < masking.yml <