├── .dockerignore ├── .eslintignore ├── .eslintrc.js ├── .github └── workflows │ ├── javascript-ci.yml │ ├── python-ci.yml │ └── rust-ci.yml ├── .gitignore ├── .husky └── pre-commit ├── .lintstagedrc.json ├── .prettierignore ├── .vscode ├── extensions.json ├── settings.json └── workspace.code-workspace ├── .vsts-ci.yml ├── .yarn ├── plugins │ └── @yarnpkg │ │ ├── plugin-interactive-tools.cjs │ │ ├── plugin-typescript.cjs │ │ ├── plugin-version.cjs │ │ └── plugin-workspace-tools.cjs ├── releases │ └── yarn-3.2.2.cjs └── sdks │ ├── eslint │ ├── bin │ │ └── eslint.js │ └── package.json │ ├── integrations.yml │ ├── prettier │ ├── index.js │ └── package.json │ └── typescript │ ├── bin │ ├── tsc │ └── tsserver │ └── package.json ├── .yarnrc.yml ├── CODE_OF_CONDUCT.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── SECURITY.md ├── docker-compose.yml ├── docs └── dp │ ├── README.md │ └── dp_marginals.pdf ├── package.json ├── packages ├── cli │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── main.rs │ │ └── multi_value_column_cmd_input.rs ├── components │ ├── package.json │ ├── src │ │ ├── components │ │ │ ├── FileDrop.context.ts │ │ │ ├── FileDrop.hooks.ts │ │ │ ├── FileDrop.index.ts │ │ │ ├── FileDrop.styles.ts │ │ │ ├── FileDrop.tsx │ │ │ ├── FileDrop.types.ts │ │ │ ├── FlexContainer.hooks.ts │ │ │ ├── FlexContainer.index.ts │ │ │ ├── FlexContainer.tsx │ │ │ ├── FlexContainer.types.ts │ │ │ ├── FlexItem.hooks.ts │ │ │ ├── FlexItem.index.ts │ │ │ ├── FlexItem.tsx │ │ │ ├── FlexItem.types.ts │ │ │ └── index.ts │ │ ├── hooks │ │ │ ├── index.ts │ │ │ └── useMergeAllClasses.ts │ │ ├── index.ts │ │ └── types │ │ │ └── expand.ts │ └── tsconfig.json ├── core │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── data_block │ │ │ ├── block.rs │ │ │ ├── csv_block_creator.rs │ │ │ ├── csv_io_error.rs │ │ │ ├── csv_record_input_values.rs │ │ │ ├── data_block_creator.rs │ │ │ ├── data_block_creator_error.rs │ │ │ ├── headers_metadata.rs │ │ │ ├── input_value.rs │ │ │ ├── mod.rs │ │ │ ├── multi_value_column_metadata.rs │ │ │ ├── raw_data.rs │ │ │ ├── record.rs │ │ │ ├── subject_id_joiner.rs │ │ │ ├── typedefs.rs │ │ │ └── value.rs │ │ ├── dp │ │ │ ├── dp_parameters.rs │ │ │ ├── mod.rs │ │ │ ├── noise_aggregator.rs │ │ │ ├── noise_parameters.rs │ │ │ ├── noisy_count_threshold.rs │ │ │ ├── percentile.rs │ │ │ ├── register_pyo3.rs │ │ │ ├── stats_error.rs │ │ │ └── typedefs.rs │ │ ├── lib.rs │ │ ├── processing │ │ │ ├── aggregator │ │ │ │ ├── aggregated_count.rs │ │ │ │ ├── aggregated_data.rs │ │ │ │ ├── data_aggregator.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── records_analysis_data.rs │ │ │ │ ├── register_pyo3.rs │ │ │ │ ├── rows_aggregator.rs │ │ │ │ ├── typedefs.rs │ │ │ │ └── value_combination.rs │ │ │ ├── evaluator │ │ │ │ ├── data_evaluator.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── preservation_bucket.rs │ │ │ │ ├── preservation_by_count.rs │ │ │ │ ├── preservation_by_length.rs │ │ │ │ ├── rare_combinations_comparison_data.rs │ │ │ │ ├── register_pyo3.rs │ │ │ │ └── typedefs.rs │ │ │ ├── generator │ │ │ │ ├── data_generator.rs │ │ │ │ ├── generated_data.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── register_pyo3.rs │ │ │ │ └── synthesizers │ │ │ │ │ ├── aggregate_seeded │ │ │ │ │ └── mod.rs │ │ │ │ │ ├── attribute_rows_sampler.rs │ │ │ │ │ ├── cache.rs │ │ │ │ │ ├── consolidate_parameters.rs │ │ │ │ │ ├── mod.rs │ │ │ │ │ ├── oversampling_parameters.rs │ │ │ │ │ ├── row_seeded │ │ │ │ │ ├── mod.rs │ │ │ │ │ └── seeded_rows_synthesizer.rs │ │ │ │ │ ├── traits │ │ │ │ │ ├── consolidate.rs │ │ │ │ │ ├── mod.rs │ │ │ │ │ ├── suppress.rs │ │ │ │ │ └── synthesis_data.rs │ │ │ │ │ ├── typedefs.rs │ │ │ │ │ ├── unseeded │ │ │ │ │ ├── mod.rs │ │ │ │ │ └── unseeded_rows_synthesizer.rs │ │ │ │ │ └── value_seeded │ │ │ │ │ └── mod.rs │ │ │ └── mod.rs │ │ └── utils │ │ │ ├── collections.rs │ │ │ ├── math.rs │ │ │ ├── mod.rs │ │ │ ├── reporting │ │ │ ├── logger_progress_reporter.rs │ │ │ ├── mod.rs │ │ │ ├── processing_stopped_error.rs │ │ │ ├── report_progress.rs │ │ │ └── sendable_progress_reporter.rs │ │ │ ├── strings.rs │ │ │ ├── threading.rs │ │ │ └── time.rs │ └── tests │ │ ├── data_block │ │ ├── block.rs │ │ ├── csv_block_creator.rs │ │ ├── csv_block_creator_duplicated_id.rs │ │ ├── csv_block_creator_multi_value.rs │ │ ├── mod.rs │ │ ├── raw_data.rs │ │ └── value.rs │ │ ├── dp │ │ ├── mod.rs │ │ ├── noise_aggregator.rs │ │ └── percentile.rs │ │ ├── lib.rs │ │ ├── processing │ │ ├── aggregator │ │ │ ├── mod.rs │ │ │ └── value_combination.rs │ │ └── mod.rs │ │ ├── resources │ │ ├── test_block.csv │ │ ├── test_data_block.csv │ │ ├── test_data_block_multi_value.csv │ │ ├── test_duplicated_id_missing_id.csv │ │ ├── test_duplicated_id_valid.csv │ │ ├── test_multi_value_column_joiner.csv │ │ └── test_noise_aggregator.csv │ │ └── utils │ │ └── mod.rs ├── lib-pacsynth │ ├── CODE_OF_CONDUCT.md │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── SECURITY.md │ ├── pyproject.toml │ ├── requirements.txt │ ├── samples │ │ ├── dp_aggregate_seeded_detailed_example.ipynb │ │ ├── dp_aggregate_seeded_short_example.ipynb │ │ └── utils.py │ └── src │ │ ├── aggregate_seeded │ │ ├── dp │ │ │ ├── accuracy_mode.rs │ │ │ ├── builder.rs │ │ │ ├── fabrication_mode.rs │ │ │ ├── mod.rs │ │ │ ├── parameters.rs │ │ │ └── synthesizer.rs │ │ └── mod.rs │ │ ├── dataset │ │ ├── dataset_data_block_creator.rs │ │ └── mod.rs │ │ ├── lib.rs │ │ └── utils │ │ ├── logger.rs │ │ ├── mod.rs │ │ ├── progress.rs │ │ └── threading.rs ├── lib-python │ ├── Cargo.toml │ ├── README.md │ ├── requirements.txt │ └── src │ │ ├── data_processor.rs │ │ └── lib.rs ├── lib-wasm │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── lib.rs │ │ ├── processing │ │ ├── aggregator │ │ │ ├── aggregate_result.rs │ │ │ ├── aggregate_stats.rs │ │ │ ├── mod.rs │ │ │ └── single_attribute_counts.rs │ │ ├── evaluator │ │ │ ├── evaluate_result.rs │ │ │ ├── microdata_data_stats.rs │ │ │ └── mod.rs │ │ ├── generator │ │ │ ├── generate_result.rs │ │ │ └── mod.rs │ │ ├── mod.rs │ │ ├── navigator │ │ │ ├── attributes_intersection.rs │ │ │ ├── mod.rs │ │ │ ├── navigate_result.rs │ │ │ └── selected_attributes.rs │ │ ├── sds_context │ │ │ ├── context.rs │ │ │ ├── errors.rs │ │ │ └── mod.rs │ │ └── sds_processor │ │ │ ├── base_synthesis_parameters.rs │ │ │ ├── csv_data_parameters.rs │ │ │ ├── dp_parameters.rs │ │ │ ├── header_names.rs │ │ │ ├── mod.rs │ │ │ ├── multi_value_columns.rs │ │ │ ├── noisy_count_threshold.rs │ │ │ ├── oversampling_parameters.rs │ │ │ └── processor.rs │ │ └── utils │ │ ├── js │ │ ├── js_progress_reporter.rs │ │ ├── mod.rs │ │ ├── set_panic_hook.rs │ │ └── ts_definitions.rs │ │ ├── logger.rs │ │ └── mod.rs ├── python-pipeline │ ├── README.md │ ├── requirements.txt │ ├── requirements_frozen.txt │ ├── src │ │ ├── aggregator.py │ │ ├── batch_run.py │ │ ├── evaluator.py │ │ ├── generator.py │ │ ├── join_batch_run_stats.py │ │ ├── navigator.py │ │ ├── showcase.py │ │ ├── showme.py │ │ └── util.py │ └── template │ │ └── data_showcase.pbit └── webapp │ ├── .env │ ├── README.md │ ├── index.html │ ├── package.json │ ├── public │ └── synthetic_data_showcase.pbit │ ├── src │ ├── App │ │ ├── App.tsx │ │ ├── Layout │ │ │ ├── Header │ │ │ │ ├── ErrorBar │ │ │ │ │ ├── ErrorBar.tsx │ │ │ │ │ └── index.ts │ │ │ │ ├── Header.tsx │ │ │ │ ├── NavBar │ │ │ │ │ ├── NavBar.tsx │ │ │ │ │ └── index.ts │ │ │ │ ├── TitleBar │ │ │ │ │ ├── TitleBar.tsx │ │ │ │ │ └── index.ts │ │ │ │ └── index.ts │ │ │ ├── Layout.tsx │ │ │ ├── Routes.tsx │ │ │ ├── hooks │ │ │ │ ├── index.ts │ │ │ │ └── useOnTableChange.ts │ │ │ └── index.ts │ │ ├── contexts │ │ │ ├── ChartContext.tsx │ │ │ ├── DataContext.tsx │ │ │ ├── FileUploader.tsx │ │ │ ├── StyleContext.tsx │ │ │ └── index.ts │ │ └── index.ts │ ├── components │ │ ├── AllSynthesisInfo │ │ │ ├── AllSynthesisInfo.hooks.tsx │ │ │ ├── AllSynthesisInfo.styles.ts │ │ │ ├── AllSynthesisInfo.tsx │ │ │ ├── AllSynthesisInfo.types.ts │ │ │ ├── StatefulAllSynthesisInfo.tsx │ │ │ └── index.ts │ │ ├── AttributeIntersectionValueChartLegend │ │ │ ├── AttributeIntersectionValueChartLegend.tsx │ │ │ ├── hooks.ts │ │ │ └── index.ts │ │ ├── AttributeSelector │ │ │ ├── ColumnAttributeSelector.tsx │ │ │ ├── ColumnAttributeSelectorGrid.tsx │ │ │ ├── HeaderSelector.tsx │ │ │ ├── SelectedAttributes.tsx │ │ │ ├── hooks.ts │ │ │ └── index.ts │ │ ├── Charts │ │ │ ├── AttributeIntersectionValueChart.tsx │ │ │ ├── MetricsChart.tsx │ │ │ ├── hooks │ │ │ │ ├── charts.ts │ │ │ │ ├── index.ts │ │ │ │ ├── useMetricsByCountLabels.ts │ │ │ │ └── useMetricsByLenLabels.ts │ │ │ └── index.ts │ │ ├── CollapsablePanel │ │ │ ├── CollapsablePanel.tsx │ │ │ ├── CollapsablePanel.types.ts │ │ │ └── index.ts │ │ ├── CsvTable │ │ │ ├── CsvTable.tsx │ │ │ └── index.ts │ │ ├── DataEvaluationInfo │ │ │ ├── DataEvaluationInfo.tsx │ │ │ └── index.ts │ │ ├── DataEvaluationInfoDownloader │ │ │ ├── DataEvaluationInfoDownloader.tsx │ │ │ ├── hooks.ts │ │ │ └── index.ts │ │ ├── ErrorMessageBar │ │ │ ├── ErrorMessageBar.tsx │ │ │ └── index.ts │ │ ├── HumanReadableSummary │ │ │ ├── HumanReadableSummary.hooks.ts │ │ │ ├── HumanReadableSummary.tsx │ │ │ └── index.ts │ │ ├── InfoTooltip │ │ │ ├── InfoTooltip.tsx │ │ │ └── index.ts │ │ ├── Markdown │ │ │ ├── Markdown.tsx │ │ │ └── index.ts │ │ ├── MetricsSummaryTable │ │ │ ├── MetricsSummaryTable.tsx │ │ │ ├── hooks.ts │ │ │ └── index.ts │ │ ├── NavBar │ │ │ ├── NavBar.tsx │ │ │ └── index.ts │ │ ├── PolicyAndCookieBanner │ │ │ ├── PolicyAndCookieBanner.hooks.ts │ │ │ ├── PolicyAndCookieBanner.tsx │ │ │ ├── PolicyAndCookieBanner.types.ts │ │ │ └── index.ts │ │ ├── SynthesisDropdown │ │ │ ├── SynthesisDropdown.hooks.ts │ │ │ ├── SynthesisDropdown.tsx │ │ │ ├── SynthesisDropdown.types.ts │ │ │ └── index.ts │ │ ├── TooltipWrapper │ │ │ ├── TooltipWrapper.tsx │ │ │ └── index.ts │ │ └── controls │ │ │ ├── DownloadButton │ │ │ ├── DownloadButton.tsx │ │ │ ├── DownloadInfo.ts │ │ │ ├── hooks.ts │ │ │ └── index.ts │ │ │ ├── FileInputButton │ │ │ ├── FileInputButton.tsx │ │ │ └── index.ts │ │ │ └── index.ts │ ├── index.tsx │ ├── models │ │ ├── aggregation │ │ │ ├── MicrodataMaxStatistics.ts │ │ │ └── index.ts │ │ ├── csv │ │ │ └── index.ts │ │ ├── index.ts │ │ ├── pipeline │ │ │ └── index.ts │ │ ├── synthesis │ │ │ ├── AccuracyMode.ts │ │ │ ├── FabricationMode.ts │ │ │ ├── OversamplingType.ts │ │ │ ├── RawSynthesisParameters.ts │ │ │ ├── UseSyntheticCounts.ts │ │ │ └── index.ts │ │ └── workers │ │ │ ├── SdsManagerInstance.ts │ │ │ └── index.ts │ ├── pages │ │ ├── Home │ │ │ ├── HomePage.tsx │ │ │ └── index.ts │ │ ├── Navigate │ │ │ ├── ChartArea │ │ │ │ └── ChartArea.tsx │ │ │ ├── Commands │ │ │ │ ├── Commands.tsx │ │ │ │ └── SelectColumns │ │ │ │ │ └── SelectColumns.tsx │ │ │ ├── NavigatePage.tsx │ │ │ ├── hooks │ │ │ │ ├── index.ts │ │ │ │ ├── useInitiallySelectedHeaders.ts │ │ │ │ ├── useOnClearSelectedAttributes.ts │ │ │ │ ├── useOnNewSelectedAttributesByColumn.ts │ │ │ │ ├── useOnRunNavigate.ts │ │ │ │ ├── useOnSetSelectedAttributes.ts │ │ │ │ └── useOnToggleSelectedHeader.ts │ │ │ └── index.ts │ │ ├── Pages.ts │ │ ├── Prepare │ │ │ ├── DataInput │ │ │ │ ├── DataInput.tsx │ │ │ │ └── index.ts │ │ │ ├── PreparePage.tsx │ │ │ └── index.ts │ │ ├── Select │ │ │ ├── AggregateStatistics │ │ │ │ ├── AggregateStatistics.hooks.ts │ │ │ │ ├── AggregateStatistics.styles.ts │ │ │ │ ├── AggregateStatistics.tsx │ │ │ │ ├── ContributionChart.styles.ts │ │ │ │ ├── ContributionChart.tsx │ │ │ │ ├── ContributionChart.types.ts │ │ │ │ └── index.ts │ │ │ ├── InfoBar │ │ │ │ ├── InfoBar.tsx │ │ │ │ └── index.ts │ │ │ ├── SelectCommands │ │ │ │ ├── AggregationControls │ │ │ │ │ └── AggregationControls.tsx │ │ │ │ ├── MultiValueColumns │ │ │ │ │ ├── MultiValueColumns.tsx │ │ │ │ │ └── index.ts │ │ │ │ ├── SelectColumns │ │ │ │ │ ├── SelectColumns.hooks.ts │ │ │ │ │ ├── SelectColumns.tsx │ │ │ │ │ └── index.ts │ │ │ │ ├── SelectCommands.tsx │ │ │ │ ├── SensitiveZeros │ │ │ │ │ ├── SensitiveZeros.hooks.tsx │ │ │ │ │ ├── SensitiveZeros.tsx │ │ │ │ │ └── index.ts │ │ │ │ ├── SubjectId │ │ │ │ │ ├── SubjectId.tsx │ │ │ │ │ └── index.ts │ │ │ │ └── index.ts │ │ │ ├── SelectPage.tsx │ │ │ ├── TablePreview │ │ │ │ ├── TablePreview.tsx │ │ │ │ └── index.ts │ │ │ └── index.ts │ │ ├── Synthesize │ │ │ ├── DataEvaluation │ │ │ │ ├── DataEvaluation.hooks.ts │ │ │ │ ├── DataEvaluation.styles.ts │ │ │ │ ├── DataEvaluation.tsx │ │ │ │ ├── DataEvaluation.types.ts │ │ │ │ └── index.ts │ │ │ ├── DataSynthesis │ │ │ │ ├── DataSynthesis.hooks.ts │ │ │ │ ├── DataSynthesis.styles.tsx │ │ │ │ ├── DataSynthesis.tsx │ │ │ │ └── index.ts │ │ │ ├── DataSynthesisParameters │ │ │ │ ├── DataSynthesisAdvancedParameters.tsx │ │ │ │ ├── DataSynthesisParameters.hooks.ts │ │ │ │ ├── DataSynthesisParameters.styles.ts │ │ │ │ ├── DataSynthesisParameters.tsx │ │ │ │ ├── DataSynthesisParameters.types.ts │ │ │ │ └── index.ts │ │ │ ├── DataSynthesisResult │ │ │ │ ├── DataSynthesisResult.hooks.ts │ │ │ │ ├── DataSynthesisResult.styles.ts │ │ │ │ ├── DataSynthesisResult.tsx │ │ │ │ ├── DataSynthesisResult.types.ts │ │ │ │ └── index.ts │ │ │ ├── Synthesize.hooks.ts │ │ │ ├── SynthesizePage.tsx │ │ │ └── index.ts │ │ ├── hooks │ │ │ ├── commands │ │ │ │ ├── index.ts │ │ │ │ └── useDownloadCommand.ts │ │ │ ├── index.ts │ │ │ ├── useCanRun.ts │ │ │ ├── useDropdownOnChange.ts │ │ │ ├── useOnGetAllAssetsDownloadInfo.ts │ │ │ └── useSpinButtonOnChange.ts │ │ └── index.ts │ ├── states │ │ ├── dataShowcaseContext │ │ │ ├── allSynthesisInfo.ts │ │ │ ├── charts.ts │ │ │ ├── files.ts │ │ │ ├── hooks.ts │ │ │ ├── index.ts │ │ │ ├── isProcessing.ts │ │ │ ├── multiValueColumns.ts │ │ │ ├── sdsManagerInstance.ts │ │ │ ├── selectedPipelineStep.ts │ │ │ ├── selectedSynthesisInfo.ts │ │ │ ├── sensitiveContent.ts │ │ │ ├── steps.ts │ │ │ └── tables.ts │ │ ├── globalErrorMessage.ts │ │ ├── index.ts │ │ └── rawSynthesisParameters.ts │ ├── ui-tooltips │ │ ├── index.ts │ │ ├── mds │ │ │ ├── ACCURACY_MODE.md │ │ │ ├── ANALYSIS_LENGTH.md │ │ │ ├── CACHE_SIZE.md │ │ │ ├── COUNT_MEAN_AND_ERROR.md │ │ │ ├── DELTA_FACTOR.md │ │ │ ├── EVALUATE.md │ │ │ ├── FABRICATED_COMBINATIONS.md │ │ │ ├── FABRICATION_MODE.md │ │ │ ├── MEAN_PROPORTIONAL_ERROR.md │ │ │ ├── NAVIGATE.md │ │ │ ├── NOISE_EPSILON.md │ │ │ ├── NUMBER_OF_RECORDS_EPSILON_PROPORTION.md │ │ │ ├── N_COUNT_MEAN_AND_ERROR.md │ │ │ ├── OVERSAMPLING.md │ │ │ ├── OVERSAMPLING_RATIO.md │ │ │ ├── OVERSAMPLING_TRIES.md │ │ │ ├── PERCENTILE_EPSILON_PROPORTION.md │ │ │ ├── PERCENTILE_PERCENTAGE.md │ │ │ ├── RARE_COMBS.md │ │ │ ├── RECORDS_WITH_RARE_COMBS.md │ │ │ ├── RECORDS_WITH_UNIQUE_COMBS.md │ │ │ ├── RECORD_EXPANSION.md │ │ │ ├── RECORD_LIMIT.md │ │ │ ├── REPORTING_LENGTH.md │ │ │ ├── RESOLUTION.md │ │ │ ├── SENSITIVE_FILE.md │ │ │ ├── SUPPRESSED_COMBINATIONS.md │ │ │ ├── SYNTHESIS_MODE.md │ │ │ ├── SYNTHESIZE.md │ │ │ ├── THRESHOLD_VALUE.md │ │ │ ├── UNIQUE_COMBS.md │ │ │ ├── USE_SYNTHETIC_COUNTS.md │ │ │ └── WEIGHT_SELECTION_PERCENTILE.md │ │ └── tooltips.ts │ ├── utils │ │ ├── arquero.ts │ │ ├── env.ts │ │ ├── hash.ts │ │ ├── index.ts │ │ └── thematic.ts │ ├── vite-env.d.ts │ └── workers │ │ ├── AggregateStatisticsGenerator.ts │ │ ├── BaseSdsWasmWorker.ts │ │ ├── SdsManager.ts │ │ ├── WasmSynthesizer.ts │ │ ├── types │ │ ├── AggregateType.ts │ │ ├── CancelablePromise.ts │ │ ├── Proxy.ts │ │ ├── SdsManagerSynthesisCallbacks.ts │ │ ├── SynthesisInfo.ts │ │ ├── SynthesisMode.ts │ │ ├── SynthesisParameters.ts │ │ ├── WasmSynthesizerWorkerInfo.ts │ │ ├── WasmSynthesizerWorkerStatus.ts │ │ ├── WorkerProgressCallback.ts │ │ └── index.ts │ │ └── utils │ │ ├── index.ts │ │ └── workerProxy.ts │ ├── tsconfig.json │ ├── types │ └── styled-components │ │ └── index.d.ts │ └── vite.config.ts ├── tsconfig.json ├── webapp.dockerfile └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | # TS/JS ignores 2 | node_modules/ 3 | build/ 4 | dist/ 5 | out/ 6 | lib/ 7 | 8 | # Yarn v2 Gitignores 9 | .yarn/* 10 | !.yarn/patches 11 | !.yarn/plugins 12 | !.yarn/releases 13 | !.yarn/sdks 14 | !.yarn/versions 15 | 16 | # Rust ignores 17 | target/ 18 | 19 | # Env ignores 20 | .env/ 21 | .vscode/ 22 | 23 | # Documentation ignore 24 | docs/ -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | storybook-static/ 3 | build/ 4 | dist/ 5 | lib/ 6 | coverage/ 7 | public/ 8 | .yarn/ 9 | .yarnrc.yaml 10 | yarn.lock 11 | .pnp.js 12 | .pnp.cjs 13 | .cache/ 14 | pkg/ 15 | target/ 16 | .eslintrc.js 17 | babel.config.js 18 | target/ 19 | bot_backups/* 20 | DATA_REVIEW.md -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | module.exports = { 6 | root: true, 7 | extends: ['@essex/eslint-config/fast'], 8 | parser: '@typescript-eslint/parser', 9 | parserOptions: { 10 | project: './tsconfig.json', 11 | }, 12 | rules: { 13 | '@typescript-eslint/no-explicit-any': 'off', 14 | 'no-redeclare': 'off', 15 | '@typescript-eslint/no-non-null-assertion': 'off', 16 | '@typescript-eslint/no-redeclare': ['warn'], 17 | '@typescript-eslint/explicit-module-boundary-types': [ 18 | 'warn', 19 | { allowArgumentsExplicitlyTypedAsAny: true }, 20 | ], 21 | 22 | // TODO: Re-enable 23 | 'jsx-a11y/click-events-have-key-events': 0, 24 | 'jsx-a11y/no-static-element-interactions': 0, 25 | '@essex/extensions': [ 26 | 'error', [ 27 | { 28 | files: ['**/*.{ts,tsx,js,jsx,mts,mjs}'], 29 | ignorePackages: true, 30 | relativeModulePrefixes: ['.'], 31 | expectedExtensions: ['.js', '.mjs', '.cjs', '.jsx', '.css'], 32 | disallowedExtensions: [], 33 | } 34 | ] 35 | ] 36 | }, 37 | } 38 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | yarn git:precommit 5 | -------------------------------------------------------------------------------- /.lintstagedrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "*": ["essex prettify --staged"], 3 | "*.{js,jsx,ts,tsx}": ["eslint --fix"] 4 | } 5 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | storybook-static/ 3 | build/ 4 | dist/ 5 | lib/ 6 | coverage/ 7 | public/ 8 | .yarn/ 9 | .yarnrc.yml 10 | yarn.lock 11 | .pnp.js 12 | .pnp.cjs 13 | .cache/ 14 | pkg/ 15 | target/ 16 | .eslintrc.js 17 | babel.config.js 18 | target/ 19 | bot_backups/* 20 | DATA_REVIEW.md -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "esbenp.prettier-vscode", 4 | "dbaeumer.vscode-eslint" 5 | ] 6 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules\\typescript\\lib", 3 | "editor.defaultFormatter": "esbenp.prettier-vscode", 4 | "editor.formatOnSave": true, 5 | "files.eol": "\n", 6 | "editor.codeActionsOnSave": { 7 | "source.fixAll": true 8 | }, 9 | "javascript.preferences.importModuleSpecifier": "relative", 10 | "javascript.preferences.importModuleSpecifierEnding": "js", 11 | "typescript.preferences.importModuleSpecifier": "relative", 12 | "typescript.preferences.importModuleSpecifierEnding": "js", 13 | "explorer.fileNesting.enabled": true, 14 | "explorer.fileNesting.patterns": { 15 | "*.ts": "${capture}.tsx, ${capture}.*.ts, ${capture}.*.tsx, ${capture}.module.scss, ${capture}.module.css, ${capture}.md", 16 | "*.js": "${capture}.js.map, ${capture}.min.js, ${capture}.d.ts", 17 | "*.jsx": "${capture}.js", 18 | "*.tsx": "${capture}.ts, ${capture}.*.ts, ${capture}.*.tsx, ${capture}.module.scss, ${capture}.module.css, ${capture}.md", 19 | "tsconfig.json": "tsconfig.*.json", 20 | "package.json": "LICENSE, package-lock.json, yarn.lock, .yarnrc.yml, .pnp.cjs, .pnp.loader.mjs, .eslintrc, .eslintignore, .prettierignore, .gitattributes, .gitignore", 21 | "README.md": "SECURITY.md, SUPPORT.md, CODE_OF_CONDUCT.md", 22 | "jest.config.js": "jest.setup.mjs" 23 | }, 24 | "[rust]": { 25 | "editor.defaultFormatter": "rust-lang.rust-analyzer" 26 | }, 27 | "[python]": { 28 | "editor.defaultFormatter": "ms-python.python" 29 | }, 30 | "rust-analyzer.check.command": "clippy" 31 | } 32 | -------------------------------------------------------------------------------- /.vsts-ci.yml: -------------------------------------------------------------------------------- 1 | name: SDS Compliance CI 2 | pool: 3 | vmImage: ubuntu-latest 4 | 5 | trigger: 6 | batch: true 7 | branches: 8 | include: 9 | - main 10 | - dev 11 | 12 | stages: 13 | - stage: Compliance 14 | dependsOn: [] 15 | jobs: 16 | - job: ComplianceJob 17 | pool: 18 | vmImage: windows-latest 19 | steps: 20 | - task: CredScan@3 21 | inputs: 22 | outputFormat: sarif 23 | debugMode: false 24 | 25 | - task: ComponentGovernanceComponentDetection@0 26 | inputs: 27 | scanType: 'Register' 28 | verbosity: 'Verbose' 29 | alertWarningLevel: 'High' 30 | 31 | - task: PublishSecurityAnalysisLogs@3 32 | inputs: 33 | ArtifactName: 'CodeAnalysisLogs' 34 | ArtifactType: 'Container' 35 | -------------------------------------------------------------------------------- /.yarn/sdks/eslint/bin/eslint.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const {existsSync} = require(`fs`); 4 | const {createRequire, createRequireFromPath} = require(`module`); 5 | const {resolve} = require(`path`); 6 | 7 | const relPnpApiPath = "../../../../.pnp.cjs"; 8 | 9 | const absPnpApiPath = resolve(__dirname, relPnpApiPath); 10 | const absRequire = (createRequire || createRequireFromPath)(absPnpApiPath); 11 | 12 | if (existsSync(absPnpApiPath)) { 13 | if (!process.versions.pnp) { 14 | // Setup the environment to be able to require eslint/bin/eslint.js 15 | require(absPnpApiPath).setup(); 16 | } 17 | } 18 | 19 | // Defer to the real eslint/bin/eslint.js your application uses 20 | module.exports = absRequire(`eslint/bin/eslint.js`); 21 | -------------------------------------------------------------------------------- /.yarn/sdks/eslint/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "eslint", 3 | "version": "7.32.0-sdk", 4 | "main": "./lib/api.js", 5 | "type": "commonjs" 6 | } 7 | -------------------------------------------------------------------------------- /.yarn/sdks/integrations.yml: -------------------------------------------------------------------------------- 1 | # This file is automatically generated by @yarnpkg/sdks. 2 | # Manual changes might be lost! 3 | 4 | integrations: 5 | - vscode 6 | -------------------------------------------------------------------------------- /.yarn/sdks/prettier/index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const {existsSync} = require(`fs`); 4 | const {createRequire, createRequireFromPath} = require(`module`); 5 | const {resolve} = require(`path`); 6 | 7 | const relPnpApiPath = "../../../.pnp.cjs"; 8 | 9 | const absPnpApiPath = resolve(__dirname, relPnpApiPath); 10 | const absRequire = (createRequire || createRequireFromPath)(absPnpApiPath); 11 | 12 | if (existsSync(absPnpApiPath)) { 13 | if (!process.versions.pnp) { 14 | // Setup the environment to be able to require prettier/index.js 15 | require(absPnpApiPath).setup(); 16 | } 17 | } 18 | 19 | // Defer to the real prettier/index.js your application uses 20 | module.exports = absRequire(`prettier/index.js`); 21 | -------------------------------------------------------------------------------- /.yarn/sdks/prettier/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "prettier", 3 | "version": "2.4.1-sdk", 4 | "main": "./index.js", 5 | "type": "commonjs" 6 | } 7 | -------------------------------------------------------------------------------- /.yarn/sdks/typescript/bin/tsc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const {existsSync} = require(`fs`); 4 | const {createRequire, createRequireFromPath} = require(`module`); 5 | const {resolve} = require(`path`); 6 | 7 | const relPnpApiPath = "../../../../.pnp.cjs"; 8 | 9 | const absPnpApiPath = resolve(__dirname, relPnpApiPath); 10 | const absRequire = (createRequire || createRequireFromPath)(absPnpApiPath); 11 | 12 | if (existsSync(absPnpApiPath)) { 13 | if (!process.versions.pnp) { 14 | // Setup the environment to be able to require typescript/bin/tsc 15 | require(absPnpApiPath).setup(); 16 | } 17 | } 18 | 19 | // Defer to the real typescript/bin/tsc your application uses 20 | module.exports = absRequire(`typescript/bin/tsc`); 21 | -------------------------------------------------------------------------------- /.yarn/sdks/typescript/bin/tsserver: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const {existsSync} = require(`fs`); 4 | const {createRequire, createRequireFromPath} = require(`module`); 5 | const {resolve} = require(`path`); 6 | 7 | const relPnpApiPath = "../../../../.pnp.cjs"; 8 | 9 | const absPnpApiPath = resolve(__dirname, relPnpApiPath); 10 | const absRequire = (createRequire || createRequireFromPath)(absPnpApiPath); 11 | 12 | if (existsSync(absPnpApiPath)) { 13 | if (!process.versions.pnp) { 14 | // Setup the environment to be able to require typescript/bin/tsserver 15 | require(absPnpApiPath).setup(); 16 | } 17 | } 18 | 19 | // Defer to the real typescript/bin/tsserver your application uses 20 | module.exports = absRequire(`typescript/bin/tsserver`); 21 | -------------------------------------------------------------------------------- /.yarn/sdks/typescript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "typescript", 3 | "version": "4.4.4-sdk", 4 | "main": "./lib/typescript.js", 5 | "type": "commonjs" 6 | } 7 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | 3 | packageExtensions: 4 | "@fluentui/font-icons-mdl2@*": 5 | peerDependencies: 6 | react: "*" 7 | "@fluentui/foundation-legacy@*": 8 | peerDependencies: 9 | "@types/react": "*" 10 | "@fluentui/react-focus@*": 11 | peerDependencies: 12 | "@types/react": "*" 13 | "@fluentui/react-hooks@*": 14 | peerDependencies: 15 | "@types/react": "*" 16 | "@fluentui/react@*": 17 | peerDependencies: 18 | "@types/react": "*" 19 | "@fluentui/style-utilities@*": 20 | peerDependencies: 21 | "@types/react": "*" 22 | react: "*" 23 | "@fluentui/theme@*": 24 | peerDependencies: 25 | "@types/react": "*" 26 | "@fluentui/utilities@*": 27 | peerDependencies: 28 | "@types/react": "*" 29 | 30 | plugins: 31 | - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs 32 | spec: "@yarnpkg/plugin-interactive-tools" 33 | - path: .yarn/plugins/@yarnpkg/plugin-version.cjs 34 | spec: "@yarnpkg/plugin-version" 35 | - path: .yarn/plugins/@yarnpkg/plugin-typescript.cjs 36 | spec: "@yarnpkg/plugin-typescript" 37 | - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs 38 | spec: "@yarnpkg/plugin-workspace-tools" 39 | 40 | yarnPath: .yarn/releases/yarn-3.2.2.cjs 41 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "packages/core", 4 | "packages/cli", 5 | "packages/lib-wasm", 6 | "packages/lib-python", 7 | "packages/lib-pacsynth" 8 | ] 9 | 10 | [profile.release] 11 | opt-level = 3 12 | 13 | [profile.test] 14 | opt-level = 3 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | webapp: 4 | build: 5 | context: . 6 | dockerfile: webapp.dockerfile 7 | ports: 8 | - 3000:80 9 | -------------------------------------------------------------------------------- /docs/dp/dp_marginals.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/synthetic-data-showcase/16151bf41956042c2970552f1e50eba9dd65f87d/docs/dp/dp_marginals.pdf -------------------------------------------------------------------------------- /packages/cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sds-cli" 3 | version = "1.9.0" 4 | license = "MIT" 5 | description = "Command line interface for the sds-core library" 6 | repository = "https://github.com/microsoft/synthetic-data-showcase" 7 | edition = "2018" 8 | 9 | [dependencies] 10 | sds-core = { path = "../core", features = ["rayon"] } 11 | log = { version = "0.4" } 12 | env_logger = { version = "0.9" } 13 | structopt = { version = "0.3" } 14 | csv = { version = "1.1" } 15 | statrs = { version = "0.16"} -------------------------------------------------------------------------------- /packages/cli/src/multi_value_column_cmd_input.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | const DELIMITER: char = ','; 4 | 5 | #[derive(Debug)] 6 | pub struct MultiValueColumnCmdInput { 7 | pub column_name: String, 8 | pub attr_delimiter: String, 9 | } 10 | 11 | impl FromStr for MultiValueColumnCmdInput { 12 | type Err = String; 13 | 14 | fn from_str(s: &str) -> Result { 15 | if let Some((column_name, delimiter)) = s.split_once(DELIMITER) { 16 | if !column_name.is_empty() && !delimiter.is_empty() { 17 | return Ok(MultiValueColumnCmdInput { 18 | column_name: column_name.to_owned(), 19 | attr_delimiter: delimiter.to_owned(), 20 | }); 21 | } 22 | } 23 | Err("wrong format, expected: ,".to_owned()) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /packages/components/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@sds/components", 3 | "version": "0.0.1", 4 | "private": true, 5 | "description": "Components for SDS", 6 | "type": "module", 7 | "main": "./dist/index.js", 8 | "module": "./dist/index.js", 9 | "types": "./dist/types/index.d.ts", 10 | "exports": { 11 | ".": { 12 | "import": "./dist/index.js" 13 | }, 14 | "./package.json": "./package.json" 15 | }, 16 | "scripts": { 17 | "clean": "shx rm -rf ./dist", 18 | "build": "tsc", 19 | "start": "run-s \"build --watch --preserveWatchOutput\"" 20 | }, 21 | "author": "", 22 | "license": "MIT", 23 | "devDependencies": { 24 | "@types/mime": "^3.0.1", 25 | "@types/node": "^16.18.16", 26 | "@types/react": "^17.0.53", 27 | "npm-run-all": "^4.1.5", 28 | "react": "^17.0.2", 29 | "shx": "^0.3.4", 30 | "ts-node": "^10.9.1", 31 | "typescript": "^5.0.2" 32 | }, 33 | "peerDependencies": { 34 | "react": "^17.0.2" 35 | }, 36 | "dependencies": { 37 | "@griffel/react": "^1.5.5", 38 | "mime": "^3.0.0", 39 | "react-dropzone": "^14.2.3" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /packages/components/src/components/FileDrop.context.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { createContext } from 'react' 6 | 7 | import type { FileDropContextType } from './FileDrop.types.js' 8 | 9 | // eslint-disable-next-line 10 | export const fileDropDefaultContext: FileDropContextType = () => {} 11 | 12 | export const FileDropContext = createContext( 13 | fileDropDefaultContext, 14 | ) 15 | -------------------------------------------------------------------------------- /packages/components/src/components/FileDrop.hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import mime from 'mime' 6 | import { useContext, useMemo } from 'react' 7 | import type { DropzoneState } from 'react-dropzone' 8 | import { useDropzone } from 'react-dropzone' 9 | 10 | import { FileDropContext } from './FileDrop.context.js' 11 | import type { FileDropContextType, FileDropProps } from './FileDrop.types.js' 12 | 13 | export function useFileDropOpen(): FileDropContextType { 14 | return useContext(FileDropContext) 15 | } 16 | 17 | export function useFileDrop(props: FileDropProps): DropzoneState { 18 | const accept = useMemo(() => { 19 | if (props.accept) { 20 | return ( 21 | Array.isArray(props.accept) ? props.accept : props.accept.split(',') 22 | ).reduce>((acc, file) => { 23 | file = file.trim() 24 | const mimeType = file.startsWith('.') ? mime.getType(file) : file 25 | if (!mimeType) { 26 | throw new Error(`Unable to determine MIME type for ${file}`) 27 | } 28 | acc[mimeType] = [ 29 | ...(Array.isArray(acc[mimeType]) ? acc[mimeType] : []), 30 | file, 31 | ] 32 | return acc 33 | }, {}) 34 | } 35 | return undefined 36 | }, [props.accept]) 37 | 38 | return useDropzone({ 39 | ...{ 40 | ...props, 41 | accept: undefined, 42 | }, 43 | ...(accept && { accept }), 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /packages/components/src/components/FileDrop.index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export { useFileDropOpen } from './FileDrop.hooks.js' 6 | export * from './FileDrop.js' 7 | export * from './FileDrop.types.js' 8 | export type { FileRejection } from 'react-dropzone' 9 | -------------------------------------------------------------------------------- /packages/components/src/components/FileDrop.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { makeStyles } from '@griffel/react' 6 | 7 | import { useMergeAllClasses } from '../hooks/useMergeAllClasses.js' 8 | import type { FileDropSlotClassNames, FileDropSlots } from './FileDrop.types.js' 9 | 10 | export const useDefaultStyles = makeStyles({ 11 | Overlay: { 12 | position: 'fixed', 13 | display: 'flex', 14 | justifyContent: 'center', 15 | alignItems: 'center', 16 | top: 0, 17 | left: 0, 18 | zIndex: 1000000, 19 | width: '100vw', 20 | height: '100vh', 21 | fontWeight: 'bold', 22 | opacity: 0.6, 23 | backgroundColor: 'var(--neutralTertiaryAlt, #c8c6c4)', 24 | color: 'var(--black, #000000)', 25 | }, 26 | OverlayMessage: {}, 27 | }) 28 | 29 | export function useFileDropStyles( 30 | slotClassNames?: FileDropSlotClassNames, 31 | ): FileDropSlotClassNames { 32 | const defaultClasses = useDefaultStyles() 33 | return useMergeAllClasses(defaultClasses, slotClassNames) 34 | } 35 | -------------------------------------------------------------------------------- /packages/components/src/components/FileDrop.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { FC } from 'react' 6 | import React, { forwardRef, memo, useImperativeHandle } from 'react' 7 | 8 | import { FileDropContext } from './FileDrop.context.js' 9 | import { useFileDrop } from './FileDrop.hooks.js' 10 | import { useFileDropStyles } from './FileDrop.styles.js' 11 | import type { FileDropProps } from './FileDrop.types.js' 12 | 13 | export const FileDrop: FC = memo( 14 | forwardRef(function FileDrop(props, ref) { 15 | const { getRootProps, getInputProps, isDragActive, open } = 16 | useFileDrop(props) 17 | const classes = useFileDropStyles(props.slotClassNames) 18 | 19 | useImperativeHandle(ref, () => ({ open }), [open]) 20 | 21 | return ( 22 | 23 | 24 | 25 | {isDragActive && ( 26 |
27 | 28 | {props.onDragMessage ?? 'Drop files.'} 29 | 30 |
31 | )} 32 | {props.children} 33 | 34 | 35 | ) 36 | }), 37 | ) 38 | FileDrop.displayName = 'FileDrop' 39 | -------------------------------------------------------------------------------- /packages/components/src/components/FileDrop.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { PropsWithChildren, RefObject } from 'react' 6 | import type { DropzoneOptions } from 'react-dropzone' 7 | 8 | import type { Expand } from '../types/expand.js' 9 | export type FileDropContextType = () => void 10 | 11 | export type FileDropRef = { 12 | open: () => void 13 | } 14 | 15 | export type FileDropProps = Expand< 16 | Omit & 17 | PropsWithChildren<{ 18 | accept?: string | string[] 19 | onDragMessage?: string 20 | ref?: RefObject 21 | slotClassNames?: FileDropSlotClassNames 22 | }> 23 | > 24 | 25 | export enum FileDropSlots { 26 | Overlay = 'Overlay', 27 | OverlayMessage = 'OverlayMessage', 28 | } 29 | 30 | export type FileDropSlotClassNames = Partial> 31 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexContainer.hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { CSSProperties } from 'react' 6 | import { useMemo } from 'react' 7 | 8 | import type { FlexContainerProps } from './FlexContainer.types.js' 9 | 10 | export function useFlexContainerStyles({ 11 | vertical = false, 12 | wrap = false, 13 | justify = 'flex-start', 14 | align = 'stretch', 15 | gap = '', 16 | style = {}, 17 | }: FlexContainerProps): CSSProperties { 18 | return useMemo( 19 | () => ({ 20 | display: 'flex', 21 | flexDirection: vertical ? 'column' : 'row', 22 | justifyContent: justify, 23 | alignItems: align, 24 | flexWrap: wrap ? 'wrap' : 'nowrap', 25 | ...(gap !== '' ? { gap } : {}), 26 | ...style, 27 | }), 28 | [vertical, justify, align, wrap, gap, style], 29 | ) 30 | } 31 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexContainer.index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export { FlexContainer } from './FlexContainer.js' 6 | export { FlexContainerProps } from './FlexContainer.types.js' 7 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexContainer.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { CSSProperties, FC } from 'react' 6 | import React, { memo } from 'react' 7 | 8 | import { useFlexContainerStyles } from './FlexContainer.hooks.js' 9 | import type { FlexContainerProps } from './FlexContainer.types.js' 10 | 11 | export const FlexContainer: FC = memo( 12 | function FlexContainer(props) { 13 | const { className, children } = props 14 | const inlineStyles: CSSProperties = useFlexContainerStyles(props) 15 | 16 | return ( 17 |
18 | {children} 19 |
20 | ) 21 | }, 22 | ) 23 | FlexContainer.displayName = 'FlexContainer' 24 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexContainer.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import type { CSSProperties, PropsWithChildren } from 'react' 7 | 8 | export type FlexContainerProps = PropsWithChildren<{ 9 | vertical?: boolean 10 | wrap?: boolean 11 | justify?: 12 | | 'flex-start' 13 | | 'flex-end' 14 | | 'center' 15 | | 'space-between' 16 | | 'space-around' 17 | | 'space-evenly' 18 | align?: 'stretch' | 'flex-start' | 'flex-end' | 'center' | 'baseline' 19 | gap?: string 20 | className?: string 21 | style?: CSSProperties 22 | }> 23 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexItem.hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import type { CSSProperties } from 'react' 7 | import { useMemo } from 'react' 8 | 9 | import type { FlexItemProps } from './FlexItem.types.js' 10 | 11 | export function useFlexItemStyles({ 12 | order = 0, 13 | grow = 0, 14 | shrink = 1, 15 | basis, 16 | align, 17 | style = {}, 18 | }: FlexItemProps): CSSProperties { 19 | const flexBasis = useMemo(() => { 20 | return basis ?? (grow > 0 ? '0%' : 'auto') 21 | }, [basis, grow]) 22 | 23 | return useMemo( 24 | () => ({ 25 | order, 26 | flexGrow: grow, 27 | flexShrink: shrink, 28 | flexBasis, 29 | ...(align != null ? { alignSelf: align } : {}), 30 | ...style, 31 | }), 32 | [order, grow, shrink, flexBasis, align, style], 33 | ) 34 | } 35 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexItem.index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export { FlexItem } from './FlexItem.js' 6 | export { FlexItemProps } from './FlexItem.types.js' 7 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexItem.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import type { CSSProperties, FC } from 'react' 7 | import React, { memo } from 'react' 8 | 9 | import { useFlexItemStyles } from './FlexItem.hooks.js' 10 | import type { FlexItemProps } from './FlexItem.types.js' 11 | 12 | export const FlexItem: FC = memo(function FlexItem(props) { 13 | const { children, className } = props 14 | const inlineStyles: CSSProperties = useFlexItemStyles(props) 15 | 16 | return ( 17 |
18 | {children} 19 |
20 | ) 21 | }) 22 | FlexItem.displayName = 'FlexItem' 23 | -------------------------------------------------------------------------------- /packages/components/src/components/FlexItem.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { CSSProperties, PropsWithChildren } from 'react' 6 | 7 | export type FlexItemProps = PropsWithChildren<{ 8 | order?: number 9 | shrink?: number 10 | basis?: string 11 | grow?: number 12 | align?: 'auto' | 'flex-start' | 'flex-end' | 'center' | 'baseline' | 'stretch' 13 | className?: string 14 | style?: CSSProperties 15 | }> 16 | -------------------------------------------------------------------------------- /packages/components/src/components/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './FileDrop.index.js' 6 | export * from './FlexContainer.index.js' 7 | export * from './FlexItem.index.js' 8 | -------------------------------------------------------------------------------- /packages/components/src/hooks/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './useMergeAllClasses.js' 6 | -------------------------------------------------------------------------------- /packages/components/src/hooks/useMergeAllClasses.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import { mergeClasses } from '@griffel/react' 7 | import { useMemo } from 'react' 8 | 9 | export function useMergeAllClasses( 10 | classNamesBySlots1?: Partial>, 11 | classNamesBySlots2?: Partial>, 12 | ): Partial> { 13 | const classNames = useMemo(() => { 14 | const keys = [ 15 | ...new Set([ 16 | ...Object.keys(classNamesBySlots1 ?? {}), 17 | ...Object.keys(classNamesBySlots2 ?? {}), 18 | ]), 19 | ] 20 | return keys.reduce>((acc, cur) => { 21 | acc[cur] = mergeClasses( 22 | classNamesBySlots1?.[cur], 23 | classNamesBySlots2?.[cur], 24 | ) 25 | 26 | return acc 27 | }, {} as Record) 28 | }, [classNamesBySlots1, classNamesBySlots2]) 29 | 30 | return classNames 31 | } 32 | -------------------------------------------------------------------------------- /packages/components/src/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './components/index.js' 6 | export * from './hooks/index.js' 7 | -------------------------------------------------------------------------------- /packages/components/src/types/expand.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | export type Expand = T extends infer O ? { [K in keyof O]: O[K] } : never 7 | 8 | // expands object types recursively 9 | export type ExpandRecursively = T extends object 10 | ? T extends infer O 11 | ? { [K in keyof O]: ExpandRecursively } 12 | : never 13 | : T 14 | 15 | export type ExpandWithFunctions = T extends (...args: infer A) => infer R 16 | ? (...args: Expand) => Expand 17 | : T extends infer O 18 | ? { [K in keyof O]: O[K] } 19 | : never 20 | 21 | export type ExpandRecursivelyWithFunctions = T extends ( 22 | ...args: infer A 23 | ) => infer R 24 | ? (...args: ExpandRecursively) => ExpandRecursively 25 | : T extends object 26 | ? T extends infer O 27 | ? { [K in keyof O]: ExpandRecursively } 28 | : never 29 | : T 30 | -------------------------------------------------------------------------------- /packages/components/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["./src/**/*"], 3 | "compilerOptions": { 4 | "outDir": "./dist", 5 | "declaration": true, 6 | "declarationDir": "./dist/types", 7 | "target": "ES2020", 8 | "lib": ["ES2020"], 9 | "jsx": "react", 10 | "module": "ES2020", 11 | "forceConsistentCasingInFileNames": true, 12 | "strict": true, 13 | "skipLibCheck": true, 14 | "noErrorTruncation": true, 15 | "esModuleInterop": true, 16 | "moduleResolution": "node" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /packages/core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sds-core" 3 | version = "1.9.0" 4 | license = "MIT" 5 | description = "Synthetic data showcase core library" 6 | repository = "https://github.com/microsoft/synthetic-data-showcase" 7 | edition = "2021" 8 | 9 | [lib] 10 | crate-type = ["rlib"] 11 | 12 | [dependencies] 13 | rand = { version = "0.8" } 14 | fnv = { version = "1.0" } 15 | itertools = { version = "0.10" } 16 | lru = { version = "0.10" } 17 | getrandom = { version = "0.2", features = ["js"] } 18 | log = { version = "0.4", features = ["std"] } 19 | csv = { version = "1.1" } 20 | instant = { version = "0.1", features = [ "stdweb", "wasm-bindgen" ] } 21 | pyo3 = { version = "0.18", features = ["extension-module"], optional = true } 22 | rayon = { version = "1.5", optional = true } 23 | serde = { version = "1.0", features = [ "derive", "rc" ] } 24 | serde_json = { version = "1.0" } 25 | statrs = { version = "0.16 "} -------------------------------------------------------------------------------- /packages/core/README.md: -------------------------------------------------------------------------------- 1 | # SDS core library 2 | 3 | Core implementation of the synthetic data showcase library for data aggregation and synthesis. This is implemented in Rust, but we offer bindings for the main capabilities to both [python](../lib-python/README.md) and [web assembly](../lib-wasm/README.md). If you are looking for a directly exposed DP Synthesizer, please refer to [`pac-synth`](../lib-pacsynth/README.md). 4 | 5 | # How to compile 6 | 7 | The project can be compiled and run on both Windows and Linux operating systems, you will need the stable Rust tolling installed in order to compile it. 8 | 9 | Please follow the steps below to compile it locally. 10 | 11 | ## I. Install Rust tooling 12 | 13 | Please follow the [steps](https://www.rust-lang.org/tools/install) according to your operating system to install the Rust tooling. 14 | 15 | ## II. Compile core library and CLI application 16 | 17 | Open a command line in the root directory of the cloned repository and run: 18 | 19 | ```bash 20 | > cargo build --release 21 | ``` 22 | 23 | This will compile the core library and CLI application in release mode - binaries will be stored in the `target/release` folder. The `sds-cli` binary (`sds-cli.exe` on Windows) should be available in there. 24 | -------------------------------------------------------------------------------- /packages/core/src/data_block/csv_block_creator.rs: -------------------------------------------------------------------------------- 1 | use super::{data_block_creator::DataBlockCreator, typedefs::CsvRecord, DataBlockCreatorError}; 2 | use csv::{Error, Reader, StringRecord}; 3 | use std::{io::Read, marker::PhantomData}; 4 | 5 | /// Creates a data block by reading a CSV file 6 | pub struct CsvDataBlockCreator { 7 | phantom: PhantomData, 8 | } 9 | 10 | impl DataBlockCreator for CsvDataBlockCreator { 11 | type InputType = Reader; 12 | type ErrorType = Error; 13 | 14 | /// Reads the headers from the CSV reader 15 | fn get_headers(reader: &mut Self::InputType) -> Result { 16 | Ok(reader 17 | .headers()? 18 | .into_iter() 19 | .map(|h| h.to_string()) 20 | .collect()) 21 | } 22 | 23 | /// Reads the records from the CSV reader 24 | fn get_records(reader: &mut Self::InputType) -> Result, Error> { 25 | reader 26 | .records() 27 | .map(|record_result: Result| { 28 | Ok(record_result? 29 | .into_iter() 30 | .map(|value| value.to_string()) 31 | .collect::()) 32 | }) 33 | .collect::, Error>>() 34 | } 35 | } 36 | 37 | /// Error that could be generated when creating a data block 38 | /// from a CSV file 39 | pub type CsvDataBlockCreatorError = DataBlockCreatorError; 40 | -------------------------------------------------------------------------------- /packages/core/src/data_block/csv_io_error.rs: -------------------------------------------------------------------------------- 1 | use csv::Error; 2 | use std::fmt::{Display, Formatter, Result}; 3 | 4 | #[cfg(feature = "pyo3")] 5 | use pyo3::exceptions::PyIOError; 6 | 7 | #[cfg(feature = "pyo3")] 8 | use pyo3::prelude::*; 9 | 10 | /// Wrapper for a csv::Error, so the from 11 | /// trait can be implemented for PyErr 12 | pub struct CsvIOError { 13 | error: Error, 14 | } 15 | 16 | impl CsvIOError { 17 | /// Creates a new CsvIOError from a csv::Error 18 | /// # Arguments 19 | /// * `error` - Related csv::Error 20 | pub fn new(error: Error) -> CsvIOError { 21 | CsvIOError { error } 22 | } 23 | } 24 | 25 | impl Display for CsvIOError { 26 | fn fmt(&self, f: &mut Formatter<'_>) -> Result { 27 | write!(f, "{}", self.error) 28 | } 29 | } 30 | 31 | #[cfg(feature = "pyo3")] 32 | impl From for PyErr { 33 | fn from(err: CsvIOError) -> PyErr { 34 | PyIOError::new_err(err.error.to_string()) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /packages/core/src/data_block/data_block_creator_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Display, Formatter, Result}; 2 | 3 | #[cfg(feature = "pyo3")] 4 | use pyo3::exceptions::PyIOError; 5 | 6 | #[cfg(feature = "pyo3")] 7 | use pyo3::prelude::*; 8 | 9 | /// Generic error generated when creating a data block 10 | #[derive(Debug)] 11 | pub enum DataBlockCreatorError 12 | where 13 | T: Display, 14 | { 15 | /// This can be generated when parsing headers and records 16 | /// on the implemented trait 17 | ParsingError(T), 18 | /// This is generated while trying to join records using the 19 | /// Subject ID 20 | JoinRecordsByIdError(String), 21 | } 22 | 23 | impl Display for DataBlockCreatorError 24 | where 25 | T: Display, 26 | { 27 | fn fmt(&self, f: &mut Formatter<'_>) -> Result { 28 | write!( 29 | f, 30 | "{}", 31 | match self { 32 | DataBlockCreatorError::ParsingError(err) => format!("{err}"), 33 | DataBlockCreatorError::JoinRecordsByIdError(err) => err.clone(), 34 | } 35 | ) 36 | } 37 | } 38 | 39 | #[cfg(feature = "pyo3")] 40 | impl From> for PyErr 41 | where 42 | T: Display, 43 | { 44 | fn from(err: DataBlockCreatorError) -> PyErr { 45 | PyIOError::new_err(match err { 46 | DataBlockCreatorError::ParsingError(err) => format!("{err}"), 47 | DataBlockCreatorError::JoinRecordsByIdError(err) => err, 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /packages/core/src/data_block/input_value.rs: -------------------------------------------------------------------------------- 1 | use fnv::FnvHashSet; 2 | use std::sync::Arc; 3 | 4 | /// Input Value parsed from the CSV/TSV file 5 | /// (can be either a single value or a multi value attribute) 6 | #[derive(Debug, Clone)] 7 | pub enum DataBlockInputValue { 8 | /// Represents a single value 9 | SingleValue(Arc), 10 | /// Represents multi values split using a delimiter 11 | MultiValue(FnvHashSet>), 12 | } 13 | -------------------------------------------------------------------------------- /packages/core/src/data_block/mod.rs: -------------------------------------------------------------------------------- 1 | mod block; 2 | mod csv_block_creator; 3 | mod csv_io_error; 4 | mod csv_record_input_values; 5 | mod data_block_creator; 6 | mod data_block_creator_error; 7 | mod headers_metadata; 8 | mod input_value; 9 | mod multi_value_column_metadata; 10 | mod raw_data; 11 | mod record; 12 | mod subject_id_joiner; 13 | mod typedefs; 14 | mod value; 15 | 16 | pub use block::*; 17 | pub use csv_block_creator::*; 18 | pub use csv_io_error::*; 19 | pub use data_block_creator::*; 20 | pub use data_block_creator_error::*; 21 | pub use multi_value_column_metadata::*; 22 | pub use raw_data::*; 23 | pub use record::*; 24 | pub use typedefs::*; 25 | pub use value::*; 26 | -------------------------------------------------------------------------------- /packages/core/src/data_block/multi_value_column_metadata.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::sync::Arc; 3 | 4 | /// Metadata about a column originated from 5 | /// parsing a column in the original input dataset 6 | #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 7 | pub struct MultiValueColumnMetadata { 8 | /// Column name in the original dataset 9 | pub src_header_name: Arc, 10 | /// Attribute name the originates the new column in the 11 | /// parsed dataset 12 | pub attribute_name: Arc, 13 | /// Delimiter for the attributes in the original column 14 | pub delimiter: String, 15 | } 16 | 17 | impl MultiValueColumnMetadata { 18 | /// Creates a new MultiValueColumnMetadata 19 | /// # Arguments 20 | /// * `src_header_name` - Column name in the original dataset 21 | /// * `attribute_name` - Attribute name the originates the new column in the 22 | /// parsed dataset 23 | /// * `delimiter` - Delimiter for the attributes in the original column 24 | #[inline] 25 | pub fn new( 26 | src_header_name: Arc, 27 | attribute_name: Arc, 28 | delimiter: String, 29 | ) -> Self { 30 | MultiValueColumnMetadata { 31 | src_header_name, 32 | attribute_name, 33 | delimiter, 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /packages/core/src/data_block/record.rs: -------------------------------------------------------------------------------- 1 | use super::value::DataBlockValue; 2 | use serde::{Deserialize, Serialize}; 3 | use std::sync::Arc; 4 | 5 | /// Represents all the values of a given row in a data block 6 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] 7 | pub struct DataBlockRecord { 8 | /// Vector of data block values for a given row indexed by column 9 | pub values: Vec>, 10 | } 11 | 12 | impl DataBlockRecord { 13 | /// Returns a new DataBlockRecord 14 | /// # Arguments 15 | /// * `values` - Vector of data block values for a given row indexed by column 16 | #[inline] 17 | pub fn new(values: Vec>) -> DataBlockRecord { 18 | DataBlockRecord { values } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /packages/core/src/dp/mod.rs: -------------------------------------------------------------------------------- 1 | mod dp_parameters; 2 | mod noise_aggregator; 3 | mod noise_parameters; 4 | mod noisy_count_threshold; 5 | mod percentile; 6 | mod stats_error; 7 | mod typedefs; 8 | 9 | #[cfg(feature = "pyo3")] 10 | mod register_pyo3; 11 | 12 | pub use dp_parameters::*; 13 | pub use noise_aggregator::*; 14 | pub use noisy_count_threshold::*; 15 | pub use percentile::*; 16 | pub use stats_error::*; 17 | pub use typedefs::*; 18 | 19 | #[cfg(feature = "pyo3")] 20 | pub use register_pyo3::*; 21 | -------------------------------------------------------------------------------- /packages/core/src/dp/noisy_count_threshold.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | use super::InputValueByLen; 4 | 5 | #[derive(Debug, Clone, Serialize, Deserialize)] 6 | #[serde(tag = "type", content = "valuesByLen")] 7 | /// Possible thresholds when adding noise with DP 8 | pub enum NoisyCountThreshold { 9 | /// Filter combinations by combination length based on a fixed threshold 10 | /// (keep only noisy counts that are `> threshold`) 11 | Fixed(InputValueByLen), 12 | /// Filter combinations by combination length based on a fraction of 13 | /// the fabricated counts distribution 14 | /// (this should be a value between 0 and 1.0) 15 | Adaptive(InputValueByLen), 16 | } 17 | -------------------------------------------------------------------------------- /packages/core/src/dp/register_pyo3.rs: -------------------------------------------------------------------------------- 1 | use super::DpParameters; 2 | use pyo3::{types::PyModule, PyResult, Python}; 3 | 4 | pub fn register_pyo3(_py: Python, m: &PyModule) -> PyResult<()> { 5 | m.add_class::()?; 6 | Ok(()) 7 | } 8 | -------------------------------------------------------------------------------- /packages/core/src/dp/stats_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Display, Formatter, Result}; 2 | 3 | #[cfg(feature = "pyo3")] 4 | use pyo3::exceptions::PyIOError; 5 | 6 | #[cfg(feature = "pyo3")] 7 | use pyo3::prelude::*; 8 | 9 | /// Wrapper for a statrs::StatsError, so the from 10 | /// trait can be implemented for PyErr 11 | pub struct StatsError { 12 | error: statrs::StatsError, 13 | } 14 | 15 | impl StatsError { 16 | /// Creates a new StatsError from a statrs::StatsError 17 | /// # Arguments 18 | /// * `error` - Related statrs::StatsError 19 | pub fn new(error: statrs::StatsError) -> StatsError { 20 | StatsError { error } 21 | } 22 | } 23 | 24 | impl Display for StatsError { 25 | fn fmt(&self, f: &mut Formatter<'_>) -> Result { 26 | write!(f, "{}", self.error) 27 | } 28 | } 29 | 30 | #[cfg(feature = "pyo3")] 31 | impl From for PyErr { 32 | fn from(err: StatsError) -> PyErr { 33 | PyIOError::new_err(err.error.to_string()) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /packages/core/src/dp/typedefs.rs: -------------------------------------------------------------------------------- 1 | use fnv::{FnvHashMap, FnvHashSet}; 2 | use std::sync::Arc; 3 | 4 | use crate::processing::aggregator::ValueCombination; 5 | 6 | /// Maps the record index to the combinations that should 7 | /// have its contribution removed 8 | pub type CombinationsToRemoveByRecord = FnvHashMap>>; 9 | 10 | /// Maps the combination length to the allowed sensitivity for each 11 | /// length 12 | pub type AllowedSensitivityByLen = FnvHashMap; 13 | 14 | /// Maps the record index to the initial combinations 15 | /// generated from the record 16 | pub type CombinationsByRecord = Vec>>; 17 | 18 | /// Maps a value combination to its count 19 | pub type CombinationsCountMap = FnvHashMap, f64>; 20 | 21 | /// Maps a value combination to its count but grouped by combination 22 | /// length 23 | pub type CombinationsCountMapByLen = FnvHashMap; 24 | 25 | /// Specifies an input value by combination length 26 | pub type InputValueByLen = FnvHashMap; 27 | -------------------------------------------------------------------------------- /packages/core/src/lib.rs: -------------------------------------------------------------------------------- 1 | /// Module with definitions related to data blocks. 2 | /// Data block is a memory block to store csv/tsv data in memory, 3 | /// including headers and records, this way we can work with references 4 | /// to the data block to avoid copying and cloning memory 5 | pub mod data_block; 6 | 7 | /// Module with helper functions and structures to handle with 8 | /// differential privacy 9 | pub mod dp; 10 | 11 | /// Module for data processing, including data aggregation, synthesis 12 | /// and evaluation 13 | pub mod processing; 14 | 15 | /// Module with some utilities useful to the project 16 | pub mod utils; 17 | -------------------------------------------------------------------------------- /packages/core/src/processing/aggregator/aggregated_count.rs: -------------------------------------------------------------------------------- 1 | use super::typedefs::RecordsSet; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[cfg(feature = "pyo3")] 5 | use pyo3::prelude::*; 6 | 7 | /// Result of data aggregation for each combination 8 | #[cfg_attr(feature = "pyo3", pyclass)] 9 | #[derive(Debug, Clone, Serialize, Deserialize, Default)] 10 | pub struct AggregatedCount { 11 | /// How many times the combination appears on the records 12 | pub count: usize, 13 | /// Which records this combinations is part of 14 | pub contained_in_records: RecordsSet, 15 | } 16 | 17 | #[cfg(feature = "pyo3")] 18 | #[cfg_attr(feature = "pyo3", pymethods)] 19 | impl AggregatedCount { 20 | /// How many times the combination appears on the records 21 | #[getter] 22 | fn count(&self) -> usize { 23 | self.count 24 | } 25 | 26 | /// Which records this combinations is part of 27 | /// This method will clone the data, so its recommended to have its result stored 28 | /// in a local variable to avoid it being called multiple times 29 | fn get_contained_in_records(&self) -> RecordsSet { 30 | self.contained_in_records.clone() 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /packages/core/src/processing/aggregator/mod.rs: -------------------------------------------------------------------------------- 1 | mod aggregated_count; 2 | mod aggregated_data; 3 | mod data_aggregator; 4 | mod records_analysis_data; 5 | mod rows_aggregator; 6 | mod typedefs; 7 | mod value_combination; 8 | 9 | #[cfg(feature = "pyo3")] 10 | mod register_pyo3; 11 | 12 | pub use aggregated_count::*; 13 | pub use aggregated_data::*; 14 | pub use data_aggregator::*; 15 | pub use records_analysis_data::*; 16 | pub use typedefs::*; 17 | pub use value_combination::*; 18 | 19 | #[cfg(feature = "pyo3")] 20 | pub use register_pyo3::*; 21 | -------------------------------------------------------------------------------- /packages/core/src/processing/aggregator/register_pyo3.rs: -------------------------------------------------------------------------------- 1 | use super::AggregatedData; 2 | use pyo3::{types::PyModule, PyResult, Python}; 3 | 4 | pub fn register_pyo3(_py: Python, m: &PyModule) -> PyResult<()> { 5 | m.add_class::()?; 6 | Ok(()) 7 | } 8 | -------------------------------------------------------------------------------- /packages/core/src/processing/evaluator/mod.rs: -------------------------------------------------------------------------------- 1 | mod data_evaluator; 2 | mod preservation_bucket; 3 | mod preservation_by_count; 4 | mod preservation_by_length; 5 | mod rare_combinations_comparison_data; 6 | mod typedefs; 7 | 8 | #[cfg(feature = "pyo3")] 9 | mod register_pyo3; 10 | 11 | pub use data_evaluator::*; 12 | pub use preservation_bucket::*; 13 | pub use preservation_by_count::*; 14 | pub use preservation_by_length::*; 15 | pub use rare_combinations_comparison_data::*; 16 | pub use typedefs::*; 17 | 18 | #[cfg(feature = "pyo3")] 19 | pub use register_pyo3::*; 20 | -------------------------------------------------------------------------------- /packages/core/src/processing/evaluator/register_pyo3.rs: -------------------------------------------------------------------------------- 1 | use super::Evaluator; 2 | use pyo3::{types::PyModule, PyResult, Python}; 3 | 4 | pub fn register_pyo3(_py: Python, m: &PyModule) -> PyResult<()> { 5 | m.add_class::()?; 6 | Ok(()) 7 | } 8 | -------------------------------------------------------------------------------- /packages/core/src/processing/evaluator/typedefs.rs: -------------------------------------------------------------------------------- 1 | use super::preservation_bucket::PreservationBucket; 2 | use fnv::FnvHashMap; 3 | 4 | /// Maps a value to its correspondent PreservationBucket 5 | pub type PreservationBucketsMap = FnvHashMap; 6 | -------------------------------------------------------------------------------- /packages/core/src/processing/generator/mod.rs: -------------------------------------------------------------------------------- 1 | mod data_generator; 2 | mod generated_data; 3 | mod synthesizers; 4 | 5 | #[cfg(feature = "pyo3")] 6 | mod register_pyo3; 7 | 8 | pub use data_generator::*; 9 | pub use generated_data::*; 10 | pub use synthesizers::*; 11 | 12 | #[cfg(feature = "pyo3")] 13 | pub use register_pyo3::*; 14 | -------------------------------------------------------------------------------- /packages/core/src/processing/generator/register_pyo3.rs: -------------------------------------------------------------------------------- 1 | use super::OversamplingParameters; 2 | use pyo3::{types::PyModule, PyResult, Python}; 3 | 4 | pub fn register_pyo3(_py: Python, m: &PyModule) -> PyResult<()> { 5 | m.add_class::()?; 6 | Ok(()) 7 | } 8 | -------------------------------------------------------------------------------- /packages/core/src/processing/generator/synthesizers/mod.rs: -------------------------------------------------------------------------------- 1 | mod aggregate_seeded; 2 | mod attribute_rows_sampler; 3 | mod cache; 4 | mod consolidate_parameters; 5 | mod oversampling_parameters; 6 | mod row_seeded; 7 | mod traits; 8 | mod typedefs; 9 | mod unseeded; 10 | mod value_seeded; 11 | 12 | pub use aggregate_seeded::*; 13 | pub use cache::*; 14 | pub use oversampling_parameters::*; 15 | pub use row_seeded::*; 16 | pub use typedefs::*; 17 | pub use unseeded::*; 18 | pub use value_seeded::*; 19 | -------------------------------------------------------------------------------- /packages/core/src/processing/generator/synthesizers/traits/mod.rs: -------------------------------------------------------------------------------- 1 | mod consolidate; 2 | mod suppress; 3 | mod synthesis_data; 4 | 5 | pub use consolidate::*; 6 | pub use suppress::*; 7 | pub use synthesis_data::*; 8 | -------------------------------------------------------------------------------- /packages/core/src/processing/generator/synthesizers/traits/synthesis_data.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | data_block::DataBlockHeaders, processing::generator::synthesizers::typedefs::AttributeCountMap, 3 | }; 4 | 5 | pub trait SynthesisData { 6 | fn get_headers(&self) -> &DataBlockHeaders; 7 | fn get_single_attr_counts(&self) -> &AttributeCountMap; 8 | fn get_resolution(&self) -> usize; 9 | } 10 | -------------------------------------------------------------------------------- /packages/core/src/processing/mod.rs: -------------------------------------------------------------------------------- 1 | /// Module to perform data aggregation 2 | pub mod aggregator; 3 | /// Module to perform data evaluation 4 | pub mod evaluator; 5 | /// Module to perform data generation (synthesis) 6 | pub mod generator; 7 | -------------------------------------------------------------------------------- /packages/core/src/utils/math.rs: -------------------------------------------------------------------------------- 1 | /// Rounds `value` down to closest multiple of `multiple`. 2 | /// The result is returned as a `isize` 3 | #[inline] 4 | pub fn iround_down(value: f64, multiple: f64) -> isize { 5 | (((value / multiple).floor()) as isize) * (multiple as isize) 6 | } 7 | 8 | /// Rounds `value` down to closest multiple of `multiple`. 9 | /// The result is returned as a `usize` 10 | #[inline] 11 | pub fn uround_down(value: f64, multiple: f64) -> usize { 12 | (((value / multiple).floor()) as usize) * (multiple as usize) 13 | } 14 | 15 | /// Calculates the percentage of processed elements up to a total 16 | #[inline] 17 | pub fn calc_percentage(n_processed: f64, total: f64) -> f64 { 18 | n_processed * 100.0 / total 19 | } 20 | -------------------------------------------------------------------------------- /packages/core/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | /// Module for reporting utilities 2 | pub mod reporting; 3 | 4 | /// Module for collection utilities 5 | pub mod collections; 6 | 7 | /// Module for math utilities 8 | pub mod math; 9 | 10 | /// Module for strings utilities 11 | pub mod strings; 12 | 13 | /// Module for threading utilities 14 | pub mod threading; 15 | 16 | /// Module for time utilities 17 | pub mod time; 18 | -------------------------------------------------------------------------------- /packages/core/src/utils/reporting/logger_progress_reporter.rs: -------------------------------------------------------------------------------- 1 | use super::{ReportProgress, StoppableResult}; 2 | use log::{log, Level}; 3 | 4 | /// Simple progress reporter using the default logger. 5 | /// * It will log progress using the configured `log_level` 6 | /// * It will only log at every 1% completed 7 | pub struct LoggerProgressReporter { 8 | progress: f64, 9 | log_level: Level, 10 | } 11 | 12 | impl LoggerProgressReporter { 13 | /// Returns a new LoggerProgressReporter 14 | /// # Arguments 15 | /// * `log_level - which log level use to log progress 16 | pub fn new(log_level: Level) -> LoggerProgressReporter { 17 | LoggerProgressReporter { 18 | progress: 0.0, 19 | log_level, 20 | } 21 | } 22 | } 23 | 24 | impl ReportProgress for LoggerProgressReporter { 25 | fn report(&mut self, new_progress: f64) -> StoppableResult<()> { 26 | let p = new_progress.floor(); 27 | 28 | if p > self.progress { 29 | self.progress = new_progress; 30 | log!( 31 | self.log_level, 32 | "Processing progress: {:.2} %", 33 | self.progress 34 | ); 35 | } 36 | Ok(()) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /packages/core/src/utils/reporting/mod.rs: -------------------------------------------------------------------------------- 1 | mod logger_progress_reporter; 2 | mod processing_stopped_error; 3 | mod report_progress; 4 | mod sendable_progress_reporter; 5 | 6 | pub use logger_progress_reporter::*; 7 | pub use processing_stopped_error::*; 8 | pub use report_progress::*; 9 | pub use sendable_progress_reporter::*; 10 | -------------------------------------------------------------------------------- /packages/core/src/utils/reporting/processing_stopped_error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Debug, Display, Formatter, Result as FmtResult}; 2 | 3 | #[cfg(feature = "pyo3")] 4 | use pyo3::exceptions::PyIOError; 5 | 6 | #[cfg(feature = "pyo3")] 7 | use pyo3::prelude::*; 8 | 9 | /// Indicates that a processing step that reports progress 10 | /// has been stopped 11 | #[derive(Default)] 12 | pub struct ProcessingStoppedError; 13 | 14 | impl Display for ProcessingStoppedError { 15 | fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { 16 | write!(f, "processing has been stopped") 17 | } 18 | } 19 | 20 | impl Debug for ProcessingStoppedError { 21 | fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { 22 | write!(f, "{}", self) 23 | } 24 | } 25 | 26 | #[cfg(feature = "pyo3")] 27 | impl From for PyErr { 28 | fn from(err: ProcessingStoppedError) -> PyErr { 29 | PyIOError::new_err(err.to_string()) 30 | } 31 | } 32 | 33 | /// Result that wraps something that can be stopped 34 | pub type StoppableResult = Result; 35 | -------------------------------------------------------------------------------- /packages/core/src/utils/reporting/report_progress.rs: -------------------------------------------------------------------------------- 1 | use super::StoppableResult; 2 | 3 | /// Implement this trait to inform progress 4 | pub trait ReportProgress { 5 | /// Receives the updated progress 6 | /// If this returns an error, this means that 7 | /// processing should be stopped 8 | fn report(&mut self, new_progress: f64) -> StoppableResult<()>; 9 | } 10 | -------------------------------------------------------------------------------- /packages/core/src/utils/strings.rs: -------------------------------------------------------------------------------- 1 | /// Normalizes reserved delimiters required by SDS to work 2 | #[inline] 3 | pub fn normalize_reserved_delimiters(value: &str) -> String { 4 | value 5 | .trim() 6 | // replace reserved delimiters 7 | .replace(';', "") 8 | .replace(':', "") 9 | } 10 | 11 | /// Transforms a string to used in a case insensitive comparison 12 | #[inline] 13 | pub fn transform_for_insensitive_cmp(value: &str) -> String { 14 | value.trim().to_lowercase() 15 | } 16 | -------------------------------------------------------------------------------- /packages/core/src/utils/threading.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "rayon")] 2 | use rayon; 3 | 4 | #[cfg(feature = "pyo3")] 5 | use pyo3::prelude::*; 6 | 7 | pub fn get_number_of_threads() -> usize { 8 | #[cfg(feature = "rayon")] 9 | { 10 | rayon::current_num_threads() 11 | } 12 | #[cfg(not(feature = "rayon"))] 13 | { 14 | 1 15 | } 16 | } 17 | 18 | #[cfg(feature = "rayon")] 19 | #[cfg_attr(feature = "pyo3", pyfunction)] 20 | #[allow(unused_must_use)] 21 | /// Sets the number of threads used for parallel processing 22 | /// # Arguments 23 | /// * `n` - number of threads 24 | pub fn set_number_of_threads(n: usize) { 25 | rayon::ThreadPoolBuilder::new() 26 | .num_threads(n) 27 | .build_global(); 28 | } 29 | 30 | #[cfg(feature = "pyo3")] 31 | pub fn register_pyo3(_py: Python, m: &PyModule) -> PyResult<()> { 32 | m.add_function(wrap_pyfunction!(set_number_of_threads, m)?)?; 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /packages/core/tests/data_block/mod.rs: -------------------------------------------------------------------------------- 1 | mod block; 2 | 3 | mod csv_block_creator_duplicated_id; 4 | 5 | mod csv_block_creator_multi_value; 6 | 7 | mod csv_block_creator; 8 | 9 | mod raw_data; 10 | 11 | mod value; 12 | -------------------------------------------------------------------------------- /packages/core/tests/data_block/value.rs: -------------------------------------------------------------------------------- 1 | use sds_core::data_block::{DataBlockHeaders, DataBlockValue}; 2 | use std::{str::FromStr, sync::Arc}; 3 | 4 | fn get_headers() -> DataBlockHeaders { 5 | ["A", "B"].map(|h| Arc::new(String::from(h))).to_vec() 6 | } 7 | 8 | fn get_value_0() -> DataBlockValue { 9 | DataBlockValue::new(0, Arc::new(String::from("a1"))) 10 | } 11 | 12 | fn get_value_1() -> DataBlockValue { 13 | DataBlockValue::new(1, Arc::new(String::from("b2"))) 14 | } 15 | 16 | #[test] 17 | fn valid_as_str_using_headers() { 18 | assert!(get_value_0().as_str_using_headers(&get_headers()) == "A:a1"); 19 | assert!(get_value_1().as_str_using_headers(&get_headers()) == "B:b2"); 20 | } 21 | 22 | #[test] 23 | fn valid_as_str() { 24 | assert!(get_value_0().to_string() == "0:a1"); 25 | assert!(get_value_1().to_string() == "1:b2"); 26 | } 27 | 28 | #[test] 29 | fn valid_from_str() { 30 | assert!(DataBlockValue::from_str(&get_value_0().to_string()).unwrap() == get_value_0()); 31 | assert!(DataBlockValue::from_str(&get_value_1().to_string()).unwrap() == get_value_1()); 32 | assert!(DataBlockValue::from_str("invalid").is_err()); 33 | assert!(DataBlockValue::from_str("invalid:a1").is_err()); 34 | } 35 | 36 | #[test] 37 | fn invalid_from_str() { 38 | assert!(DataBlockValue::from_str("invalid").is_err()); 39 | assert!(DataBlockValue::from_str("invalid:a1").is_err()); 40 | } 41 | -------------------------------------------------------------------------------- /packages/core/tests/dp/mod.rs: -------------------------------------------------------------------------------- 1 | mod noise_aggregator; 2 | 3 | mod percentile; 4 | -------------------------------------------------------------------------------- /packages/core/tests/lib.rs: -------------------------------------------------------------------------------- 1 | mod data_block; 2 | 3 | mod dp; 4 | 5 | mod processing; 6 | 7 | mod utils; 8 | -------------------------------------------------------------------------------- /packages/core/tests/processing/aggregator/mod.rs: -------------------------------------------------------------------------------- 1 | mod value_combination; 2 | -------------------------------------------------------------------------------- /packages/core/tests/processing/mod.rs: -------------------------------------------------------------------------------- 1 | mod aggregator; 2 | -------------------------------------------------------------------------------- /packages/core/tests/resources/test_block.csv: -------------------------------------------------------------------------------- 1 | A,B,C,D 2 | a1,b1,c1,d1 3 | a2,b2,,d2 4 | a1,b2,,d3 -------------------------------------------------------------------------------- /packages/core/tests/resources/test_data_block.csv: -------------------------------------------------------------------------------- 1 | A,B,C,D;h,E:h 2 | a1,0,c1,d1, 3 | a2,b2,0,d3;D,e2 4 | a3:A,b3,,, 5 | ,0,,d4, -------------------------------------------------------------------------------- /packages/core/tests/resources/test_data_block_multi_value.csv: -------------------------------------------------------------------------------- 1 | RowID,A,B,C,F:SenseZeros,G:SenseZeros,D 2 | 1,a1,b1;b4;b3,c1;c2;0;c3,0,0,d1|d3 3 | 2,a1,0,c1;0,0,0,d1||d4 4 | 3,a1,b1;0,,0," Value;ToNormalize ",d1 -------------------------------------------------------------------------------- /packages/core/tests/resources/test_duplicated_id_missing_id.csv: -------------------------------------------------------------------------------- 1 | ID,A,B,C,F,G,D 2 | 1,a1,b1;b4;0; b3,c1;c2;c3,0,0,d1|d3 3 | 1,a1,b1,c1,0,0,d1 4 | 2,a1,b1;b2;b3,c2,1,0,d2 5 | 3,a2,b1,c2,0,0,d2 6 | ,a2,b2;b3;b2,c1,1,0,d1|d4 7 | 5,a2,b2,c2,0,0,d3 8 | 6,a2,b2,c1;c4; ;c3,0,0,d2 9 | 7,a2,,c2,0,0,d2 10 | 8,a3,b2,c2,0,0,d2 11 | ,a2;b2,b2,c2,0,0,d1 12 | 9,a1;b2,b1,c2,0,0,d3|d10 13 | 9,a2,b2,c2,0,1,d7 -------------------------------------------------------------------------------- /packages/core/tests/resources/test_duplicated_id_valid.csv: -------------------------------------------------------------------------------- 1 | ID,A,B,C,F,G,D 2 | 2,a1,b1;b2;b3,c2,1,0,d2 3 | 1,a1,b1;b4;0; b3,c1;c2;c3,0,0,d1|d3 4 | 3,a2,b1,c2,0,0,d2 5 | 4,a2,b2;b3;b2,c1,1,0,d1|d4 6 | 9,a1;b2,b1,c2,0,0,d3|d10 7 | 5,a2,b2,c2,0,0,d3 8 | 1,a1,b1,c1,0,0,d1 9 | 6,a2,b2,c1;c4; ;c3,0,0,d2 10 | 9,a2;b2,b2,c2,0,0,d1 11 | 8,a3,b2,c2,0,0,d2 12 | 7,a2,,c2,0,0,d2 13 | 9,a2,b2,c2,0,1,d7 -------------------------------------------------------------------------------- /packages/core/tests/resources/test_multi_value_column_joiner.csv: -------------------------------------------------------------------------------- 1 | ID,A,B,C,F,G,D 2 | 1,a1,b1;b4;0; b3,c1;c2;c3,0,0,d1|d3 3 | 2,a1,b1,c1,0,0,d1 4 | 3,a1,b1,c1,0,0,d1 5 | 4,a1,b1;b2;b3,c2,1,0,d2 6 | 5,a2,b1,c2,0,0,d2 7 | 6,a2,b2;b3;b2,c1,0,0,d1|d4 8 | 7,a2,b2,c2,1,0,d2 9 | 8,a2,b2,c2,0,0,d3 10 | 9,a2,b2,c1;c4; ;c3,0,0,d2 11 | 10,a2,,c2,0,0,d2 12 | 11,a3,b2,c2,0,0,d2 -------------------------------------------------------------------------------- /packages/core/tests/resources/test_noise_aggregator.csv: -------------------------------------------------------------------------------- 1 | A,B,C,D 2 | a1,b1,c1,d1 3 | a2,b2,,d2 4 | a1,,c1, -------------------------------------------------------------------------------- /packages/lib-pacsynth/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pac-synth" 3 | version = "0.0.8" 4 | license = "MIT" 5 | description = "Private Accurate Combination (PAC) Synthesizers" 6 | repository = "https://github.com/microsoft/synthetic-data-showcase" 7 | edition = "2021" 8 | 9 | [lib] 10 | name = "pacsynth" 11 | crate-type = ["cdylib"] 12 | 13 | [dependencies] 14 | log = { version = "0.4", features = ["std"] } 15 | pyo3 = { version = "0.18", features = ["extension-module", "abi3-py37"] } 16 | sds-core = { path = "../core", features = ["pyo3", "rayon"] } 17 | serde = { version = "1.0", features = [ "derive", "rc" ] } 18 | serde_json = { version = "1.0" } -------------------------------------------------------------------------------- /packages/lib-pacsynth/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=0.13,<0.14"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "pac-synth" 7 | description = "Private Accurate Combination (PAC) Synthesizers" 8 | readme = "README.md" 9 | license = { file = "LICENSE" } 10 | authors=[ 11 | { name = "Synthetic Data Showcase Team", email="sds-team@microsoft.com" }, 12 | { name = "Darren Edge", email = "darren.edge@microsoft.com" }, 13 | { name = "Derek Worthen", email = "derek.worthen@microsoft.com" }, 14 | { name = "Nathan Evans", email = "nathan.evans@microsoft.com" }, 15 | { name = "Rodrigo Racanicci", email = "rracanicci@microsoft.com" }, 16 | ] 17 | maintainers = [ 18 | { name = "Rodrigo Racanicci", email = "rracanicci@microsoft.com" }, 19 | ] 20 | requires-python = ">=3.7" 21 | classifiers = [ 22 | "Programming Language :: Rust", 23 | "Programming Language :: Python :: Implementation :: CPython", 24 | "Programming Language :: Python :: Implementation :: PyPy", 25 | ] 26 | 27 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/requirements.txt: -------------------------------------------------------------------------------- 1 | # dependencies 2 | maturin 3 | pandas 4 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/src/aggregate_seeded/dp/mod.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | mod accuracy_mode; 4 | mod builder; 5 | mod fabrication_mode; 6 | mod parameters; 7 | mod synthesizer; 8 | 9 | pub use accuracy_mode::*; 10 | pub use builder::*; 11 | pub use fabrication_mode::*; 12 | pub use parameters::*; 13 | pub use synthesizer::*; 14 | 15 | pub(crate) fn register(py: Python<'_>, parent_module: &PyModule) -> PyResult<()> { 16 | accuracy_mode::register(py, parent_module)?; 17 | builder::register(py, parent_module)?; 18 | fabrication_mode::register(py, parent_module)?; 19 | parameters::register(py, parent_module)?; 20 | synthesizer::register(py, parent_module)?; 21 | Ok(()) 22 | } 23 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/src/aggregate_seeded/mod.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | mod dp; 4 | 5 | pub use dp::*; 6 | 7 | pub(crate) fn register(py: Python<'_>, parent_module: &PyModule) -> PyResult<()> { 8 | dp::register(py, parent_module)?; 9 | Ok(()) 10 | } 11 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/src/dataset/dataset_data_block_creator.rs: -------------------------------------------------------------------------------- 1 | use super::DatasetRawData; 2 | use pyo3::exceptions::PyValueError; 3 | use pyo3::prelude::*; 4 | use sds_core::data_block::{CsvRecord, DataBlockCreator}; 5 | 6 | pub struct DatasetDataBlockCreator; 7 | 8 | impl DataBlockCreator for DatasetDataBlockCreator { 9 | type InputType = DatasetRawData; 10 | type ErrorType = PyErr; 11 | 12 | fn get_headers(input: &mut Self::InputType) -> Result { 13 | if input.is_empty() { 14 | return Err(PyValueError::new_err("dataset missing headers")); 15 | } 16 | // for the headers, we just clone, without reusing the memory 17 | Ok(input[0].clone()) 18 | } 19 | 20 | fn get_records(input: &mut Self::InputType) -> Result, Self::ErrorType> { 21 | let headers_len = input.get(0).map(|headers| headers.len()).unwrap_or(0); 22 | // this consumes from the input data to reuse the same memory 23 | let records: Vec = input.drain(1..).collect(); 24 | 25 | if records.iter().any(|r| r.len() != headers_len) { 26 | return Err(PyValueError::new_err( 27 | "the number of headers must be the same as the number of elements in every record", 28 | )); 29 | } 30 | Ok(records) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/src/lib.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | mod aggregate_seeded; 4 | mod dataset; 5 | mod utils; 6 | 7 | #[pymodule] 8 | /// Private Accurate Combination (PAC) Synthesizers 9 | /// 10 | /// Library to generate synthetic data for privacy-preserving data sharing and analysis. 11 | /// 12 | /// The synthesizers aim to privately keep the accuracy of the attribute combinations counts 13 | /// from the original dataset, as well as the statistical distributions of the original data. 14 | fn pacsynth(py: Python, m: &PyModule) -> PyResult<()> { 15 | aggregate_seeded::register(py, m)?; 16 | dataset::register(py, m)?; 17 | utils::register(py, m)?; 18 | Ok(()) 19 | } 20 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | mod logger; 4 | mod progress; 5 | mod threading; 6 | 7 | pub use logger::*; 8 | pub use progress::*; 9 | pub use threading::*; 10 | 11 | pub(crate) fn register(py: Python<'_>, m: &PyModule) -> PyResult<()> { 12 | logger::register(py, m)?; 13 | threading::register(py, m)?; 14 | Ok(()) 15 | } 16 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/src/utils/progress.rs: -------------------------------------------------------------------------------- 1 | use log::{log_enabled, Level::Debug}; 2 | use sds_core::utils::reporting::LoggerProgressReporter; 3 | 4 | #[inline] 5 | pub fn create_progress_reporter() -> Option { 6 | if log_enabled!(Debug) { 7 | Some(LoggerProgressReporter::new(Debug)) 8 | } else { 9 | None 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /packages/lib-pacsynth/src/utils/threading.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | use sds_core::utils::threading; 3 | 4 | #[pyfunction] 5 | #[pyo3(text_signature = "(n)")] 6 | /// Sets the number of threads used for the algorithms that provide a parallel implementation. 7 | /// If not called, the default is one thread per CPU core. 8 | /// 9 | /// Arguments: 10 | /// * n: int - desired number of threads 11 | pub fn set_number_of_threads(n: usize) { 12 | threading::set_number_of_threads(n); 13 | } 14 | 15 | pub(crate) fn register(_py: Python, m: &PyModule) -> PyResult<()> { 16 | m.add_function(wrap_pyfunction!(set_number_of_threads, m)?)?; 17 | Ok(()) 18 | } 19 | -------------------------------------------------------------------------------- /packages/lib-python/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sds-pyo3" 3 | version = "1.9.0" 4 | license = "MIT" 5 | description = "Python bindings for the sds-core library" 6 | repository = "https://github.com/microsoft/synthetic-data-showcase" 7 | edition = "2021" 8 | 9 | [lib] 10 | name = "sds" 11 | crate-type = ["cdylib"] 12 | 13 | [dependencies] 14 | log = { version = "0.4", features = ["std"] } 15 | csv = { version = "1.1" } 16 | pyo3 = { version = "0.18", features = ["extension-module"] } 17 | sds-core = { path = "../core", features = ["pyo3", "rayon"] } 18 | env_logger = { version = "0.10" } -------------------------------------------------------------------------------- /packages/lib-python/README.md: -------------------------------------------------------------------------------- 1 | # SDS Python library 2 | 3 | Python bindings around the core library for data aggregation and synthesis. 4 | 5 | # How to compile 6 | 7 | To compile the python bindings please follow these steps. 8 | 9 | ## I. Install Python and Pip 10 | 11 | You will need _python_ and _pip_ installed on your system. 12 | 13 | ## II. Install Rust tooling 14 | 15 | The python bindings depends on the core library. You will need the Rust tooling installed in order to compile it. To install it, please refer to [here](../core/README.md#i.-install-rust-tooling). 16 | 17 | ## III. Create a virtual env 18 | 19 | Go to the folder where you want to store your virtual environment and run: 20 | 21 | ```bash 22 | > python -m venv .env 23 | > source .env/bin/activate 24 | ``` 25 | 26 | ## IV. Install the required dependencies 27 | 28 | Install _Maturin_ to build the bindings. 29 | 30 | ```bash 31 | > pip install maturin 32 | ``` 33 | 34 | ## V. Generate the python bindings 35 | 36 | Browse to the `lib-python` package, build and install it as a python module in the current virtual environment. 37 | 38 | ```bash 39 | > cd packages/lib-python 40 | > maturin develop --release 41 | ``` 42 | -------------------------------------------------------------------------------- /packages/lib-python/requirements.txt: -------------------------------------------------------------------------------- 1 | # dependencies 2 | maturin 3 | -------------------------------------------------------------------------------- /packages/lib-python/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate will generate python bindings for the main features 2 | //! of the `sds_core` library. 3 | use data_processor::SDSProcessor; 4 | use pyo3::prelude::*; 5 | use sds_core::{ 6 | dp, 7 | processing::{aggregator, evaluator, generator}, 8 | utils::threading, 9 | }; 10 | 11 | /// Module that exposes the main processor 12 | pub mod data_processor; 13 | 14 | /// A Python module implemented in Rust. The name of this function must match 15 | /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to 16 | /// import the module. 17 | #[pymodule] 18 | fn sds(py: Python, m: &PyModule) -> PyResult<()> { 19 | env_logger::init(); 20 | m.add_class::()?; 21 | threading::register_pyo3(py, m)?; 22 | dp::register_pyo3(py, m)?; 23 | aggregator::register_pyo3(py, m)?; 24 | evaluator::register_pyo3(py, m)?; 25 | generator::register_pyo3(py, m)?; 26 | Ok(()) 27 | } 28 | -------------------------------------------------------------------------------- /packages/lib-wasm/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sds-wasm" 3 | version = "1.9.0" 4 | license = "MIT" 5 | description = "Web Assembly bindings for the sds-core library" 6 | repository = "https://github.com/microsoft/synthetic-data-showcase" 7 | edition = "2021" 8 | 9 | [lib] 10 | crate-type = ["cdylib"] 11 | 12 | [features] 13 | default = ["console_error_panic_hook"] 14 | 15 | [dependencies] 16 | wasm-bindgen = { version = "0.2", features = ["serde-serialize"] } 17 | serde_json = { version = "1.0" } 18 | log = { version = "0.4", features = ["std"] } 19 | csv = { version = "1.1" } 20 | web-sys = { version = "0.3", features = [ "console" ]} 21 | sds-core = { path = "../core" } 22 | js-sys = { version = "0.3" } 23 | serde = { version = "1.0", features = [ "derive", "rc" ] } 24 | 25 | # The `console_error_panic_hook` crate provides better debugging of panics by 26 | # logging them with `console.error`. This is great for development, but requires 27 | # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for 28 | # code size when deploying. 29 | console_error_panic_hook = { version = "0.1", optional = true } 30 | 31 | [dev-dependencies] 32 | wasm-bindgen-test = { version = "0.3" } -------------------------------------------------------------------------------- /packages/lib-wasm/README.md: -------------------------------------------------------------------------------- 1 | # SDS Wasm library 2 | 3 | Web assembly bindings around the core library for data aggregation, synthesis and evaluation. 4 | 5 | # How to compile 6 | 7 | The project can be compiled and run on both Windows and Linux operating systems, you will need the stable Rust tolling installed in order to compile it. 8 | 9 | ## I. Install Rust tooling 10 | 11 | This will be the same base tooling required to build the core library, so please refer to [here](../core/README.md#i.-install-rust-tooling). 12 | 13 | ## II. Install Wasm Pack 14 | 15 | Besides the base Rust tooling, Wasm Pack is used to compile the Web Assembly bindings and JavaScript glue. This allows the Rust implementation to directly run on the browser without the need of a server. Please follow these [steps](https://rustwasm.github.io/wasm-pack/installer/) to install it according to your operating system. 16 | 17 | ## III. Compile Wasm bindings 18 | 19 | Open a command line in the root directory of the cloned repository and run: 20 | 21 | ```bash 22 | > npm install replace -g 23 | > cd packages/lib-wasm 24 | > wasm-pack build --release --target web --out-dir ../../target/wasm 25 | > replace "sds-wasm" "@essex/sds-core" ../../target/wasm/package.json 26 | ``` 27 | 28 | This will build the wasm code and JS glue and put it under `target/wasm`. 29 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod processing; 2 | 3 | pub mod utils; 4 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/aggregator/aggregate_stats.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::js::{to_js_value, JsAggregateStatistics, JsResult}; 2 | use sds_core::processing::aggregator::AggregatedMetricByString; 3 | use serde::{Deserialize, Serialize}; 4 | use wasm_bindgen::{prelude::wasm_bindgen, JsCast, JsValue}; 5 | 6 | #[derive(Serialize, Deserialize)] 7 | #[serde(rename_all = "camelCase")] 8 | #[wasm_bindgen] 9 | pub struct WasmAggregateStatistics { 10 | pub(crate) number_of_records_with_rare_combinations: usize, 11 | pub(crate) percentage_of_records_with_rare_combinations_per_column: AggregatedMetricByString, 12 | pub(crate) percentage_of_records_with_rare_combinations_per_attribute: AggregatedMetricByString, 13 | pub(crate) number_of_records: usize, 14 | } 15 | 16 | #[wasm_bindgen] 17 | impl WasmAggregateStatistics { 18 | #[wasm_bindgen(js_name = "toJs")] 19 | pub fn to_js(&self) -> JsResult { 20 | to_js_value(self) 21 | .map(|r| r.unchecked_into()) 22 | .map_err(|err| JsValue::from(err.to_string())) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/aggregator/mod.rs: -------------------------------------------------------------------------------- 1 | mod aggregate_result; 2 | mod aggregate_stats; 3 | mod single_attribute_counts; 4 | 5 | pub use aggregate_result::*; 6 | pub use aggregate_stats::*; 7 | pub use single_attribute_counts::*; 8 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/aggregator/single_attribute_counts.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | pub type SingleAttributeCounts = HashMap; 4 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/evaluator/mod.rs: -------------------------------------------------------------------------------- 1 | mod evaluate_result; 2 | mod microdata_data_stats; 3 | 4 | pub use evaluate_result::*; 5 | pub use microdata_data_stats::*; 6 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/generator/mod.rs: -------------------------------------------------------------------------------- 1 | mod generate_result; 2 | 3 | pub use generate_result::*; 4 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod aggregator; 2 | 3 | pub mod evaluator; 4 | 5 | pub mod generator; 6 | 7 | pub mod navigator; 8 | 9 | pub mod sds_processor; 10 | 11 | pub mod sds_context; 12 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/navigator/mod.rs: -------------------------------------------------------------------------------- 1 | mod attributes_intersection; 2 | mod navigate_result; 3 | mod selected_attributes; 4 | 5 | pub use attributes_intersection::*; 6 | pub use navigate_result::*; 7 | pub use selected_attributes::*; 8 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_context/errors.rs: -------------------------------------------------------------------------------- 1 | pub const MISSING_SENSITIVE_DATA_ERROR: &str = "missing sensitive data"; 2 | 3 | pub const MISSING_SENSITIVE_AGGREGATE_RESULT_ERROR: &str = "missing sensitive aggregates"; 4 | 5 | pub const MISSING_REPORTABLE_AGGREGATE_RESULT_ERROR: &str = "missing reportable aggregates"; 6 | 7 | pub const MISSING_SYNTHETIC_AGGREGATE_RESULT_ERROR: &str = "missing synthetic aggregates"; 8 | 9 | pub const MISSING_GENERATE_RESULT_ERROR: &str = "missing synthesized data"; 10 | 11 | pub const MISSING_SYNTHETIC_PROCESSOR_ERROR: &str = "missing synthetic processor"; 12 | 13 | pub const MISSING_EVALUATE_RESULT_ERROR: &str = "missing evaluate result"; 14 | 15 | pub const MISSING_NAVIGATE_RESULT_ERROR: &str = "missing navigate result"; 16 | 17 | pub const INVALID_REPORTABLE_REPORTING_LENGTH_ERROR: &str = 18 | "reportable aggregates computed with reporting length of {0}, trying to evaluate with {1}"; 19 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_context/mod.rs: -------------------------------------------------------------------------------- 1 | mod context; 2 | mod errors; 3 | 4 | pub use context::*; 5 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/base_synthesis_parameters.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use wasm_bindgen::{prelude::wasm_bindgen, JsValue}; 3 | 4 | use crate::utils::js::{from_js_value, JsBaseSynthesisParameters}; 5 | 6 | #[derive(Serialize, Deserialize)] 7 | #[serde(rename_all = "camelCase")] 8 | #[wasm_bindgen] 9 | pub struct WasmBaseSynthesisParameters { 10 | pub(crate) resolution: usize, 11 | pub(crate) cache_max_size: Option, 12 | pub(crate) empty_value: Option, 13 | } 14 | 15 | impl TryFrom for WasmBaseSynthesisParameters { 16 | type Error = JsValue; 17 | 18 | fn try_from(js_base_params: JsBaseSynthesisParameters) -> Result { 19 | from_js_value(&js_base_params).map_err(|err| JsValue::from(err.to_string())) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/csv_data_parameters.rs: -------------------------------------------------------------------------------- 1 | use super::{header_names::HeaderNames, MultiValueColumns}; 2 | use serde::{Deserialize, Serialize}; 3 | use wasm_bindgen::{prelude::wasm_bindgen, JsValue}; 4 | 5 | use crate::utils::js::{from_js_value, JsCsvDataParameters}; 6 | 7 | #[derive(Serialize, Deserialize, Clone)] 8 | #[serde(rename_all = "camelCase")] 9 | #[wasm_bindgen] 10 | pub struct WasmCsvDataParameters { 11 | pub(crate) delimiter: char, 12 | pub(crate) subject_id: Option, 13 | pub(crate) use_columns: HeaderNames, 14 | pub(crate) multi_value_columns: MultiValueColumns, 15 | pub(crate) sensitive_zeros: HeaderNames, 16 | pub(crate) record_limit: usize, 17 | } 18 | 19 | impl TryFrom for WasmCsvDataParameters { 20 | type Error = JsValue; 21 | 22 | fn try_from(js_csv_data_params: JsCsvDataParameters) -> Result { 23 | from_js_value(&js_csv_data_params).map_err(|err| JsValue::from(err.to_string())) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/dp_parameters.rs: -------------------------------------------------------------------------------- 1 | use sds_core::dp::DpParameters; 2 | use wasm_bindgen::JsValue; 3 | 4 | use crate::utils::js::{from_js_value, JsDpParameters}; 5 | 6 | impl TryFrom for DpParameters { 7 | type Error = JsValue; 8 | 9 | fn try_from(js_dp_params: JsDpParameters) -> Result { 10 | from_js_value(&js_dp_params).map_err(|err| JsValue::from(err.to_string())) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/header_names.rs: -------------------------------------------------------------------------------- 1 | use std::convert::TryFrom; 2 | use wasm_bindgen::JsValue; 3 | 4 | use crate::utils::js::{from_js_value, JsHeaderNames}; 5 | 6 | pub type HeaderNames = Vec; 7 | 8 | impl TryFrom for HeaderNames { 9 | type Error = JsValue; 10 | 11 | fn try_from(js_header_names: JsHeaderNames) -> Result { 12 | from_js_value(&js_header_names).map_err(|err| JsValue::from(err.to_string())) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/mod.rs: -------------------------------------------------------------------------------- 1 | mod base_synthesis_parameters; 2 | mod csv_data_parameters; 3 | mod dp_parameters; 4 | mod header_names; 5 | mod multi_value_columns; 6 | mod noisy_count_threshold; 7 | mod oversampling_parameters; 8 | mod processor; 9 | 10 | pub use base_synthesis_parameters::*; 11 | pub use csv_data_parameters::*; 12 | pub use dp_parameters::*; 13 | pub use header_names::*; 14 | pub use multi_value_columns::*; 15 | pub use noisy_count_threshold::*; 16 | pub use oversampling_parameters::*; 17 | pub use processor::*; 18 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/multi_value_columns.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, convert::TryFrom}; 2 | use wasm_bindgen::JsValue; 3 | 4 | use crate::utils::js::{from_js_value, JsMultiValueColumns}; 5 | 6 | pub type MultiValueColumns = HashMap; 7 | 8 | impl TryFrom for MultiValueColumns { 9 | type Error = JsValue; 10 | 11 | fn try_from(js_multi_value_columns: JsMultiValueColumns) -> Result { 12 | from_js_value(&js_multi_value_columns).map_err(|err| JsValue::from(err.to_string())) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/noisy_count_threshold.rs: -------------------------------------------------------------------------------- 1 | use sds_core::dp::NoisyCountThreshold; 2 | use wasm_bindgen::JsValue; 3 | 4 | use crate::utils::js::{from_js_value, JsNoisyCountThreshold}; 5 | 6 | impl TryFrom for NoisyCountThreshold { 7 | type Error = JsValue; 8 | 9 | fn try_from(js_threshold: JsNoisyCountThreshold) -> Result { 10 | from_js_value(&js_threshold).map_err(|err| JsValue::from(err.to_string())) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/processing/sds_processor/oversampling_parameters.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use wasm_bindgen::{prelude::wasm_bindgen, JsValue}; 3 | 4 | use crate::utils::js::{from_js_value, JsOversamplingParameters}; 5 | 6 | #[derive(Serialize, Deserialize)] 7 | #[serde(rename_all = "camelCase")] 8 | #[wasm_bindgen] 9 | pub struct WasmOversamplingParameters { 10 | pub(crate) oversampling_ratio: Option, 11 | pub(crate) oversampling_tries: Option, 12 | } 13 | 14 | impl TryFrom for WasmOversamplingParameters { 15 | type Error = JsValue; 16 | 17 | fn try_from(js_oversampling_params: JsOversamplingParameters) -> Result { 18 | from_js_value(&js_oversampling_params).map_err(|err| JsValue::from(err.to_string())) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/utils/js/mod.rs: -------------------------------------------------------------------------------- 1 | mod js_progress_reporter; 2 | mod set_panic_hook; 3 | mod ts_definitions; 4 | 5 | pub use js_progress_reporter::*; 6 | pub use set_panic_hook::*; 7 | pub use ts_definitions::*; 8 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/utils/js/set_panic_hook.rs: -------------------------------------------------------------------------------- 1 | pub fn set_panic_hook() { 2 | // When the `console_error_panic_hook` feature is enabled, we can call the 3 | // `set_panic_hook` function at least once during initialization, and then 4 | // we will get better error messages if our code ever panics. 5 | // 6 | // For more details see 7 | // https://github.com/rustwasm/console_error_panic_hook#readme 8 | #[cfg(feature = "console_error_panic_hook")] 9 | console_error_panic_hook::set_once(); 10 | } 11 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/utils/logger.rs: -------------------------------------------------------------------------------- 1 | use super::js::set_panic_hook; 2 | use log::{set_boxed_logger, set_max_level, LevelFilter, Log, Metadata, Record}; 3 | use std::str::FromStr; 4 | use wasm_bindgen::prelude::*; 5 | use web_sys::console; 6 | 7 | pub struct ConsoleLogger { 8 | pub level: LevelFilter, 9 | } 10 | 11 | impl ConsoleLogger { 12 | pub fn new(level: LevelFilter) -> ConsoleLogger { 13 | ConsoleLogger { level } 14 | } 15 | } 16 | 17 | impl Log for ConsoleLogger { 18 | #[inline] 19 | fn enabled(&self, metadata: &Metadata) -> bool { 20 | metadata.level() <= self.level 21 | } 22 | 23 | #[inline] 24 | fn log(&self, record: &Record) { 25 | if self.enabled(record.metadata()) { 26 | let l_str = format!("[{}] - {}", record.level(), record.args()); 27 | console::log_1(&l_str.into()); 28 | } 29 | } 30 | 31 | fn flush(&self) {} 32 | } 33 | 34 | #[wasm_bindgen] 35 | pub fn init_logger(level_str: &str) -> bool { 36 | set_panic_hook(); 37 | if let Ok(level) = LevelFilter::from_str(level_str) { 38 | return match set_boxed_logger(Box::new(ConsoleLogger::new(level))) { 39 | Ok(_) => { 40 | set_max_level(level); 41 | true 42 | } 43 | _ => false, 44 | }; 45 | } 46 | false 47 | } 48 | -------------------------------------------------------------------------------- /packages/lib-wasm/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod js; 2 | 3 | pub mod logger; 4 | -------------------------------------------------------------------------------- /packages/python-pipeline/requirements.txt: -------------------------------------------------------------------------------- 1 | # dependencies 2 | pandas 3 | matplotlib 4 | seaborn 5 | psutil 6 | joblib 7 | -------------------------------------------------------------------------------- /packages/python-pipeline/requirements_frozen.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/synthetic-data-showcase/16151bf41956042c2970552f1e50eba9dd65f87d/packages/python-pipeline/requirements_frozen.txt -------------------------------------------------------------------------------- /packages/python-pipeline/template/data_showcase.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/synthetic-data-showcase/16151bf41956042c2970552f1e50eba9dd65f87d/packages/python-pipeline/template/data_showcase.pbit -------------------------------------------------------------------------------- /packages/webapp/.env: -------------------------------------------------------------------------------- 1 | VITE_SDS_WASM_LOG_LEVEL=trace -------------------------------------------------------------------------------- /packages/webapp/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | SDS - Synthetic Data Showcase 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /packages/webapp/public/synthetic_data_showcase.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/synthetic-data-showcase/16151bf41956042c2970552f1e50eba9dd65f87d/packages/webapp/public/synthetic_data_showcase.pbit -------------------------------------------------------------------------------- /packages/webapp/src/App/App.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { memo, StrictMode } from 'react' 6 | import { HashRouter } from 'react-router-dom' 7 | 8 | import { 9 | ChartContext, 10 | DataContext, 11 | FileUploader, 12 | StyleContext, 13 | } from './contexts/index.js' 14 | import { Routes } from './Layout/Routes.js' 15 | 16 | export const App: React.FC = memo(function App() { 17 | return ( 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | ) 32 | }) 33 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/Header/ErrorBar/ErrorBar.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { FC } from 'react' 6 | import { memo } from 'react' 7 | 8 | import { ErrorMessageBar } from '~components/ErrorMessageBar' 9 | import { useGlobalErrorMessage } from '~states' 10 | 11 | export const ErrorBar: FC = memo(function ErrorBar() { 12 | const [globalError, setGlobalError] = useGlobalErrorMessage() 13 | return ( 14 | setGlobalError(undefined)} 17 | /> 18 | ) 19 | }) 20 | ErrorBar.displayName = 'ErrorBar' 21 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/Header/ErrorBar/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './ErrorBar.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/Header/Header.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { FlexContainer } from '@sds/components' 6 | import type { FC } from 'react' 7 | import { memo } from 'react' 8 | 9 | import { ErrorBar } from './ErrorBar/index.js' 10 | import { NavBar } from './NavBar/index.js' 11 | import { TitleBar } from './TitleBar/index.js' 12 | 13 | export const Header: FC = memo(function Header() { 14 | return ( 15 | 16 | 17 | 18 | 19 | 20 | ) 21 | }) 22 | Header.displayName = 'Header' 23 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/Header/NavBar/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './NavBar.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/Header/TitleBar/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './TitleBar.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/Header/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './Header.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/Routes.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { Spinner } from '@fluentui/react' 6 | import type { FC, ReactNode } from 'react' 7 | import { memo, Suspense } from 'react' 8 | import { Route, Routes as Switch } from 'react-router-dom' 9 | import styled from 'styled-components' 10 | 11 | import type { PageDetails } from '~pages' 12 | import { Pages } from '~pages' 13 | 14 | import { Layout } from './Layout.js' 15 | 16 | function toRoute(page: PageDetails): ReactNode { 17 | const Component = page.component 18 | return ( 19 | }> 24 | 25 | 26 | } 27 | /> 28 | ) 29 | } 30 | 31 | export const Routes: FC = memo(function Routes() { 32 | const layoutRoutes = Object.values(Pages) 33 | .filter(page => page.useLayout) 34 | .map(toRoute) 35 | 36 | const independentRoutes = Object.values(Pages) 37 | .filter(page => !page.useLayout) 38 | .map(toRoute) 39 | 40 | return ( 41 | 42 | }> 43 | {layoutRoutes} 44 | 45 | {independentRoutes} 46 | 47 | ) 48 | }) 49 | 50 | const StyledSpinner = styled(Spinner)` 51 | margin-top: 20px; 52 | ` 53 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/hooks/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './useOnTableChange.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/hooks/useOnTableChange.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { introspect } from '@data-wrangling-components/core' 6 | import { useEffect } from 'react' 7 | 8 | import { 9 | useClearSensitiveData, 10 | useIsProcessingSetter, 11 | usePreparedTable, 12 | useRawSynthesisParametersPropertySetter, 13 | useSensitiveContent, 14 | } from '~states' 15 | import { columnIndexesWithZeros, tableHeaders } from '~utils' 16 | 17 | export function useOnTableChange(): void { 18 | const [preparedTable] = usePreparedTable() 19 | const [, setSensitiveContent] = useSensitiveContent() 20 | const setIsProcessing = useIsProcessingSetter() 21 | const clearSensitiveData = useClearSensitiveData() 22 | const setRecordLimit = useRawSynthesisParametersPropertySetter('recordLimit') 23 | 24 | useEffect(() => { 25 | async function run() { 26 | if (preparedTable !== undefined) { 27 | const t = preparedTable.table! 28 | setIsProcessing(true) 29 | await clearSensitiveData() 30 | 31 | setSensitiveContent({ 32 | table: t, 33 | headers: tableHeaders(t), 34 | columnsWithZeros: columnIndexesWithZeros(t), 35 | delimiter: ',', 36 | metadata: introspect(t, true), 37 | }) 38 | setRecordLimit(t.numRows()) 39 | setIsProcessing(false) 40 | } 41 | } 42 | void run() 43 | }, [ 44 | preparedTable, 45 | setSensitiveContent, 46 | setIsProcessing, 47 | clearSensitiveData, 48 | setRecordLimit, 49 | ]) 50 | } 51 | -------------------------------------------------------------------------------- /packages/webapp/src/App/Layout/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | export * from './Routes.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/App/contexts/DataContext.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { FC, PropsWithChildren } from 'react' 6 | import { memo } from 'react' 7 | import { RecoilRoot } from 'recoil' 8 | 9 | export const DataContext: FC< 10 | PropsWithChildren<{ 11 | /* nothing */ 12 | }> 13 | > = memo(function DataContext({ children }) { 14 | return {children} 15 | }) 16 | -------------------------------------------------------------------------------- /packages/webapp/src/App/contexts/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './ChartContext.js' 6 | export * from './DataContext.js' 7 | export * from './FileUploader.js' 8 | export * from './StyleContext.js' 9 | -------------------------------------------------------------------------------- /packages/webapp/src/App/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './App.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AllSynthesisInfo/AllSynthesisInfo.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { Spinner } from '@fluentui/react' 6 | import styled from 'styled-components' 7 | 8 | export const WrappedText = styled.span` 9 | white-space: normal; 10 | ` 11 | 12 | export const Container = styled.div` 13 | .ms-DetailsHeader { 14 | padding-top: 0px; 15 | } 16 | 17 | .ms-DetailsRow-cell { 18 | margin: auto; 19 | } 20 | ` 21 | 22 | export const StyledSpinner = styled(Spinner)` 23 | align-items: baseline; 24 | ` 25 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AllSynthesisInfo/AllSynthesisInfo.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ISynthesisInfo } from '~workers/types' 6 | 7 | export type SelectSynthesisInfoCallback = ( 8 | newSelection: ISynthesisInfo, 9 | ) => void | Promise 10 | 11 | export type DeleteSynthesisInfoCallback = ( 12 | selection: ISynthesisInfo, 13 | ) => void | Promise 14 | 15 | export interface AllSynthesisInfoProps { 16 | allSynthesisInfo: ISynthesisInfo[] 17 | selectedSynthesisInfo: ISynthesisInfo | null 18 | onSelected?: SelectSynthesisInfoCallback 19 | onDelete?: DeleteSynthesisInfoCallback 20 | } 21 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AllSynthesisInfo/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './AllSynthesisInfo.js' 6 | export * from './StatefulAllSynthesisInfo.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AttributeIntersectionValueChartLegend/AttributeIntersectionValueChartLegend.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IStackTokens } from '@fluentui/react' 6 | import { Label, Stack, useTheme } from '@fluentui/react' 7 | import { memo } from 'react' 8 | import styled from 'styled-components' 9 | 10 | import { useActualNominalColor, useEstimatedNominalColor } from './hooks.js' 11 | 12 | export const AttributeIntersectionValueChartLegend: React.FC = memo( 13 | function AttributeIntersectionValueChartLegend() { 14 | const theme = useTheme() 15 | const stackTokens: IStackTokens = { 16 | childrenGap: theme.spacing.s1, 17 | } 18 | const estimatedColor = useEstimatedNominalColor() 19 | const actualColor = useActualNominalColor() 20 | 21 | return ( 22 | 28 | 29 | Synthetic 30 | 31 | Aggregate 32 | 33 | ) 34 | }, 35 | ) 36 | 37 | const ColorLegend = styled.div` 38 | width: 40px; 39 | height: 12px; 40 | background-color: ${props => props.color}; 41 | ` 42 | 43 | const ColorLabel = styled(Label)` 44 | font-weight: normal; 45 | ` 46 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AttributeIntersectionValueChartLegend/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './AttributeIntersectionValueChartLegend.js' 6 | export * from './hooks.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AttributeSelector/HeaderSelector.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { HeaderNames } from '@essex/sds-core' 6 | import type { IStackTokens } from '@fluentui/react' 7 | import { Checkbox, Stack, useTheme } from '@fluentui/react' 8 | import { memo } from 'react' 9 | 10 | export interface HeaderSelectorProps { 11 | headers: HeaderNames 12 | selectedHeaders: boolean[] 13 | onToggle: (columnIndex: number) => void 14 | } 15 | 16 | export const HeaderSelector: React.FC = memo( 17 | function HeaderSelector({ 18 | headers, 19 | selectedHeaders, 20 | onToggle, 21 | }: HeaderSelectorProps) { 22 | const theme = useTheme() 23 | 24 | const stackTokens: IStackTokens = { 25 | childrenGap: theme.spacing.s1, 26 | } 27 | 28 | return ( 29 | 30 | {headers.map((h, i) => ( 31 | onToggle(i)} 36 | /> 37 | ))} 38 | 39 | ) 40 | }, 41 | ) 42 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AttributeSelector/hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { 6 | IAttributesIntersection, 7 | ISelectedAttributesByColumn, 8 | } from '@essex/sds-core' 9 | import _ from 'lodash' 10 | import { useMemo } from 'react' 11 | 12 | export function useMaxCount(items: IAttributesIntersection[]): number { 13 | return useMemo( 14 | () => 15 | Number( 16 | _.max([ 17 | _.maxBy(items, item => item.estimatedCount)?.estimatedCount, 18 | _.maxBy(items, item => item.actualCount ?? 0)?.actualCount, 19 | ]), 20 | ) ?? 1, 21 | [items], 22 | ) 23 | } 24 | 25 | export function useSelectedAttributesByColumnEntries( 26 | selectedAttributesByColumn: ISelectedAttributesByColumn, 27 | ): [string, Set][] { 28 | return useMemo( 29 | () => Object.entries(selectedAttributesByColumn), 30 | [selectedAttributesByColumn], 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /packages/webapp/src/components/AttributeSelector/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './ColumnAttributeSelector.js' 6 | export * from './ColumnAttributeSelectorGrid.js' 7 | export * from './HeaderSelector.js' 8 | export * from './SelectedAttributes.js' 9 | -------------------------------------------------------------------------------- /packages/webapp/src/components/Charts/hooks/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './charts.js' 6 | export * from './useMetricsByCountLabels.js' 7 | export * from './useMetricsByLenLabels.js' 8 | -------------------------------------------------------------------------------- /packages/webapp/src/components/Charts/hooks/useMetricsByCountLabels.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IMetricByKey } from '@essex/sds-core' 6 | import { useMemo } from 'react' 7 | 8 | export function getMetricsByCountLabels( 9 | metricsByKey?: IMetricByKey | null, 10 | ): number[] { 11 | return Object.keys(metricsByKey ?? {}) 12 | .reverse() 13 | .map(bin => Number(bin)) 14 | } 15 | 16 | export function useMetricsByCountLabels( 17 | metricsByKey?: IMetricByKey | null, 18 | ): number[] { 19 | return useMemo(() => getMetricsByCountLabels(metricsByKey), [metricsByKey]) 20 | } 21 | -------------------------------------------------------------------------------- /packages/webapp/src/components/Charts/hooks/useMetricsByLenLabels.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import _ from 'lodash' 6 | import { useMemo } from 'react' 7 | 8 | export function getMetricsByLenLabels( 9 | reportingLength?: number | null, 10 | ): number[] { 11 | if (!reportingLength) { 12 | return [] 13 | } 14 | return _.range(1, Number(reportingLength) + 1) 15 | } 16 | 17 | export function useMetricsByLenLabels( 18 | reportingLength?: number | null, 19 | ): number[] { 20 | return useMemo( 21 | () => getMetricsByLenLabels(reportingLength), 22 | [reportingLength], 23 | ) 24 | } 25 | -------------------------------------------------------------------------------- /packages/webapp/src/components/Charts/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './AttributeIntersectionValueChart.js' 6 | export * from './MetricsChart.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/CollapsablePanel/CollapsablePanel.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { IconButton, useTheme } from '@fluentui/react' 6 | import { FlexContainer } from '@sds/components' 7 | import { memo, useState } from 'react' 8 | 9 | import type { CollapsablePanelProps } from './CollapsablePanel.types.js' 10 | 11 | export const CollapsablePanel: React.FC = memo( 12 | function CollapsablePanel({ header, children, defaultCollapsed }) { 13 | const theme = useTheme() 14 | const [isCollapsed, setIsCollapsed] = useState(defaultCollapsed ?? false) 15 | 16 | return ( 17 | 18 | 25 | {header} 26 | setIsCollapsed(prev => !prev)} 32 | /> 33 | 34 | {isCollapsed && children} 35 | 36 | ) 37 | }, 38 | ) 39 | -------------------------------------------------------------------------------- /packages/webapp/src/components/CollapsablePanel/CollapsablePanel.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type React from 'react' 6 | 7 | export interface CollapsablePanelProps { 8 | header: React.ReactNode 9 | children: React.ReactNode 10 | defaultCollapsed?: boolean 11 | } 12 | -------------------------------------------------------------------------------- /packages/webapp/src/components/CollapsablePanel/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './CollapsablePanel.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/CsvTable/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './CsvTable.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/DataEvaluationInfo/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DataEvaluationInfo.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/DataEvaluationInfoDownloader/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DataEvaluationInfoDownloader.js' 6 | export * from './hooks.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/ErrorMessageBar/ErrorMessageBar.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { MessageBar, MessageBarType } from '@fluentui/react' 6 | import type { FC } from 'react' 7 | import { memo } from 'react' 8 | 9 | export interface ErrorMessageBarProps { 10 | message?: string 11 | onDismiss?: () => void | Promise 12 | } 13 | export const ErrorMessageBar: FC = memo( 14 | function ErrorMessageBar({ message, onDismiss }) { 15 | return ( 16 | <> 17 | {message !== undefined && ( 18 | 24 | {' '} 25 | {message}{' '} 26 | 27 | )} 28 | 29 | ) 30 | }, 31 | ) 32 | -------------------------------------------------------------------------------- /packages/webapp/src/components/ErrorMessageBar/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './ErrorMessageBar.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/HumanReadableSummary/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './HumanReadableSummary.hooks.js' 6 | export * from './HumanReadableSummary.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/InfoTooltip/InfoTooltip.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IIconProps, ITooltipHostProps } from '@fluentui/react' 6 | import { IconButton, TooltipHost } from '@fluentui/react' 7 | import { memo, useCallback } from 'react' 8 | 9 | export interface InfoTooltipProps { 10 | children?: JSX.Element 11 | title?: string 12 | } 13 | 14 | const infoIcon: IIconProps = { iconName: 'Info' } 15 | 16 | export const InfoTooltip: React.FC = memo( 17 | function InfoTooltip({ title, children, ...props }) { 18 | const renderChildren = useCallback(() => { 19 | return children ?? null 20 | }, [children]) 21 | 22 | if (!props.tooltipProps) { 23 | props.tooltipProps = { 24 | onRenderContent: renderChildren, 25 | } 26 | } 27 | 28 | return ( 29 | 30 | 31 | 32 | ) 33 | }, 34 | ) 35 | -------------------------------------------------------------------------------- /packages/webapp/src/components/InfoTooltip/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './InfoTooltip.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/Markdown/Markdown.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import DOMPurify from 'dompurify' 6 | import { marked } from 'marked' 7 | import { memo } from 'react' 8 | 9 | export interface MarkdownProps { 10 | children: string 11 | } 12 | 13 | export const Markdown: React.FC = memo(function InfoTooltip({ 14 | children, 15 | }: MarkdownProps) { 16 | return ( 17 |
20 | ) 21 | }) 22 | 23 | export function fromMarkdownStr(markdown: string): JSX.Element { 24 | return {markdown} 25 | } 26 | -------------------------------------------------------------------------------- /packages/webapp/src/components/Markdown/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './Markdown.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/MetricsSummaryTable/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './hooks.js' 6 | export * from './MetricsSummaryTable.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/NavBar/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './NavBar.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/PolicyAndCookieBanner/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './PolicyAndCookieBanner.js' 6 | export * from './PolicyAndCookieBanner.types.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/SynthesisDropdown/SynthesisDropdown.hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IDropdownOption } from '@fluentui/react' 6 | import { useCallback, useMemo } from 'react' 7 | 8 | import type { ISynthesisInfo } from '~workers/types' 9 | 10 | import type { OnSynthesisSelectedCallback } from './SynthesisDropdown.types.js' 11 | 12 | export function useAllSynthesisOptions( 13 | allSynthesisInfo: ISynthesisInfo[], 14 | ): IDropdownOption[] { 15 | return useMemo(() => { 16 | return allSynthesisInfo.map(p => ({ key: p.key, text: p.key })) 17 | }, [allSynthesisInfo]) 18 | } 19 | 20 | export function useSynthesisDropdownOnChange( 21 | allSynthesisInfo: ISynthesisInfo[], 22 | onSynthesisSelected?: OnSynthesisSelectedCallback, 23 | ): (event: React.FormEvent, item?: IDropdownOption) => void { 24 | return useCallback( 25 | (event: React.FormEvent, item?: IDropdownOption) => { 26 | if (item) { 27 | const synthesis = allSynthesisInfo.find(p => p.key === item.key) 28 | 29 | if (synthesis) { 30 | onSynthesisSelected?.(synthesis) 31 | } 32 | } 33 | }, 34 | [allSynthesisInfo, onSynthesisSelected], 35 | ) 36 | } 37 | -------------------------------------------------------------------------------- /packages/webapp/src/components/SynthesisDropdown/SynthesisDropdown.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { Dropdown } from '@fluentui/react' 6 | import { memo } from 'react' 7 | 8 | import { 9 | useAllSynthesisOptions, 10 | useSynthesisDropdownOnChange, 11 | } from './SynthesisDropdown.hooks.js' 12 | import type { SynthesisDropdownProps } from './SynthesisDropdown.types.js' 13 | 14 | export const SynthesisDropdown: React.FC = memo( 15 | function SynthesisDropdown({ 16 | allSynthesisInfo, 17 | selectedSynthesis, 18 | onChange, 19 | disabled, 20 | }: SynthesisDropdownProps) { 21 | const allSynthesisOptions = useAllSynthesisOptions(allSynthesisInfo) 22 | const handleOnChange = useSynthesisDropdownOnChange( 23 | allSynthesisInfo, 24 | onChange, 25 | ) 26 | 27 | return ( 28 | 47 | ) 48 | }, 49 | ) 50 | -------------------------------------------------------------------------------- /packages/webapp/src/components/SynthesisDropdown/SynthesisDropdown.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ISynthesisInfo } from '~workers/types' 6 | 7 | export type OnSynthesisSelectedCallback = (synthesis: ISynthesisInfo) => void 8 | 9 | export interface SynthesisDropdownProps { 10 | allSynthesisInfo: ISynthesisInfo[] 11 | selectedSynthesis: ISynthesisInfo | null 12 | onChange?: OnSynthesisSelectedCallback 13 | disabled?: boolean 14 | } 15 | -------------------------------------------------------------------------------- /packages/webapp/src/components/SynthesisDropdown/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SynthesisDropdown.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/TooltipWrapper/TooltipWrapper.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { Label, Stack } from '@fluentui/react' 6 | import { memo } from 'react' 7 | 8 | import { InfoTooltip } from '~components/InfoTooltip' 9 | 10 | export interface TooltipWrapperProps { 11 | label?: string 12 | tooltip?: JSX.Element 13 | children?: JSX.Element 14 | } 15 | 16 | export const TooltipWrapper: React.FC = memo( 17 | function SpinButtonWithTooltip({ 18 | label, 19 | tooltip, 20 | children, 21 | }: TooltipWrapperProps) { 22 | return ( 23 | 24 | 25 | {label && } 26 | {tooltip && {tooltip}} 27 | 28 | {children} 29 | 30 | ) 31 | }, 32 | ) 33 | -------------------------------------------------------------------------------- /packages/webapp/src/components/TooltipWrapper/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './TooltipWrapper.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/controls/DownloadButton/DownloadButton.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IIconProps } from '@fluentui/react' 6 | import { ActionButton, Stack, useTheme } from '@fluentui/react' 7 | import type { FC } from 'react' 8 | import { useRef } from 'react' 9 | import styled from 'styled-components' 10 | 11 | import type { DownloadInfo } from './DownloadInfo.js' 12 | import { useDownloadOnClick } from './hooks.js' 13 | 14 | const downloadIcon: IIconProps = { iconName: 'Download' } 15 | 16 | export interface DownloadButtonProps { 17 | onGetDownloadInfo: () => Promise 18 | title?: string 19 | label?: string 20 | disabled?: boolean 21 | } 22 | 23 | export const DownloadButton: FC = ({ 24 | onGetDownloadInfo, 25 | title, 26 | label, 27 | disabled, 28 | }) => { 29 | const downloadAnchorRef = useRef(null) 30 | const theme = useTheme() 31 | const downloadOnClick = useDownloadOnClick( 32 | downloadAnchorRef, 33 | onGetDownloadInfo, 34 | ) 35 | 36 | return ( 37 | 38 | 44 | {label} 45 | 46 | 47 | 48 | ) 49 | } 50 | 51 | const DownloadAnchor = styled.a` 52 | display: none; 53 | ` 54 | -------------------------------------------------------------------------------- /packages/webapp/src/components/controls/DownloadButton/DownloadInfo.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export interface DownloadInfo { 6 | url: string 7 | alias: string 8 | } 9 | -------------------------------------------------------------------------------- /packages/webapp/src/components/controls/DownloadButton/hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { RefObject } from 'react' 6 | import { useCallback } from 'react' 7 | 8 | import type { DownloadInfo } from './DownloadInfo.js' 9 | 10 | export function useDownloadOnClick( 11 | downloadAnchorRef: RefObject, 12 | getDownloadInfo: () => Promise, 13 | ): () => void { 14 | return useCallback(async () => { 15 | if (downloadAnchorRef.current) { 16 | const info = await getDownloadInfo() 17 | 18 | if (info) { 19 | downloadAnchorRef.current.href = info.url 20 | downloadAnchorRef.current.download = info.alias 21 | downloadAnchorRef.current.click() 22 | } 23 | } 24 | }, [downloadAnchorRef, getDownloadInfo]) 25 | } 26 | -------------------------------------------------------------------------------- /packages/webapp/src/components/controls/DownloadButton/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DownloadButton.js' 6 | export * from './DownloadInfo.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/components/controls/FileInputButton/FileInputButton.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IIconProps } from '@fluentui/react' 6 | import { ActionButton } from '@fluentui/react' 7 | import type { ChangeEvent, FC, PropsWithChildren } from 'react' 8 | import { useRef } from 'react' 9 | 10 | const openFileIcon: IIconProps = { iconName: 'FabricOpenFolderHorizontal' } 11 | 12 | export type FileInputButtonProps = PropsWithChildren<{ 13 | onChange: (e: ChangeEvent) => void 14 | disabled?: boolean 15 | }> 16 | 17 | export const FileInputButton: FC = ({ 18 | onChange, 19 | disabled, 20 | children, 21 | }) => { 22 | const inputFile = useRef(null) 23 | return ( 24 | <> 25 | { 30 | inputFile.current?.click() 31 | }} 32 | > 33 | {children} 34 | 35 | 43 | 44 | ) 45 | } 46 | -------------------------------------------------------------------------------- /packages/webapp/src/components/controls/FileInputButton/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './FileInputButton.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/components/controls/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './FileInputButton/index.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/index.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { initializeIcons } from '@fluentui/font-icons-mdl2' 6 | import { render } from 'react-dom' 7 | 8 | import { App } from './App/index.js' 9 | 10 | initializeIcons() 11 | 12 | function mount(): void { 13 | try { 14 | const root = document.getElementById('root') 15 | render(, root) 16 | } catch (err) { 17 | console.error('error rendering application', err) 18 | } 19 | } 20 | mount() 21 | -------------------------------------------------------------------------------- /packages/webapp/src/models/aggregation/MicrodataMaxStatistics.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IMicrodataStatistics } from '@essex/sds-core' 6 | 7 | export type IMicrodataMaxStatistics = Partial<{ 8 | [k in keyof IMicrodataStatistics]: number 9 | }> 10 | -------------------------------------------------------------------------------- /packages/webapp/src/models/aggregation/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './MicrodataMaxStatistics.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/models/csv/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { TableMetadata } from '@data-wrangling-components/core' 6 | import { table } from 'arquero' 7 | import type ColumnTable from 'arquero/dist/types/table/column-table' 8 | 9 | export interface ICsvTableHeader { 10 | name: string 11 | fieldName: string 12 | use: boolean 13 | hasSensitiveZeros: boolean 14 | spreadWithDelimiter: string | null 15 | } 16 | 17 | export interface ICsvContent { 18 | headers: ICsvTableHeader[] 19 | columnsWithZeros?: number[] 20 | delimiter: string 21 | table: ColumnTable 22 | metadata?: TableMetadata 23 | subjectId?: string 24 | } 25 | 26 | export const defaultCsvContent: ICsvContent = { 27 | headers: [], 28 | columnsWithZeros: undefined, 29 | delimiter: ',', 30 | table: table({}), 31 | subjectId: undefined, 32 | } 33 | -------------------------------------------------------------------------------- /packages/webapp/src/models/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './aggregation/index.js' 6 | export * from './csv/index.js' 7 | export * from './pipeline/index.js' 8 | export * from './synthesis/index.js' 9 | export * from './workers/index.js' 10 | -------------------------------------------------------------------------------- /packages/webapp/src/models/pipeline/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum PipelineStep { 6 | Prepare = 'Prepare', 7 | Select = 'Select', 8 | Synthesize = 'Synthesize', 9 | Evaluate = 'Evaluate', 10 | Navigate = 'Navigate', 11 | } 12 | -------------------------------------------------------------------------------- /packages/webapp/src/models/synthesis/AccuracyMode.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum AccuracyMode { 6 | PrioritizeLongCombinations = 'Prioritize Long Combinations', 7 | PrioritizeShortCombinations = 'Prioritize Short Combinations', 8 | Balanced = 'Balanced', 9 | } 10 | -------------------------------------------------------------------------------- /packages/webapp/src/models/synthesis/FabricationMode.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum FabricationMode { 6 | Balanced = 'Balanced', 7 | Progressive = 'Progressive', 8 | Minimize = 'Minimize', 9 | Uncontrolled = 'Uncontrolled', 10 | Custom = 'Custom', 11 | } 12 | 13 | export const defaultThreshold = 1.0 14 | -------------------------------------------------------------------------------- /packages/webapp/src/models/synthesis/OversamplingType.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum OversamplingType { 6 | Controlled = 'Controlled', 7 | Unlimited = 'Unlimited', 8 | } 9 | -------------------------------------------------------------------------------- /packages/webapp/src/models/synthesis/UseSyntheticCounts.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum UseSyntheticCounts { 6 | No = 'No', 7 | Yes = 'Yes', 8 | } 9 | -------------------------------------------------------------------------------- /packages/webapp/src/models/synthesis/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './AccuracyMode.js' 6 | export * from './FabricationMode.js' 7 | export * from './OversamplingType.js' 8 | export * from './RawSynthesisParameters.js' 9 | export * from './UseSyntheticCounts.js' 10 | -------------------------------------------------------------------------------- /packages/webapp/src/models/workers/SdsManagerInstance.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { Remote } from 'comlink' 6 | 7 | import type { SdsManager } from '~workers/SdsManager' 8 | import type { IWorkerProxy } from '~workers/utils' 9 | 10 | export interface ISdsManagerInstance { 11 | instance: Remote 12 | workerProxy: IWorkerProxy 13 | } 14 | -------------------------------------------------------------------------------- /packages/webapp/src/models/workers/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SdsManagerInstance.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Home/HomePage.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { FC } from 'react' 6 | import { memo } from 'react' 7 | import { Navigate } from 'react-router-dom' 8 | 9 | import { Pages } from '~pages' 10 | 11 | export const HomePage: FC = memo(function HomePage() { 12 | return 13 | }) 14 | HomePage.displayName = 'HomePage' 15 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Home/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './HomePage.js' 6 | export { HomePage as default } from './HomePage.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Navigate/hooks/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './useInitiallySelectedHeaders.js' 6 | export * from './useOnClearSelectedAttributes.js' 7 | export * from './useOnNewSelectedAttributesByColumn.js' 8 | export * from './useOnRunNavigate.js' 9 | export * from './useOnSetSelectedAttributes.js' 10 | export * from './useOnToggleSelectedHeader.js' 11 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Navigate/hooks/useInitiallySelectedHeaders.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { HeaderNames } from '@essex/sds-core' 6 | import { useMemo } from 'react' 7 | 8 | const initiallySelectedHeaders = 6 9 | 10 | export function calcInitiallySelectedHeaders(headers: HeaderNames): boolean[] { 11 | return headers.map((_, i) => i < initiallySelectedHeaders) 12 | } 13 | 14 | export function useInitiallySelectedHeaders(headers: HeaderNames): boolean[] { 15 | return useMemo(() => calcInitiallySelectedHeaders(headers), [headers]) 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Navigate/hooks/useOnClearSelectedAttributes.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ISelectedAttributesByColumn } from '@essex/sds-core' 6 | import { useCallback } from 'react' 7 | 8 | export function useOnClearSelectedAttributes( 9 | setNewSelectedAttributesByColumn: ( 10 | value: ISelectedAttributesByColumn, 11 | ) => void, 12 | ): () => void { 13 | return useCallback(() => { 14 | setNewSelectedAttributesByColumn({}) 15 | }, [setNewSelectedAttributesByColumn]) 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Navigate/hooks/useOnNewSelectedAttributesByColumn.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ISelectedAttributesByColumn } from '@essex/sds-core' 6 | import type { MutableRefObject } from 'react' 7 | import { useCallback } from 'react' 8 | 9 | import type { ISdsManagerInstance } from '~models' 10 | 11 | export function useOnNewSelectedAttributesByColumn( 12 | contextKey: string | undefined, 13 | setIsLoading: (value: boolean) => void, 14 | isMounted: MutableRefObject, 15 | setSelectedAttributesByColumn: (value: ISelectedAttributesByColumn) => void, 16 | manager: ISdsManagerInstance | null, 17 | ): ( 18 | newSelectedAttributesByColumn: ISelectedAttributesByColumn, 19 | ) => Promise { 20 | return useCallback( 21 | async (newSelectedAttributesByColumn: ISelectedAttributesByColumn) => { 22 | if (manager && contextKey) { 23 | try { 24 | await manager.instance.selectAttributes( 25 | contextKey, 26 | newSelectedAttributesByColumn, 27 | ) 28 | 29 | if (isMounted.current) { 30 | setSelectedAttributesByColumn(newSelectedAttributesByColumn) 31 | } 32 | } catch { 33 | setIsLoading(false) 34 | setSelectedAttributesByColumn({}) 35 | } 36 | } 37 | }, 38 | [ 39 | manager, 40 | setIsLoading, 41 | isMounted, 42 | setSelectedAttributesByColumn, 43 | contextKey, 44 | ], 45 | ) 46 | } 47 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Navigate/hooks/useOnSetSelectedAttributes.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { 6 | IAttributesIntersection, 7 | ISelectedAttributesByColumn, 8 | } from '@essex/sds-core' 9 | import { useCallback } from 'react' 10 | 11 | export function useOnSetSelectedAttributes( 12 | setNewSelectedAttributesByColumn: ( 13 | value: ISelectedAttributesByColumn, 14 | ) => void, 15 | selectedAttributesByColumn: ISelectedAttributesByColumn, 16 | ): ( 17 | headerIndex: number, 18 | item: IAttributesIntersection | undefined, 19 | ) => Promise { 20 | return useCallback( 21 | async (headerIndex: number, item: IAttributesIntersection | undefined) => { 22 | await setNewSelectedAttributesByColumn({ 23 | ...selectedAttributesByColumn, 24 | [headerIndex]: 25 | item !== undefined 26 | ? new Set([item.value]) 27 | : new Set(), 28 | }) 29 | }, 30 | [setNewSelectedAttributesByColumn, selectedAttributesByColumn], 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Navigate/hooks/useOnToggleSelectedHeader.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useCallback } from 'react' 6 | 7 | export function useOnToggleSelectedHeader( 8 | selectedHeaders: boolean[], 9 | setSelectedHeaders: (value: boolean[]) => void, 10 | ): (index: number) => Promise { 11 | return useCallback( 12 | async index => { 13 | const newSelectedHeaders = [...selectedHeaders] 14 | newSelectedHeaders[index] = !newSelectedHeaders[index] 15 | await setSelectedHeaders(newSelectedHeaders) 16 | }, 17 | [setSelectedHeaders, selectedHeaders], 18 | ) 19 | } 20 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Navigate/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './NavigatePage.js' 6 | export { NavigatePage as default } from './NavigatePage.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Prepare/DataInput/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DataInput.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Prepare/PreparePage.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { FC } from 'react' 6 | import { memo } from 'react' 7 | import styled from 'styled-components' 8 | 9 | import { DataInput } from './DataInput/index.js' 10 | 11 | export const PreparePage: FC = memo(function PreparePage() { 12 | return ( 13 | 14 | 15 | 16 | ) 17 | }) 18 | PreparePage.displayName = 'PreparePage' 19 | 20 | const Container = styled.div` 21 | height: 100%; 22 | overflow-y: auto; 23 | ` 24 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Prepare/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './PreparePage.js' 6 | export { PreparePage as default } from './PreparePage.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/AggregateStatistics/AggregateStatistics.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { FlexContainer, FlexItem } from '@sds/components' 6 | import styled from 'styled-components' 7 | 8 | export const Container = styled(FlexContainer)` 9 | width: 100%; 10 | padding: ${p => p.theme.spacing.m}; 11 | font-size: ${p => p.theme.fonts.large.fontSize}; 12 | ` 13 | 14 | export const StyledReport = styled.div` 15 | margin-bottom: ${p => p.theme.spacing.m}; 16 | text-align: center; 17 | ` 18 | 19 | export const ChartsContainer = styled(FlexContainer)` 20 | width: 100%; 21 | ` 22 | 23 | export const ChartItem = styled(FlexItem)` 24 | width: calc(50% - ${p => p.theme.spacing.s1}); 25 | ` 26 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/AggregateStatistics/ContributionChart.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { FlexItem } from '@sds/components' 6 | import styled from 'styled-components' 7 | 8 | export const ChartContainer = styled(FlexItem)` 9 | overflow-y: auto; 10 | overflow-x: hidden; 11 | padding: ${p => p.theme.spacing.s1}; 12 | direction: ltr; 13 | & > div { 14 | direction: ltr; 15 | } 16 | &::-webkit-scrollbar { 17 | width: 4px; 18 | background: ${p => p.theme.palette.neutralLight}; 19 | } 20 | &::-webkit-scrollbar-thumb { 21 | background-color: ${p => p.theme.palette.themePrimary}; 22 | border-radius: 20px; 23 | } 24 | ` 25 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/AggregateStatistics/ContributionChart.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IAggregatedMetricByString } from '@essex/sds-core' 6 | import type { TooltipItem } from 'chart.js' 7 | 8 | export interface ContributionChartProps { 9 | selectedKey?: string 10 | valuePerKey: IAggregatedMetricByString 11 | label: string 12 | containerHeight: number | string 13 | barHeight: number 14 | tooltipFormatter?: (item: TooltipItem<'bar'>) => string 15 | onClick?: (key: string) => void 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/AggregateStatistics/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './AggregateStatistics.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/InfoBar/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './InfoBar.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/MultiValueColumns/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './MultiValueColumns.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/SelectColumns/SelectColumns.hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useCallback } from 'react' 6 | import type { SetterOrUpdater } from 'recoil' 7 | 8 | import type { ICsvContent } from '~models' 9 | 10 | export function useOnUseColumnCheckToggle( 11 | setter: SetterOrUpdater, 12 | ): (index: number) => void { 13 | return useCallback( 14 | index => { 15 | setter(previous => ({ 16 | ...previous, 17 | headers: [ 18 | ...previous.headers.slice(0, index), 19 | { 20 | ...previous.headers[index], 21 | use: !previous.headers[index].use, 22 | }, 23 | ...previous.headers.slice(index + 1), 24 | ], 25 | })) 26 | }, 27 | [setter], 28 | ) 29 | } 30 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/SelectColumns/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SelectColumns.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/SelectCommands.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { FlexContainer } from '@sds/components' 6 | import type { FC } from 'react' 7 | import { memo } from 'react' 8 | import styled from 'styled-components' 9 | 10 | import { AggregationControls } from './AggregationControls/AggregationControls.js' 11 | import { MultiValueColumns } from './MultiValueColumns/index.js' 12 | import { SelectColumns } from './SelectColumns/index.js' 13 | import { SensitiveZeros } from './SensitiveZeros/index.js' 14 | import { SubjectId } from './SubjectId/index.js' 15 | 16 | export const SelectCommands: FC = memo(function SelectCommand() { 17 | return ( 18 | 19 | 20 | | 21 | 22 | | 23 | 24 | | 25 | 26 | 27 | 28 | ) 29 | }) 30 | SelectCommands.displayName = 'SelectCommands' 31 | 32 | const Container = styled(FlexContainer)` 33 | border-bottom: 1px solid ${p => p.theme.palette.neutralLight}; 34 | box-shadow: ${p => p.theme.effects.elevation4}; 35 | ` 36 | 37 | const Divider = styled.span` 38 | font-size: ${p => p.theme.fonts.smallPlus.fontSize}; 39 | color: ${p => p.theme.palette.neutralTertiary}; 40 | ` 41 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/SensitiveZeros/SensitiveZeros.hooks.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useCallback, useMemo } from 'react' 6 | import type { SetterOrUpdater } from 'recoil' 7 | 8 | import type { ICsvContent } from '~models' 9 | 10 | export function useColumnsWithZeros( 11 | content: ICsvContent, 12 | ): string[] | undefined { 13 | return useMemo( 14 | () => 15 | content.columnsWithZeros 16 | ?.map(i => content.headers[i]) 17 | .filter(h => h.use) 18 | .map(h => h.name), 19 | [content], 20 | ) 21 | } 22 | 23 | export function useOnSensitiveZeroCheckToggle( 24 | setter: SetterOrUpdater, 25 | ): (index: number) => void { 26 | return useCallback( 27 | index => { 28 | setter(previous => ({ 29 | ...previous, 30 | headers: [ 31 | ...previous.headers.slice(0, index), 32 | { 33 | ...previous.headers[index], 34 | hasSensitiveZeros: !previous.headers[index].hasSensitiveZeros, 35 | }, 36 | ...previous.headers.slice(index + 1), 37 | ], 38 | })) 39 | }, 40 | [setter], 41 | ) 42 | } 43 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/SensitiveZeros/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SensitiveZeros.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/SubjectId/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SubjectId.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectCommands/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SelectCommands.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/SelectPage.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { FlexContainer } from '@sds/components' 6 | import type { FC } from 'react' 7 | import { memo } from 'react' 8 | import styled from 'styled-components' 9 | 10 | import { PageDescription, Pages } from '~pages' 11 | 12 | import { AggregateStatistics } from './AggregateStatistics/index.js' 13 | import { InfoBar } from './InfoBar/index.js' 14 | import { SelectCommands } from './SelectCommands/index.js' 15 | import { TablePreview } from './TablePreview/index.js' 16 | 17 | export const SelectPage: FC = memo(function SelectPage() { 18 | return ( 19 | 20 | {Pages.Select.description} 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | ) 29 | }) 30 | SelectPage.displayName = 'SelectPage' 31 | 32 | const Container = styled(FlexContainer)` 33 | height: 100%; 34 | overflow-y: auto; 35 | &::-webkit-scrollbar { 36 | display: none; 37 | } 38 | ` 39 | 40 | const MainContent = styled(FlexContainer)` 41 | height: 100%; 42 | overflow-y: auto; 43 | ` 44 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/TablePreview/TablePreview.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { ArqueroDetailsList } from '@essex/arquero-react' 6 | import { useTheme } from '@fluentui/react' 7 | import type { FC } from 'react' 8 | import { memo, useMemo } from 'react' 9 | import styled from 'styled-components' 10 | 11 | import { useSensitiveContent } from '~states' 12 | 13 | export const TablePreview: FC = memo(function TablePreview() { 14 | const [sensitiveContent] = useSensitiveContent() 15 | const theme = useTheme() 16 | 17 | const visibleColumns = useMemo(() => { 18 | return sensitiveContent.headers.filter(h => h.use).map(h => h.name) 19 | }, [sensitiveContent]) 20 | 21 | if (!visibleColumns.length) return <> 22 | 23 | return ( 24 | 25 | 43 | 44 | ) 45 | }) 46 | TablePreview.displayName = 'TablePreview' 47 | 48 | const Container = styled.div` 49 | height: 100%; 50 | min-height: 150px; 51 | ` 52 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/TablePreview/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './TablePreview.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Select/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SelectPage.js' 6 | export { SelectPage as default } from './SelectPage.js' 7 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataEvaluation/DataEvaluation.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { FlexItem } from '@sds/components' 6 | import styled from 'styled-components' 7 | 8 | export const FlexCentered = styled(FlexItem)` 9 | text-align: center; 10 | overflow-x: auto; 11 | ` 12 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataEvaluation/DataEvaluation.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ISynthesisInfo } from '~workers/types' 6 | 7 | export interface DataEvaluationProps { 8 | selectedSynthesis: ISynthesisInfo | null 9 | } 10 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataEvaluation/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DataEvaluation.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesis/DataSynthesis.styles.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import { FlexContainer } from '@sds/components' 7 | import styled from 'styled-components' 8 | 9 | export const Container = styled(FlexContainer)` 10 | margin-top: ${p => p.theme.spacing.s2}; 11 | margin-left: ${p => p.theme.spacing.l1}; 12 | margin-right: ${p => p.theme.spacing.l1}; 13 | ` 14 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesis/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DataSynthesis.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesisParameters/DataSynthesisParameters.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { Dropdown, SpinButton } from '@fluentui/react' 6 | import styled from 'styled-components' 7 | 8 | export const StyledDropdown = styled(Dropdown)` 9 | width: 205px; 10 | ` 11 | 12 | export const StyledSpinButton = styled(SpinButton)` 13 | width: 205px; 14 | ` 15 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesisParameters/DataSynthesisParameters.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ICsvContent, IRawSynthesisParameters } from '~models' 6 | 7 | export interface DataSynthesisParametersProps { 8 | enableRun: boolean 9 | sensitiveCsvContent: ICsvContent 10 | onRun: (rawParameters: IRawSynthesisParameters) => Promise 11 | } 12 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesisParameters/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DataSynthesisParameters.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesisResult/DataSynthesisResult.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import styled from 'styled-components' 6 | 7 | export const StyledItem = styled.div` 8 | margin-top: ${p => p.theme.spacing.s1}; 9 | ` 10 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesisResult/DataSynthesisResult.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import type { ISynthesisInfo } from '~workers/types' 7 | 8 | export interface DataSynthesisResultProps { 9 | selectedSynthesis: ISynthesisInfo 10 | } 11 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/DataSynthesisResult/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './DataSynthesisResult.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/SynthesizePage.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { FC } from 'react' 6 | import { memo } from 'react' 7 | import styled from 'styled-components' 8 | 9 | import { PageDescription, Pages } from '~pages' 10 | 11 | import { DataSynthesis } from './DataSynthesis/index.js' 12 | 13 | export const SynthesizePage: FC = memo(function SynthesizePage() { 14 | return ( 15 | 16 | {Pages.Synthesize.description} 17 | 18 | 19 | ) 20 | }) 21 | SynthesizePage.displayName = 'SynthesizePage' 22 | 23 | const Container = styled.div` 24 | height: 100%; 25 | overflow-y: auto; 26 | ` 27 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/Synthesize/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './Synthesize.hooks.js' 6 | export * from './SynthesizePage.js' 7 | export { SynthesizePage as default } from './SynthesizePage.js' 8 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/hooks/commands/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './useDownloadCommand.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/hooks/commands/useDownloadCommand.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { downloadCommand } from '@data-wrangling-components/react' 6 | import type { ICommandBarItemProps } from '@fluentui/react' 7 | import { useMemo } from 'react' 8 | 9 | import type { ICsvContent } from '~models' 10 | 11 | export function useDownloadCommand( 12 | content: ICsvContent, 13 | filename: string, 14 | ): ICommandBarItemProps { 15 | return useMemo( 16 | () => downloadCommand(content.table, filename), 17 | [content, filename], 18 | ) 19 | } 20 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/hooks/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './commands/index.js' 6 | export * from './useCanRun.js' 7 | export * from './useDropdownOnChange.js' 8 | export * from './useOnGetAllAssetsDownloadInfo.js' 9 | export * from './useSpinButtonOnChange.js' 10 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/hooks/useCanRun.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useMemo } from 'react' 6 | 7 | import { useIsProcessingValue, useSensitiveContentValue } from '~states' 8 | 9 | export function useCanRun(): boolean { 10 | const content = useSensitiveContentValue() 11 | const isProcessing = useIsProcessingValue() 12 | 13 | return useMemo(() => { 14 | return content.table.totalRows() > 0 && !isProcessing 15 | }, [content, isProcessing]) 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/hooks/useDropdownOnChange.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IDropdownOption } from '@fluentui/react' 6 | import { useCallback } from 'react' 7 | 8 | export function useDropdownOnChange( 9 | setter: (val: T) => void, 10 | ): (event: React.FormEvent, item?: IDropdownOption) => void { 11 | return useCallback( 12 | (event: React.FormEvent, item?: IDropdownOption) => { 13 | if (item) { 14 | setter(item.key as unknown as T) 15 | } 16 | }, 17 | [setter], 18 | ) 19 | } 20 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/hooks/useSpinButtonOnChange.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useCallback } from 'react' 6 | 7 | /** 8 | * onChange handler that only fires if the input is a valid number 9 | * @param setter 10 | * @returns 11 | */ 12 | export function useSpinButtonOnChange( 13 | setter: (val: number) => void, 14 | ): (event: React.SyntheticEvent, newValue?: string) => void { 15 | return useCallback( 16 | (event, newValue?) => { 17 | const val = +(newValue || 0) 18 | if (!isNaN(val)) { 19 | setter(val) 20 | } 21 | }, 22 | [setter], 23 | ) 24 | } 25 | -------------------------------------------------------------------------------- /packages/webapp/src/pages/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './Pages.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/allSynthesisInfo.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState, useResetRecoilState } from 'recoil' 7 | 8 | import type { ISynthesisInfo } from '~workers/types' 9 | 10 | const state = atom({ 11 | key: 'all-synthesis-info', 12 | default: [], 13 | }) 14 | 15 | export function useAllSynthesisInfo(): [ 16 | ISynthesisInfo[], 17 | SetterOrUpdater, 18 | ] { 19 | return useRecoilState(state) 20 | } 21 | 22 | export function useResetAllSynthesisInfo(): () => void { 23 | return useResetRecoilState(state) 24 | } 25 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/charts.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { HeaderNames, ISelectedAttributesByColumn } from '@essex/sds-core' 6 | import type { SetterOrUpdater } from 'recoil' 7 | import { atom, useRecoilState } from 'recoil' 8 | 9 | const headersState = atom({ 10 | key: 'headers', 11 | default: [], 12 | }) 13 | 14 | export function useHeaders(): [HeaderNames, SetterOrUpdater] { 15 | return useRecoilState(headersState) 16 | } 17 | 18 | const selectedHeadersState = atom({ 19 | key: 'selectedHeaders', 20 | default: [], 21 | }) 22 | 23 | export function useSelectedHeaders(): [boolean[], SetterOrUpdater] { 24 | return useRecoilState(selectedHeadersState) 25 | } 26 | 27 | const selectedAttributesByColumnState = atom({ 28 | key: 'selectedAttributesByColumn', 29 | default: {}, 30 | }) 31 | 32 | export function useSelectedAttributesByColumn(): [ 33 | ISelectedAttributesByColumn, 34 | SetterOrUpdater, 35 | ] { 36 | return useRecoilState(selectedAttributesByColumnState) 37 | } 38 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/files.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState } from 'recoil' 7 | 8 | const fileUploadErrorState = atom({ 9 | key: 'file-upload-state', 10 | default: undefined, 11 | }) 12 | 13 | export function useFileUploadErrorMessage(): [ 14 | string | undefined, 15 | SetterOrUpdater, 16 | ] { 17 | return useRecoilState(fileUploadErrorState) 18 | } 19 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useCallback } from 'react' 6 | 7 | import { useResetSensitiveContent } from '~states' 8 | 9 | import { useResetAllSynthesisInfo } from './allSynthesisInfo.js' 10 | import { useSdsManagerInstance } from './sdsManagerInstance.js' 11 | import { useResetSelectedSynthesisInfo } from './selectedSynthesisInfo.js' 12 | 13 | export type DataClearer = () => Promise 14 | 15 | export function useClearSensitiveData(): DataClearer { 16 | const resetSensitiveContent = useResetSensitiveContent() 17 | 18 | return useCallback(async () => { 19 | resetSensitiveContent() 20 | }, [resetSensitiveContent]) 21 | } 22 | 23 | export function useClearContexts(): DataClearer { 24 | const [manager] = useSdsManagerInstance() 25 | const resetAllSynthesisInfo = useResetAllSynthesisInfo() 26 | const resetSelectedSynthesisInfo = useResetSelectedSynthesisInfo() 27 | 28 | return useCallback(async () => { 29 | await manager?.instance?.terminateAllSynthesizers() 30 | resetAllSynthesisInfo() 31 | resetSelectedSynthesisInfo() 32 | }, [manager, resetAllSynthesisInfo, resetSelectedSynthesisInfo]) 33 | } 34 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './allSynthesisInfo.js' 6 | export * from './charts.js' 7 | export * from './files.js' 8 | export * from './hooks.js' 9 | export * from './isProcessing.js' 10 | export * from './multiValueColumns.js' 11 | export * from './sdsManagerInstance.js' 12 | export * from './selectedPipelineStep.js' 13 | export * from './selectedSynthesisInfo.js' 14 | export * from './sensitiveContent.js' 15 | export * from './steps.js' 16 | export * from './tables.js' 17 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/isProcessing.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil' 7 | 8 | const state = atom({ 9 | key: 'is-processing', 10 | default: false, 11 | }) 12 | 13 | export function useIsProcessing(): [boolean, SetterOrUpdater] { 14 | return useRecoilState(state) 15 | } 16 | 17 | export function useIsProcessingValue(): boolean { 18 | return useRecoilValue(state) 19 | } 20 | 21 | export function useIsProcessingSetter(): SetterOrUpdater { 22 | return useSetRecoilState(state) 23 | } 24 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/multiValueColumns.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState } from 'recoil' 7 | 8 | const multiValueColumnsState = atom>({ 9 | key: 'multi-value-columns', 10 | default: {}, 11 | dangerouslyAllowMutability: true, 12 | }) 13 | 14 | export function useMultiValueColumns(): [ 15 | Record, 16 | SetterOrUpdater>, 17 | ] { 18 | return useRecoilState(multiValueColumnsState) 19 | } 20 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/sdsManagerInstance.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState } from 'recoil' 7 | 8 | import type { ISdsManagerInstance } from '~models' 9 | 10 | const state = atom({ 11 | key: 'sds-manager-instance', 12 | default: null, 13 | }) 14 | 15 | export function useSdsManagerInstance(): [ 16 | ISdsManagerInstance | null, 17 | SetterOrUpdater, 18 | ] { 19 | return useRecoilState(state) 20 | } 21 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/selectedPipelineStep.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil' 7 | 8 | import { PipelineStep } from '~models' 9 | 10 | const state = atom({ 11 | key: 'selected-pipeline-step', 12 | default: PipelineStep.Prepare, 13 | }) 14 | 15 | export function useSelectedPipelineStep(): [string, SetterOrUpdater] { 16 | return useRecoilState(state) 17 | } 18 | 19 | export function useSelectedPipelineStepValue(): string { 20 | return useRecoilValue(state) 21 | } 22 | 23 | export function useSelectedPipelineStepSetter(): SetterOrUpdater { 24 | return useSetRecoilState(state) 25 | } 26 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/selectedSynthesisInfo.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState, useResetRecoilState } from 'recoil' 7 | 8 | import type { ISynthesisInfo } from '~workers/types' 9 | 10 | const state = atom({ 11 | key: 'selected-synthesis-info', 12 | default: null, 13 | }) 14 | 15 | export function useSelectedSynthesisInfo(): [ 16 | ISynthesisInfo | null, 17 | SetterOrUpdater, 18 | ] { 19 | return useRecoilState(state) 20 | } 21 | 22 | export function useResetSelectedSynthesisInfo(): () => void { 23 | return useResetRecoilState(state) 24 | } 25 | -------------------------------------------------------------------------------- /packages/webapp/src/states/dataShowcaseContext/steps.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { Step } from '@data-wrangling-components/core' 6 | import type { SetterOrUpdater } from 'recoil' 7 | import { atom, useRecoilState } from 'recoil' 8 | 9 | const stepsState = atom({ 10 | key: 'steps', 11 | default: [], 12 | }) 13 | 14 | export function useSteps(): [Step[], SetterOrUpdater] { 15 | return useRecoilState(stepsState) 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/states/globalErrorMessage.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { SetterOrUpdater } from 'recoil' 6 | import { atom, useRecoilState } from 'recoil' 7 | 8 | const globalErrorMessage = atom({ 9 | key: 'global-error-message', 10 | default: undefined, 11 | }) 12 | 13 | export function useGlobalErrorMessage(): [ 14 | string | undefined, 15 | SetterOrUpdater, 16 | ] { 17 | return useRecoilState(globalErrorMessage) 18 | } 19 | -------------------------------------------------------------------------------- /packages/webapp/src/states/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './dataShowcaseContext/index.js' 6 | export * from './globalErrorMessage.js' 7 | export * from './rawSynthesisParameters.js' 8 | -------------------------------------------------------------------------------- /packages/webapp/src/states/rawSynthesisParameters.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useMemo } from 'react' 6 | import type { SetterOrUpdater } from 'recoil' 7 | import { atom, useRecoilState } from 'recoil' 8 | 9 | import type { IRawSynthesisParameters } from '~models/synthesis/RawSynthesisParameters' 10 | import { defaultRawSynthesisParameters } from '~models/synthesis/RawSynthesisParameters' 11 | 12 | const state = atom({ 13 | key: 'raw-synthesis-parameters', 14 | default: defaultRawSynthesisParameters, 15 | }) 16 | 17 | export function useRawSynthesisParameters(): [ 18 | IRawSynthesisParameters, 19 | SetterOrUpdater, 20 | ] { 21 | return useRecoilState(state) 22 | } 23 | 24 | export function useRawSynthesisParametersPropertySetter( 25 | prop: keyof IRawSynthesisParameters, 26 | ): (val: T) => void { 27 | const [, setRawSynthesisParams] = useRawSynthesisParameters() 28 | return useMemo( 29 | () => (val: T) => { 30 | setRawSynthesisParams(prev => ({ 31 | ...prev, 32 | [prop]: val, 33 | })) 34 | }, 35 | [prop, setRawSynthesisParams], 36 | ) 37 | } 38 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './tooltips.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/ACCURACY_MODE.md: -------------------------------------------------------------------------------- 1 | Choose how to split the privacy budget between the combination lengths 2 | 3 | **`Prioritize Long Combinations`**: 4 | Use this if you want to prioritize accurate counts for longer attribute combinations (for example, 3-counts being more accurate than 2-counts). 5 | 6 | **`Prioritize Short Combinations`**: 7 | Use this if you want to prioritize accurate for shorter attribute combinations (for example, 1-counts being more accurate than 2-counts). 8 | 9 | **`Balanced`**: 10 | Evenly split the privacy budget (epsilon) between 1-counts, 2-counts, and so on up to the aggregation limit. 11 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/ANALYSIS_LENGTH.md: -------------------------------------------------------------------------------- 1 | The maximum combination length used to compute aggregate counts and evaluate the quality of the synthetic data. 2 | 3 | Cannot be changed if the synthesis mode requires aggregate counts to be computed before data synthesis. 4 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/CACHE_SIZE.md: -------------------------------------------------------------------------------- 1 | The data synthesis process computes how many times attribute combinations occur in the sensitive data and uses this information to generate the synthetic data. The **`cache size`** parameter indicates the maximum number of attribute combinations that will have their count cached to speed up the processing time. If a certain combination is not part of the cache, its count will be recomputed. 2 | 3 | Naturally, there is a balance between performance and memory utilization – the bigger the cache, the faster the data synthesis at the cost of greater memory use. 4 | 5 | The default value is `100000`, which is generally a good balance between performance and memory usage, but this **can and should** be tuned depending on the input dataset. 6 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/COUNT_MEAN_AND_ERROR.md: -------------------------------------------------------------------------------- 1 | ### Mean value 2 | 3 | Mean count of all the attribute combinations in the **sensitive data** (up to the `aggregation limit`). 4 | 5 | 6 | ### Mean error 7 | 8 | The **absolute difference between the aggregate or synthetic count and the sensitive count** (original value). -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/DELTA_FACTOR.md: -------------------------------------------------------------------------------- 1 | Factor used to calculate the delta DP parameter. 2 | 3 | `Delta = 1 / ([delta factor] * [record limit])` 4 | 5 | If set to `0`, then will default at runtime to `ln(record limit)`, resulting in: 6 | 7 | `Delta = 1 / (ln(record limit) * [record limit])` 8 | 9 | When set to '0', the record limit will be also protected with differential privacy, consuming a portion of the privacy budget. Look at the `Number of records epsilon proportion` parameter. 10 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/EVALUATE.md: -------------------------------------------------------------------------------- 1 | This will run an analysis both on the sensitive data and the synthesized data, enabling a comparison between the two. 2 | 3 | The **`analysis length`** determines the maximum length of attribute combination for which aggregate counts are precomputed and reported. In the **`navigate`** step, this value determines how many attribute value selections a user may make while retaining the ability to compare estimated (synthetic) vs actual (sensitive) values. 4 | 5 | **Caution**: analysis results may contain metrics and information directly related to the sensitive data, and if shared, may violate the intended privacy guarantee. 6 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/FABRICATED_COMBINATIONS.md: -------------------------------------------------------------------------------- 1 | Percentage of attribute combinations that **do not exist in the sensitive microdata**, but **were created in the aggregate or synthetic data** (when analyzed up to the defined **`analysis length`**). 2 | 3 | By design, this should always be zero (under k-anonymity) or as small as possible (under differential privacy). However, for the **`Aggregate Seeded`** and **`DP Aggregate Seeded`** methods, this guarantee only holds for combination lengths up to and including the aggregation limit – longer attribute combinations are not available and so cannot be compared. -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/MEAN_PROPORTIONAL_ERROR.md: -------------------------------------------------------------------------------- 1 | Each aggregated combination can be organized into buckets based on their counts. For example, the bucket 10 contains the combinations that have counts from `(0, 10]`, the bucket 20 from `(10, 20]`, and so on. The bucket number therefore communicates the upper bound of the range. 2 | 3 | For every bucket, the mean proportional error can be computed as the mean of: 4 | 5 | ``` 6 | if sensitive_count > 0 { 7 | |synthetic_count - sensitive_count| / 8 | sensitive_count 9 | } else { 10 | 1.0 11 | } 12 | ``` 13 | 14 | The average of the mean proportional error for all buckets defines the overall mean proportional error. 15 | 16 | The closer this is to 0, the closer the aggregate/synthetic counts are from the sensitive counts for each bucket (see the chart below). 17 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/NAVIGATE.md: -------------------------------------------------------------------------------- 1 | This page displays the comparison between the actual aggregate counts (rounded down to the nearest multiple of **`privacy resolution`** under k-anonymity, or otherwise protected with DP guarantees) and the estimated counts obtained from the generated synthetic data. 2 | 3 | Queries for counts of records matching specific combinations can be created by selecting the corresponding bars and filtering the data accordingly. Customize the visible attributes by using **`Select Columns`**. 4 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/NOISE_EPSILON.md: -------------------------------------------------------------------------------- 1 | Epsilon (privacy budget) used to calculate the scale of the noise that needs to be added to aggregate counts. 2 | 3 | Higher epsilon values will lead to less noise, but will also make the final output less private. 4 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/NUMBER_OF_RECORDS_EPSILON_PROPORTION.md: -------------------------------------------------------------------------------- 1 | When using `DP Aggregate Seeded`, the protected number of records in the aggregate data is reported with differential privacy. This defines the proportion of the overall epsilon dedicated to this stage (0.005 means 0.5%). 2 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/N_COUNT_MEAN_AND_ERROR.md: -------------------------------------------------------------------------------- 1 | ### Mean value 2 | 3 | Mean count of the `k`-length attribute combinations in the **sensitive data**. 4 | 5 | For example: 6 | 7 | ``` 8 | A:a1 - 7 9 | A:a2 - 3 10 | B:b2 - 5 11 | ``` 12 | 13 | Mean of the 1-counts attribute combinations is `(10 + 5 + 5) / 3 = 5` 14 | 15 | ### Mean error 16 | 17 | The **absolute difference between the aggregate or synthetic count and the sensitive count** (original value). -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/OVERSAMPLING.md: -------------------------------------------------------------------------------- 1 | **`Value Seeded`** synthesis proceeds by sampling attributes from single attribute counts. When the sampling is performed, there might be some attribute combinations that are oversampled (synthetic counts greater than reported aggregate counts). 2 | 3 | This allows the oversampling to be `Unlimited` or `Controlled` with some parameters. 4 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/OVERSAMPLING_RATIO.md: -------------------------------------------------------------------------------- 1 | The allowed oversampling proportion for each individual aggregate count (e.g., `0.1` means that synthetic aggregate counts are allowed to exceed the reported aggregates by up to 10%). -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/OVERSAMPLING_TRIES.md: -------------------------------------------------------------------------------- 1 | When a particular attribute is sampled, adding it to the record being synthesized might lead to oversampling. This parameter sets how many times we should try resampling a new attribute to avoid oversampling. -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/PERCENTILE_EPSILON_PROPORTION.md: -------------------------------------------------------------------------------- 1 | Percentile selection is performed with differential privacy. This defines the proportion of the overall epsilon dedicated to this stage (0.01 means 1%). 2 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/PERCENTILE_PERCENTAGE.md: -------------------------------------------------------------------------------- 1 | The record sensitivity is defined as the number of different combinations (of lengths 1 up to and including the **`aggregation limit`**) that can be generated from each record. 2 | 3 | To reduce the maximimum noise added to reportable aggregate counts, we can artificially limit the number of combination counts that any one record is allowed to affect. This is achieved by randomly selecting which combinations to update, up to a given limit. 4 | 5 | This parameter sets the percentile used to randomly filter attribute combinations from records. The lower the value, the more combinations might have their counts suppressed on the reportable aggregates. -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/RARE_COMBS.md: -------------------------------------------------------------------------------- 1 | The percentage of rare combinations (`count < privacy resolution`) across all combinations generated up to the specified **`analysis length`**. 2 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/RECORDS_WITH_RARE_COMBS.md: -------------------------------------------------------------------------------- 1 | Percentage of records containing attribute combinations that appear fewer times than the specified **`privacy resolution`**. This includes combinations generated up to the **`analysis length`**. 2 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/RECORDS_WITH_UNIQUE_COMBS.md: -------------------------------------------------------------------------------- 1 | Percentage of records containing attribute combinations that appear only once across the entire sensitive data set. 2 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/RECORD_EXPANSION.md: -------------------------------------------------------------------------------- 1 | ``` 2 | [ 3 | ( Number of records in the synthetic dataset / 4 | Number of records in the sensitive dataset ) 5 | - 1 6 | ] * 100 7 | ``` 8 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/RECORD_LIMIT.md: -------------------------------------------------------------------------------- 1 | **`Record limit`** may be used to limit data synthesis to the specified number of records, taken from the start of the sensitive data. By default, all records will be processed. 2 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/REPORTING_LENGTH.md: -------------------------------------------------------------------------------- 1 | To complement the synthetic data, the tool also pre-computes reportable counts of sensitive records containing all short combinations of attributes. The **`aggregation limit`** parameter specifies the maximum combination length that should be used to compute aggregate counts (e.g., an aggregation limit of 3 will lead to 1, 2 and 3 length attribute combinations being computed). 2 | 3 | Some synthesis modes, such as **`Value Seeded`**, **`Aggregate Seeded`** and **`DP Aggregate Seeded`**, require such pre-computed aggregates counts as the foundations of data synthesis. 4 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/RESOLUTION.md: -------------------------------------------------------------------------------- 1 | ### K-Anonymity 2 | 3 | With k-anonymity, the main privacy control offered by the tool is based on the numbers of individuals described by different combinations of attributes. The **`privacy resolution`** determines the minimum group size that will be (a) reported explicitly in the aggregate data and (b) represented implicitly by the records of the synthetic data. This makes it possible to offer privacy guarantees in clearly understandable terms, e.g.: 4 | 5 | "All attribute combinations in this synthetic dataset describe groups of 10 or more individuals in the original sensitive dataset, therefore may never be used to infer the presence of individuals or groups smaller than 10." 6 | 7 | Under such guarantees, it is impossible for attackers to infer the presence of groups whose size is below the **`privacy resolution`**. For groups at or above this resolution, the 'safety in numbers' principle applies – the higher the limit, the harder it becomes to make inferences about the presence of known individuals. 8 | 9 | ### DP 10 | 11 | With differential privacy, this parameter is only used for evaluation purposes. The aggregate counts are protected in a probabilistic way rather than through strict thresholding. 12 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/SUPPRESSED_COMBINATIONS.md: -------------------------------------------------------------------------------- 1 | Percentage of attribute combinations that exist in the sensitive data, but were suppressed in the aggregate or synthetic data (when analyzed up to the defined **`analysis length`**). -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/SYNTHESIZE.md: -------------------------------------------------------------------------------- 1 | Runs the data synthesis based on the configured parameters. The resulting synthetic records will be presented as a table sorted by the number of non-empty attribute values. 2 | 3 | Processing time ranges from a few seconds up to several minutes, depending on the size of the dataset, number of columns, and configured parameters. 4 | 5 | There is also a **`download button`** on the table, so the synthetic data can be downloaded as a csv file. 6 | 7 | This will also run an analysis both on the sensitive data and the synthesized data, enabling a comparison between the two. 8 | 9 | The **`aggregation limit`** determines the maximum length of attribute combination for which aggregate counts are precomputed and reported. In the **`navigate`** step, this value determines how many attribute value selections a user may make while retaining the ability to compare estimated (synthetic) vs actual (sensitive) values. 10 | 11 | **Caution**: analysis results may contain metrics and information directly related to the sensitive data, and if shared, may violate the intended privacy guarantee. 12 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/THRESHOLD_VALUE.md: -------------------------------------------------------------------------------- 1 | Threshold value between 0 and 1.0 to be specified per combination length: 2 | 3 | - **0**: Minimizes fabrication, resulting in the minimum possible fabrication. 4 | 5 | - **1.0**: Does not control fabrication. 6 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/UNIQUE_COMBS.md: -------------------------------------------------------------------------------- 1 | The percentage of unique attribute combinations across all combinations generated up to and including the specified **`analysis length`**. 2 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/USE_SYNTHETIC_COUNTS.md: -------------------------------------------------------------------------------- 1 | Indicates whether attribute combination counts of already-synthesized records should influence the subsequent behavior of the sampling process: 2 | 3 | - If **No**: the sampling process will take into account only the reported aggregate counts to balance the sampling process 4 | 5 | - If **Yes**: the sampling process will take into account both the reported aggregate counts and the already-synthesized counts to balance the sampling process 6 | -------------------------------------------------------------------------------- /packages/webapp/src/ui-tooltips/mds/WEIGHT_SELECTION_PERCENTILE.md: -------------------------------------------------------------------------------- 1 | During the `Aggregated Seeded` and `DP Aggregated Seeded` synthesis, aggregate counts are used as weights to balance the attribute selection. This parameter specifies the percentile used to select the weight for every attribute candidate. 2 | -------------------------------------------------------------------------------- /packages/webapp/src/utils/env.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export function getBaseUrl(): string { 6 | let base = (import.meta.env.VITE_BASE_URL as string) || '' 7 | 8 | if (!base.endsWith('/')) { 9 | base += '/' 10 | } 11 | return base 12 | } 13 | 14 | export function getSdsWasmLogLevel(): string { 15 | return (import.meta.env.VITE_SDS_WASM_LOG_LEVEL as string) || 'warn' 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/utils/hash.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export async function hash(contents: string): Promise { 6 | const uint8Msg = new TextEncoder().encode(contents) 7 | const hashBuffer = await crypto.subtle.digest('SHA-1', uint8Msg) 8 | const hashArray = Array.from(new Uint8Array(hashBuffer)) 9 | const hash = hashArray.map(b => b.toString(16).padStart(2, '0')).join('') 10 | return hash 11 | } 12 | -------------------------------------------------------------------------------- /packages/webapp/src/utils/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './arquero.js' 6 | export * from './env.js' 7 | export * from './thematic.js' 8 | -------------------------------------------------------------------------------- /packages/webapp/src/utils/thematic.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useThematic } from '@thematic/react' 6 | import { useMemo } from 'react' 7 | 8 | export function useNominalScale(): string[] { 9 | const themetic = useThematic() 10 | 11 | return useMemo(() => { 12 | // adding blue from previous versions to keep consistence 13 | return ['rgb(128 172 247)', ...themetic.scales().nominal().toArray()] 14 | }, [themetic]) 15 | } 16 | 17 | export function useNominalBoldScale(): string[] { 18 | const themetic = useThematic() 19 | 20 | return useMemo(() => { 21 | // adding blue from previous versions to keep consistence 22 | return ['rgb(0 95 174)', ...themetic.scales().nominalBold().toArray()] 23 | }, [themetic]) 24 | } 25 | 26 | export function useNominalMutedScale(): string[] { 27 | const themetic = useThematic() 28 | 29 | return useMemo(() => { 30 | // adding blue from previous versions to keep consistence 31 | return ['rgb(207 212 228)', ...themetic.scales().nominalMuted().toArray()] 32 | }, [themetic]) 33 | } 34 | -------------------------------------------------------------------------------- /packages/webapp/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | /// 6 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/AggregateStatisticsGenerator.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { IAggregateStatistics, ICsvDataParameters } from '@essex/sds-core' 6 | import { expose } from 'comlink' 7 | 8 | /* eslint-disable */ 9 | import { BaseSdsWasmWorker } from './BaseSdsWasmWorker' 10 | import type { Proxy, WorkerProgressCallback } from './types' 11 | /* eslint-enable */ 12 | 13 | export class AggregateStatisticsGenerator extends BaseSdsWasmWorker { 14 | public async generateAggregateStatistics( 15 | csvData: string, 16 | csvDataParameters: ICsvDataParameters, 17 | reportingLength: number, 18 | resolution: number, 19 | progressCallback?: Proxy, 20 | ): Promise { 21 | const context = this.getContext() 22 | 23 | if (csvDataParameters.useColumns.length !== 0) { 24 | context.setSensitiveData(csvData, csvDataParameters) 25 | return context.sensitiveAggregateStatistics( 26 | reportingLength, 27 | resolution, 28 | p => { 29 | progressCallback?.(p) 30 | return true 31 | }, 32 | ) 33 | } else { 34 | context.clear() 35 | return null 36 | } 37 | } 38 | } 39 | 40 | expose(AggregateStatisticsGenerator) 41 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/BaseSdsWasmWorker.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import init, { init_logger, WasmSdsContext } from '@essex/sds-core' 6 | import wasmPath from '@essex/sds-core/sds_wasm_bg.wasm?url' 7 | 8 | import { getSdsWasmLogLevel } from '../utils/env.js' 9 | 10 | export class BaseSdsWasmWorker { 11 | private _context: WasmSdsContext | null 12 | protected _name: string 13 | 14 | constructor(name: string) { 15 | this._context = null 16 | this._name = name 17 | } 18 | 19 | public async init(): Promise { 20 | await init(wasmPath) 21 | init_logger(getSdsWasmLogLevel()) 22 | this._context = new WasmSdsContext() 23 | } 24 | 25 | public async terminate(): Promise { 26 | this._context?.free() 27 | } 28 | 29 | protected getContext(): WasmSdsContext { 30 | if (this._context === null) { 31 | throw new Error( 32 | `"${this._name}" worker has not been properly initialized, did you call init?`, 33 | ) 34 | } 35 | return this._context 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/AggregateType.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum AggregateType { 6 | Sensitive = 'sensitive', 7 | Aggregated = 'aggregate', 8 | Synthetic = 'synthetic', 9 | } 10 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/CancelablePromise.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export interface ICancelablePromise { 6 | id: string 7 | promise: Promise 8 | cancel: () => void 9 | } 10 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/Proxy.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ProxyMarked } from 'comlink' 6 | 7 | export type Proxy = T & ProxyMarked 8 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/SdsManagerSynthesisCallbacks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ISynthesisInfo } from './SynthesisInfo.js' 6 | 7 | export type SynthesisCallback = (synthesisInfo: ISynthesisInfo) => Promise 8 | 9 | export interface ISdsManagerSynthesisCallbacks { 10 | started?: SynthesisCallback 11 | finished?: SynthesisCallback 12 | progressUpdated?: SynthesisCallback 13 | terminating?: SynthesisCallback 14 | terminated?: SynthesisCallback 15 | } 16 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/SynthesisInfo.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ISynthesisParameters } from './SynthesisParameters.js' 6 | import type { IWasmSynthesizerWorkerStatus } from './WasmSynthesizerWorkerStatus.js' 7 | 8 | export interface ISynthesisInfo { 9 | key: string 10 | parameters: ISynthesisParameters 11 | status: IWasmSynthesizerWorkerStatus 12 | startedAt: Date 13 | finishedAt?: Date 14 | progress: number 15 | errorMessage?: string 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/SynthesisMode.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum SynthesisMode { 6 | Unseeded = 'Unseeded', 7 | RowSeeded = 'Row Seeded', 8 | ValueSeeded = 'Value Seeded', 9 | AggregateSeeded = 'Aggregate Seeded', 10 | DP = 'Differential Privacy', 11 | } 12 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/SynthesisParameters.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { 6 | IBaseSynthesisParameters, 7 | ICsvDataParameters, 8 | IDpParameters, 9 | INoisyCountThreshold, 10 | IOversamplingParameters, 11 | } from '@essex/sds-core' 12 | 13 | import type { SynthesisMode } from './SynthesisMode.js' 14 | 15 | export interface ISynthesisParameters { 16 | mode: SynthesisMode 17 | csvDataParameters: ICsvDataParameters 18 | baseSynthesisParameters: IBaseSynthesisParameters 19 | reportingLength: number 20 | } 21 | 22 | export interface IUnseededSynthesisParameters extends ISynthesisParameters { 23 | mode: SynthesisMode.Unseeded 24 | } 25 | 26 | export interface IRowSeededSynthesisParameters extends ISynthesisParameters { 27 | mode: SynthesisMode.RowSeeded 28 | } 29 | 30 | export interface IValueSeededSynthesisParameters extends ISynthesisParameters { 31 | mode: SynthesisMode.ValueSeeded 32 | oversampling?: IOversamplingParameters 33 | } 34 | 35 | export interface IAggregateSeededSynthesisParameters 36 | extends ISynthesisParameters { 37 | mode: SynthesisMode.AggregateSeeded 38 | useSyntheticCounts: boolean 39 | weightSelectionPercentile?: number 40 | } 41 | 42 | export interface IDpSynthesisParameters extends ISynthesisParameters { 43 | mode: SynthesisMode.DP 44 | dpParameters: IDpParameters 45 | noiseThreshold: INoisyCountThreshold 46 | useSyntheticCounts: boolean 47 | weightSelectionPercentile?: number 48 | } 49 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/WasmSynthesizerWorkerInfo.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { Remote } from 'comlink' 6 | 7 | import type { IWorkerProxy } from '~workers/utils' 8 | import type { WasmSynthesizer } from '~workers/WasmSynthesizer' 9 | 10 | import type { ISynthesisInfo } from './SynthesisInfo.js' 11 | 12 | export interface IWasmSynthesizerWorkerInfo { 13 | synthesisInfo: ISynthesisInfo 14 | synthesizerWorkerProxy: IWorkerProxy 15 | synthesizer: Remote 16 | } 17 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/WasmSynthesizerWorkerStatus.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export enum IWasmSynthesizerWorkerStatus { 6 | RUNNING = 'RUNNING', 7 | FINISHED = 'FINISHED', 8 | ERROR = 'ERROR', 9 | TERMINATING = 'TERMINATING', 10 | TERMINATED = 'TERMINATED', 11 | } 12 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/WorkerProgressCallback.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export type WorkerProgressCallback = (p: number) => void 6 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/types/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './AggregateType.js' 6 | export * from './CancelablePromise.js' 7 | export * from './Proxy.js' 8 | export * from './SdsManagerSynthesisCallbacks.js' 9 | export * from './SynthesisInfo.js' 10 | export * from './SynthesisMode.js' 11 | export * from './SynthesisParameters.js' 12 | export * from './WasmSynthesizerWorkerInfo.js' 13 | export * from './WasmSynthesizerWorkerStatus.js' 14 | export * from './WorkerProgressCallback.js' 15 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/utils/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './workerProxy.js' 6 | -------------------------------------------------------------------------------- /packages/webapp/src/workers/utils/workerProxy.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { Remote } from 'comlink' 6 | import { releaseProxy, wrap } from 'comlink' 7 | 8 | export interface IWorkerProxy { 9 | ProxyConstructor: Remote 10 | terminate: () => void 11 | } 12 | 13 | export function createWorkerProxy(worker: Worker): IWorkerProxy { 14 | const ProxyConstructor = wrap(worker) 15 | const terminate = () => { 16 | ProxyConstructor[releaseProxy]() 17 | worker.terminate() 18 | } 19 | return { ProxyConstructor, terminate } 20 | } 21 | -------------------------------------------------------------------------------- /packages/webapp/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "moduleResolution": "Node", 4 | "module": "ESNext", 5 | "baseUrl": ".", 6 | "paths": { 7 | "~utils": ["./src/utils/index.ts"], 8 | "~utils/*": ["./src/utils/*"], 9 | "~components": ["./src/components/index.ts"], 10 | "~components/*": ["./src/components/*"], 11 | "~models": ["src/models/index.ts"], 12 | "~models/*": ["src/models/*"], 13 | "~states": ["src/states/index.ts"], 14 | "~states/*": ["src/states/*"], 15 | "~workers/*": ["./src/workers/*"], 16 | "~ui-tooltips": ["src/ui-tooltips/index.ts"], 17 | "~ui-tooltips/*": ["src/ui-tooltips/*"], 18 | "~pages": ["src/pages/index.ts"], 19 | "~pages/*": ["src/pages/*"] 20 | }, 21 | "allowJs": true, 22 | "lib": ["ESNext", "DOM"], 23 | "outDir": "lib", 24 | "jsx": "react-jsx", 25 | "resolveJsonModule": true, 26 | "forceConsistentCasingInFileNames": true, 27 | "strict": true, 28 | "esModuleInterop": true, 29 | "allowSyntheticDefaultImports": true, 30 | "skipLibCheck": true, 31 | "noImplicitAny": false, 32 | "typeRoots": ["./node_modules/@types/", "./types"], 33 | "noErrorTruncation": true, 34 | "downlevelIteration": true 35 | }, 36 | "include": ["src/**/*"], 37 | "exclude": ["**/__tests__/**"] 38 | } 39 | -------------------------------------------------------------------------------- /packages/webapp/types/styled-components/index.d.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import 'styled-components' 6 | 7 | import type { FluentTheme } from '@thematic/fluent' 8 | 9 | // and extend them! 10 | declare module 'styled-components' { 11 | // eslint-disable-next-line @typescript-eslint/no-empty-interface 12 | export interface DefaultTheme extends FluentTheme {} 13 | } 14 | -------------------------------------------------------------------------------- /packages/webapp/vite.config.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import react from '@vitejs/plugin-react' 6 | import { defineConfig } from 'vite' 7 | import tsconfigPaths from 'vite-tsconfig-paths' 8 | 9 | // https://vitejs.dev/config/ 10 | 11 | // export default defineConfig(essexViteConfig) 12 | export default defineConfig(({ command, mode }) => { 13 | return { 14 | base: process.env.VITE_BASE_URL || '/', 15 | build: { 16 | target: 'es2020', 17 | sourcemap: true, 18 | rollupOptions: { 19 | input: { 20 | main: './index.html', 21 | }, 22 | }, 23 | }, 24 | optimizeDeps: { 25 | esbuildOptions: { 26 | target: 'es2020', 27 | }, 28 | }, 29 | server: { 30 | port: 3000, 31 | }, 32 | plugins: [tsconfigPaths(), react()], 33 | } 34 | }) 35 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "ESNext", 4 | "target": "ESNext", 5 | "lib": ["ESNext"], 6 | "declaration": true, 7 | "sourceMap": true, 8 | "strict": true, 9 | "moduleResolution": "node", 10 | "esModuleInterop": true, 11 | "skipLibCheck": true, 12 | "useDefineForClassFields": true, 13 | "forceConsistentCasingInFileNames": true, 14 | "noErrorTruncation": true 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /webapp.dockerfile: -------------------------------------------------------------------------------- 1 | # Registry to pull images from 2 | ARG REGISTRY 3 | 4 | # --- compile wasm bindings from rust --- 5 | FROM ${REGISTRY}rust:1.64 as wasm-builder 6 | 7 | # install wasm-pack to build wasm bindings 8 | RUN cargo install wasm-pack 9 | 10 | WORKDIR /usr/src/sds 11 | 12 | COPY . . 13 | 14 | # build the wasm bindings for webapp to use 15 | RUN cd packages/lib-wasm && wasm-pack build --release --target web --out-dir ../../target/wasm 16 | 17 | # --- compile application from typescript --- 18 | FROM ${REGISTRY}node:16 as app-builder 19 | 20 | WORKDIR /usr/src/sds 21 | 22 | # copy from rust build 23 | COPY --from=wasm-builder /usr/src/sds ./ 24 | 25 | # setting sds env vars 26 | ENV VITE_SDS_WASM_LOG_LEVEL=warn 27 | 28 | # install dependencies and build 29 | RUN npm install replace -g 30 | RUN replace "sds-wasm" "@essex/sds-core" ./target/wasm/package.json 31 | RUN yarn install 32 | RUN yarn build 33 | 34 | # --- statically serve built application with nginx --- 35 | FROM ${REGISTRY}nginx:1.21 36 | 37 | COPY --from=app-builder /usr/src/sds/packages/webapp/dist /usr/share/nginx/html --------------------------------------------------------------------------------