2 |
8 | OneAIFW - Local Presidio Architecture
9 |
10 |
11 | Browser Extension (MV3)
12 | - Select text, Ctrl+Shift+A
13 | - Calls local /api/anonymize
14 |
15 |
16 | Desktop UI (Tkinter)
17 | - Calls local service
18 | - Displays placeholdersMap
19 |
20 |
21 | Presidio Service (FastAPI)
22 | - presidio-analyzer: AnalyzerEngine (spaCy + PatternRecognizer)
23 | - presidio-anonymizer: AnonymizerEngine
24 | - Endpoints: /api/analyze, /api/anonymize, /api/restore
25 | - Generates translation-safe placeholders: __PII_*__
26 |
27 |
28 | Models and Recognizers
29 | - spaCy models (optional)
30 | - PatternRecognizers (regex)
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 | Note: Placeholders are safe for MT/LLM round trips. Consider storing placeholdersMap in a session store (Redis) for large workflows.
42 |
43 |
--------------------------------------------------------------------------------
/py-origin/services/app/test_restore.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import os, sys
3 | try:
4 | # When executed as a package (recommended)
5 | from .anonymizer import AnonymizerWrapper
6 | except Exception:
7 | # Fallback: allow running from this directory via `python -m unittest test_restore.py`
8 | PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
9 | if PROJECT_ROOT not in sys.path:
10 | sys.path.insert(0, PROJECT_ROOT)
11 | from services.presidio_service.app.anonymizer import AnonymizerWrapper
12 |
13 |
14 | class DummyAnalyzer:
15 | def analyze(self, text: str, language: str = 'en'):
16 | # Return no entities; we'll test restore directly with crafted placeholders
17 | return []
18 |
19 |
20 | class TestRestore(unittest.TestCase):
21 | def setUp(self):
22 | self.wrapper = AnonymizerWrapper(DummyAnalyzer())
23 |
24 | def test_exact_placeholder_restore(self):
25 | placeholders = {"__PII_EMAIL_ADDRESS_761b3e66__": "test@example.com"}
26 | text = "我的邮箱是 __PII_EMAIL_ADDRESS_761b3e66__"
27 | out = self.wrapper.restore(text, placeholders)
28 | self.assertEqual(out, "我的邮箱是 test@example.com")
29 |
30 | def test_missing_underscores_variant(self):
31 | placeholders = {"__PII_EMAIL_ADDRESS_761b3e66__": "test@example.com"}
32 | text = "我的邮箱是 PII_EMAIL_ADDRESS_761b3e66"
33 | out = self.wrapper.restore(text, placeholders)
34 | self.assertEqual(out, "我的邮箱是 test@example.com")
35 |
36 | def test_leaked_suffix_after_original(self):
37 | placeholders = {"__PII_EMAIL_ADDRESS_0b9df4b0__": "test@example.com"}
38 | text = "我的邮箱是 test@example.com0b9df4b0__"
39 | out = self.wrapper.restore(text, placeholders)
40 | self.assertEqual(out, "我的邮箱是 test@example.com")
41 |
42 | def test_overlapping_entities_prefer_longer(self):
43 | # Ensure independent of restore, the function does not break when overlapping-like patterns appear
44 | placeholders = {
45 | "__PII_URL_a37ec55b__": "example.com",
46 | "__PII_EMAIL_ADDRESS_6fbb5771__": "test@example.com",
47 | }
48 | text = "站点 example.com 和邮箱 __PII_EMAIL_ADDRESS_6fbb5771__"
49 | out = self.wrapper.restore(text, placeholders)
50 | self.assertEqual(out, "站点 example.com 和邮箱 test@example.com")
51 |
52 |
53 | if __name__ == "__main__":
54 | unittest.main()
55 |
56 |
57 |
--------------------------------------------------------------------------------
/cli/python/services/app/test_restore.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import os, sys
3 | try:
4 | # When executed as a package (recommended)
5 | from .anonymizer import AnonymizerWrapper
6 | except Exception:
7 | # Fallback: allow running from this directory via `python -m unittest test_restore.py`
8 | PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
9 | if PROJECT_ROOT not in sys.path:
10 | sys.path.insert(0, PROJECT_ROOT)
11 | from services.presidio_service.app.anonymizer import AnonymizerWrapper
12 |
13 |
14 | class DummyAnalyzer:
15 | def analyze(self, text: str, language: str = 'en'):
16 | # Return no entities; we'll test restore directly with crafted placeholders
17 | return []
18 |
19 |
20 | class TestRestore(unittest.TestCase):
21 | def setUp(self):
22 | self.wrapper = AnonymizerWrapper(DummyAnalyzer())
23 |
24 | def test_exact_placeholder_restore(self):
25 | placeholders = {"__PII_EMAIL_ADDRESS_761b3e66__": "test@example.com"}
26 | text = "我的邮箱是 __PII_EMAIL_ADDRESS_761b3e66__"
27 | out = self.wrapper.restore(text, placeholders)
28 | self.assertEqual(out, "我的邮箱是 test@example.com")
29 |
30 | def test_missing_underscores_variant(self):
31 | placeholders = {"__PII_EMAIL_ADDRESS_761b3e66__": "test@example.com"}
32 | text = "我的邮箱是 PII_EMAIL_ADDRESS_761b3e66"
33 | out = self.wrapper.restore(text, placeholders)
34 | self.assertEqual(out, "我的邮箱是 test@example.com")
35 |
36 | def test_leaked_suffix_after_original(self):
37 | placeholders = {"__PII_EMAIL_ADDRESS_0b9df4b0__": "test@example.com"}
38 | text = "我的邮箱是 test@example.com0b9df4b0__"
39 | out = self.wrapper.restore(text, placeholders)
40 | self.assertEqual(out, "我的邮箱是 test@example.com")
41 |
42 | def test_overlapping_entities_prefer_longer(self):
43 | # Ensure independent of restore, the function does not break when overlapping-like patterns appear
44 | placeholders = {
45 | "__PII_URL_a37ec55b__": "example.com",
46 | "__PII_EMAIL_ADDRESS_6fbb5771__": "test@example.com",
47 | }
48 | text = "站点 example.com 和邮箱 __PII_EMAIL_ADDRESS_6fbb5771__"
49 | out = self.wrapper.restore(text, placeholders)
50 | self.assertEqual(out, "站点 example.com 和邮箱 test@example.com")
51 |
52 |
53 | if __name__ == "__main__":
54 | unittest.main()
55 |
56 |
57 |
--------------------------------------------------------------------------------
/core/SpanMerger.zig:
--------------------------------------------------------------------------------
1 | const std = @import("std");
2 | const entity = @import("recog_entity.zig");
3 |
4 | pub const RecogEntity = entity.RecogEntity;
5 | pub const EntityType = entity.EntityType;
6 |
7 | pub const Config = struct {
8 | whitelist: []const EntityType, // accept only these if non-empty
9 | blacklist: []const EntityType, // drop these if present
10 | threshold: f32, // min score
11 | };
12 |
13 | fn inSet(set: []const EntityType, t: EntityType) bool {
14 | var i: usize = 0;
15 | while (i < set.len) : (i += 1) {
16 | if (set[i] == t) return true;
17 | }
18 | return false;
19 | }
20 |
21 | pub fn merge(allocator: std.mem.Allocator, a: []const RecogEntity, b: []const RecogEntity, cfg: Config) ![]RecogEntity {
22 | var tmp = try std.ArrayList(RecogEntity).initCapacity(allocator, a.len + b.len);
23 | defer tmp.deinit(allocator);
24 | for (a) |e| try tmp.append(allocator, e);
25 | for (b) |e| try tmp.append(allocator, e);
26 |
27 | var spans = try tmp.toOwnedSlice(allocator);
28 | errdefer allocator.free(spans);
29 |
30 | var filtered = try std.ArrayList(RecogEntity).initCapacity(allocator, spans.len);
31 | defer filtered.deinit(allocator);
32 | for (spans) |e| {
33 | if (e.score < cfg.threshold) continue;
34 | if (cfg.whitelist.len > 0 and !inSet(cfg.whitelist, e.entity_type)) continue;
35 | if (cfg.blacklist.len > 0 and inSet(cfg.blacklist, e.entity_type)) continue;
36 | try filtered.append(allocator, e);
37 | }
38 | allocator.free(spans);
39 | spans = try filtered.toOwnedSlice(allocator);
40 |
41 | std.sort.block(RecogEntity, spans, {}, struct {
42 | fn lessThan(_: void, a: RecogEntity, b: RecogEntity) bool {
43 | return if (a.start == b.start) a.end < b.end else a.start < b.start;
44 | }
45 | }.lessThan);
46 |
47 | var out = try std.ArrayList(RecogEntity).initCapacity(allocator, spans.len);
48 | defer out.deinit(allocator);
49 | var i: usize = 0;
50 | while (i < spans.len) : (i += 1) {
51 | const cur = spans[i];
52 | if (out.items.len == 0) {
53 | try out.append(allocator, cur);
54 | } else {
55 | const last = out.items[out.items.len - 1];
56 | if (cur.start == last.start and cur.end == last.end) {
57 | if (cur.score > last.score) out.items[out.items.len - 1] = cur;
58 | } else {
59 | try out.append(allocator, cur);
60 | }
61 | }
62 | }
63 | allocator.free(spans);
64 | return try out.toOwnedSlice(allocator);
65 | }
66 |
--------------------------------------------------------------------------------
/apps/webapp/scripts/serve-coi.mjs:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | import http from 'node:http'
3 | import fs from 'node:fs'
4 | import path from 'node:path'
5 | import url from 'node:url'
6 |
7 | const __filename = url.fileURLToPath(import.meta.url)
8 | const __dirname = path.dirname(__filename)
9 |
10 | const root = path.resolve(__dirname, '..', 'public')
11 | const port = process.env.PORT ? Number(process.env.PORT) : 5500
12 |
13 | const mime = {
14 | '.html': 'text/html; charset=utf-8',
15 | '.js': 'application/javascript; charset=utf-8',
16 | '.mjs': 'application/javascript; charset=utf-8',
17 | '.css': 'text/css; charset=utf-8',
18 | '.json': 'application/json; charset=utf-8',
19 | '.wasm': 'application/wasm',
20 | '.onnx': 'application/octet-stream',
21 | '.txt': 'text/plain; charset=utf-8',
22 | }
23 |
24 | function send(res, status, body, ext) {
25 | res.statusCode = status
26 | res.setHeader('Cross-Origin-Opener-Policy', 'same-origin')
27 | res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp')
28 | if (ext && mime[ext]) res.setHeader('Content-Type', mime[ext])
29 | res.end(body)
30 | }
31 |
32 | function safeJoin(rootDir, reqPath) {
33 | const p = path.normalize(decodeURIComponent(reqPath.split('?')[0]))
34 | const full = path.join(rootDir, p)
35 | if (!full.startsWith(rootDir)) return null
36 | return full
37 | }
38 |
39 | const server = http.createServer((req, res) => {
40 | const urlPath = req.url || '/'
41 | let filePath = safeJoin(root, urlPath)
42 | if (!filePath) return send(res, 403, 'Forbidden')
43 |
44 | fs.stat(filePath, (err, stat) => {
45 | if (err) {
46 | // default file
47 | const fallback = path.join(root, 'offline.html')
48 | return fs.readFile(fallback, (e2, buf) => {
49 | if (e2) return send(res, 404, 'Not found')
50 | send(res, 200, buf, '.html')
51 | })
52 | }
53 | if (stat.isDirectory()) {
54 | const indexFile = path.join(filePath, 'index.html')
55 | fs.readFile(indexFile, (e3, buf) => {
56 | if (e3) {
57 | const fallback = path.join(filePath, 'offline.html')
58 | return fs.readFile(fallback, (e4, buf2) => {
59 | if (e4) return send(res, 404, 'Not found')
60 | send(res, 200, buf2, '.html')
61 | })
62 | }
63 | send(res, 200, buf, '.html')
64 | })
65 | } else {
66 | fs.readFile(filePath, (e5, buf) => {
67 | if (e5) return send(res, 404, 'Not found')
68 | send(res, 200, buf, path.extname(filePath))
69 | })
70 | }
71 | })
72 | })
73 |
74 | server.listen(port, () => {
75 | console.log(`Serving ${root} with COOP/COEP at http://127.0.0.1:${port}/offline.html`)
76 | })
77 |
--------------------------------------------------------------------------------
/apps/webapp/README.md:
--------------------------------------------------------------------------------
1 | # OneAIFW WebApp
2 |
3 | A browser demo based on aifw-js. It supports:
4 | - Online development with Vite
5 | - An offline demo page served with COOP/COEP (enables ORT threads/SIMD)
6 | - Production build
7 |
8 | ## Prerequisites
9 | - Monorepo managed by pnpm. This webapp depends on the local package `@oneaifw/aifw-js` via `workspace:*`.
10 | - Node.js 18+ and pnpm 8+.
11 |
12 | ## Build aifw-js (in workspace)
13 | From the repository root (skip if already built):
14 | ```bash
15 | pnpm -w --filter @oneaifw/aifw-js build
16 | ```
17 |
18 | ## Online development (Vite)
19 | From `apps/webapp`:
20 | ```bash
21 | pnpm run dev
22 | ```
23 | Open the URL printed in the terminal (typically `http://127.0.0.1:5173/`).
24 |
25 | Notes:
26 | - Calling `await init()` uses the managed mode by default, which fetches NER models and ORT wasm from the GitHub-hosted assets and caches them.
27 | - To enable ORT threads/SIMD you need cross-origin isolation (COOP/COEP). Vite dev server doesn’t enable it by default; functionality works but might run with reduced performance. For full performance testing, use the “Offline demo” section below.
28 |
29 | ## Offline demo (with COOP/COEP)
30 | The offline page is `aifw-offline.html`. Copy assets into `public/` and serve with the built-in COOP/COEP server:
31 | ```bash
32 | cd apps/webapp
33 | pnpm run offline # copy @oneaifw/aifw-js dist to public/vendor/aifw-js, and copy aifw-offline.html into public/
34 | pnpm run serve:coi # start the local static server with COOP/COEP (default port 5500)
35 | ```
36 | Then open:
37 | ```
38 | http://127.0.0.1:5500/aifw-offline.html
39 | ```
40 |
41 | Troubleshooting:
42 | - If `http://127.0.0.1:5500/offline.html` returns 404, use `aifw-offline.html`, or run `pnpm run offline` to ensure the file has been copied into `public/`.
43 |
44 | ## Production build
45 | From `apps/webapp`:
46 | ```bash
47 | pnpm run build
48 | ```
49 | Serve the generated `dist/` as static assets. It’s recommended to enable COOP/COEP response headers in production to fully leverage ORT threads/SIMD. You can adapt your own server or follow the idea from the offline demo server.
50 |
51 | ## Managed assets (at runtime)
52 | - `@oneaifw/aifw-js` uses managed mode in `init()` by default: on first run it downloads models and ORT wasm from the hosted repository, verifies integrity (SHA3-256), and warms up browser Cache Storage for faster subsequent loads.
53 | - Resource hosting repository on Hugginface
54 |
55 | ## Scripts
56 | - `pnpm run dev`: start the Vite dev server.
57 | - `pnpm run offline`: prepare offline demo assets into `public/`.
58 | - `pnpm run serve:coi`: start a local static server with COOP/COEP (default port 5500).
59 | - `pnpm run build`: production build into `dist/`.
60 |
--------------------------------------------------------------------------------
/web/README.md:
--------------------------------------------------------------------------------
1 | # AIFW Web Module
2 |
3 | AIFW Web 模块提供了一个基于 Web 的界面来演示 OneAIFW 项目的隐私保护功能。
4 |
5 | ## 功能特性
6 |
7 | - 🌐 **Web 界面**：直观的 Web 界面介绍 AIFW 项目
8 | - 🔍 **敏感信息分析**：检测文本中的敏感信息实体
9 | - 🎭 **匿名化处理**：将敏感信息替换为占位符
10 | - 🔄 **文本恢复**：将匿名化文本恢复为原始内容
11 | - 🌍 **多语言支持**：支持中文和英文文本处理
12 | - 📱 **响应式设计**：适配桌面和移动设备
13 |
14 | ## 快速开始
15 |
16 | ### 1. 安装依赖
17 |
18 | ```bash
19 | pip install -r ../py-origin/services/requirements.txt
20 | pip install -r requirements.txt
21 | ```
22 |
23 | ### 2. 启动服务
24 |
25 | ```bash
26 | python run.py
27 | ```
28 |
29 | 或者直接运行：
30 |
31 | ```bash
32 | python app.py
33 | ```
34 |
35 | ### 3. 访问界面
36 |
37 | 打开浏览器访问：http://localhost:5000
38 |
39 | ## API 接口
40 |
41 | ### 健康检查
42 | ```
43 | GET /api/health
44 | ```
45 |
46 | ### 分析敏感信息
47 | ```
48 | POST /api/analyze
49 | Content-Type: application/json
50 |
51 | {
52 | "text": "要分析的文本",
53 | "language": "zh"
54 | }
55 | ```
56 |
57 | ### 匿名化处理
58 | ```
59 | POST /api/mask
60 | Content-Type: application/json
61 |
62 | {
63 | "text": "要匿名化的文本",
64 | "language": "zh"
65 | }
66 | ```
67 |
68 | ### 恢复文本
69 | ```
70 | POST /api/restore
71 | Content-Type: application/json
72 |
73 | {
74 | "text": "匿名化文本",
75 | "placeholders_map": {
76 | "PII_EMAIL_12345678__": "test@example.com"
77 | }
78 | }
79 | ```
80 |
81 | ### 调用 LLM（需要配置 API 密钥）
82 | ```
83 | POST /api/call
84 | Content-Type: application/json
85 |
86 | {
87 | "text": "要处理的文本",
88 | "api_key_file": "/path/to/api-key.json",
89 | "model": "gpt-4o-mini",
90 | "temperature": 0.0
91 | }
92 | ```
93 |
94 | ## 项目结构
95 |
96 | ```
97 | web/
98 | ├── app.py # Flask 应用主文件
99 | ├── run.py # 启动脚本
100 | ├── requirements.txt # Python 依赖
101 | ├── README.md # 说明文档
102 | ├── templates/ # HTML 模板
103 | │ └── index.html # 主页面
104 | └── static/ # 静态资源
105 | ├── css/
106 | │ └── style.css # 样式文件
107 | └── js/
108 | └── app.js # JavaScript 文件
109 | ```
110 |
111 | ## 依赖说明
112 |
113 | - **Flask**: Web 框架
114 | - **requests**: HTTP 请求库
115 | - **py-origin 模块**: AIFW 核心功能（需要从上级目录导入）
116 |
117 | ## 注意事项
118 |
119 | 1. 确保 `py-origin` 目录在项目根目录下
120 | 2. 首次运行可能需要安装 spaCy 语言模型
121 | 3. LLM 功能需要配置有效的 API 密钥文件
122 | 4. 建议在虚拟环境中运行
123 |
124 | ## 故障排除
125 |
126 | ### 导入错误
127 | 如果遇到 `ImportError`，请确保：
128 | - 在正确的目录下运行
129 | - `py-origin` 目录存在且可访问
130 | - 已安装所有必要的依赖
131 |
132 | ### 服务不可用
133 | 如果 AIFW 服务不可用：
134 | - 检查 `py-origin` 目录结构
135 | - 确保所有依赖已正确安装
136 | - 查看控制台错误信息
137 |
138 | ## 开发说明
139 |
140 | ### 添加新功能
141 | 1. 在 `app.py` 中添加新的路由
142 | 2. 在 `templates/index.html` 中添加 UI 元素
143 | 3. 在 `static/js/app.js` 中添加前端逻辑
144 | 4. 在 `static/css/style.css` 中添加样式
145 |
146 | ### 自定义样式
147 | 修改 `static/css/style.css` 文件来自定义界面样式。
148 |
149 | ### 添加新的 API 端点
150 | 在 `app.py` 中添加新的路由函数，遵循现有的模式。
151 |
--------------------------------------------------------------------------------
/tests/transformer-js/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | OneAIFW NER (Transformers.js)
7 |
17 |
18 |
19 | OneAIFW NER (Transformers.js)
20 |
21 |
22 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
(waiting)
54 |