├── .gitignore ├── Labfiles ├── 02-custom-document-intelligence │ ├── Python │ │ ├── readme.txt │ │ ├── .env │ │ └── test-model.py │ ├── test1.jpg │ ├── C-Sharp │ │ ├── appsettings.json │ │ ├── test-model.csproj │ │ └── Program.cs │ ├── sample-forms │ │ ├── Form_1.jpg │ │ ├── Form_2.jpg │ │ ├── Form_3.jpg │ │ ├── Form_4.jpg │ │ ├── Form_5.jpg │ │ └── fields.json │ ├── setup.sh │ └── setup.cmd ├── 04-custom-skill │ ├── customskill │ │ ├── AnalyzeForm │ │ │ ├── sample.dat │ │ │ ├── field_mappings.json │ │ │ ├── function.json │ │ │ └── __init__.py │ │ ├── AnalyzeInvoice │ │ │ ├── sample.dat │ │ │ ├── function.json │ │ │ └── __init__.py │ │ ├── ExtractTables │ │ │ ├── sample.dat │ │ │ ├── function.json │ │ │ └── __init__.py │ │ ├── .funcignore │ │ ├── proxies.json │ │ ├── .vscode │ │ │ ├── extensions.json │ │ │ ├── settings.json │ │ │ ├── launch.json │ │ │ └── tasks.json │ │ ├── Test │ │ │ └── Ignite.png │ │ ├── Train │ │ │ ├── Commercial Motor Vehicle - Trey.pdf │ │ │ ├── Commercial Motor Vehicle - Adatum.pdf │ │ │ ├── Commercial Motor Vehicle - Fincher.pdf │ │ │ ├── Commercial Motor Vehicle - Lamna.pdf │ │ │ ├── Commercial Motor Vehicle - Liberty.pdf │ │ │ └── Commercial Motor Vehicle - Adatum.json │ │ ├── requirements.txt │ │ ├── host.json │ │ ├── .gitignore │ │ ├── FormRecognizerTrainModel.ipynb │ │ ├── azuredeploy.json │ │ └── Readme.md │ ├── SampleInvoices │ │ ├── Invoice_1.pdf │ │ ├── Invoice_2.pdf │ │ ├── Invoice_3.pdf │ │ ├── Invoice_4.pdf │ │ └── Invoice_5.pdf │ └── setup.sh ├── 05-content-understanding │ ├── code │ │ ├── requirements.txt │ │ ├── .env │ │ ├── invoice-1234.pdf │ │ ├── invoice-1235.pdf │ │ ├── invoice-1236.pdf │ │ └── analyze_invoice.py │ └── forms │ │ ├── rest-form.pdf │ │ ├── test-form.pdf │ │ ├── invoice-1234.pdf │ │ ├── invoice-1235.pdf │ │ ├── invoice-1236.pdf │ │ └── train-form.pdf ├── 01-prebuild-models │ ├── sample-invoice │ │ └── sample-invoice.pdf │ ├── CSharp │ │ ├── invoicereader.csproj │ │ └── Program.cs │ └── Python │ │ └── document-analysis.py └── 03-composed-model │ ├── trainingdata │ ├── TestDoc │ │ └── f1040_7.pdf │ ├── 1040examples │ │ ├── f1040_1.pdf │ │ ├── f1040_2.pdf │ │ ├── f1040_3.pdf │ │ ├── f1040_4.pdf │ │ └── f1040_5.pdf │ ├── ClassifyExamples │ │ ├── f1040_1.pdf │ │ ├── f1040_2.pdf │ │ ├── f1040_3.pdf │ │ ├── f1040_4.pdf │ │ ├── f1040_5.pdf │ │ ├── f1099msc_payer.pdf │ │ ├── f1099msc_state.pdf │ │ ├── f1099msc_payer_2.pdf │ │ ├── f1099msc_recipient.pdf │ │ └── f1099msc_recipient_2.pdf │ └── 1099examples │ │ ├── f1099msc_payer.pdf │ │ ├── f1099msc_state.pdf │ │ ├── f1099msc_payer_2.pdf │ │ ├── f1099msc_recipient.pdf │ │ └── f1099msc_recipient_2.pdf │ ├── setup.sh │ └── setup.ps1 ├── Instructions ├── media │ ├── Form_1.jpg │ ├── train-form.png │ ├── cloud-shell.png │ ├── code-editor.png │ ├── switch-bash.png │ ├── test-analyzer.png │ ├── ai-foundry-home.png │ ├── azure-resources.png │ ├── create-storage.png │ ├── invoice-schema.png │ ├── sample-invoice.png │ ├── 5-create-resource.png │ ├── 5-studio-home-page.png │ ├── ai-foundry-portal.png │ ├── ai-foundry-project.png │ ├── analysis-results.png │ ├── cloud-shell-ready.png │ ├── completed-schema.png │ ├── keys-and-endpoint.png │ ├── language-detection.png │ ├── open-cloud-shell.png │ ├── read-german-sample.png │ ├── test-form-results.png │ ├── 5-find-endpoint-key.png │ ├── 5-create-resource-details.png │ ├── cloudshell-launch-portal.png │ ├── resource_group_variables.png │ └── content-understanding-project.png ├── Exercises │ ├── 03-composed-model.md │ ├── 01-use-prebuilt-models.md │ ├── 02-custom-document-intelligence.md │ └── 05-content-understanding.md └── Labs │ ├── 02-custom-document-intelligence.md │ ├── 01-use-prebuilt-models.md │ ├── 05-content-understanding.md │ └── 03-composed-model.md ├── README.md ├── _config.yml ├── .github └── ISSUE_TEMPLATE │ └── bug_report.md ├── LICENSE ├── index.md └── _build.yml /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | obj 3 | *.sln -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/Python/readme.txt: -------------------------------------------------------------------------------- 1 | This folder contains Python code -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/AnalyzeForm/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure" 3 | } -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/AnalyzeInvoice/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure" 3 | } -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/ExtractTables/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure" 3 | } -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/code/requirements.txt: -------------------------------------------------------------------------------- 1 | dotenv 2 | azure-identity 3 | azure-ai-projects -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/.funcignore: -------------------------------------------------------------------------------- 1 | .git* 2 | .vscode 3 | local.settings.json 4 | test 5 | .venv -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/proxies.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/proxies", 3 | "proxies": {} 4 | } 5 | -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/code/.env: -------------------------------------------------------------------------------- 1 | PROJECT_CONNECTION="YOUR_PROJECT_CONNECTION_STRING" 2 | ANALYZER="contoso-invoice-analyzer" -------------------------------------------------------------------------------- /Instructions/media/Form_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/Form_1.jpg -------------------------------------------------------------------------------- /Instructions/media/train-form.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/train-form.png -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/Python/.env: -------------------------------------------------------------------------------- 1 | DOC_INTELLIGENCE_ENDPOINT=your_endpoint 2 | DOC_INTELLIGENCE_KEY=your_key 3 | MODEL_ID=your_model_id -------------------------------------------------------------------------------- /Instructions/media/cloud-shell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/cloud-shell.png -------------------------------------------------------------------------------- /Instructions/media/code-editor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/code-editor.png -------------------------------------------------------------------------------- /Instructions/media/switch-bash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/switch-bash.png -------------------------------------------------------------------------------- /Instructions/media/test-analyzer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/test-analyzer.png -------------------------------------------------------------------------------- /Instructions/media/ai-foundry-home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/ai-foundry-home.png -------------------------------------------------------------------------------- /Instructions/media/azure-resources.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/azure-resources.png -------------------------------------------------------------------------------- /Instructions/media/create-storage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/create-storage.png -------------------------------------------------------------------------------- /Instructions/media/invoice-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/invoice-schema.png -------------------------------------------------------------------------------- /Instructions/media/sample-invoice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/sample-invoice.png -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/AnalyzeForm/field_mappings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Vehicle make:": "VehicleMake", 3 | "Vehicle year:": "VehicleYear" 4 | } 5 | -------------------------------------------------------------------------------- /Instructions/media/5-create-resource.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/5-create-resource.png -------------------------------------------------------------------------------- /Instructions/media/5-studio-home-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/5-studio-home-page.png -------------------------------------------------------------------------------- /Instructions/media/ai-foundry-portal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/ai-foundry-portal.png -------------------------------------------------------------------------------- /Instructions/media/ai-foundry-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/ai-foundry-project.png -------------------------------------------------------------------------------- /Instructions/media/analysis-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/analysis-results.png -------------------------------------------------------------------------------- /Instructions/media/cloud-shell-ready.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/cloud-shell-ready.png -------------------------------------------------------------------------------- /Instructions/media/completed-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/completed-schema.png -------------------------------------------------------------------------------- /Instructions/media/keys-and-endpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/keys-and-endpoint.png -------------------------------------------------------------------------------- /Instructions/media/language-detection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/language-detection.png -------------------------------------------------------------------------------- /Instructions/media/open-cloud-shell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/open-cloud-shell.png -------------------------------------------------------------------------------- /Instructions/media/read-german-sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/read-german-sample.png -------------------------------------------------------------------------------- /Instructions/media/test-form-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/test-form-results.png -------------------------------------------------------------------------------- /Instructions/media/5-find-endpoint-key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/5-find-endpoint-key.png -------------------------------------------------------------------------------- /Instructions/media/5-create-resource-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/5-create-resource-details.png -------------------------------------------------------------------------------- /Instructions/media/cloudshell-launch-portal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/cloudshell-launch-portal.png -------------------------------------------------------------------------------- /Instructions/media/resource_group_variables.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/resource_group_variables.png -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-python.python" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/test1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/02-custom-document-intelligence/test1.jpg -------------------------------------------------------------------------------- /Instructions/media/content-understanding-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Instructions/media/content-understanding-project.png -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/SampleInvoices/Invoice_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/SampleInvoices/Invoice_1.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/SampleInvoices/Invoice_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/SampleInvoices/Invoice_2.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/SampleInvoices/Invoice_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/SampleInvoices/Invoice_3.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/SampleInvoices/Invoice_4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/SampleInvoices/Invoice_4.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/SampleInvoices/Invoice_5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/SampleInvoices/Invoice_5.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Test/Ignite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/customskill/Test/Ignite.png -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/forms/rest-form.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/forms/rest-form.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/forms/test-form.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/forms/test-form.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/code/invoice-1234.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/code/invoice-1234.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/code/invoice-1235.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/code/invoice-1235.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/code/invoice-1236.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/code/invoice-1236.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/forms/invoice-1234.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/forms/invoice-1234.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/forms/invoice-1235.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/forms/invoice-1235.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/forms/invoice-1236.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/forms/invoice-1236.pdf -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/forms/train-form.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/05-content-understanding/forms/train-form.pdf -------------------------------------------------------------------------------- /Labfiles/01-prebuild-models/sample-invoice/sample-invoice.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/01-prebuild-models/sample-invoice/sample-invoice.pdf -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/C-Sharp/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "DocIntelligenceEndpoint": "", 3 | "DocIntelligenceKey": "", 4 | "ModelId": "" 5 | } 6 | -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/TestDoc/f1040_7.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/TestDoc/f1040_7.pdf -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/sample-forms/Form_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/02-custom-document-intelligence/sample-forms/Form_1.jpg -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/sample-forms/Form_2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/02-custom-document-intelligence/sample-forms/Form_2.jpg -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/sample-forms/Form_3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/02-custom-document-intelligence/sample-forms/Form_3.jpg -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/sample-forms/Form_4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/02-custom-document-intelligence/sample-forms/Form_4.jpg -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/sample-forms/Form_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/02-custom-document-intelligence/sample-forms/Form_5.jpg -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1040examples/f1040_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1040examples/f1040_1.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1040examples/f1040_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1040examples/f1040_2.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1040examples/f1040_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1040examples/f1040_3.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1040examples/f1040_4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1040examples/f1040_4.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1040examples/f1040_5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1040examples/f1040_5.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_1.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_2.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_3.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_4.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1040_5.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_payer.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_payer.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_state.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_state.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_payer_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_payer_2.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_recipient.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_recipient.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_payer.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_payer.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_state.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_state.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_recipient_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/1099examples/f1099msc_recipient_2.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_payer_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_payer_2.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Trey.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Trey.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_recipient.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_recipient.pdf -------------------------------------------------------------------------------- /Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_recipient_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/03-composed-model/trainingdata/ClassifyExamples/f1099msc_recipient_2.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Adatum.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Adatum.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Fincher.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Fincher.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Lamna.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Lamna.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Liberty.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MicrosoftLearning/mslearn-ai-document-intelligence/HEAD/Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Liberty.pdf -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/requirements.txt: -------------------------------------------------------------------------------- 1 | # DO NOT include azure-functions-worker in this file 2 | # The Python Worker is managed by Azure Functions platform 3 | # Manually managing azure-functions-worker may cause unexpected issues 4 | 5 | azure-functions 6 | azure-ai-formrecognizer 7 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": ".", 3 | "azureFunctions.scmDoBuildDuringDeployment": true, 4 | "azureFunctions.pythonVenv": ".venv", 5 | "azureFunctions.projectLanguage": "Python", 6 | "azureFunctions.projectRuntime": "~2", 7 | "debug.internalConsoleOptions": "neverOpen" 8 | } -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | 5 | { 6 | "name": "Attach to Python Functions", 7 | "type": "python", 8 | "request": "attach", 9 | "port": 9091, 10 | "preLaunchTask": "func: host start" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | }, 11 | "extensionBundle": { 12 | "id": "Microsoft.Azure.Functions.ExtensionBundle", 13 | "version": "[2.*, 3.0.0)" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > **NOTE**: This repo is being archived, and is no longer maintained. Please visit the [Develop AI information extraction solutions](https://learn.microsoft.com/training/paths/ai-extract-information/) learning path and [associated lab repo](https://github.com/MicrosoftLearning/mslearn-ai-information-extraction), or [Azure AI Foundry documentation](https://learn.microsoft.com/azure/ai-foundry/) 2 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/AnalyzeForm/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "__init__.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "function", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "post" 11 | ] 12 | }, 13 | { 14 | "type": "http", 15 | "direction": "out", 16 | "name": "$return" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /Labfiles/01-prebuild-models/CSharp/invoicereader.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/AnalyzeInvoice/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "__init__.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "function", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "get", 11 | "post" 12 | ] 13 | }, 14 | { 15 | "type": "http", 16 | "direction": "out", 17 | "name": "$return" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/ExtractTables/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "__init__.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "function", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "get", 11 | "post" 12 | ] 13 | }, 14 | { 15 | "type": "http", 16 | "direction": "out", 17 | "name": "$return" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | remote_theme: MicrosoftLearning/Jekyll-Theme 2 | exclude: 3 | - readme.md 4 | - .github/ 5 | header_pages: 6 | - index.html 7 | author: Microsoft Learning 8 | twitter_username: mslearning 9 | github_username: MicrosoftLearning 10 | plugins: 11 | - jekyll-sitemap 12 | - jekyll-mentions 13 | - jemoji 14 | markdown: kramdown 15 | kramdown: 16 | syntax_highlighter_opts: 17 | disable : true 18 | title: Azure AI Document Intelligence Exercises 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: ivorb 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Additional context** 27 | Add any other context about the problem here. 28 | -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/C-Sharp/test-model.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | PreserveNewest 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "func", 6 | "command": "host start", 7 | "problemMatcher": "$func-python-watch", 8 | "isBackground": true, 9 | "dependsOn": "pipInstall" 10 | }, 11 | { 12 | "label": "pipInstall", 13 | "type": "shell", 14 | "osx": { 15 | "command": "${config:azureFunctions.pythonVenv}/bin/python -m pip install -r requirements.txt" 16 | }, 17 | "windows": { 18 | "command": "${config:azureFunctions.pythonVenv}\\Scripts\\python -m pip install -r requirements.txt" 19 | }, 20 | "linux": { 21 | "command": "${config:azureFunctions.pythonVenv}/bin/python -m pip install -r requirements.txt" 22 | }, 23 | "problemMatcher": [] 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Microsoft Learning 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Labfiles/01-prebuild-models/Python/document-analysis.py: -------------------------------------------------------------------------------- 1 | from azure.core.credentials import AzureKeyCredential 2 | from azure.ai.formrecognizer import DocumentAnalysisClient 3 | 4 | # Store connection information 5 | endpoint = "" 6 | key = "" 7 | 8 | fileUri = "https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence/blob/main/Labfiles/01-prebuild-models/sample-invoice/sample-invoice.pdf?raw=true" 9 | fileLocale = "en-US" 10 | fileModelId = "prebuilt-invoice" 11 | 12 | print(f"\nConnecting to Forms Recognizer at: {endpoint}") 13 | print(f"Analyzing invoice at: {fileUri}") 14 | 15 | # Create the client 16 | 17 | # Analyse the invoice 18 | 19 | # Display invoice information to the user 20 | 21 | 22 | 23 | 24 | customer_name = receipt.fields.get("CustomerName") 25 | if customer_name: 26 | print(f"Customer Name: '{customer_name.value}, with confidence {customer_name.confidence}.") 27 | 28 | 29 | invoice_total = receipt.fields.get("InvoiceTotal") 30 | if invoice_total: 31 | print(f"Invoice Total: '{invoice_total.value.symbol}{invoice_total.value.amount}, with confidence {invoice_total.confidence}.") 32 | 33 | print("\nAnalysis complete.\n") -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Azure AI Document Intelligence Exercises 3 | permalink: index.html 4 | layout: home 5 | --- 6 | 7 | The following exercises are designed to provide you with a hands-on learning experience in which you'll explore common tasks that developers do when building document intelligence solutions on Microsoft Azure. 8 | 9 | > **Note**: To complete the exercises, you'll need an Azure subscription in which you have sufficient permissions and quota to provision the necessary Azure resources. If you don't already have one, you can sign up for an [Azure account](https://azure.microsoft.com/free). There's a free trial option for new users that includes credits for the first 30 days. 10 | 11 | ## Exercises 12 | 13 | {% assign labs = site.pages | where_exp:"page", "page.url contains '/Instructions'" %} 14 | {% for activity in labs %} 15 | {% if activity.url contains 'ai-foundry' %} 16 | {% continue %} 17 | {% endif %} 18 |
19 | ### [{{ activity.lab.title }}]({{ site.github.url }}{{ activity.url }}) 20 | 21 | {{activity.lab.description}} 22 | 23 | {% endfor %} 24 | 25 | > **Note**: While you can complete these exercises on their own, they're designed to complement modules on [Microsoft Learn](https://learn.microsoft.com/training/paths/extract-data-from-forms-document-intelligence/); in which you'll find a deeper dive into some of the underlying concepts on which these exercises are based. 26 | -------------------------------------------------------------------------------- /_build.yml: -------------------------------------------------------------------------------- 1 | name: '$(Date:yyyyMMdd)$(Rev:.rr)' 2 | jobs: 3 | - job: build_markdown_content 4 | displayName: 'Build Markdown Content' 5 | workspace: 6 | clean: all 7 | pool: 8 | vmImage: 'Ubuntu 16.04' 9 | container: 10 | image: 'microsoftlearning/markdown-build:latest' 11 | steps: 12 | - task: Bash@3 13 | displayName: 'Build Content' 14 | inputs: 15 | targetType: inline 16 | script: | 17 | cp /{attribution.md,template.docx,package.json,package.js} . 18 | npm install 19 | node package.js --version $(Build.BuildNumber) 20 | - task: GitHubRelease@0 21 | displayName: 'Create GitHub Release' 22 | inputs: 23 | gitHubConnection: 'github-microsoftlearning-organization' 24 | repositoryName: '$(Build.Repository.Name)' 25 | tagSource: manual 26 | tag: 'v$(Build.BuildNumber)' 27 | title: 'Version $(Build.BuildNumber)' 28 | releaseNotesSource: input 29 | releaseNotes: '# Version $(Build.BuildNumber) Release' 30 | assets: '$(Build.SourcesDirectory)/out/*.zip' 31 | assetUploadMode: replace 32 | - task: PublishBuildArtifacts@1 33 | displayName: 'Publish Output Files' 34 | inputs: 35 | pathtoPublish: '$(Build.SourcesDirectory)/out/' 36 | artifactName: 'Lab Files' 37 | -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/Python/test-model.py: -------------------------------------------------------------------------------- 1 | from azure.core.credentials import AzureKeyCredential 2 | from azure.ai.formrecognizer import DocumentAnalysisClient 3 | from dotenv import load_dotenv 4 | import os 5 | 6 | # Get configuration settings 7 | load_dotenv() 8 | endpoint = os.getenv("DOC_INTELLIGENCE_ENDPOINT") 9 | key = os.getenv("DOC_INTELLIGENCE_KEY") 10 | model_id = os.getenv("MODEL_ID") 11 | 12 | formUrl = "https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence/blob/main/Labfiles/02-custom-document-intelligence/test1.jpg?raw=true" 13 | 14 | document_analysis_client = DocumentAnalysisClient( 15 | endpoint=endpoint, credential=AzureKeyCredential(key) 16 | ) 17 | 18 | # Make sure your document's type is included in the list of document types the custom model can analyze 19 | response = document_analysis_client.begin_analyze_document_from_url(model_id, formUrl) 20 | result = response.result() 21 | 22 | for idx, document in enumerate(result.documents): 23 | print("--------Analyzing document #{}--------".format(idx + 1)) 24 | print("Document has type {}".format(document.doc_type)) 25 | print("Document has confidence {}".format(document.confidence)) 26 | print("Document was analyzed by model with ID {}".format(result.model_id)) 27 | for name, field in document.fields.items(): 28 | field_value = field.value if field.value else field.content 29 | print("Found field '{}' with value '{}' and with confidence {}".format(name, field_value, field.confidence)) 30 | 31 | print("-----------------------------------") 32 | -------------------------------------------------------------------------------- /Labfiles/01-prebuild-models/CSharp/Program.cs: -------------------------------------------------------------------------------- 1 | using Azure; 2 | using Azure.AI.FormRecognizer.DocumentAnalysis; 3 | 4 | // Store connection information 5 | string endpoint = ""; 6 | string apiKey = ""; 7 | 8 | Uri fileUri = new Uri("https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence/blob/main/Labfiles/01-prebuild-models/sample-invoice/sample-invoice.pdf?raw=true"); 9 | 10 | Console.WriteLine("\nConnecting to Forms Recognizer at: {0}", endpoint); 11 | Console.WriteLine("Analyzing invoice at: {0}\n", fileUri.ToString()); 12 | 13 | // Create the client 14 | 15 | // Analyze the invoice 16 | 17 | // Display invoice information to the user 18 | 19 | 20 | 21 | 22 | 23 | if (invoice.Fields.TryGetValue("CustomerName", out DocumentField? customerNameField)) 24 | { 25 | if (customerNameField.FieldType == DocumentFieldType.String) 26 | { 27 | string customerName = customerNameField.Value.AsString(); 28 | Console.WriteLine($"Customer Name: '{customerName}', with confidence {customerNameField.Confidence}."); 29 | } 30 | } 31 | 32 | if (invoice.Fields.TryGetValue("InvoiceTotal", out DocumentField? invoiceTotalField)) 33 | { 34 | if (invoiceTotalField.FieldType == DocumentFieldType.Currency) 35 | { 36 | CurrencyValue invoiceTotal = invoiceTotalField.Value.AsCurrency(); 37 | Console.WriteLine($"Invoice Total: '{invoiceTotal.Symbol}{invoiceTotal.Amount}', with confidence {invoiceTotalField.Confidence}."); 38 | } 39 | } 40 | } 41 | Console.WriteLine("\nAnalysis complete.\n"); -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/setup.sh: -------------------------------------------------------------------------------- 1 | #~!/bin/bash 2 | 3 | # Store color codes 4 | NORMAL=$(tput sgr0) 5 | GREEN=$(tput setaf 2) 6 | 7 | # Set up the resource group 8 | resourceGroupName=FormsRecognizerResources 9 | printf "${GREEN}Setting up the $resourceGroupName resource group. \n${NORMAL}" 10 | az group create --location westus --name FormsRecognizerResources 11 | 12 | # Get random numbers to create unique resource names 13 | unique_id=$((1 + RANDOM % 99999)) 14 | 15 | # Create the Forms Recognizer resource 16 | printf "${GREEN}Setting up the Forms Recognizer resource. \n${NORMAL}" 17 | # First, purge it in case there's a recently deleted one 18 | SubID=$(az account show --query id --output tsv) 19 | az resource delete --ids "/subscriptions/${SubID}/providers/Microsoft.CognitiveServices/locations/westus/resourceGroups/${resourceGroupName}/deletedAccounts/FormsRecognizer" 20 | # Now, create the new one 21 | az cognitiveservices account create --kind FormRecognizer --location westus --name FormsRecognizer --resource-group $resourceGroupName --sku F0 --yes 22 | 23 | # Create the Cognitive Search resource 24 | printf "${GREEN}Setting up Azure Cognitive Search. \n${NORMAL}" 25 | # Purge any existing search resource with the same name 26 | az resource delete --ids "/subscriptions/${SubID}/resourceGroups/${resourceGroupName}/providers/Microsoft.Search/searchServices/enrichedcognitivesearch" 27 | # Now, create the new one 28 | az search service create --name "enrichedcognitivesearch$unique_id" --location westus --resource-group $resourceGroupName --sku Free --partition-count 1 --replica-count 1 29 | 30 | printf "${GREEN}Setup completed. \n${NORMAL}" 31 | -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/C-Sharp/Program.cs: -------------------------------------------------------------------------------- 1 | using Azure; 2 | using Azure.AI.FormRecognizer.DocumentAnalysis; 3 | using Microsoft.Extensions.Configuration; 4 | 5 | // Get configuration settings from AppSettings 6 | IConfiguration configuration = new ConfigurationBuilder() 7 | .AddJsonFile("appsettings.json") 8 | .Build(); 9 | string endpoint = configuration["DocIntelligenceEndpoint"]; 10 | string apiKey = configuration["DocIntelligenceKey"]; 11 | AzureKeyCredential credential = new AzureKeyCredential(apiKey); 12 | DocumentAnalysisClient client = new DocumentAnalysisClient(new Uri(endpoint), credential); 13 | 14 | string modelId = configuration["ModelId"]; 15 | Uri fileUri = new Uri("https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence/blob/main/Labfiles/02-custom-document-intelligence/test1.jpg?raw=true"); 16 | Console.WriteLine($"Analyzing document from Uri: {fileUri.AbsoluteUri}"); 17 | 18 | AnalyzeDocumentOperation operation = await client.AnalyzeDocumentFromUriAsync(WaitUntil.Completed, modelId, fileUri); 19 | AnalyzeResult result = operation.Value; 20 | 21 | Console.WriteLine($"Document was analyzed with model with ID: {result.ModelId}"); 22 | 23 | foreach (AnalyzedDocument document in result.Documents) 24 | { 25 | Console.WriteLine($"Document of type: {document.DocumentType}"); 26 | 27 | foreach (KeyValuePair fieldKvp in document.Fields) 28 | { 29 | string fieldName = fieldKvp.Key; 30 | DocumentField field = fieldKvp.Value; 31 | 32 | Console.WriteLine($"Field '{fieldName}': "); 33 | 34 | Console.WriteLine($" Content: '{field.Content}'"); 35 | Console.WriteLine($" Confidence: '{field.Confidence}'"); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Set variable values 4 | subscription_id="YOUR_SUBSCRIPTION_ID" 5 | resource_group="YOUR_RESOURCE_GROUP" 6 | location="YOUR_LOCATION_NAME" 7 | expiry_date="2026-01-01T00:00:00Z" 8 | 9 | # Get random numbers to create unique resource names 10 | unique_id=$((1 + RANDOM % 99999)) 11 | 12 | # Create a storage account in your Azure resource group 13 | echo "Creating storage..." 14 | az storage account create --name "ai102form$unique_id" --subscription "$subscription_id" --resource-group "$resource_group" --location "$location" --sku Standard_LRS --encryption-services blob --default-action Allow --allow-blob-public-access true --only-show-errors --output none 15 | 16 | echo "Uploading files..." 17 | # Get storage key to create a container in the storage account 18 | key_json=$(az storage account keys list --subscription "$subscription_id" --resource-group "$resource_group" --account-name "ai102form$unique_id" --query "[?keyName=='key1'].{keyName:keyName, permissions:permissions, value:value}") 19 | key_string=$(echo "$key_json" | jq -r '.[0].value') 20 | AZURE_STORAGE_KEY=${key_string} 21 | 22 | # Create a container 23 | az storage container create --account-name "ai102form$unique_id" --name sampleforms --public-access blob --auth-mode key --account-key "$AZURE_STORAGE_KEY" --output none 24 | 25 | # Upload files from your local sampleforms folder to a container called sampleforms in the storage account 26 | # Each file is uploaded as a blob 27 | az storage blob upload-batch -d sampleforms -s ./sample-forms --account-name "ai102form$unique_id" --auth-mode key --account-key "$AZURE_STORAGE_KEY" --output none 28 | 29 | # Set a variable value for future use 30 | STORAGE_ACCT_NAME="ai102form$unique_id" 31 | 32 | # Get a Shared Access Signature (a signed URI that points to one or more storage resources) for the blobs in sampleforms 33 | SAS_TOKEN=$(az storage container generate-sas --account-name "ai102form$unique_id" --name sampleforms --expiry "$expiry_date" --permissions rwl --auth-mode key --account-key "$AZURE_STORAGE_KEY") 34 | URI="https://$STORAGE_ACCT_NAME.blob.core.windows.net/sampleforms?$SAS_TOKEN" 35 | 36 | # Print the generated Shared Access Signature URI, which is used by Azure Storage to authorize access to the storage resource 37 | echo "-------------------------------------" 38 | echo "SAS URI: $URI" 39 | -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/sample-forms/fields.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "fieldKey": "Merchant", 5 | "fieldType": "string", 6 | "fieldFormat": "not-specified" 7 | }, 8 | { 9 | "fieldKey": "PhoneNumber", 10 | "fieldType": "string", 11 | "fieldFormat": "not-specified" 12 | }, 13 | { 14 | "fieldKey": "Website", 15 | "fieldType": "string", 16 | "fieldFormat": "not-specified" 17 | }, 18 | { 19 | "fieldKey": "Email", 20 | "fieldType": "string", 21 | "fieldFormat": "not-specified" 22 | }, 23 | { 24 | "fieldKey": "PurchaseOrderNumber", 25 | "fieldType": "string", 26 | "fieldFormat": "not-specified" 27 | }, 28 | { 29 | "fieldKey": "DatedAs", 30 | "fieldType": "string", 31 | "fieldFormat": "not-specified" 32 | }, 33 | { 34 | "fieldKey": "VendorName", 35 | "fieldType": "string", 36 | "fieldFormat": "not-specified" 37 | }, 38 | { 39 | "fieldKey": "CompanyName", 40 | "fieldType": "string", 41 | "fieldFormat": "not-specified" 42 | }, 43 | { 44 | "fieldKey": "CompanyAddress", 45 | "fieldType": "string", 46 | "fieldFormat": "not-specified" 47 | }, 48 | { 49 | "fieldKey": "CompanyPhoneNumber", 50 | "fieldType": "string", 51 | "fieldFormat": "not-specified" 52 | }, 53 | { 54 | "fieldKey": "Subtotal", 55 | "fieldType": "string", 56 | "fieldFormat": "not-specified" 57 | }, 58 | { 59 | "fieldKey": "Tax", 60 | "fieldType": "string", 61 | "fieldFormat": "not-specified" 62 | }, 63 | { 64 | "fieldKey": "Total", 65 | "fieldType": "string", 66 | "fieldFormat": "not-specified" 67 | }, 68 | { 69 | "fieldKey": "Signature", 70 | "fieldType": "string", 71 | "fieldFormat": "not-specified" 72 | }, 73 | { 74 | "fieldKey": "Quantity", 75 | "fieldType": "number", 76 | "fieldFormat": "not-specified" 77 | } 78 | ] 79 | } -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # pipenv 86 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 87 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 88 | # having no cross-platform support, pipenv may install dependencies that don’t work, or not 89 | # install all needed dependencies. 90 | #Pipfile.lock 91 | 92 | # celery beat schedule file 93 | celerybeat-schedule 94 | 95 | # SageMath parsed files 96 | *.sage.py 97 | 98 | # Environments 99 | .env 100 | .venv 101 | env/ 102 | venv/ 103 | ENV/ 104 | env.bak/ 105 | venv.bak/ 106 | 107 | # Spyder project settings 108 | .spyderproject 109 | .spyproject 110 | 111 | # Rope project settings 112 | .ropeproject 113 | 114 | # mkdocs documentation 115 | /site 116 | 117 | # mypy 118 | .mypy_cache/ 119 | .dmypy.json 120 | dmypy.json 121 | 122 | # Pyre type checker 123 | .pyre/ 124 | 125 | # Azure Functions artifacts 126 | bin 127 | obj 128 | appsettings.json 129 | local.settings.json 130 | .python_packages -------------------------------------------------------------------------------- /Labfiles/02-custom-document-intelligence/setup.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | SETLOCAL ENABLEDELAYEDEXPANSION 3 | 4 | rem Set variable values 5 | set subscription_id=YOUR_SUBSCRIPTION_ID 6 | set resource_group=YOUR_RESOURCE_GROUP 7 | set location=YOUR_LOCATION_NAME 8 | set expiry_date=2026-01-01T00:00:00Z 9 | 10 | rem Get random numbers to create unique resource names 11 | set unique_id=!random!!random! 12 | 13 | rem Create a storage account in your Azure resource group 14 | echo Creating storage... 15 | call az storage account create --name ai102form!unique_id! --subscription !subscription_id! --resource-group !resource_group! --location !location! --sku Standard_LRS --encryption-services blob --default-action Allow --allow-blob-public-access true --only-show-errors --output none 16 | 17 | echo Uploading files... 18 | rem Get storage key to create a container in the storage account 19 | for /f "tokens=*" %%a in ( 20 | 'az storage account keys list --subscription !subscription_id! --resource-group !resource_group! --account-name ai102form!unique_id! --query "[?keyName=='key1'].{keyName:keyName, permissions:permissions, value:value}"' 21 | ) do ( 22 | set key_json=!key_json!%%a 23 | ) 24 | set key_string=!key_json:[ { "keyName": "key1", "permissions": "Full", "value": "=! 25 | set AZURE_STORAGE_KEY=!key_string:" } ]=! 26 | rem Create container 27 | call az storage container create --account-name ai102form!unique_id! --name sampleforms --public-access blob --auth-mode key --account-key %AZURE_STORAGE_KEY% --output none 28 | rem Upload files from your local sampleforms folder to a container called sampleforms in the storage account 29 | rem Each file is uploaded as a blob 30 | call az storage blob upload-batch -d sampleforms -s ./sample-forms --account-name ai102form!unique_id! --auth-mode key --account-key %AZURE_STORAGE_KEY% --output none 31 | rem Set a variable value for future use 32 | set STORAGE_ACCT_NAME=ai102form!unique_id! 33 | 34 | rem Get a Shared Access Signature (a signed URI that points to one or more storage resources) for the blobs in sampleforms 35 | for /f "tokens=*" %%a in ( 36 | 'az storage container generate-sas --account-name ai102form!unique_id! --name sampleforms --expiry !expiry_date! --permissions rwl' 37 | ) do ( 38 | set SAS_TOKEN=%%a 39 | set SAS_TOKEN=!SAS_TOKEN:~1,-1! 40 | ) 41 | set URI=https://!STORAGE_ACCT_NAME!.blob.core.windows.net/sampleforms?!SAS_TOKEN! 42 | 43 | rem Print the generated Shared Access Signature URI, which is used by Azure Storage to authorize access to the storage resource 44 | echo ------------------------------------- 45 | echo SAS URI: !URI! 46 | -------------------------------------------------------------------------------- /Labfiles/03-composed-model/setup.sh: -------------------------------------------------------------------------------- 1 | #~!/bin/bash 2 | 3 | # Store color codes 4 | NORMAL=$(tput sgr0) 5 | GREEN=$(tput setaf 2) 6 | 7 | # Set up the resource group 8 | resourceGroupName=DocumentIntelligenceResources 9 | printf "${GREEN}Setting up the DocumentIntelligenceResources resource group. \n${NORMAL}" 10 | az group create --location westus2 --name DocumentIntelligenceResources 11 | 12 | # Create a name for the storage account 13 | storageAccName=docintelstorage$((10000 + RANDOM % 99999)) 14 | 15 | # Set up the Azure Storage account 16 | printf "${GREEN}Setting up the $storageAccName storage account. \n${NORMAL}" 17 | az storage account create --name $storageAccName --resource-group $resourceGroupName --kind StorageV2 --sku Standard_LRS 18 | 19 | # Get the connection string for the new storage account 20 | connectionString=$(az storage account show-connection-string --name $storageAccName --key key1 --resource-group $resourceGroupName --query connectionString) 21 | 22 | # Enable CORS on the storage account 23 | az storage cors add --methods DELETE GET HEAD MERGE OPTIONS POST PUT --origins * --services b --allowed-headers * --max-age 200 --exposed-headers * --connection-string $connectionString 24 | 25 | # Create the storage containers 26 | printf "${GREEN}Creating containers for the sample forms. \n${NORMAL}" 27 | az storage container create --account-name $storageAccName --name 1040examples --auth-mode login 28 | az storage container create --account-name $storageAccName --name 1099examples --auth-mode login 29 | az storage container create --account-name $storageAccName --name testdoc --auth-mode login 30 | 31 | # Upload the sample data 32 | printf "${GREEN}Uploading the sample forms to the storage account. \n${NORMAL}" 33 | az storage blob upload-batch -d 1040examples --account-name $storageAccName --connection-string $connectionString -s "trainingdata/1040examples" --pattern *.pdf 34 | az storage blob upload-batch -d 1099examples --account-name $storageAccName --connection-string $connectionString -s "trainingdata/1099examples" --pattern *.pdf 35 | az storage blob upload-batch -d testdoc --account-name $storageAccName --connection-string $connectionString -s "trainingdata/TestDoc" --pattern *.pdf 36 | 37 | # Create the Forms Recognizer resource 38 | printf "${GREEN} Setting up the Document Intelligence resource. \n${NORMAL}" 39 | # First, purge it in case there's a recently deleted one 40 | SubID=$(az account show --query id --output tsv) 41 | az resource delete --ids "/subscriptions/${SubID}/providers/Microsoft.CognitiveServices/locations/westus2/resourceGroups/${resourceGroupName}/deletedAccounts/FormsRecognizer" 42 | # Now, create the new one 43 | az cognitiveservices account create --kind FormRecognizer --location westus2 --name FormsRecognizer --resource-group $resourceGroupName --sku F0 --yes 44 | -------------------------------------------------------------------------------- /Labfiles/03-composed-model/setup.ps1: -------------------------------------------------------------------------------- 1 | # Variables 2 | $randomIdentifier = (New-Guid).ToString().Substring(0,4) 3 | $resourceGroupName = "DocumentIntelligenceResources$randomIdentifier" 4 | $storageAccName = "docintelstorage$randomIdentifier" 5 | $docIntelligenceName = "DocumentIntelligence$randomIdentifier" 6 | 7 | # Set up the resource group 8 | Write-Output "Setting up the $resourceGroupName resource group." 9 | az group create --location westus2 ` 10 | --name $resourceGroupName 11 | 12 | # Set up the Azure Storage account 13 | Write-Output "Setting up the $storageAccName storage account." 14 | az storage account create --name $storageAccName ` 15 | --resource-group $resourceGroupName ` 16 | --kind StorageV2 ` 17 | --sku Standard_LRS ` 18 | --allow-blob-public-access true 19 | 20 | # Get the connection string for the new storage account 21 | $connectionString=$(az storage account show-connection-string --name $storageAccName --key key1 --query connectionString) 22 | 23 | # Enable CORS on the storage account 24 | az storage cors add --methods DELETE GET HEAD MERGE OPTIONS POST PUT --origins * --services b --allowed-headers * --max-age 200 --exposed-headers * --connection-string $connectionString 25 | 26 | # Create the storage containers 27 | Write-Output "Creating containers for the sample forms." 28 | az storage container create --account-name $storageAccName --name 1040examples --connection-string $connectionString 29 | az storage container create --account-name $storageAccName --name 1099examples --connection-string $connectionString 30 | az storage container create --account-name $storageAccName --name classifyexamples --connection-string $connectionString 31 | az storage container create --account-name $storageAccName --name testdoc --connection-string $connectionString 32 | # Upload the sample data 33 | Write-Output "Uploading the sample forms to the storage account." 34 | az storage blob upload-batch -d 1040examples --account-name $storageAccName --connection-string $connectionString -s "trainingdata/1040examples" --pattern *.pdf 35 | az storage blob upload-batch -d 1099examples --account-name $storageAccName --connection-string $connectionString -s "trainingdata/1099examples" --pattern *.pdf 36 | az storage blob upload-batch -d testdoc --account-name $storageAccName --connection-string $connectionString -s "trainingdata/ClassifyExamples" --pattern *.pdf 37 | az storage blob upload-batch -d testdoc --account-name $storageAccName --connection-string $connectionString -s "trainingdata/TestDoc" --pattern *.pdf 38 | Write-Output "Uploaded sample data." 39 | 40 | # Create the Forms Recognizer resource 41 | Write-Output "Setting up the Document Intelligence resource." 42 | az cognitiveservices account create --kind FormRecognizer --location westus2 --name $docIntelligenceName --resource-group $resourceGroupName --sku F0 --yes -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/AnalyzeForm/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | import logging 5 | import pathlib 6 | from azure.core.exceptions import ResourceNotFoundError 7 | from azure.ai.formrecognizer import FormRecognizerClient 8 | from azure.ai.formrecognizer import FormTrainingClient 9 | from azure.core.credentials import AzureKeyCredential 10 | import azure.functions as func 11 | 12 | def main(req: func.HttpRequest) -> func.HttpResponse: 13 | logging.info('Invoked AnalyzeForm Skill.') 14 | try: 15 | body = json.dumps(req.get_json()) 16 | if body: 17 | # For testing uncomment the following line to log the incoming request 18 | #logging.info(body) 19 | result = compose_response(body) 20 | return func.HttpResponse(result, mimetype="application/json") 21 | else: 22 | return func.HttpResponse( 23 | "The body of the request could not be parsed", 24 | status_code=400 25 | ) 26 | except ValueError: 27 | return func.HttpResponse( 28 | "The body of the request could not be parsed", 29 | status_code=400 30 | ) 31 | except KeyError: 32 | return func.HttpResponse( 33 | "Skill configuration error. Endpoint, key and model_id required.", 34 | status_code=400 35 | ) 36 | except AssertionError as error: 37 | return func.HttpResponse( 38 | "Request format is not a valid custom skill input", 39 | status_code=400 40 | ) 41 | 42 | def compose_response(json_data): 43 | body = json.loads(json_data) 44 | assert ('values' in body), "request does not implement the custom skill interface" 45 | values = body['values'] 46 | # Prepare the Output before the loop 47 | results = {} 48 | results["values"] = [] 49 | mappings = None 50 | with open(pathlib.Path(__file__).parent / 'field_mappings.json') as file: 51 | mappings = json.loads(file.read()) 52 | endpoint = os.environ["FORMS_RECOGNIZER_ENDPOINT"] 53 | key = os.environ["FORMS_RECOGNIZER_KEY"] 54 | model_id = os.environ["FORMS_RECOGNIZER_MODEL_ID"] 55 | form_recognizer_client = FormRecognizerClient(endpoint, AzureKeyCredential(key)) 56 | for value in values: 57 | output_record = transform_value(value, mappings, form_recognizer_client, model_id) 58 | if output_record != None: 59 | results["values"].append(output_record) 60 | break 61 | return json.dumps(results, ensure_ascii=False) 62 | 63 | ## Perform an operation on a record 64 | def transform_value(value, mappings, form_recognizer_client,model_id): 65 | try: 66 | recordId = value['recordId'] 67 | except AssertionError as error: 68 | return None 69 | try: 70 | assert ('data' in value), "'data' field is required." 71 | data = value['data'] 72 | formUrl = data['formUrl'] 73 | formSasToken = data ['formSasToken'] 74 | formUrl = formUrl + formSasToken 75 | poller = form_recognizer_client.begin_recognize_custom_forms_from_url( 76 | model_id=model_id, form_url=formUrl) 77 | result = poller.result() 78 | recognized = {} 79 | for recognized_form in result: 80 | print("Form type: {}".format(recognized_form.form_type)) 81 | for name, field in recognized_form.fields.items(): 82 | label = field.label_data.text if field.label_data else name 83 | for (k, v) in mappings.items(): 84 | if(label == k): 85 | recognized[v] = field.value 86 | 87 | except AssertionError as error: 88 | return ( 89 | { 90 | "recordId": recordId, 91 | "errors": [ { "message": "Error:" + error.args[0] } ] 92 | }) 93 | except Exception as error: 94 | return ( 95 | { 96 | "recordId": recordId, 97 | "errors": [ { "message": "Error:" + str(error) } ] 98 | }) 99 | return ({ 100 | "recordId": recordId, 101 | "data": { 102 | "recognized": recognized 103 | } 104 | }) 105 | -------------------------------------------------------------------------------- /Labfiles/05-content-understanding/code/analyze_invoice.py: -------------------------------------------------------------------------------- 1 | from dotenv import load_dotenv 2 | from datetime import datetime 3 | import os 4 | import sys 5 | import requests 6 | import json 7 | 8 | # Import namespaces 9 | from azure.ai.projects.models import ConnectionType 10 | from azure.identity import DefaultAzureCredential 11 | from azure.core.credentials import AzureKeyCredential 12 | from azure.ai.projects import AIProjectClient 13 | 14 | 15 | def main(): 16 | 17 | # Clear the console 18 | os.system('cls' if os.name=='nt' else 'clear') 19 | 20 | # Get invoice 21 | invoice_file = 'invoice-1236.pdf' 22 | if len(sys.argv) > 1: 23 | invoice_file = sys.argv[1] 24 | 25 | try: 26 | global speech_config 27 | 28 | # Get config settings 29 | load_dotenv() 30 | project_connection = os.getenv('PROJECT_CONNECTION') 31 | analyzer = os.getenv('ANALYZER') 32 | 33 | # Get AI Services endpoint and key from the project 34 | project_client = AIProjectClient.from_connection_string( 35 | conn_str=project_connection, 36 | credential=DefaultAzureCredential()) 37 | 38 | ai_svc_connection = project_client.connections.get_default( 39 | connection_type=ConnectionType.AZURE_AI_SERVICES, 40 | include_credentials=True, 41 | ) 42 | 43 | ai_svc_endpoint = ai_svc_connection.endpoint_url 44 | ai_svc_key = ai_svc_connection.key 45 | 46 | # Analyze invoice 47 | analyze_invoice (invoice_file, analyzer, ai_svc_endpoint, ai_svc_key) 48 | 49 | except Exception as ex: 50 | print(ex) 51 | 52 | def analyze_invoice (invoice_file, analyzer, endpoint, key): 53 | print (f"Analyzing {invoice_file}") 54 | 55 | CU_VERSION = "2024-12-01-preview"; 56 | 57 | with open(invoice_file, "rb") as file: 58 | data = file.read() 59 | 60 | headers = { 61 | "Ocp-Apim-Subscription-Key": key, 62 | "Content-Type": "application/octet-stream"} 63 | 64 | 65 | url = endpoint + f'/contentunderstanding/analyzers/{analyzer}:analyze?api-version={CU_VERSION}' 66 | 67 | print ('Analyzing document...') 68 | response = requests.post(url, headers=headers, data=data) 69 | 70 | print(response.status_code) 71 | response_json = response.json() 72 | 73 | # Extract the "id" value from the response 74 | id_value = response_json.get("id") 75 | 76 | # Perform a GET request tO get the results 77 | print ('Getting results...') 78 | result_url = f'{endpoint}/contentunderstanding/analyzers/{analyzer}/results/{id_value}?api-version={CU_VERSION}' 79 | result_response = requests.get(result_url, headers=headers) 80 | print(result_response.status_code) 81 | 82 | status = result_response.json().get("status") 83 | while status == "Running": 84 | result_response = requests.get(result_url, headers=headers) 85 | status = result_response.json().get("status") 86 | 87 | if status == "Succeeded": 88 | print("Analysis succeeded.") 89 | result_json = result_response.json() 90 | contents = result_json["result"]["contents"] 91 | 92 | # Iterate through the fields and extract the names and values 93 | for content in contents: 94 | if "fields" in content: 95 | fields = content["fields"] 96 | for field_name, field_data in fields.items(): 97 | if "valueNumber" in field_data: 98 | print(f"{field_name}: {field_data['valueNumber']}") 99 | elif "valueString" in field_data: 100 | print(f"{field_name}: {field_data['valueString']}") 101 | elif field_name == "Items": 102 | print("Items:") 103 | item_list = field_data["valueArray"] 104 | for item in item_list: 105 | print(" Item:") 106 | item_values = item["valueObject"] 107 | for item_field_name, item_field_data in item_values.items(): 108 | if "valueNumber" in item_field_data: 109 | print(f" {item_field_name}: {item_field_data['valueNumber']}") 110 | elif "valueString" in item_field_data: 111 | print(f" {item_field_name}: {item_field_data['valueString']}") 112 | 113 | 114 | if __name__ == "__main__": 115 | main() 116 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/ExtractTables/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | import logging 5 | import datetime 6 | from json import JSONEncoder 7 | import azure.functions as func 8 | from azure.ai.formrecognizer import FormRecognizerClient 9 | from azure.ai.formrecognizer import FormTrainingClient 10 | from azure.core.credentials import AzureKeyCredential 11 | 12 | class DateTimeEncoder(JSONEncoder): 13 | #Override the default method 14 | def default(self, obj): 15 | if isinstance(obj, (datetime.date, datetime.datetime)): 16 | return obj.isoformat() 17 | 18 | 19 | def main(req: func.HttpRequest) -> func.HttpResponse: 20 | logging.info('Python HTTP trigger function processed a request.') 21 | try: 22 | body = json.dumps(req.get_json()) 23 | if body: 24 | logging.info(body) 25 | result = compose_response(body) 26 | return func.HttpResponse(result, mimetype="application/json") 27 | else: 28 | return func.HttpResponse( 29 | "Invalid body", 30 | status_code=400 31 | ) 32 | except ValueError: 33 | return func.HttpResponse( 34 | "Invalid body", 35 | status_code=400 36 | ) 37 | except KeyError: 38 | return func.HttpResponse( 39 | "Skill configuration error. Endpoint, key and model_id required.", 40 | status_code=400 41 | ) 42 | 43 | 44 | def compose_response(json_data): 45 | values = json.loads(json_data)['values'] 46 | # Prepare the Output before the loop 47 | results = {} 48 | results["values"] = [] 49 | endpoint = os.environ["FORMS_RECOGNIZER_ENDPOINT"] 50 | key = os.environ["FORMS_RECOGNIZER_KEY"] 51 | form_recognizer_client = FormRecognizerClient(endpoint, AzureKeyCredential(key)) 52 | for value in values: 53 | output_record = transform_value(value, form_recognizer_client) 54 | if output_record != None: 55 | results["values"].append(output_record) 56 | 57 | return json.dumps(results, ensure_ascii=False, cls=DateTimeEncoder) 58 | 59 | 60 | ## Perform an operation on a record 61 | def transform_value(value, form_recognizer_client): 62 | try: 63 | recordId = value['recordId'] 64 | except AssertionError as error: 65 | return None 66 | # Validate the inputs 67 | try: 68 | assert ('data' in value), "'data' field is required." 69 | data = value['data'] 70 | print(data) 71 | form_url = data["formUrl"] + data["formSasToken"] 72 | print(form_url) 73 | poller = form_recognizer_client.begin_recognize_content_from_url(form_url) 74 | pages = poller.result() 75 | tables = [] 76 | if not pages: 77 | print("No pages found in doc") 78 | else: 79 | for page in pages: 80 | if not page.tables: 81 | print("No tables on page") 82 | else: 83 | for table in page.tables: 84 | cells = [] 85 | print("Table found on page {}:".format(table.page_number)) 86 | for cell in table.cells: 87 | cells.append( 88 | { 89 | "text": cell.text, 90 | "rowIndex": cell.row_index, 91 | "colIndex": cell.column_index, 92 | "confidence": cell.confidence, 93 | "is_header": cell.is_header 94 | } 95 | ) 96 | tables.append( 97 | { 98 | "page_number": table.page_number, 99 | "row_count": table.row_count, 100 | "column_count": table.column_count, 101 | "cells": cells 102 | } 103 | ) 104 | except AssertionError as error: 105 | return ( 106 | { 107 | "recordId": recordId, 108 | "errors": [ { "message": "Error:" + error.args[0] } ] 109 | }) 110 | except Exception as error: 111 | return ( 112 | { 113 | "recordId": recordId, 114 | "errors": [ { "message": "Error:" + str(error) } ] 115 | }) 116 | return ({ 117 | "recordId": recordId, 118 | "data": { 119 | "tables": tables 120 | } 121 | }) -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/FormRecognizerTrainModel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "!pip install azure-ai-formrecognizer" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import os\n", 19 | "from azure.core.exceptions import ResourceNotFoundError\n", 20 | "from azure.ai.formrecognizer import FormRecognizerClient\n", 21 | "from azure.ai.formrecognizer import FormTrainingClient\n", 22 | "from azure.core.credentials import AzureKeyCredential" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 3, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "endpoint = \"YourFormsRecognizerEndpoint\"\n", 32 | "key = \"YourFormsRecognizerKey\"\n", 33 | "\n", 34 | "form_recognizer_client = FormRecognizerClient(endpoint, AzureKeyCredential(key))\n", 35 | "form_training_client = FormTrainingClient(endpoint, AzureKeyCredential(key))\n", 36 | "\n" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "# To train a model you need an Azure Storage account.\n", 46 | "# Use the SAS URL to access your training files.\n", 47 | "# Upload the files from the train folder to a storage container and generate a SAS token\n", 48 | "trainingDataUrl = \"\"\n", 49 | "\n", 50 | "poller = form_training_client.begin_training(trainingDataUrl, use_training_labels=False)\n", 51 | "model = poller.result()\n", 52 | "\n", 53 | "print(\"Model ID: {}\".format(model.model_id))\n", 54 | "print(\"Status: {}\".format(model.status))\n", 55 | "print(\"Training started on: {}\".format(model.training_started_on))\n", 56 | "print(\"Training completed on: {}\".format(model.training_completed_on))\n", 57 | "\n", 58 | "print(\"\\nRecognized fields:\")\n", 59 | "for submodel in model.submodels:\n", 60 | " print(\n", 61 | " \"The submodel with form type '{}' has recognized the following fields: {}\".format(\n", 62 | " submodel.form_type,\n", 63 | " \", \".join(\n", 64 | " [\n", 65 | " field.label if field.label else name\n", 66 | " for name, field in submodel.fields.items()\n", 67 | " ]\n", 68 | " ),\n", 69 | " )\n", 70 | " )\n", 71 | "\n", 72 | "# Training result information\n", 73 | "for doc in model.training_documents:\n", 74 | " print(\"Document name: {}\".format(doc.name))\n", 75 | " print(\"Document status: {}\".format(doc.status))\n", 76 | " print(\"Document page count: {}\".format(doc.page_count))\n", 77 | " print(\"Document errors: {}\".format(doc.errors))" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "# Model ID from when you trained your model. Look for the model id in the output from the previous cell\n", 87 | "# Upload the image from the test folder to a storage container and generate a SAS url \n", 88 | "model_id = \"modelid\"\n", 89 | "formUrl = \"test image SAS url\"\n", 90 | "poller = form_recognizer_client.begin_recognize_custom_forms_from_url(\n", 91 | " model_id=model_id, form_url=formUrl)\n", 92 | "result = poller.result()\n", 93 | "\n", 94 | "for recognized_form in result:\n", 95 | " print(\"Form type: {}\".format(recognized_form.form_type))\n", 96 | " for name, field in recognized_form.fields.items():\n", 97 | " print(\"Field '{}' has label '{}' with value '{}' and a confidence score of {}\".format(\n", 98 | " name,\n", 99 | " field.label_data.text if field.label_data else name,\n", 100 | " field.value,\n", 101 | " field.confidence\n", 102 | " ))" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [] 111 | } 112 | ], 113 | "metadata": { 114 | "kernelspec": { 115 | "display_name": "Python 3", 116 | "language": "python", 117 | "name": "python3" 118 | }, 119 | "language_info": { 120 | "codemirror_mode": { 121 | "name": "ipython", 122 | "version": 3 123 | }, 124 | "file_extension": ".py", 125 | "mimetype": "text/x-python", 126 | "name": "python", 127 | "nbconvert_exporter": "python", 128 | "pygments_lexer": "ipython3", 129 | "version": "3.7.4" 130 | } 131 | }, 132 | "nbformat": 4, 133 | "nbformat_minor": 2 134 | } 135 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/AnalyzeInvoice/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | import logging 5 | import datetime 6 | from json import JSONEncoder 7 | import azure.functions as func 8 | from azure.ai.formrecognizer import FormRecognizerClient 9 | from azure.ai.formrecognizer import FormTrainingClient 10 | from azure.core.credentials import AzureKeyCredential 11 | 12 | class DateTimeEncoder(JSONEncoder): 13 | #Override the default method 14 | def default(self, obj): 15 | if isinstance(obj, (datetime.date, datetime.datetime)): 16 | return obj.isoformat() 17 | 18 | def main(req: func.HttpRequest) -> func.HttpResponse: 19 | logging.info('Invoked AnalyzeInvoice Skill.') 20 | try: 21 | body = json.dumps(req.get_json()) 22 | if body: 23 | logging.info(body) 24 | result = compose_response(body) 25 | return func.HttpResponse(result, mimetype="application/json") 26 | else: 27 | return func.HttpResponse( 28 | "Invalid body", 29 | status_code=400 30 | ) 31 | except ValueError: 32 | return func.HttpResponse( 33 | "Invalid body", 34 | status_code=400 35 | ) 36 | def compose_response(json_data): 37 | values = json.loads(json_data)['values'] 38 | 39 | # Prepare the Output before the loop 40 | results = {} 41 | results["values"] = [] 42 | endpoint = os.environ["FORMS_RECOGNIZER_ENDPOINT"] 43 | key = os.environ["FORMS_RECOGNIZER_KEY"] 44 | form_recognizer_client = FormRecognizerClient(endpoint, AzureKeyCredential(key)) 45 | for value in values: 46 | output_record = transform_value(value, form_recognizer_client) 47 | if output_record != None: 48 | results["values"].append(output_record) 49 | return json.dumps(results, ensure_ascii=False, cls=DateTimeEncoder) 50 | 51 | ## Perform an operation on a record 52 | def transform_value(value, form_recognizer_client): 53 | try: 54 | recordId = value['recordId'] 55 | except AssertionError as error: 56 | return None 57 | # Validate the inputs 58 | try: 59 | assert ('data' in value), "'data' field is required." 60 | data = value['data'] 61 | print(data) 62 | form_url = data["formUrl"] + data["formSasToken"] 63 | print(form_url) 64 | poller = form_recognizer_client.begin_recognize_invoices_from_url(form_url) 65 | invoices = poller.result() 66 | invoiceResults = [] 67 | 68 | for idx, invoice in enumerate(invoices): 69 | invoiceResult = {} 70 | amount_due = invoice.fields.get("AmountDue") 71 | if amount_due: 72 | invoiceResult["AmountDue"] = amount_due.value 73 | billing_address = invoice.fields.get("BillingAddress") 74 | if billing_address: 75 | invoiceResult["BillingAddress"] = billing_address.value 76 | billing_address_recipient = invoice.fields.get("BillingAddressRecipient") 77 | if billing_address_recipient: 78 | invoiceResult["BillingAddressRecipient"] = billing_address_recipient.value 79 | customer_address = invoice.fields.get("CustomerAddress") 80 | if customer_address: 81 | invoiceResult["CustomerAddress"] = customer_address.value 82 | customer_address_recipient = invoice.fields.get("CustomerAddressRecipient") 83 | if customer_address_recipient: 84 | invoiceResult["CustomerAddressRecipient"] = customer_address_recipient.value 85 | due_date = invoice.fields.get("DueDate") 86 | if due_date: 87 | invoiceResult["DueDate"] = due_date.value 88 | invoice_date = invoice.fields.get("InvoiceDate") 89 | if invoice_date: 90 | invoiceResult["InvoiceDate"] = invoice_date.value 91 | invoice_id = invoice.fields.get("InvoiceId") 92 | if invoice_id: 93 | invoiceResult["InvoiceId"] = invoice_id.value 94 | invoice_total = invoice.fields.get("InvoiceTotal") 95 | if invoice_total: 96 | invoiceResult["InvoiceTotal"] = invoice_total.value 97 | vendor_address = invoice.fields.get("VendorAddress") 98 | if vendor_address: 99 | invoiceResult["VendorAddress"] = vendor_address.value 100 | vendor_name = invoice.fields.get("VendorName") 101 | if vendor_name: 102 | invoiceResult["VendorName"] = vendor_name.value 103 | sub_total = invoice.fields.get("SubTotal") 104 | if sub_total: 105 | invoiceResult["SubTotal"] = sub_total.value 106 | total_tax = invoice.fields.get("TotalTax") 107 | if sub_total: 108 | invoiceResult["TotalTax"] = total_tax.value 109 | 110 | invoiceResults.append(invoiceResult) 111 | #BillingAddress,BillingAddressRecipient,AmountDue,SubTotal,TotalTax, 112 | 113 | except AssertionError as error: 114 | return ( 115 | { 116 | "recordId": recordId, 117 | "errors": [ { "message": "Error:" + error.args[0] } ] 118 | }) 119 | except Exception as error: 120 | return ( 121 | { 122 | "recordId": recordId, 123 | "errors": [ { "message": "Error:" + str(error) } ] 124 | }) 125 | return ({ 126 | "recordId": recordId, 127 | "data": { 128 | "invoices": invoiceResults 129 | } 130 | }) 131 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Train/Commercial Motor Vehicle - Adatum.json: -------------------------------------------------------------------------------- 1 | Response status code: 200 2 | url: http://51.143.58.183:15678/formrecognizer/v2.0-preview/custom/models/df8d2b93-b0d0-4e25-a8ce-05db392e5b2d/analyzeresults/87e7e141-db63-435b-8cf0-9c8de2c88262 3 | Response body: {"status": "succeeded", "analyzeResult": {"pageResults": [{"page": 1, "clusterId": 0, "keyValuePairs": [{"key": {"text": "vehicle make:", "boundingBox": [0.8875, 3.2292, 2.0, 3.2292, 2.0, 3.3847, 0.8875, 3.3847]}, "value": {"text": "Adatum", "boundingBox": [2.0472, 3.2292, 2.6194, 3.2292, 2.6194, 3.3847, 2.0472, 3.3847]}, "confidence": 0.36}, {"key": {"text": "company vehicle number if any:", "boundingBox": [4.3222, 3.2292, 6.8903, 3.2292, 6.8903, 3.3847, 4.3222, 3.3847]}, "value": {"text": "18270", "boundingBox": [6.9361, 3.2292, 7.4, 3.2292, 7.4, 3.3847, 6.9361, 3.3847]}, "confidence": 0.36}, {"key": {"text": "vehicle year:", "boundingBox": [0.8875, 3.6125, 1.9167, 3.6125, 1.9167, 3.7694, 0.8875, 3.7694]}, "value": {"text": "2011", "boundingBox": [1.9625, 3.6125, 2.3319, 3.6125, 2.3319, 3.7694, 1.9625, 3.7694]}, "confidence": 0.36}, {"key": {"text": "tire size:", "boundingBox": [4.3222, 3.6125, 5.0458, 3.6125, 5.0458, 3.7694, 4.3222, 3.7694]}, "value": {"text": "32X12R16.5", "boundingBox": [5.0931, 3.6125, 6.0181, 3.6125, 6.0181, 3.7694, 5.0931, 3.7694]}, "confidence": 0.36}, {"key": {"text": "vehicle id number:", "boundingBox": [0.8875, 3.9958, 2.3792, 3.9958, 2.3792, 4.1528, 0.8875, 4.1528]}, "value": {"text": "LQVV737", "boundingBox": [2.4264, 3.9958, 3.1472, 3.9958, 3.1472, 4.1528, 2.4264, 4.1528]}, "confidence": 0.36}, {"key": {"text": "owner, if leased:", "boundingBox": [4.3222, 3.9958, 5.6583, 3.9958, 5.6583, 4.1528, 4.3222, 4.1528]}, "value": {"text": "VanArsdel, Ltd.", "boundingBox": [5.7042, 3.9958, 6.8431, 3.9958, 6.8431, 4.1528, 5.7042, 4.1528]}, "confidence": 0.36}, {"key": {"text": "__Tokens__1"}, "value": {"text": "Commercial Motor Vehicle", "boundingBox": [2.7236, 1.0833, 5.8639, 1.0833, 5.8639, 1.3181, 2.7236, 1.3181]}, "confidence": 1.0}, {"key": {"text": "__Tokens__2"}, "value": {"text": "Maintenance Record", "boundingBox": [3.0708, 1.3694, 5.5139, 1.3694, 5.5139, 1.6042, 3.0708, 1.6042]}, "confidence": 1.0}, {"key": {"text": "__Tokens__3"}, "value": {"text": "VEHICLE IDENTIFYING INFORMATION", "boundingBox": [2.4722, 2.5806, 6.0264, 2.5806, 6.0264, 2.7625, 2.4722, 2.7625]}, "confidence": 1.0}], "tables": [{"rows": 7, "columns": 4, "cells": [{"rowIndex": 0, "columnIndex": 0, "isHeader": true, "text": "Date of Maintenance", "boundingBox": [1.0792, 5.2847, 2.0056, 5.2847, 2.0056, 5.6028, 1.0792, 5.6028], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 0, "columnIndex": 1, "isHeader": true, "text": "Operation Performed", "boundingBox": [2.775, 5.2847, 4.3056, 5.2847, 4.3056, 5.4278, 2.775, 5.4278], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 0, "columnIndex": 2, "isHeader": true, "text": "Vehicle Mileage", "boundingBox": [5.2208, 5.2847, 5.7833, 5.2847, 5.7833, 5.6028, 5.2208, 5.6028], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 0, "columnIndex": 3, "isHeader": true, "text": "Date Next Maintenance Due", "boundingBox": [6.1208, 5.2847, 7.3806, 5.2847, 7.3806, 5.6028, 6.1208, 5.6028], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 1, "columnIndex": 0, "isHeader": false, "text": "12/07/2012", "boundingBox": [1.1583, 5.7542, 1.925, 5.7542, 1.925, 5.8986, 1.1583, 5.8986], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 1, "columnIndex": 1, "isHeader": false, "text": "Engine check. Brakes check.", "boundingBox": [2.1542, 5.7542, 4.1167, 5.7542, 4.1167, 5.8986, 2.1542, 5.8986], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 1, "columnIndex": 2, "isHeader": false, "text": "15,837", "boundingBox": [5.2681, 5.7542, 5.7361, 5.7542, 5.7361, 5.8986, 5.2681, 5.8986], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 1, "columnIndex": 3, "isHeader": false, "text": "09/2014", "boundingBox": [6.4736, 5.7542, 7.0264, 5.7542, 7.0264, 5.8986, 6.4736, 5.8986], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 2, "columnIndex": 0, "isHeader": false, "text": "24/08/2014", "boundingBox": [1.1583, 6.1611, 1.9264, 6.1611, 1.9264, 6.3042, 1.1583, 6.3042], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 2, "columnIndex": 1, "isHeader": false, "text": "Oil change.", "boundingBox": [2.1542, 6.1611, 2.9292, 6.1611, 2.9292, 6.3042, 2.1542, 6.3042], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 2, "columnIndex": 2, "isHeader": false, "text": "36,112", "boundingBox": [5.2681, 6.1611, 5.7361, 6.1611, 5.7361, 6.3042, 5.2681, 6.3042], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 2, "columnIndex": 3, "isHeader": false, "text": "09/2016", "boundingBox": [6.4736, 6.1611, 7.0264, 6.1611, 7.0264, 6.3042, 6.4736, 6.3042], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 3, "columnIndex": 0, "isHeader": false, "text": "15/09/2016", "boundingBox": [1.1583, 6.5694, 1.9264, 6.5694, 1.9264, 6.7125, 1.1583, 6.7125], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 3, "columnIndex": 1, "isHeader": false, "text": "Battery Replacement.", "boundingBox": [2.1542, 6.5694, 3.625, 6.5694, 3.625, 6.7125, 2.1542, 6.7125], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 3, "columnIndex": 2, "isHeader": false, "text": "76,172", "boundingBox": [5.2681, 6.5694, 5.7361, 6.5694, 5.7361, 6.7125, 5.2681, 6.7125], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 3, "columnIndex": 3, "isHeader": false, "text": "09/2017", "boundingBox": [6.4736, 6.5694, 7.0264, 6.5694, 7.0264, 6.7125, 6.4736, 6.7125], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 4, "columnIndex": 0, "isHeader": false, "text": "29/09/2017", "boundingBox": [1.1583, 6.9764, 1.9264, 6.9764, 1.9264, 7.1194, 1.1583, 7.1194], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 4, "columnIndex": 1, "isHeader": false, "text": "Steering Review. Oil change.", "boundingBox": [2.1542, 6.9764, 4.125, 6.9764, 4.125, 7.1194, 2.1542, 7.1194], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 4, "columnIndex": 2, "isHeader": false, "text": "98,212", "boundingBox": [5.2681, 6.9764, 5.7361, 6.9764, 5.7361, 7.1194, 5.2681, 7.1194], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 4, "columnIndex": 3, "isHeader": false, "text": "09/2018", "boundingBox": [6.4736, 6.9764, 7.0264, 6.9764, 7.0264, 7.1194, 6.4736, 7.1194], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 5, "columnIndex": 0, "isHeader": false, "text": "25/09/2018", "boundingBox": [1.1583, 7.3833, 1.9264, 7.3833, 1.9264, 7.5264, 1.1583, 7.5264], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 5, "columnIndex": 1, "isHeader": false, "text": "Brakes check.", "boundingBox": [2.1542, 7.3833, 3.1139, 7.3833, 3.1139, 7.5264, 2.1542, 7.5264], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 5, "columnIndex": 2, "isHeader": false, "text": "122,981", "boundingBox": [5.2236, 7.3833, 5.7778, 7.3833, 5.7778, 7.5264, 5.2236, 7.5264], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 5, "columnIndex": 3, "isHeader": false, "text": "09/2019", "boundingBox": [6.4736, 7.3833, 7.0264, 7.3833, 7.0264, 7.5264, 6.4736, 7.5264], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 6, "columnIndex": 0, "isHeader": false, "text": "30/09/2019", "boundingBox": [1.1583, 7.7889, 1.9264, 7.7889, 1.9264, 7.9333, 1.1583, 7.9333], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 6, "columnIndex": 1, "isHeader": false, "text": "Light change.", "boundingBox": [2.1542, 7.7889, 3.0708, 7.7889, 3.0708, 7.9333, 2.1542, 7.9333], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 6, "columnIndex": 2, "isHeader": false, "text": "153,233", "boundingBox": [5.2236, 7.7889, 5.7778, 7.7889, 5.7778, 7.9333, 5.2236, 7.9333], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}, {"rowIndex": 6, "columnIndex": 3, "isHeader": false, "text": "09/2020", "boundingBox": [6.4736, 7.7889, 7.0264, 7.7889, 7.0264, 7.9333, 6.4736, 7.9333], "confidence": 1.0, "columnSpan": 1, "rowSpan": 1}]}]}], "documentResults": [], "errors": [], "version": "2.0.0", "readResults": [{"page": 1, "width": 8.5, "height": 11.0, "angle": 0, "unit": "inch"}]}, "createdDateTime": "2019-10-30T19:56:07.1874046Z", "lastUpdatedDateTime": "2019-10-30T19:56:22Z"} 4 | -------------------------------------------------------------------------------- /Instructions/Exercises/03-composed-model.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Create a composed Document Intelligence model' 4 | module: 'Module 6 - Document Intelligence' 5 | --- 6 | 7 | # Create a composed Document Intelligence model 8 | 9 | In this exercise, you'll create and train two custom models that analyze different tax forms. Then, you'll create a composed model that includes both of these custom models. You'll test the model by submitting a form and you'll check that it recognizes the document type and labeled fields correctly. 10 | 11 | ## Set up resources 12 | 13 | We'll use a script to create the Azure AI Document Intelligence resource, a storage account with sample forms, and a resource group: 14 | 15 | 1. Start Visual Studio Code. 16 | 1. Open the palette (SHIFT+CTRL+P) and run a **Git: Clone** command to clone the `https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence` repository to a local folder (it doesn't matter which folder). 17 | 1. When the repository has been cloned, open the folder in Visual Studio Code. 18 | 19 | > **Note**: If Visual Studio Code shows you a pop-up message to prompt you to trust the code you are opening, click on **Yes, I trust the authors** option in the pop-up. 20 | 21 | > **Note**: If you are prompted to add required assets to build and debug, select **Not Now**. If there are any other pop ups from Visual Studio Code, you can safely dismiss them. 22 | 23 | 1. Expand the **Labfiles** folder in the left pane, and right click on the **03-composed-model** directory. Select the option to open in the integrated terminal, and execute the following script: 24 | 25 | ```powershell 26 | az login --output none 27 | ``` 28 | 29 | > **Note**: If you get an error about no active subscriptions and have MFA enabled, you may need to login to Azure portal at `https://portal.azure.com` first, then rerun the `az login`. 30 | 31 | 1. When prompted, sign into your Azure subscription. Then return to Visual Studio Code and wait for the sign-in process to complete. 32 | 1. In the integrated terminal, run the following command to set up resources: 33 | 34 | ```powershell 35 | ./setup.ps1 36 | ``` 37 | 38 | > **IMPORTANT**: The last resource created by the script is your Azure AI Document Intelligence service. If that command fails due to you already having an F0 tier resource, either use that resource for this lab or create one manually using the S0 tier in the Azure portal. 39 | 40 | ## Create the 1040 Forms custom model 41 | 42 | To create a composed model, we must first create two or more custom models. To create the first custom model: 43 | 44 | 1. In a new browser tab, start the **Azure AI Document Intelligence Studio** at `https://documentintelligence.ai.azure.com/studio` 45 | 1. Scroll down, and then under **Custom models**, select **Custom extraction model**. 46 | 1. If you're asked to sign into your account, use your Azure credentials. 47 | 1. If you're asked which Azure AI Document Intelligence resource to use, select the subscription and resource name you used when you created the Azure AI Document Intelligence resource. 48 | 1. Under **My Projects**, select **+ Create a project**. 49 | 1. In the **Project name** textbox, type **1040 Forms**, and then select **Continue**. 50 | 1. On the **Configure service resource** page, in the **Subscription** drop-down list, select your Azure subscription. 51 | 1. In the **Resource group** drop-down list, select the **DocumentIntelligenceResources<xxxx>** created for you. 52 | 1. In the **Document Intelligence or Cognitive Service Resource** drop-down list, select **DocumentIntelligence<xxxx>**. 53 | 1. In the **API version** drop-down list, ensure that **2024-07-31 (Preview)** is selected and then select **Continue**. 54 | 1. On the **Connect training data source** page, in the **Subscription** drop-down list, select your Azure subscription. 55 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 56 | 1. In the **Storage account** drop-down list, select the only storage account listed. If you have multiple storage accounts in your subscription, choose the one starting with *docintelstorage* 57 | 1. In the **Blob container** drop-down list, select **1040examples**, and then select **Continue**. 58 | 1. In the **Review and create** page, select **Create project**. 59 | 1. Select **Run now** under *Run layout* in the *Start labeling now* pop up, and wait for the analysis to complete. 60 | 61 | ## Label the 1040 Forms custom model 62 | 63 | Now, let's label the fields in the example forms: 64 | 65 | 1. In the **Label data** page, in the top-right of the page, select **+ Add a field**, and then select **Field**. 66 | 1. Type **FirstName** and then press *Enter*. 67 | 1. Select the document called **f1040_1.pdf** on the left list, select **John** and then select **FirstName**. 68 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 69 | 1. Type **LastName** and then press *Enter*. 70 | 1. In the document, select **Doe** and then select **LastName**. 71 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 72 | 1. Type **City** and then press *Enter*. 73 | 1. In the document, select **Los Angeles** and then select **City**. 74 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 75 | 1. Type **State** and then press *Enter*. 76 | 1. In the document, select **CA** and then select **State**. 77 | 1. Repeat the labeling process for the remaining forms in the list on the left, using the labels you created. Label the same four fields: *FirstName*, *LastName*, *City*, and *State*. Notice that one of the documents does not have city or state data. 78 | 79 | > **IMPORTANT** 80 | > For the purposes of this exercise, we're using only five example forms and labeling only four fields. In your real-world models, you should use as many samples as possible to maximize the accuracy and confidence of your predictions. You should also label all the available fields in the forms, rather than just four fields. 81 | 82 | ## Train the 1040 Forms custom model 83 | 84 | Now that the sample forms are labeled, we can train the first custom model: 85 | 86 | 1. In the Azure AI Document Intelligence Studio,on the top right of the screen, select **Train**. 87 | 1. In the **Train a new model** dialog, in the **Model ID** textbox, type **1040FormsModel**. 88 | 1. In the **Build mode** drop-down list, select **Template**, and then select **Train**. 89 | 1. In the **Training in progress** dialog, select **Go to Models**. 90 | 91 | ## Create the 1099 Forms custom model 92 | 93 | Now, you must create a second model, which you'll train on example 1099 tax forms: 94 | 95 | 1. In Azure AI Document Intelligence Studio, select **Custom extraction model**. 96 | 1. Under **My Projects**, select **+ Create a project**. 97 | 1. In the **Project name** textbox, type **1099 Forms**, and then select **Continue**. 98 | 1. On the **Configure service resource** page, in the **Subscription** drop-down list, select your Azure subscription. 99 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 100 | 1. In the **Document Intelligence or Cognitive Service Resource** drop-down list, select **DocumentIntelligence<xxxx>**. 101 | 1. In the **API version** drop-down list, ensure that **2024-07-31 (Preview)** is selected and then select **Continue**. 102 | 1. On the **Connnect training data source** page, in the **Subscription** drop-down list, select your Azure subscription. 103 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 104 | 1. In the **Storage account** drop-down list, select the only storage account listed. 105 | 1. In the **Blob container** drop-down list, select **1099examples**, and then select **Continue**. 106 | 1. In the **Review and create** page, select **Create project**. 107 | 1. Select the drop-down button for **Run layout** and select **Unanalyzed documents**. 108 | 1. Wait for the analysis to complete. 109 | 110 | ## Label the 1099 Forms custom model 111 | 112 | Now, label the example forms with some fields: 113 | 114 | 1. In the **Label data** page, in the top-right of the page, select **+ Add a field**, and then select **Field**. 115 | 1. Type **FirstName** and then press *Enter*. 116 | 1. Select the document called **f1099msc_payer.pdf**, select **John** and then select **FirstName**. 117 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 118 | 1. Type **LastName** and then press *Enter*. 119 | 1. In the document, select **Doe** and then select **LastName**. 120 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 121 | 1. Type **City** and then press *Enter*. 122 | 1. In the document, select **New Haven** and then select **City**. 123 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 124 | 1. Type **State** and then press *Enter*. 125 | 1. In the document, select **CT** and then select **State**. 126 | 1. Repeat the labeling process for the remaining forms in the list on the left. Label the same four fields: *FirstName*, *LastName*, *City*, and *State*. Notice that two of the documents don't have any name data to label. 127 | 128 | ## Train the 1099 Forms custom model 129 | 130 | You can now train the second custom model: 131 | 132 | 1. In the Azure AI Document Intelligence Studio,on the top right, select **Train**. 133 | 1. In the **Train a new model** dialog, in the **Model ID** textbox, type **1099FormsModel**. 134 | 1. In the **Build mode** drop-down list, select **Template**, and then select **Train**. 135 | 1. In the **Training in progress** dialog, select **Go to Models**. 136 | 1. The training process can take a few minutes. Refresh the browser occasionally until both models display the **succeeded** status. 137 | 138 | ## Use the model 139 | 140 | Now that the model is complete, let's test it with an example form: 141 | 142 | 1. In the Azure AI Document Intelligence Studio, select the **Models** page, select the **1040FormsModel**. 143 | 1. Select **Test**. 144 | 1. Select **Browse for files** and then browse to the location where you cloned the repository. 145 | 1. Select **03-composed-model/trainingdata/TestDoc/f1040_7.pdf**, and then select **Open**. 146 | 1. Select **Run analysis**. Azure AI Document Intelligence analyses the form by using the composed model. 147 | 1. The document you analyzed is an example of the 1040 tax form. Check the **DocType** property to see if the correct custom model has been used. Also check the **FirstName**, **LastName**, **City**, and **State** values identified by the model. 148 | 149 | ## Clean up resources 150 | 151 | Now that you've seen how composed models work, let's remove the resources you created in your Azure subscription. 152 | 153 | 1. In the **Azure portal** at `https://portal.azure.com/`, select **Resource groups**. 154 | 1. In the list of **Resource groups**, select the **DocumentIntelligenceResources<xxxx>** that you created, and then select **Delete resource group**. 155 | 1. In the **TYPE THE RESOURCE GROUP NAME** textbox, type the name of the resource group, and then select **Delete** to delete the Document Intelligence resource and the storage account. 156 | 157 | ## Learn more 158 | 159 | - [Compose custom models](/azure/ai-services/document-intelligence/concept-composed-models) 160 | -------------------------------------------------------------------------------- /Instructions/Exercises/01-use-prebuilt-models.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Use prebuilt Document Intelligence models' 4 | module: 'Module 6 - Document Intelligence' 5 | --- 6 | 7 | # Use prebuilt Document Intelligence models 8 | 9 | In this exercise, you'll set up an Azure AI Document Intelligence resource in your Azure subscription. You'll use both the Azure AI Document Intelligence Studio and C# or Python to submit forms to that resource for analysis. 10 | 11 | ## Create an Azure AI Document Intelligence resource 12 | 13 | Before you can call the Azure AI Document Intelligence service, you must create a resource to host that service in Azure: 14 | 15 | 1. In a browser tab, open the Azure portal at [https://portal.azure.com](https://portal.azure.com?azure-portal=true), signing in with the Microsoft account associated with your Azure subscription. 16 | 1. On the Azure portal home page, navigate to the top search box and type **Document Intelligence** and then press **Enter**. 17 | 1. On the **Document Intelligence** page, select **Create Document Intelligence**. 18 | 1. On the **Create Document Intelligence** page, use the following to configure your resource: 19 | - **Subscription**: Your Azure subscription. 20 | - **Resource group**: Select or create a resource group with a unique name such as *DocIntelligenceResources*. 21 | - **Region**: select a region near you. 22 | - **Name**: Enter a globally unique name. 23 | - **Pricing tier**: select **Free F0** (if you don't have a Free tier available, select **Standard S0**). 24 | 1. Then select **Review + create**, and **Create**. Wait while Azure creates the Azure AI Document Intelligence resource. 25 | 1. When the deployment is complete, select **Go to resource**. Keep this page open for the rest of this exercise. 26 | 27 | ## Use the Read model 28 | 29 | Let's start by using the **Azure AI Document Intelligence Studio** and the Read model to analyze a document with multiple languages. You'll connect Azure AI Document Intelligence Studio to the resource you just created to perform the analysis: 30 | 31 | 1. Open a new browser tab and go to the **Azure AI Document Intelligence Studio** at [https://documentintelligence.ai.azure.com/studio](https://documentintelligence.ai.azure.com/studio). 32 | 1. Under **Document Analysis**, select the **Read** tile. 33 | 1. If you are asked to sign into your account, use your Azure credentials. 34 | 1. If you are asked which Azure AI Document Intelligence resource to use, select the subscription and resource name you used when you created the Azure AI Document Intelligence resource. 35 | 1. In the list of documents on the left, select **read-german.pdf**. 36 | 37 | ![Screenshot showing the Read page in Azure AI Document Intelligence Studio.](../media/read-german-sample.png#lightbox) 38 | 39 | 1. At the top-left, select **Analyze options**, then enable the **Language** check-box (under **Optional detection**) in the **Analyze options** pane and click on **Save**. 40 | 1. At the top-left, select **Run Analysis**. 41 | 1. When the analysis is complete, the text extracted from the image is shown on the right in the **Content** tab. Review this text and compare it to the text in the original image for accuracy. 42 | 1. Select the **Result** tab. This tab displays the extracted JSON code. 43 | 1. Scroll to the bottom of the JSON code in the **Result** tab. Notice that the read model has detected the language of each span indicated by `locale`. Most spans are in German (language code `de`) but you can find other language codes in the spans (e.g. English - language code `en` - in one of the last span). 44 | 45 | ![Screenshot showing the detection of language for two spans in the results from the read model in Azure AI Document Intelligence Studio.](../media/language-detection.png#lightbox) 46 | 47 | ## Prepare to develop an app in Visual Studio Code 48 | 49 | Now let's explore the app that uses the Azure Document Intelligence service SDK. You'll develop your app using Visual Studio Code. The code files for your app have been provided in a GitHub repo. 50 | 51 | > **Tip**: If you have already cloned the **mslearn-ai-document-intelligence** repo, open it in Visual Studio code. Otherwise, follow these steps to clone it to your development environment. 52 | 53 | 1. Start Visual Studio Code. 54 | 1. Open the palette (SHIFT+CTRL+P) and run a **Git: Clone** command to clone the `https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence` repository to a local folder (it doesn't matter which folder). 55 | 1. When the repository has been cloned, open the folder in Visual Studio Code. 56 | 57 | > **Note**: If Visual Studio Code shows you a pop-up message to prompt you to trust the code you are opening, click on **Yes, I trust the authors** option in the pop-up. 58 | 59 | 1. Wait while additional files are installed to support the C# code projects in the repo. 60 | 61 | > **Note**: If you are prompted to add required assets to build and debug, select **Not Now**. If you are prompted with the Message *Detected an Azure Function Project in folder*, you can safely close that message. 62 | 63 | ## Configure your application 64 | 65 | Applications for both C# and Python have been provided, as well as a sample pdf file you'll use to test Document Intelligence. Both apps feature the same functionality. First, you'll complete some key parts of the application to enable using your Azure Document Intelligence resource. 66 | 67 | 1. Examine the following invoice and note some of its fields and values. This is the invoice that your code will analyze. 68 | 69 | ![Screenshot showing a sample invoice document.](../media/sample-invoice.png#lightbox) 70 | 71 | 1. In Visual Studio Code, in the **Explorer** pane, browse to the **Labfiles/01-prebuild-models** folder and expand the **CSharp** or **Python** folder depending on your language preference. Each folder contains the language-specific files for an app into which you're you're going to integrate Azure Document Intelligence functionality. 72 | 73 | 1. Right-click the **CSharp** or **Python** folder containing your code files and select **Open an integrated terminal**. Then install the Azure Form Recognizer (the previous name for Document Intelligence) SDK package by running the appropriate command for your language preference: 74 | 75 | **C#**: 76 | 77 | ```powershell 78 | dotnet add package Azure.AI.FormRecognizer --version 4.1.0 79 | ``` 80 | 81 | **Python**: 82 | 83 | ```powershell 84 | pip install azure-ai-formrecognizer==3.3.3 85 | ``` 86 | 87 | ## Add code to use the Azure Document Intelligence service 88 | 89 | Now you're ready to use the SDK to evaluate the pdf file. 90 | 91 | 1. Switch to the browser tab that displays the Azure AI Document Intelligence overview in the Azure portal. On the left pane, under *Resource Management*, select **Keys and Endpoint**. To the right of the **Endpoint** value, click the **Copy to clipboard** button. 92 | 1. In the **Explorer** pane, in the **CSharp** or **Python** folder, open the code file for your preferred language, and replace `` with the string you just copied: 93 | 94 | **C#**: ***Program.cs*** 95 | 96 | ```csharp 97 | string endpoint = ""; 98 | ``` 99 | 100 | **Python**: ***document-analysis.py*** 101 | 102 | ```python 103 | endpoint = "" 104 | ``` 105 | 106 | 1. Switch to the browser tab that displays the Azure AI Document Intelligence **Keys and Endpoint** in the Azure portal. To the right of the **KEY 1** value, click the *Copy to clipboard** button. 107 | 1. In the code file in Visual Studio Code, locate this line and replace `` with the string you just copied: 108 | 109 | **C#** 110 | 111 | ```csharp 112 | string apiKey = ""; 113 | ``` 114 | 115 | **Python** 116 | 117 | ```python 118 | key = "" 119 | ``` 120 | 121 | 1. Locate the comment `Create the client`. Following that, on new lines, enter the following code: 122 | 123 | **C#** 124 | 125 | ```csharp 126 | var cred = new AzureKeyCredential(apiKey); 127 | var client = new DocumentAnalysisClient(new Uri(endpoint), cred); 128 | ``` 129 | 130 | **Python** 131 | 132 | ```python 133 | document_analysis_client = DocumentAnalysisClient( 134 | endpoint=endpoint, credential=AzureKeyCredential(key) 135 | ) 136 | ``` 137 | 138 | 1. Locate the comment `Analyze the invoice`. Following that, on new lines, enter the following code: 139 | 140 | **C#** 141 | 142 | ```csharp 143 | AnalyzeDocumentOperation operation = await client.AnalyzeDocumentFromUriAsync(WaitUntil.Completed, "prebuilt-invoice", fileUri); 144 | ``` 145 | 146 | **Python** 147 | 148 | ```python 149 | poller = document_analysis_client.begin_analyze_document_from_url( 150 | fileModelId, fileUri, locale=fileLocale 151 | ) 152 | ``` 153 | 154 | 1. Locate the comment `Display invoice information to the user`. Following that, on news lines, enter the following code: 155 | 156 | **C#** 157 | 158 | ```csharp 159 | AnalyzeResult result = operation.Value; 160 | 161 | foreach (AnalyzedDocument invoice in result.Documents) 162 | { 163 | if (invoice.Fields.TryGetValue("VendorName", out DocumentField? vendorNameField)) 164 | { 165 | if (vendorNameField.FieldType == DocumentFieldType.String) 166 | { 167 | string vendorName = vendorNameField.Value.AsString(); 168 | Console.WriteLine($"Vendor Name: '{vendorName}', with confidence {vendorNameField.Confidence}."); 169 | } 170 | } 171 | ``` 172 | 173 | **Python** 174 | 175 | ```python 176 | receipts = poller.result() 177 | 178 | for idx, receipt in enumerate(receipts.documents): 179 | 180 | vendor_name = receipt.fields.get("VendorName") 181 | if vendor_name: 182 | print(f"\nVendor Name: {vendor_name.value}, with confidence {vendor_name.confidence}.") 183 | ``` 184 | 185 | > [!NOTE] 186 | > You've added code to display the vendor name. The starter project also includes code to display the *customer name* and *invoice total*. 187 | 188 | 1. Save the changes to the code file. 189 | 190 | 1. In the interactive terminal pane, ensure the folder context is the folder for your preferred language. Then enter the following command to run the application. 191 | 192 | 1. ***For C# only***, to build your project, enter this command: 193 | 194 | **C#**: 195 | 196 | ```powershell 197 | dotnet build 198 | ``` 199 | 200 | 1. To run your code, enter this command: 201 | 202 | **C#**: 203 | 204 | ```powershell 205 | dotnet run 206 | ``` 207 | 208 | **Python**: 209 | 210 | ```powershell 211 | python document-analysis.py 212 | ``` 213 | 214 | The program displays the vendor name, customer name, and invoice total with confidence levels. Compare the values it reports with the sample invoice you opened at the start of this section. 215 | 216 | ## Clean up 217 | 218 | If you're done with your Azure resource, remember to delete the resource in the [Azure portal](https://portal.azure.com/?azure-portal=true) to avoid further charges. 219 | 220 | ## More information 221 | 222 | For more information about the Document Intelligence service, see the [Document Intelligence documentation](https://learn.microsoft.com/azure/ai-services/document-intelligence/?azure-portal=true). 223 | -------------------------------------------------------------------------------- /Instructions/Exercises/02-custom-document-intelligence.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Extract Data from Forms' 4 | module: 'Module 6 - Document Intelligence' 5 | --- 6 | 7 | # Extract Data from Forms 8 | 9 | Suppose a company currently requires employees to manually purchase order sheets and enter the data into a database. They would like you to utilize AI services to improve the data entry process. You decide to build a machine learning model that will read the form and produce structured data that can be used to automatically update a database. 10 | 11 | **Azure AI Document Intelligence** is an Azure AI service that enables users to build automated data processing software. This software can extract text, key/value pairs, and tables from form documents using optical character recognition (OCR). Azure AI Document Intelligence has pre-built models for recognizing invoices, receipts, and business cards. The service also provides the capability to train custom models. In this exercise, we will focus on building custom models. 12 | 13 | ## Prepare to develop an app in Visual Studio Code 14 | 15 | Now let's use the service SDK to develop an app using Visual Studio Code. The code files for your app have been provided in a GitHub repo. 16 | 17 | > **Tip**: If you have already cloned the **mslearn-ai-document-intelligence** repo, open it in Visual Studio code. Otherwise, follow these steps to clone it to your development environment. 18 | 19 | 1. Start Visual Studio Code. 20 | 1. Open the palette (SHIFT+CTRL+P) and run a **Git: Clone** command to clone the `https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence` repository to a local folder (it doesn't matter which folder). 21 | 1. When the repository has been cloned, open the folder in Visual Studio Code. 22 | 23 | > **Note**: If Visual Studio Code shows you a pop-up message to prompt you to trust the code you are opening, click on **Yes, I trust the authors** option in the pop-up. 24 | 25 | 1. Wait while additional files are installed to support the C# code projects in the repo. 26 | 27 | > **Note**: If you are prompted to add required assets to build and debug, select **Not Now**. If you are prompted with the Message *Detected an Azure Function Project in folder*, you can safely close that message. 28 | 29 | ## Create a Azure AI Document Intelligence resource 30 | 31 | To use the Azure AI Document Intelligence service, you need a Azure AI Document Intelligence or Azure AI Services resource in your Azure subscription. You'll use the Azure portal to create a resource. 32 | 33 | 1. In a browser tab, open the Azure portal at `https://portal.azure.com`, signing in with the Microsoft account associated with your Azure subscription. 34 | 1. On the Azure portal home page, navigate to the top search box and type **Document Intelligence** and then press **Enter**. 35 | 1. On the **Document Intelligence** page, select **Create**. 36 | 1. On the **Create Document Intelligence** page, use the following to configure your resource: 37 | - **Subscription**: Your Azure subscription. 38 | - **Resource group**: Select or create a resource group with a unique name such as *DocIntelligenceResources*. 39 | - **Region**: select a region near you. 40 | - **Name**: Enter a globally unique name. 41 | - **Pricing tier**: select **Free F0** (if you don't have a Free tier available, select **Standard S0**). 42 | 1. Then select **Review + create**, and **Create**. Wait while Azure creates the Azure AI Document Intelligence resource. 43 | 1. When the deployment is complete, select **Go to resource** to view the resource's **Overview** page. 44 | 45 | ## Gather documents for training 46 | 47 | You'll use the sample forms such as this one to train a test a model: 48 | 49 | ![An image of an invoice used in this project.](../media/Form_1.jpg) 50 | 51 | 1. Return to **Visual Studio Code**. In the *Explorer* pane, open the **Labfiles/02-custom-document-intelligence** folder and expand the **sample-forms** folder. Notice there are files ending in **.json** and **.jpg** in the folder. 52 | 53 | You will use the **.jpg** files to train your model. 54 | 55 | The **.json** files have been generated for you and contain label information. The files will be uploaded into your blob storage container alongside the forms. 56 | 57 | You can view the images we are using in the *sample-forms* folder by selecting them on Visual Studio Code. 58 | 59 | 1. Return to the **Azure portal** and navigate to your resource's **Overview** page if you're not already there. Under the *Essentials* section, view the **Resource group** in which you created the Document Intelligence resource. 60 | 61 | 1. On the **Overview** page for your resource group, note the **Subscription ID** and **Location**. You will need these values, along with your **resource group** name in subsequent steps. 62 | 63 | ![An example of the resource group page.](../media/resource_group_variables.png) 64 | 65 | 1. In Visual Studio Code, in the *Explorer* pane, browse to the **Labfiles/02-custom-document-intelligence** folder and expand the **CSharp** or **Python** folder depending on your language preference. Each folder contains the language-specific files for an app into which you're you're going to integrate Azure OpenAI functionality. 66 | 67 | 1. Right-click the **CSharp** or **Python** folder containing your code files and select **Open an integrated terminal**. 68 | 69 | 1. In the terminal, run the following command to login to Azure. The **az login** command will open the Microsoft Edge browser, login with the same account you used to create the Azure AI Document Intelligence resource. Once you are logged in, close the browser window. 70 | 71 | ```powershell 72 | az login 73 | ``` 74 | 75 | 1. Return to Visual Studio Code. In the terminal pane, run the following command to list the Azure locations. 76 | 77 | ```powershell 78 | az account list-locations -o table 79 | ``` 80 | 81 | 1. In the output, find the **Name** value that corresponds with the location of your resource group (for example, for *East US* the corresponding name is *eastus*). 82 | 83 | > **Important**: Record the **Name** value and use it in Step 11. 84 | 85 | 1. In Visual Studio Code, in the **Labfiles/02-custom-document-intelligence** folder, select **setup.cmd**. You will use this script to run the Azure command line interface (CLI) commands required to create the other Azure resources you need. 86 | 87 | 1. In the **setup.cmd** script, review the commands. The program will: 88 | - Create a storage account in your Azure resource group 89 | - Upload files from your local *sampleforms* folder to a container called *sampleforms* in the storage account 90 | - Print a Shared Access Signature URI 91 | 92 | 1. Modify the **subscription_id**, **resource_group**, and **location** variable declarations with the appropriate values for the subscription, resource group, and location name where you deployed the Document Intelligence resource. 93 | Then **save** your changes. 94 | 95 | Leave the **expiry_date** variable as it is for the exercise. This variable is used when generating the Shared Access Signature (SAS) URI. In practice, you will want to set an appropriate expiry date for your SAS. You can learn more about SAS [here](https://docs.microsoft.com/azure/storage/common/storage-sas-overview#how-a-shared-access-signature-works). 96 | 97 | 1. In the terminal for the **Labfiles/02-custom-document-intelligence** folder, enter the following command to run the script: 98 | 99 | ```PowerShell 100 | ./setup.cmd 101 | ``` 102 | 103 | 1. When the script completes, review the displayed output. 104 | 105 | 1. In the Azure portal, refresh your resource group and verify that it contains the Azure Storage account just created. Open the storage account and in the pane on the left, select **Storage browser**. Then in Storage Browser, expand **Blob containers** and select the **sampleforms** container to verify that the files have been uploaded from your local **02-custom-document-intelligence/sample-forms** folder. 106 | 107 | ## Train the model using Document Intelligence Studio 108 | 109 | Now you will train the model using the files uploaded to the storage account. 110 | 111 | 1. In your browser, navigate to the Document Intelligence Studio at `https://documentintelligence.ai.azure.com/studio`. 112 | 1. Scroll down to the **Custom models** section and select the **Custom extraction model** tile. 113 | 1. If you are asked to sign into your account, use your Azure credentials. 114 | 1. If you are asked which Azure AI Document Intelligence resource to use, select the subscription and resource name you used when you created the Azure AI Document Intelligence resource. 115 | 1. Under **My Projects**, select **Create a project**. Use the following configurations: 116 | 117 | - **Project name**: Enter a unique name. 118 | - Select *Continue*. 119 | - **Configure service resource**: Select the subscription, resource group, and document intelligence resource you created previously in this lab. Check the *Set as default* box. Keep the default API version. 120 | - Select *Continue*. 121 | - **Connect training data source**: Select the subscription, resource group, and storage account that was created by the setup script. Check the *Set as default* box. Select the `sampleforms` blob container, and leave the folder path blank. 122 | - Select *Continue*. 123 | - Select *Create project* 124 | 125 | 1. Once your project is created, on the top right of the screen, select **Train** to train your model. Use the following configurations: 126 | - **Model ID**: *Provide a globally unique name (you'll need the model ID name in the next step)*. 127 | - **Build Mode**: Template. 128 | 1. Select **Go to Models**. 129 | 1. Training can take some time. You'll see a notification when it's complete. 130 | 131 | ## Test your custom Document Intelligence model 132 | 133 | 1. Return to Visual Studio Code. In the terminal, install the Document Intelligence package by running the appropriate command for your language preference: 134 | 135 | **C#**: 136 | 137 | ```powershell 138 | dotnet add package Azure.AI.FormRecognizer --version 4.1.0 139 | ``` 140 | 141 | **Python**: 142 | 143 | ```powershell 144 | pip install azure-ai-formrecognizer==3.3.3 145 | ``` 146 | 147 | 1. In Visual Studio Code, in the **Labfiles/02-custom-document-intelligence** folder, select the language you are using. Edit the configuration file (**appsettings.json** or **.env**, depending on your language preference) with the following values: 148 | - Your Document Intelligence endpoint. 149 | - Your Document Intelligence key. 150 | - The Model ID generated you provided when training your model. You can find this on the **Models** page of the Document Intelligence Studio. **Save** your changes. 151 | 152 | 1. In Visual Studio Code, open the code file for your client application (*Program.cs* for C#, *test-model.py* for Python) and review the code it contains, particularly that the image in the URL refers to the file in this GitHub repo on the web. 153 | 154 | 1. Return the terminal, and enter the following command to run the program: 155 | 156 | **C#** 157 | 158 | ```powershell 159 | dotnet build 160 | dotnet run 161 | ``` 162 | 163 | **Python** 164 | 165 | ```powershell 166 | python test-model.py 167 | ``` 168 | 169 | 1. View the output and observe how the output for the model provides field names like `Merchant` and `CompanyPhoneNumber`. 170 | 171 | ## Clean up 172 | 173 | If you're done with your Azure resource, remember to delete the resource in the [Azure portal](https://portal.azure.com/?azure-portal=true) to avoid further charges. 174 | 175 | ## More information 176 | 177 | For more information about the Document Intelligence service, see the [Document Intelligence documentation](https://learn.microsoft.com/azure/ai-services/document-intelligence/?azure-portal=true). 178 | -------------------------------------------------------------------------------- /Instructions/Labs/02-custom-document-intelligence.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Extract Data from Forms' 4 | module: 'Module 6 - Document Intelligence' 5 | --- 6 | 7 | # Extract Data from Forms 8 | 9 | Suppose a company currently requires employees to manually purchase order sheets and enter the data into a database. They would like you to utilize AI services to improve the data entry process. You decide to build a machine learning model that will read the form and produce structured data that can be used to automatically update a database. 10 | 11 | **Azure AI Document Intelligence** is an Azure AI service that enables users to build automated data processing software. This software can extract text, key/value pairs, and tables from form documents using optical character recognition (OCR). Azure AI Document Intelligence has pre-built models for recognizing invoices, receipts, and business cards. The service also provides the capability to train custom models. In this exercise, we will focus on building custom models. 12 | 13 | ## Create a Azure AI Document Intelligence resource 14 | 15 | To use the Azure AI Document Intelligence service, you need a Azure AI Document Intelligence or Azure AI Services resource in your Azure subscription. You'll use the Azure portal to create a resource. 16 | 17 | 1. In a browser tab, open the Azure portal at `https://portal.azure.com`, signing in with the Microsoft account associated with your Azure subscription. 18 | 1. On the Azure portal home page, navigate to the top search box and type **Document Intelligence** and then press **Enter**. 19 | 1. On the **Document Intelligence** page, select **Create**. 20 | 1. On the **Create Document Intelligence** page, use the following to configure your resource: 21 | - **Subscription**: Your Azure subscription. 22 | - **Resource group**: Select or create a resource group with a unique name such as *DocIntelligenceResources*. 23 | - **Region**: select a region near you. 24 | - **Name**: Enter a globally unique name. 25 | - **Pricing tier**: select **Free F0** (if you don't have a Free tier available, select **Standard S0**). 26 | 1. Then select **Review + create**, and **Create**. Wait while Azure creates the Azure AI Document Intelligence resource. 27 | 1. When the deployment is complete, select **Go to resource** to view the resource's **Overview** page. 28 | 29 | ## Prepare to develop an app in Cloud Shell 30 | 31 | You'll develop your text translation app using Cloud Shell. The code files for your app have been provided in a GitHub repo. 32 | 33 | > **Tip**: If you have already cloned the **mslearn-ai-document-intelligence** repo, you can skip this task. Otherwise, follow these steps to clone it to your development environment. 34 | 35 | 1. In the Azure Portal, use the **[\>_]** button to the right of the search bar at the top of the page to create a new Cloud Shell in the Azure portal, selecting a ***PowerShell*** environment. The cloud shell provides a command line interface in a pane at the bottom of the Azure portal. 36 | 37 | > **Note**: If you have previously created a cloud shell that uses a *Bash* environment, switch it to ***PowerShell***. 38 | 39 | 1. In the cloud shell toolbar, in the **Settings** menu, select **Go to Classic version** (this is required to use the code editor). 40 | 41 | > **Tip**: As you paste commands into the cloudshell, the ouput may take up a large amount of the screen buffer. You can clear the screen by entering the `cls` command to make it easier to focus on each task. 42 | 43 | 1. In the PowerShell pane, enter the following commands to clone the GitHub repo for this exercise: 44 | 45 | ``` 46 | rm -r mslearn-ai-document-intelligence -f 47 | git clone https://github.com/microsoftlearning/mslearn-ai-document-intelligence mslearn-ai-document-intelligence 48 | ``` 49 | 50 | 1. After the repo has been cloned, navigate to the folder containing the application code files: 51 | 52 | ``` 53 | cd mslearn-ai-document-intelligence/Labfiles/02-custom-document-intelligence 54 | ``` 55 | 56 | ## Gather documents for training 57 | 58 | You'll use the sample forms such as this one to train a test a model: 59 | 60 | ![An image of an invoice used in this project.](../media/Form_1.jpg) 61 | 62 | 1. In the command line, run `ls ./sample-forms` to list the content in the **sample-forms** folder. Notice there are files ending in **.json** and **.jpg** in the folder. 63 | 64 | You will use the **.jpg** files to train your model. 65 | 66 | The **.json** files have been generated for you and contain label information. The files will be uploaded into your blob storage container alongside the forms. 67 | 68 | 1. Return to the **Azure portal** and navigate to your resource's **Overview** page if you're not already there. Under the *Essentials* section, view the **Resource group** in which you created the Document Intelligence resource. 69 | 70 | 1. On the **Overview** page for your resource group, note the **Subscription ID** and **Location**. You will need these values, along with your **resource group** name in subsequent steps. 71 | 72 | ![An example of the resource group page.](../media/resource_group_variables.png) 73 | 74 | 1. In the command line, run the following command to list the Azure locations. 75 | 76 | ```powershell 77 | az account list-locations -o table 78 | ``` 79 | 80 | 1. In the output, find the **Name** value that corresponds with the location of your resource group (for example, for *East US* the corresponding name is *eastus*). 81 | 82 | > **Important**: Record the **Name** value and use it in Step 11. 83 | 84 | 1. Run the command `code setup.sh` to open **setup.sh** in a code editor. You will use this script to run the Azure command line interface (CLI) commands required to create the other Azure resources you need. 85 | 86 | 1. In the **setup.sh** script, review the commands. The program will: 87 | - Create a storage account in your Azure resource group 88 | - Upload files from your local *sampleforms* folder to a container called *sampleforms* in the storage account 89 | - Print a Shared Access Signature URI 90 | 91 | 1. Modify the **subscription_id**, **resource_group**, and **location** variable declarations with the appropriate values for the subscription, resource group, and location name where you deployed the Document Intelligence resource. 92 | 93 | Leave the **expiry_date** variable as it is for the exercise. This variable is used when generating the Shared Access Signature (SAS) URI. In practice, you will want to set an appropriate expiry date for your SAS. You can learn more about SAS [here](https://docs.microsoft.com/azure/storage/common/storage-sas-overview#how-a-shared-access-signature-works). 94 | 95 | 1. After you've replaced the placeholders, within the code editor, use the **CTRL+S** command or **Right-click > Save** to save your changes and then use the **CTRL+Q** command or **Right-click > Quit** to close the code editor while keeping the cloud shell command line open. 96 | 97 | 1. Enter the following commands to make the script executable and to run it: 98 | 99 | ```PowerShell 100 | chmod +x ./setup.sh 101 | ./setup.sh 102 | ``` 103 | 104 | 1. When the script completes, review the displayed output. 105 | 106 | 1. In the Azure portal, refresh your resource group and verify that it contains the Azure Storage account just created. Open the storage account and in the pane on the left, select **Storage browser**. Then in Storage Browser, expand **Blob containers** and select the **sampleforms** container to verify that the files have been uploaded from your local **02-custom-document-intelligence/sample-forms** folder. 107 | 108 | ## Train the model using Document Intelligence Studio 109 | 110 | Now you will train the model using the files uploaded to the storage account. 111 | 112 | 1. In your browser, navigate to the Document Intelligence Studio at `https://documentintelligence.ai.azure.com/studio`. 113 | 1. Scroll down to the **Custom models** section and select the **Custom extraction model** tile. 114 | 1. If you are asked to sign into your account, use your Azure credentials. 115 | 1. If you are asked which Azure AI Document Intelligence resource to use, select the subscription and resource name you used when you created the Azure AI Document Intelligence resource. 116 | 1. Under **My Projects**, select **Create a project**. Use the following configurations: 117 | 118 | - **Project name**: Enter a unique name. 119 | - Select *Continue*. 120 | - **Configure service resource**: Select the subscription, resource group, and document intelligence resource you created previously in this lab. Check the *Set as default* box. Keep the default API version. 121 | - Select *Continue*. 122 | - **Connect training data source**: Select the subscription, resource group, and storage account that was created by the setup script. Check the *Set as default* box. Select the `sampleforms` blob container, and leave the folder path blank. 123 | - Select *Continue*. 124 | - Select *Create project* 125 | 126 | 1. Once your project is created, on the top right of the screen, select **Train** to train your model. Use the following configurations: 127 | - **Model ID**: *Provide a globally unique name (you'll need the model ID name in the next step)*. 128 | - **Build Mode**: Template. 129 | 1. Select **Go to Models**. 130 | 1. Training can take some time. You'll see a notification when it's complete. 131 | 132 | ## Test your custom Document Intelligence model 133 | 134 | 1. Return to the Azure Portal. In the command line, run `cd C-Sharp` or `cd Python` depending on your language preference. Each folder contains the language-specific files for an app into which you're you're going to integrate Azure OpenAI functionality. 135 | 1. In the command line, install the Document Intelligence package by running the appropriate command for your language preference: 136 | 137 | **C#**: 138 | 139 | ```powershell 140 | dotnet add package Azure.AI.FormRecognizer --version 4.1.0 141 | ``` 142 | 143 | **Python**: 144 | 145 | ```powershell 146 | pip install dotenv azure-ai-formrecognizer==3.3.3 147 | ``` 148 | 149 | 1. Using the `ls` command, you can view the contents of the **C-Sharp** or **Python** folder. Note that it contains a file for configuration settings: 150 | 151 | - **C#**: appsettings.json 152 | - **Python**: .env 153 | 154 | 1. Enter the following command to edit the configuration file that has been provided: 155 | 156 | **C#** 157 | 158 | ``` 159 | code appsettings.json 160 | ``` 161 | 162 | **Python** 163 | 164 | ``` 165 | code .env 166 | ``` 167 | 168 | 1. Edit the configuration file with the following values: 169 | - Your Document Intelligence endpoint. 170 | - Your Document Intelligence key. 171 | - The Model ID generated you provided when training your model. You can find this on the **Models** page of the Document Intelligence Studio. **Save** your changes and close the code editor. 172 | 173 | 1. Open the code file for your client application (`code Program.cs` for C#, `code test-model.py` for Python) and review the code it contains, particularly that the image in the URL refers to the file in this GitHub repo on the web. Close the file without making any changes. 174 | 175 | 1. In the command line, and enter the following command to run the program: 176 | 177 | **C#** 178 | 179 | ```powershell 180 | dotnet build 181 | dotnet run 182 | ``` 183 | 184 | **Python** 185 | 186 | ```powershell 187 | python test-model.py 188 | ``` 189 | 190 | 1. View the output and observe how the output for the model provides field names like `Merchant` and `CompanyPhoneNumber`. 191 | 192 | ## Clean up 193 | 194 | If you're done with your Azure resource, remember to delete the resource in the [Azure portal](https://portal.azure.com/?azure-portal=true) to avoid further charges. 195 | 196 | ## More information 197 | 198 | For more information about the Document Intelligence service, see the [Document Intelligence documentation](https://learn.microsoft.com/azure/ai-services/document-intelligence/?azure-portal=true). 199 | -------------------------------------------------------------------------------- /Instructions/Labs/01-use-prebuilt-models.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Use prebuilt Document Intelligence models' 4 | module: 'Module 6 - Document Intelligence' 5 | --- 6 | 7 | # Use prebuilt Document Intelligence models 8 | 9 | In this exercise, you'll set up an Azure AI Foundry project with all the necessary resources for document analysis. You'll use both the Azure AI Foundry and C# or Python to submit forms to that resource for analysis. 10 | 11 | ## Create an Azure AI Foundry project 12 | 13 | Let's start by creating an Azure AI Foundry project. 14 | 15 | 1. In a web browser, open the [Azure AI Foundry portal](https://ai.azure.com) at `https://ai.azure.com` and sign in using your Azure credentials. Close any tips or quick start panes that are opened the first time you sign in, and if necessary use the **Azure AI Foundry** logo at the top left to navigate to the home page, which looks similar to the following image: 16 | 17 | ![Screenshot of Azure AI Foundry portal.](../media/ai-foundry-home.png) 18 | 19 | 1. In the home page, select **+ Create project**. 20 | 1. In the **Create a project** wizard, enter a suitable project name for (for example, `my-ai-project`) then review the Azure resources that will be automatically created to support your project. 21 | 1. Select **Customize** and specify the following settings for your hub: 22 | - **Hub name**: *A unique name - for example `my-ai-hub`* 23 | - **Subscription**: *Your Azure subscription* 24 | - **Resource group**: *Create a new resource group with a unique name (for example, `my-ai-resources`), or select an existing one* 25 | - **Location**: Choose any available region 26 | - **Connect Azure AI Services or Azure OpenAI**: *Create a new AI Services resource with an appropriate name (for example, `my-ai-services`) or use an existing one* 27 | - **Connect Azure AI Search**: Skip connecting 28 | 29 | 1. Select **Next** and review your configuration. Then select **Create** and wait for the process to complete. 30 | 1. When your project is created, close any tips that are displayed and review the project page in Azure AI Foundry portal, which should look similar to the following image: 31 | 32 | ![Screenshot of a Azure AI project details in Azure AI Foundry portal.](../media/ai-foundry-project.png) 33 | 34 | ## Use the Read model 35 | 36 | Let's start by using the **Azure AI Foundry** portal and the Read model to analyze a document with multiple languages: 37 | 38 | 1. In the navigation panel on the left, select **AI Services**. 39 | 1. In the **Azure AI Services** page, select the **Vision + Document** tile. 40 | 1. In the **Vision + Document** page, verify that the **Document** tab is selected, then select the **OCR/Read** tile. 41 | 42 | In the **Read** page, the Azure AI Services resource created with your project should already be connected. 43 | 44 | 1. In the list of documents on the left, select **read-german.pdf**. 45 | 46 | ![Screenshot showing the Read page in Azure AI Document Intelligence Studio.](../media/read-german-sample.png#lightbox) 47 | 48 | 1. At the top toolbar, select **Analyze options**, then enable the **Language** check-box (under **Optional detection**) in the **Analyze options** pane and click on **Save**. 49 | 1. At the top-left, select **Run Analysis**. 50 | 1. When the analysis is complete, the text extracted from the image is shown on the right in the **Content** tab. Review this text and compare it to the text in the original image for accuracy. 51 | 1. Select the **Result** tab. This tab displays the extracted JSON code. 52 | 1. Scroll to the bottom of the JSON code in the **Result** tab. Notice that the read model has detected the language of each span indicated by `locale`. Most spans are in German (language code `de`) but you can find other language codes in the spans (e.g. English - language code `en` - in one of the first span). 53 | 54 | ![Screenshot showing the detection of language for two spans in the results from the read model in Azure AI Document Intelligence Studio.](../media/language-detection.png#lightbox) 55 | 56 | ## Prepare to develop an app in Cloud Shell 57 | 58 | Now let's explore the app that uses the Azure Document Intelligence service SDK. You'll develop your app using Cloud Shell. The code files for your app have been provided in a GitHub repo. 59 | 60 | > **Tip**: If you have already cloned the **mslearn-ai-document-intelligence** repo, you can skip this task. Otherwise, follow these steps to clone it to your development environment. 61 | 62 | 1. In the Azure AI Foundry portal, view the **Overview** page for your project. 63 | 1. In the **Endpoints and keys** area, note the **API Key** and **Azure AI Services endpoint** under the **Azure AI Services** option. You'll use these credentials to connect to your Azure AI Services in a client application. 64 | 1. Open a new browser tab (keeping the Azure AI Foundry portal open in the existing tab). Then in the new tab, browse to the [Azure portal](https://portal.azure.com) at `https://portal.azure.com`; signing in with your Azure credentials if prompted. 65 | 1. Use the **[\>_]** button to the right of the search bar at the top of the page to create a new Cloud Shell in the Azure portal, selecting a ***PowerShell*** environment. The cloud shell provides a command line interface in a pane at the bottom of the Azure portal. 66 | 67 | > **Note**: If you have previously created a cloud shell that uses a *Bash* environment, switch it to ***PowerShell***. 68 | 69 | 1. In the cloud shell toolbar, in the **Settings** menu, select **Go to Classic version** (this is required to use the code editor). 70 | 71 | > **Tip**: As you paste commands into the cloudshell, the ouput may take up a large amount of the screen buffer. You can clear the screen by entering the `cls` command to make it easier to focus on each task. 72 | 73 | 1. In the PowerShell pane, enter the following commands to clone the GitHub repo for this exercise: 74 | 75 | ``` 76 | rm -r mslearn-ai-document-intelligence -f 77 | git clone https://github.com/microsoftlearning/mslearn-ai-document-intelligence mslearn-ai-document-intelligence 78 | ``` 79 | 80 | Applications for both C# and Python have been provided, as well as a sample pdf file you'll use to test Document Intelligence. Both apps feature the same functionality. First, you'll complete some key parts of the application to enable using your Azure Document Intelligence resource. 81 | 82 | 1. Examine the following invoice and note some of its fields and values. This is the invoice that your code will analyze. 83 | 84 | ![Screenshot showing a sample invoice document.](../media/sample-invoice.png#lightbox) 85 | 86 | ***Now follow the steps for your chosen programming language.*** 87 | 88 | 1. After the repo has been cloned, navigate to the folder containing the code files: 89 | 90 | **C#** 91 | 92 | ``` 93 | cd mslearn-ai-document-intelligence/Labfiles/01-prebuild-models/C-Sharp 94 | ``` 95 | 96 | **Python** 97 | 98 | ``` 99 | cd mslearn-ai-document-intelligence/Labfiles/01-prebuild-models/Python 100 | ``` 101 | 102 | 1. In the cloud shell command line pane, enter the following command to install the libraries you'll use: 103 | 104 | **C#** 105 | 106 | ``` 107 | dotnet add package Azure.AI.FormRecognizer --version 4.1.0 108 | ``` 109 | 110 | **Python** 111 | 112 | ``` 113 | pip install azure-ai-formrecognizer==3.3.3 114 | ``` 115 | 116 | ## Add code to use the Azure Document Intelligence service 117 | 118 | Now you're ready to use the SDK to evaluate the pdf file. 119 | 120 | 1. Enter the following command to edit the app file that has been provided: 121 | 122 | **C#** 123 | 124 | ``` 125 | code Program.cs 126 | ``` 127 | 128 | **Python** 129 | 130 | ``` 131 | code document-analysis.py 132 | ``` 133 | 134 | The file is opened in a code editor. 135 | 136 | 1. In the code file, replace the `` and `` placeholders with the **Azure AI Services endpoint** and **API Key** for your project (copied from the project **Overview** page, in the **Azure AI Services** capability option in the Azure AI Foundry portal): 137 | 138 | **C#**: ***Program.cs*** 139 | 140 | ```csharp 141 | string endpoint = ""; 142 | string apiKey = ""; 143 | ``` 144 | 145 | **Python**: ***document-analysis.py*** 146 | 147 | ```python 148 | endpoint = "" 149 | key = "" 150 | ``` 151 | 152 | 1. Locate the comment `Create the client`. Following that, on new lines, enter the following code: 153 | 154 | **C#** 155 | 156 | ```csharp 157 | var cred = new AzureKeyCredential(apiKey); 158 | var client = new DocumentAnalysisClient(new Uri(endpoint), cred); 159 | ``` 160 | 161 | **Python** 162 | 163 | ```python 164 | document_analysis_client = DocumentAnalysisClient( 165 | endpoint=endpoint, credential=AzureKeyCredential(key) 166 | ) 167 | ``` 168 | 169 | 1. Locate the comment `Analyze the invoice`. Following that, on new lines, enter the following code: 170 | 171 | **C#** 172 | 173 | ```csharp 174 | AnalyzeDocumentOperation operation = await client.AnalyzeDocumentFromUriAsync(WaitUntil.Completed, "prebuilt-invoice", fileUri); 175 | ``` 176 | 177 | **Python** 178 | 179 | ```python 180 | poller = document_analysis_client.begin_analyze_document_from_url( 181 | fileModelId, fileUri, locale=fileLocale 182 | ) 183 | ``` 184 | 185 | 1. Locate the comment `Display invoice information to the user`. Following that, on news lines, enter the following code: 186 | 187 | **C#** 188 | 189 | ```csharp 190 | AnalyzeResult result = operation.Value; 191 | 192 | foreach (AnalyzedDocument invoice in result.Documents) 193 | { 194 | if (invoice.Fields.TryGetValue("VendorName", out DocumentField? vendorNameField)) 195 | { 196 | if (vendorNameField.FieldType == DocumentFieldType.String) 197 | { 198 | string vendorName = vendorNameField.Value.AsString(); 199 | Console.WriteLine($"Vendor Name: '{vendorName}', with confidence {vendorNameField.Confidence}."); 200 | } 201 | } 202 | ``` 203 | 204 | **Python** 205 | 206 | ```python 207 | receipts = poller.result() 208 | 209 | for idx, receipt in enumerate(receipts.documents): 210 | 211 | vendor_name = receipt.fields.get("VendorName") 212 | if vendor_name: 213 | print(f"\nVendor Name: {vendor_name.value}, with confidence {vendor_name.confidence}.") 214 | ``` 215 | 216 | > [!NOTE] 217 | > You've added code to display the vendor name. The starter project also includes code to display the *customer name* and *invoice total*. 218 | 219 | 220 | 1. In the code editor, use the **CTRL+S** command or **Right-click > Save** to save your changes and then use the **CTRL+Q** command or **Right-click > Quit** to close the code editor while keeping the cloud shell command line open. 221 | 222 | 1. In the command line pane, enter the following command to run the application. 223 | 224 | 1. ***For C# only***, to build your project, enter this command: 225 | 226 | **C#**: 227 | 228 | ```powershell 229 | dotnet build 230 | ``` 231 | 232 | 1. To run your code, enter this command: 233 | 234 | **C#**: 235 | 236 | ```powershell 237 | dotnet run 238 | ``` 239 | 240 | **Python**: 241 | 242 | ```powershell 243 | python document-analysis.py 244 | ``` 245 | 246 | The program displays the vendor name, customer name, and invoice total with confidence levels. Compare the values it reports with the sample invoice you opened at the start of this section. 247 | 248 | ## Clean up 249 | 250 | If you're done with your Azure resource, remember to delete the resource in the [Azure portal](https://portal.azure.com/?azure-portal=true) to avoid further charges. 251 | 252 | ## More information 253 | 254 | For more information about the Document Intelligence service, see the [Document Intelligence documentation](https://learn.microsoft.com/azure/ai-services/document-intelligence/?azure-portal=true). 255 | -------------------------------------------------------------------------------- /Instructions/Labs/05-content-understanding.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Analyze content with Azure AI Content Understanding' 4 | module: 'Multimodal analysis with Content Understanding' 5 | --- 6 | 7 | # Analyze content with Azure AI Content Understanding 8 | 9 | In this exercise, you use Azure AI Foundry portal to create a Content Understanding project that can extract information from invoices. You'll then test your content analyzer in Azure AI Foundry Portal and consume it through the Content Understanding REST interface. 10 | 11 | This exercise takes approximately **30** minutes. 12 | 13 | ## Create an Azure AI Foundry project 14 | 15 | Let's start by creating an Azure AI Foundry project. 16 | 17 | 1. In a web browser, open the [Azure AI Foundry portal](https://ai.azure.com) at `https://ai.azure.com` and sign in using your Azure credentials. Close any tips or quick start panes that are opened the first time you sign in, and if necessary use the **Azure AI Foundry** logo at the top left to navigate to the home page, which looks similar to the following image: 18 | 19 | ![Screenshot of Azure AI Foundry portal.](../media/ai-foundry-portal.png) 20 | 21 | 1. In the home page, select **+ Create project**. 22 | 1. In the **Create a project** wizard, enter a valid name for your project, and if an existing hub is suggested, choose the option to create a new one. Then review the Azure resources that will be automatically created to support your hub and project. 23 | 1. Select **Customize** and specify the following settings for your hub: 24 | - **Hub name**: *A valid name for your hub* 25 | - **Subscription**: *Your Azure subscription* 26 | - **Resource group**: *Create or select a resource group* 27 | - **Location**: Choose one of the following regions\* 28 | - West US 29 | - Sweden Central 30 | - Australia East 31 | - **Connect Azure AI Services or Azure OpenAI**: *Create a new AI Services resource* 32 | - **Connect Azure AI Search**: *Create a new Azure AI Search resource with a unique name* 33 | 34 | > \*At the time of writing, Azure AI Content understanding is only avilable in these regions. 35 | 36 | 1. Select **Next** and review your configuration. Then select **Create** and wait for the process to complete. 37 | 1. When your project is created, close any tips that are displayed and review the project page in Azure AI Foundry portal, which should look similar to the following image: 38 | 39 | ![Screenshot of a Azure AI project details in Azure AI Foundry portal.](../media/ai-foundry-project.png) 40 | 41 | ## Create a Content Understanding analyzer 42 | 43 | You are going to build an analyzer that can extract information from invoices. You'll start by defining a schema based on a sample invoice. 44 | 45 | 1. In a new browser tab, download the [invoice-1234.pdf](https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1234.pdf) sample form from `https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1234.pdf` and save it in a local folder. 46 | 1. Return to the tab containing the home page for your Azure AI Foundry project; and in the navigation pane on the left, select **Content Understanding**. 47 | 1. On the **Content Understanding** page, select the **Custom analyzer** tab at the top. 48 | 1. On the Content Understanding custom analyzer page, select **+ Create**, and create a task with the following settings: 49 | - **Task name**: Invoice analysis 50 | - **Description**: Extract data from an invoice 51 | - **Azure AI services connection**: *the Azure AI Services resource in your Azure AI Foundry hub* 52 | - **Azure Blob Storage account**: *The default storage account in your Azure AI Foundry hub* 53 | 1. Wait for the task to be created. 54 | 55 | > **Tip**: If an error accessing storage occurs, wait a minute and try again. 56 | 57 | 1. On the **Define schema** page, upload the **invoice-1234.pdf** file you just downloaded. 58 | 1. Select the **Invoice analysis** template and then select **Create**. 59 | 60 | The *Invoice analysis* tenplate includes common fields that are found in invoices. You can use the schema editor to delete any of thre suggested fields thatyou don't need, and add any custom fields that you do. 61 | 62 | 1. In the list of suggested fields, select **BillingAddress**. This field is not needed for the invoice format you have uploaded, so use the **Delete field** (**🗑**) icon that appears to delete it. 63 | 1. Now delete the following suggested fields, which aren't needed: 64 | - BillingAddressRecipient 65 | - CustomerAddressRecipient 66 | - CustomerId 67 | - CustomerTaxId 68 | - DueDate 69 | - InvoiceTotal 70 | - PaymentTerm 71 | - PreviousUnpaidBalance 72 | - PurchaseOrder 73 | - RemittanceAddress 74 | - RemittanceAddressRecipient 75 | - ServiceAddress 76 | - ServiceAddressRecipient 77 | - ShippingAddress 78 | - ShippingAddressRecipient 79 | - TotalDiscount 80 | - VendorAddressRecipient 81 | - VendorTaxId 82 | - TaxDetails 83 | 1. Use **+ Add new field** button to add the following fields: 84 | 85 | | Field name | Field description | Value type | Method | 86 | |--|--|--|--| 87 | | `VendorPhone` | `Vendor telephone number` | String | Extract | 88 | | `ShippingFee` | `Fee for shipping` | Number | Extract | 89 | 90 | 1. verify that your completed schema looks like this, and select **Save**. 91 | ![Screenshot of a schema for an invoice.](../media/invoice-schema.png) 92 | 93 | 1. On the **Test Analyzer** page, if analysis does not begin automatically, select **Run analysis**. Then wait for analysis to complete and review the text values on the invoice that are identified as matching the fields in the schema. 94 | 1. Review the analysis results, which should look similar to this: 95 | 96 | ![Screenshot of analyzer test results.](../media/analysis-results.png) 97 | 98 | 1. View the details of the fields that were identified in the **Fields** pane, and then view the **Result** tab to see the JSON representation. 99 | 100 | ## Build and test an analyzer 101 | 102 | Now that you have trained a model to extract fields from invoices, you can build an analyzer to use with similar forms. 103 | 104 | 1. Select the **Build analyzer** page, and then select **+ Build analyzer** and build a new analyzer with the following properties (typed exactly as shown here): 105 | - **Name**: `contoso-invoice-analyzer` 106 | - **Description**: `Contoso invoice analyzer` 107 | 1. Wait for the new analyzer to be ready (use the **Refresh** button to check). 108 | 1. Download [invoice-1235.pdf](https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1235.pdf) from `https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1235.pdf` and save it in a local folder. 109 | 1. Return to the **Build analyzer** page and select the **contoso-invoice-analyzer** link. The fields defined in the analyzer's schema will be displayed. 110 | 1. In the **contoso-invoice-analyzer** page, select the **Test** tab. 111 | 1. Use the **+ Upload test files** button to upload **invoice-1235.pdf** and run the analysis to extract field data from the test form. 112 | 1. Review the results of the test, and verify that the analyzer extracted the correct fields from the test invoice. 113 | 1. Close the **contoso-invoice-analyzer*** page. 114 | 115 | ## Use the Content Understanding REST API 116 | 117 | Now that you've created an analyzer, you can consume it from a client application through the Content Understanding REST API. 118 | 119 | 1. In the **Project details** area, note the **Project connection string**. You'll use this connection string to connect to your project in a client application. 120 | 1. Open a new browser tab (keeping the Azure AI Foundry portal open in the existing tab). Then in the new tab, browse to the [Azure portal](https://portal.azure.com) at `https://portal.azure.com`; signing in with your Azure credentials if prompted. 121 | 122 | Close any welcome notifications to see the Azure portal home page. 123 | 124 | 1. Use the **[\>_]** button to the right of the search bar at the top of the page to create a new Cloud Shell in the Azure portal, selecting a ***PowerShell*** environment with no storage in your subscription. 125 | 126 | The cloud shell provides a command-line interface in a pane at the bottom of the Azure portal. You can resize or maximize this pane to make it easier to work in. 127 | 128 | > **Note**: If you have previously created a cloud shell that uses a *Bash* environment, switch it to ***PowerShell***. 129 | 130 | 1. In the cloud shell toolbar, in the **Settings** menu, select **Go to Classic version** (this is required to use the code editor). 131 | 132 | **Ensure you've switched to the classic version of the cloud shell before continuing.** 133 | 134 | 1. In the cloud shell pane, enter the following commands to clone the GitHub repo containing the code files for this exercise (type the command, or copy it to the clipboard and then right-click in the command line and paste as plain text): 135 | 136 | ``` 137 | rm -r mslearn-ai-foundry -f 138 | git clone https://github.com/microsoftlearning/mslearn-ai-document-intelligence mslearn-ai-doc 139 | ``` 140 | 141 | > **Tip**: As you enter commands into the cloudshell, the output may take up a large amount of the screen buffer. You can clear the screen by entering the `cls` command to make it easier to focus on each task. 142 | 143 | 1. After the repo has been cloned, navigate to the folder containing the code files for your app: 144 | 145 | ``` 146 | cd mslearn-ai-doc/Labfiles/05-content-understanding/code 147 | ls -a -l 148 | ``` 149 | 150 | 1. In the cloud shell command-line pane, enter the following command to install the libraries you'll use: 151 | 152 | ``` 153 | python -m venv labenv 154 | ./labenv/bin/Activate.ps1 155 | pip install dotenv azure-identity azure-ai-projects 156 | ``` 157 | 158 | 1. Enter the following command to edit the configuration file that has been provided: 159 | 160 | ``` 161 | code .env 162 | ``` 163 | 164 | The file is opened in a code editor. 165 | 166 | 1. In the code file, replace the **YOUR_PROJECT_CONNECTION_STRING** placeholder with the connection string for your project (copied from the project **Overview** page in the Azure AI Foundry portal), and ensure that **ANALYZER** is set to the name you assigned to your analyzer (which should be *contoso-invoice-analyzer*) 167 | 1. After you've replaced the placeholders, within the code editor, use the **CTRL+S** command to save your changes and then use the **CTRL+Q** command to close the code editor while keeping the cloud shell command line open. 168 | 169 | 1. In the cloud shell command line, enter the following command to edit the **analyze_invoice.py** Python code file that has been provided: 170 | 171 | ``` 172 | code analyze_invoice.py 173 | ``` 174 | The Python code file is opened in a code editor: 175 | 176 | 1. Review the code, which: 177 | - Identifies the invoice file to be analyzed, with a default of **invoice-1236.pdf**. 178 | - Retrieves the endpoint and key for your Azure AI Services resource from the project. 179 | - Submits an HTTP POST request to your Content Understanding endpoint, instructing the to analyze the image. 180 | - Checks the response from the POST operation to retrieve an ID for the analysis operation. 181 | - Repeatedly submits an HTTP GET request to your Content Understanding service until the operation is no longer running. 182 | - If the operation has succeeded, parses the JSON response and displays the values retrieved 183 | 1. Use the **CTRL+Q** command to close the code editor while keeping the cloud shell command line open. 184 | 1. In the cloud shell command line pane, enter the following command to run the Python code: 185 | 186 | ``` 187 | python analyze_invoice.py invoice-1236.pdf 188 | ``` 189 | 190 | 1. Review the output from the program. 191 | 1. Use the following command to run the program with a different invoice: 192 | 193 | ``` 194 | python analyze_invoice.py invoice-1235.pdf 195 | ``` 196 | 197 | > **Tip**: There are three invoice files in the code folder that you can use (invoice-1234.pdf, invoice-1235.pdf, and invoice-1236.pdf) 198 | 199 | ## Clean up 200 | 201 | If you've finished working with the Content Understanding service, you should delete the resources you have created in this exercise to avoid incurring unnecessary Azure costs. 202 | 203 | 1. In the Azure AI Foundry portal, navigate to the **travel-insurance** project and delete it. 204 | 1. In the Azure portal, delete the resource group you created in this exercise. 205 | 206 | -------------------------------------------------------------------------------- /Instructions/Exercises/05-content-understanding.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Analyze content with Azure AI Content Understanding' 4 | module: 'Multimodal analysis with Content Understanding' 5 | --- 6 | 7 | # Analyze content with Azure AI Content Understanding 8 | 9 | In this exercise, you use Azure AI Foundry portal to create a Content Understanding project that can extract information from invoices. You'll then test your content analyzer in Azure AI Foundry Portal and consume it through the Content Understanding REST interface. 10 | 11 | This exercise takes approximately **30** minutes. 12 | 13 | ## Create an Azure AI Foundry project 14 | 15 | Let's start by creating an Azure AI Foundry project. 16 | 17 | 1. In a web browser, open the [Azure AI Foundry portal](https://ai.azure.com) at `https://ai.azure.com` and sign in using your Azure credentials. Close any tips or quick start panes that are opened the first time you sign in, and if necessary use the **Azure AI Foundry** logo at the top left to navigate to the home page, which looks similar to the following image: 18 | 19 | ![Screenshot of Azure AI Foundry portal.](../media/ai-foundry-portal.png) 20 | 21 | 1. In the home page, select **+ Create project**. 22 | 1. In the **Create a project** wizard, enter a valid name for your project, and if an existing hub is suggested, choose the option to create a new one. Then review the Azure resources that will be automatically created to support your hub and project. 23 | 1. Select **Customize** and specify the following settings for your hub: 24 | - **Hub name**: *A valid name for your hub* 25 | - **Subscription**: *Your Azure subscription* 26 | - **Resource group**: *Create or select a resource group* 27 | - **Location**: Choose one of the following regions\* 28 | - West US 29 | - Sweden Central 30 | - Australia East 31 | - **Connect Azure AI Services or Azure OpenAI**: *Create a new AI Services resource* 32 | - **Connect Azure AI Search**: *Create a new Azure AI Search resource with a unique name* 33 | 34 | > \*At the time of writing, Azure AI Content understanding is only avilable in these regions. 35 | 36 | 1. Select **Next** and review your configuration. Then select **Create** and wait for the process to complete. 37 | 1. When your project is created, close any tips that are displayed and review the project page in Azure AI Foundry portal, which should look similar to the following image: 38 | 39 | ![Screenshot of a Azure AI project details in Azure AI Foundry portal.](../media/ai-foundry-project.png) 40 | 41 | ## Create a Content Understanding analyzer 42 | 43 | You are going to build an analyzer that can extract information from invoices. You'll start by defining a schema based on a sample invoice. 44 | 45 | 1. In a new browser tab, download the [invoice-1234.pdf](https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1234.pdf) sample form from `https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1234.pdf` and save it in a local folder. 46 | 1. Return to the tab containing the home page for your Azure AI Foundry project; and in the navigation pane on the left, select **Content Understanding**. 47 | 1. On the **Content Understanding** page, select the **Custom analyzer** tab at the top. 48 | 1. On the Content Understanding custom analyzer page, select **+ Create**, and create a task with the following settings: 49 | - **Task name**: Invoice analysis 50 | - **Description**: Extract data from an invoice 51 | - **Azure AI services connection**: *the Azure AI Services resource in your Azure AI Foundry hub* 52 | - **Azure Blob Storage account**: *The default storage account in your Azure AI Foundry hub* 53 | 1. Wait for the task to be created. 54 | 55 | > **Tip**: If an error accessing storage occurs, wait a minute and try again. 56 | 57 | 1. On the **Define schema** page, upload the **invoice-1234.pdf** file you just downloaded. 58 | 1. Select the **Invoice analysis** template and then select **Create**. 59 | 60 | The *Invoice analysis* tenplate includes common fields that are found in invoices. You can use the schema editor to delete any of thre suggested fields thatyou don't need, and add any custom fields that you do. 61 | 62 | 1. In the list of suggested fields, select **BillingAddress**. This field is not needed for the invoice format you have uploaded, so use the **Delete field** (**🗑**) icon that appears to delete it. 63 | 1. Now delete the following suggested fields, which aren't needed: 64 | - BillingAddressRecipient 65 | - CustomerAddressRecipient 66 | - CustomerId 67 | - CustomerTaxId 68 | - DueDate 69 | - InvoiceTotal 70 | - PaymentTerm 71 | - PreviousUnpaidBalance 72 | - PurchaseOrder 73 | - RemittanceAddress 74 | - RemittanceAddressRecipient 75 | - ServiceAddress 76 | - ServiceAddressRecipient 77 | - ShippingAddress 78 | - ShippingAddressRecipient 79 | - TotalDiscount 80 | - VendorAddressRecipient 81 | - VendorTaxId 82 | - TaxDetails 83 | 1. Use **+ Add new field** button to add the following fields: 84 | 85 | | Field name | Field description | Value type | Method | 86 | |--|--|--|--| 87 | | `VendorPhone` | `Vendor telephone number` | String | Extract | 88 | | `ShippingFee` | `Fee for shipping` | Number | Extract | 89 | 90 | 1. verify that your completed schema looks like this, and select **Save**. 91 | ![Screenshot of a schema for an invoice.](../media/invoice-schema.png) 92 | 93 | 1. On the **Test Analyzer** page, if analysis does not begin automatically, select **Run analysis**. Then wait for analysis to complete and review the text values on the invoice that are identified as matching the fields in the schema. 94 | 1. Review the analysis results, which should look similar to this: 95 | 96 | ![Screenshot of analyzer test results.](../media/analysis-results.png) 97 | 98 | 1. View the details of the fields that were identified in the **Fields** pane, and then view the **Result** tab to see the JSON representation. 99 | 100 | ## Build and test an analyzer 101 | 102 | Now that you have trained a model to extract fields from invoices, you can build an analyzer to use with similar forms. 103 | 104 | 1. Select the **Build analyzer** page, and then select **+ Build analyzer** and build a new analyzer with the following properties (typed exactly as shown here): 105 | - **Name**: `contoso-invoice-analyzer` 106 | - **Description**: `Contoso invoice analyzer` 107 | 1. Wait for the new analyzer to be ready (use the **Refresh** button to check). 108 | 1. Download [invoice-1235.pdf](https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1235.pdf) from `https://github.com/microsoftlearning/mslearn-ai-document-intelligence/raw/main/Labfiles/05-content-understanding/forms/invoice-1235.pdf` and save it in a local folder. 109 | 1. Return to the **Build analyzer** page and select the **contoso-invoice-analyzer** link. The fields defined in the analyzer's schema will be displayed. 110 | 1. In the **contoso-invoice-analyzer** page, select the **Test** tab. 111 | 1. Use the **+ Upload test files** button to upload **invoice-1235.pdf** and run the analysis to extract field data from the test form. 112 | 1. Review the results of the test, and verify that the analyzer extracted the correct fields from the test invoice. 113 | 1. Close the **contoso-invoice-analyzer*** page. 114 | 115 | ## Use the Content Understanding REST API 116 | 117 | Now that you've created an analyzer, you can consume it from a client application through the Content Understanding REST API. 118 | 119 | 1. In the **Project details** area, note the **Project connection string**. You'll use this connection string to connect to your project in a client application. 120 | 1. Open a new browser tab (keeping the Azure AI Foundry portal open in the existing tab). Then in the new tab, browse to the [Azure portal](https://portal.azure.com) at `https://portal.azure.com`; signing in with your Azure credentials if prompted. 121 | 122 | Close any welcome notifications to see the Azure portal home page. 123 | 124 | 1. Use the **[\>_]** button to the right of the search bar at the top of the page to create a new Cloud Shell in the Azure portal, selecting a ***PowerShell*** environment with no storage in your subscription. 125 | 126 | The cloud shell provides a command-line interface in a pane at the bottom of the Azure portal. You can resize or maximize this pane to make it easier to work in. 127 | 128 | > **Note**: If you have previously created a cloud shell that uses a *Bash* environment, switch it to ***PowerShell***. 129 | 130 | 1. In the cloud shell toolbar, in the **Settings** menu, select **Go to Classic version** (this is required to use the code editor). 131 | 132 | **Ensure you've switched to the classic version of the cloud shell before continuing.** 133 | 134 | 1. In the cloud shell pane, enter the following commands to clone the GitHub repo containing the code files for this exercise (type the command, or copy it to the clipboard and then right-click in the command line and paste as plain text): 135 | 136 | ``` 137 | rm -r mslearn-ai-foundry -f 138 | git clone https://github.com/microsoftlearning/mslearn-ai-document-intelligence mslearn-ai-doc 139 | ``` 140 | 141 | > **Tip**: As you enter commands into the cloudshell, the output may take up a large amount of the screen buffer. You can clear the screen by entering the `cls` command to make it easier to focus on each task. 142 | 143 | 1. After the repo has been cloned, navigate to the folder containing the code files for your app: 144 | 145 | ``` 146 | cd mslearn-ai-doc/Labfiles/05-content-understanding/code 147 | ls -a -l 148 | ``` 149 | 150 | 1. In the cloud shell command-line pane, enter the following command to install the libraries you'll use: 151 | 152 | ``` 153 | python -m venv labenv 154 | ./labenv/bin/Activate.ps1 155 | pip install dotenv azure-identity azure-ai-projects 156 | ``` 157 | 158 | 1. Enter the following command to edit the configuration file that has been provided: 159 | 160 | ``` 161 | code .env 162 | ``` 163 | 164 | The file is opened in a code editor. 165 | 166 | 1. In the code file, replace the **YOUR_PROJECT_CONNECTION_STRING** placeholder with the connection string for your project (copied from the project **Overview** page in the Azure AI Foundry portal), and ensure that **ANALYZER** is set to the name you assigned to your analyzer (which should be *contoso-invoice-analyzer*) 167 | 1. After you've replaced the placeholders, within the code editor, use the **CTRL+S** command to save your changes and then use the **CTRL+Q** command to close the code editor while keeping the cloud shell command line open. 168 | 169 | 1. In the cloud shell command line, enter the following command to edit the **analyze_invoice.py** Python code file that has been provided: 170 | 171 | ``` 172 | code analyze_invoice.py 173 | ``` 174 | The Python code file is opened in a code editor: 175 | 176 | 1. Review the code, which: 177 | - Identifies the invoice file to be analyzed, with a default of **invoice-1236.pdf**. 178 | - Retrieves the endpoint and key for your Azure AI Services resource from the project. 179 | - Submits an HTTP POST request to your Content Understanding endpoint, instructing the to analyze the image. 180 | - Checks the response from the POST operation to retrieve an ID for the analysis operation. 181 | - Repeatedly submits an HTTP GET request to your Content Understanding service until the operation is no longer running. 182 | - If the operation has succeeded, parses the JSON response and displays the values retrieved 183 | 1. Use the **CTRL+Q** command to close the code editor while keeping the cloud shell command line open. 184 | 1. In the cloud shell command line pane, enter the following command to run the Python code: 185 | 186 | ``` 187 | python analyze_invoice.py invoice-1236.pdf 188 | ``` 189 | 190 | 1. Review the output from the program. 191 | 1. Use the following command to run the program with a different invoice: 192 | 193 | ``` 194 | python analyze_invoice.py invoice-1235.pdf 195 | ``` 196 | 197 | > **Tip**: There are three invoice files in the code folder that you can use (invoice-1234.pdf, invoice-1235.pdf, and invoice-1236.pdf) 198 | 199 | ## Clean up 200 | 201 | If you've finished working with the Content Understanding service, you should delete the resources you have created in this exercise to avoid incurring unnecessary Azure costs. 202 | 203 | 1. In the Azure AI Foundry portal, navigate to the **travel-insurance** project and delete it. 204 | 1. In the Azure portal, delete the resource group you created in this exercise. 205 | 206 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/azuredeploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "skill_name": { 6 | "defaultValue": "analyzeformv22", 7 | "type": "String" 8 | }, 9 | "repoURL": { 10 | "type": "string", 11 | "defaultValue": "https://github.com/AzureBytes/functionshttpecho.git", 12 | "metadata": { 13 | "description": "The URL for the GitHub repository that contains the project to deploy." 14 | } 15 | }, 16 | "branch": { 17 | "type": "string", 18 | "defaultValue": "master", 19 | "metadata": { 20 | "description": "The branch of the GitHub repository to use." 21 | } 22 | }, 23 | "location": { 24 | "type": "string", 25 | "defaultValue": "[resourceGroup().location]", 26 | "metadata": { 27 | "description": "Location for all resources." 28 | } 29 | } 30 | }, 31 | "variables": {}, 32 | "resources": [ 33 | { 34 | "type": "Microsoft.CognitiveServices/accounts", 35 | "apiVersion": "2017-04-18", 36 | "name": "[parameters('skill_name')]", 37 | "location": "[resourceGroup().location]", 38 | "sku": { 39 | "name": "S0" 40 | }, 41 | "kind": "FormRecognizer", 42 | "properties": { 43 | "customSubDomainName": "[parameters('skill_name')]", 44 | "privateEndpointConnections": [], 45 | "publicNetworkAccess": "Enabled" 46 | } 47 | }, 48 | { 49 | "type": "microsoft.insights/components", 50 | "apiVersion": "2018-05-01-preview", 51 | "name": "[parameters('skill_name')]", 52 | "location": "[resourceGroup().location]", 53 | "kind": "web", 54 | "properties": { 55 | "Application_Type": "web", 56 | "Flow_Type": "Bluefield", 57 | "Request_Source": "rest", 58 | "RetentionInDays": 90, 59 | "publicNetworkAccessForIngestion": "Enabled", 60 | "publicNetworkAccessForQuery": "Enabled" 61 | } 62 | }, 63 | { 64 | "type": "Microsoft.Storage/storageAccounts", 65 | "apiVersion": "2020-08-01-preview", 66 | "name": "[parameters('skill_name')]", 67 | "location": "[resourceGroup().location]", 68 | "sku": { 69 | "name": "Standard_LRS", 70 | "tier": "Standard" 71 | }, 72 | "kind": "Storage", 73 | "properties": { 74 | "networkAcls": { 75 | "bypass": "AzureServices", 76 | "virtualNetworkRules": [], 77 | "ipRules": [], 78 | "defaultAction": "Allow" 79 | }, 80 | "supportsHttpsTrafficOnly": true, 81 | "encryption": { 82 | "services": { 83 | "file": { 84 | "keyType": "Account", 85 | "enabled": true 86 | }, 87 | "blob": { 88 | "keyType": "Account", 89 | "enabled": true 90 | } 91 | }, 92 | "keySource": "Microsoft.Storage" 93 | } 94 | } 95 | }, 96 | { 97 | "type": "Microsoft.Web/serverfarms", 98 | "apiVersion": "2018-02-01", 99 | "name": "[parameters('skill_name')]", 100 | "location": "[resourceGroup().location]", 101 | "sku": { 102 | "name": "Y1", 103 | "tier": "Dynamic", 104 | "size": "Y1", 105 | "family": "Y", 106 | "capacity": 0 107 | }, 108 | "kind": "functionapp", 109 | "properties": { 110 | "perSiteScaling": false, 111 | "maximumElasticWorkerCount": 1, 112 | "isSpot": false, 113 | "reserved": true, 114 | "isXenon": false, 115 | "hyperV": false, 116 | "targetWorkerCount": 0, 117 | "targetWorkerSizeId": 0 118 | } 119 | }, 120 | { 121 | "type": "Microsoft.Web/sites", 122 | "apiVersion": "2018-11-01", 123 | "name": "[parameters('skill_name')]", 124 | "location": "[resourceGroup().location]", 125 | "dependsOn": [ 126 | "[resourceId('Microsoft.Web/serverfarms', parameters('skill_name'))]" 127 | ], 128 | "kind": "functionapp,linux", 129 | "properties": { 130 | "enabled": true, 131 | "hostNameSslStates": [ 132 | { 133 | "name": "[concat(parameters('skill_name'), '.azurewebsites.net')]", 134 | "sslState": "Disabled", 135 | "hostType": "Standard" 136 | }, 137 | { 138 | "name": "[concat(parameters('skill_name'), '.scm.azurewebsites.net')]", 139 | "sslState": "Disabled", 140 | "hostType": "Repository" 141 | } 142 | ], 143 | "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('skill_name'))]", 144 | "reserved": true, 145 | "isXenon": false, 146 | "hyperV": false, 147 | "siteConfig": {}, 148 | "scmSiteAlsoStopped": false, 149 | "clientAffinityEnabled": false, 150 | "clientCertEnabled": false, 151 | "hostNamesDisabled": false, 152 | "containerSize": 0, 153 | "dailyMemoryTimeQuota": 0, 154 | "httpsOnly": false, 155 | "redundancyMode": "None" 156 | } 157 | }, 158 | { 159 | "type": "Microsoft.Web/sites/config", 160 | "apiVersion": "2018-11-01", 161 | "name": "[concat(parameters('skill_name'), '/web')]", 162 | "location": "[resourceGroup().location]", 163 | "dependsOn": [ 164 | "[resourceId('Microsoft.Web/sites', parameters('skill_name'))]" 165 | ], 166 | "properties": { 167 | "numberOfWorkers": 1, 168 | "defaultDocuments": [ 169 | "Default.htm", 170 | "Default.html", 171 | "Default.asp", 172 | "index.htm", 173 | "index.html", 174 | "iisstart.htm", 175 | "default.aspx", 176 | "index.php" 177 | ], 178 | "netFrameworkVersion": "v4.0", 179 | "linuxFxVersion": "Python|3.6", 180 | "requestTracingEnabled": false, 181 | "remoteDebuggingEnabled": false, 182 | "httpLoggingEnabled": false, 183 | "logsDirectorySizeLimit": 35, 184 | "detailedErrorLoggingEnabled": false, 185 | "publishingUsername": "$analyzeformv22", 186 | "azureStorageAccounts": {}, 187 | "scmType": "None", 188 | "use32BitWorkerProcess": false, 189 | "webSocketsEnabled": false, 190 | "alwaysOn": false, 191 | "managedPipelineMode": "Integrated", 192 | "virtualApplications": [ 193 | { 194 | "virtualPath": "/", 195 | "physicalPath": "site\\wwwroot", 196 | "preloadEnabled": false 197 | } 198 | ], 199 | "loadBalancing": "LeastRequests", 200 | "experiments": { 201 | "rampUpRules": [] 202 | }, 203 | "autoHealEnabled": false, 204 | "cors": { 205 | "allowedOrigins": [ 206 | "https://functions.azure.com", 207 | "https://functions-staging.azure.com", 208 | "https://functions-next.azure.com" 209 | ], 210 | "supportCredentials": false 211 | }, 212 | "localMySqlEnabled": false, 213 | "ipSecurityRestrictions": [ 214 | { 215 | "ipAddress": "Any", 216 | "action": "Allow", 217 | "priority": 1, 218 | "name": "Allow all", 219 | "description": "Allow all access" 220 | } 221 | ], 222 | "scmIpSecurityRestrictions": [ 223 | { 224 | "ipAddress": "Any", 225 | "action": "Allow", 226 | "priority": 1, 227 | "name": "Allow all", 228 | "description": "Allow all access" 229 | } 230 | ], 231 | "scmIpSecurityRestrictionsUseMain": false, 232 | "http20Enabled": false, 233 | "minTlsVersion": "1.2", 234 | "ftpsState": "AllAllowed", 235 | "reservedInstanceCount": 0 236 | } 237 | }, 238 | { 239 | "type": "Microsoft.Web/sites/functions", 240 | "apiVersion": "2018-11-01", 241 | "name": "[concat(parameters('skill_name'), '/AnalyzeForm')]", 242 | "location": "[resourceGroup().location]", 243 | "dependsOn": [ 244 | "[resourceId('Microsoft.Web/sites', parameters('skill_name'))]" 245 | ], 246 | "properties": { 247 | "script_root_path_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/AnalyzeForm/", 248 | "script_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/AnalyzeForm/__init__.py", 249 | "config_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/AnalyzeForm/function.json", 250 | "href": "https://analyzeformv22.azurewebsites.net/admin/functions/AnalyzeForm", 251 | "config": {}, 252 | "appSettings": [ 253 | { 254 | "name": "FORMS_RECOGNIZER_ENDPOINT", 255 | "value": "[concat('https://',parameters('skill_name'), '.cognitiveservices.azure.com/')]" 256 | } 257 | ] 258 | } 259 | }, 260 | { 261 | "type": "Microsoft.Web/sites/functions", 262 | "apiVersion": "2018-11-01", 263 | "name": "[concat(parameters('skill_name'), '/AnalyzeInvoice')]", 264 | "location": "[resourceGroup().location]", 265 | "dependsOn": [ 266 | "[resourceId('Microsoft.Web/sites', parameters('skill_name'))]" 267 | ], 268 | "properties": { 269 | "script_root_path_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/AnalyzeInvoice/", 270 | "script_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/AnalyzeInvoice/__init__.py", 271 | "config_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/AnalyzeInvoice/function.json", 272 | "href": "https://analyzeformv22.azurewebsites.net/admin/functions/AnalyzeInvoice", 273 | "config": {} 274 | } 275 | }, 276 | { 277 | "type": "Microsoft.Web/sites/functions", 278 | "apiVersion": "2018-11-01", 279 | "name": "[concat(parameters('skill_name'), '/ExtractTables')]", 280 | "location": "[resourceGroup().location]", 281 | "dependsOn": [ 282 | "[resourceId('Microsoft.Web/sites', parameters('skill_name'))]" 283 | ], 284 | "properties": { 285 | "script_root_path_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/ExtractTables/", 286 | "script_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/ExtractTables/__init__.py", 287 | "config_href": "https://analyzeformv22.azurewebsites.net/admin/vfs/home/site/wwwroot/ExtractTables/function.json", 288 | "href": "https://analyzeformv22.azurewebsites.net/admin/functions/ExtractTables", 289 | "config": {} 290 | } 291 | }, 292 | { 293 | "type": "Microsoft.Web/sites/hostNameBindings", 294 | "apiVersion": "2018-11-01", 295 | "name": "[concat(parameters('skill_name'), '/', parameters('skill_name'), '.azurewebsites.net')]", 296 | "location": "[resourceGroup().location]", 297 | "dependsOn": [ 298 | "[resourceId('Microsoft.Web/sites', parameters('skill_name'))]" 299 | ], 300 | "properties": { 301 | "siteName": "analyzeformv22", 302 | "hostNameType": "Verified" 303 | } 304 | }, 305 | { 306 | "type": "Microsoft.Storage/storageAccounts/blobServices/containers", 307 | "apiVersion": "2020-08-01-preview", 308 | "name": "[concat(parameters('skill_name'), '/default/azure-webjobs-hosts')]", 309 | "dependsOn": [ 310 | "[resourceId('Microsoft.Storage/storageAccounts/blobServices', parameters('skill_name'), 'default')]", 311 | "[resourceId('Microsoft.Storage/storageAccounts', parameters('skill_name'))]" 312 | ], 313 | "properties": { 314 | "defaultEncryptionScope": "$account-encryption-key", 315 | "denyEncryptionScopeOverride": false, 316 | "publicAccess": "None" 317 | } 318 | }, 319 | { 320 | "type": "Microsoft.Storage/storageAccounts/blobServices/containers", 321 | "apiVersion": "2020-08-01-preview", 322 | "name": "[concat(parameters('skill_name'), '/default/azure-webjobs-secrets')]", 323 | "dependsOn": [ 324 | "[resourceId('Microsoft.Storage/storageAccounts/blobServices', parameters('skill_name'), 'default')]", 325 | "[resourceId('Microsoft.Storage/storageAccounts', parameters('skill_name'))]" 326 | ], 327 | "properties": { 328 | "defaultEncryptionScope": "$account-encryption-key", 329 | "denyEncryptionScopeOverride": false, 330 | "publicAccess": "None" 331 | } 332 | }, 333 | { 334 | "type": "Microsoft.Storage/storageAccounts/blobServices/containers", 335 | "apiVersion": "2020-08-01-preview", 336 | "name": "[concat(parameters('skill_name'), '/default/scm-releases')]", 337 | "dependsOn": [ 338 | "[resourceId('Microsoft.Storage/storageAccounts/blobServices', parameters('skill_name'), 'default')]", 339 | "[resourceId('Microsoft.Storage/storageAccounts', parameters('skill_name'))]" 340 | ], 341 | "properties": { 342 | "defaultEncryptionScope": "$account-encryption-key", 343 | "denyEncryptionScopeOverride": false, 344 | "publicAccess": "None" 345 | } 346 | } 347 | ] 348 | } -------------------------------------------------------------------------------- /Instructions/Labs/03-composed-model.md: -------------------------------------------------------------------------------- 1 | --- 2 | lab: 3 | title: 'Create a composed Document Intelligence model' 4 | module: 'Module 6 - Document Intelligence' 5 | --- 6 | 7 | # Create a composed Document Intelligence model 8 | 9 | In this exercise, you'll create and train two custom models that analyze different tax forms. Then, you'll create a composed model that includes both of these custom models. You'll test the model by submitting a form and you'll check that it recognizes the document type and labeled fields correctly. 10 | 11 | ## Set up resources 12 | 13 | We'll use a script to create the Azure AI Document Intelligence resource, a storage account with sample forms, and a resource group: 14 | 15 | 1. In a browser tab, open the Azure portal at `https://portal.azure.com`, signing in with the Microsoft account associated with your Azure subscription. 16 | 1. Once signed in, use the **[\>_]** button to the right of the search bar at the top of the page to create a new Cloud Shell in the Azure portal, selecting a ***PowerShell*** environment. The cloud shell provides a command line interface in a pane at the bottom of the Azure portal. 17 | 18 | > **Note**: If you have previously created a cloud shell that uses a *Bash* environment, switch it to ***PowerShell***. 19 | 20 | 1. In the cloud shell toolbar, in the **Settings** menu, select **Go to Classic version** (this is required to use the code editor). 21 | 22 | > **Tip**: As you paste commands into the cloudshell, the ouput may take up a large amount of the screen buffer. You can clear the screen by entering the `cls` command to make it easier to focus on each task. 23 | 24 | 1. In the PowerShell pane, enter the following commands to clone the GitHub repo for this exercise: 25 | 26 | ``` 27 | rm -r mslearn-ai-document-intelligence -f 28 | git clone https://github.com/microsoftlearning/mslearn-ai-document-intelligence mslearn-ai-document-intelligence 29 | ``` 30 | 31 | 1. After the repo has been cloned, navigate to the folder containing this exercise's files: 32 | 33 | ``` 34 | cd mslearn-ai-document-intelligence/Labfiles/03-composed-model 35 | ``` 36 | 37 | 1. Run the following command to set up resources: 38 | 39 | ```powershell 40 | ./setup.ps1 41 | ``` 42 | 43 | > **IMPORTANT**: The last resource created by the script is your Azure AI Document Intelligence service. If that command fails due to you already having an F0 tier resource, either use that resource for this lab or create one manually using the S0 tier in the Azure portal. 44 | 45 | ## Create the 1040 Forms custom model 46 | 47 | To create a composed model, we must first create two or more custom models. To create the first custom model: 48 | 49 | 1. In a new browser tab, start the **Azure AI Document Intelligence Studio** at `https://documentintelligence.ai.azure.com/studio` 50 | 1. Scroll down, and then under **Custom models**, select **Custom extraction model**. 51 | 1. If you're asked to sign into your account, use your Azure credentials. 52 | 1. If you're asked which Azure AI Document Intelligence resource to use, select the subscription and resource name you used when you created the Azure AI Document Intelligence resource. 53 | 1. Under **My Projects**, select **+ Create a project**. 54 | 1. In the **Project name** textbox, type **1040 Forms**, and then select **Continue**. 55 | 1. On the **Configure service resource** page, in the **Subscription** drop-down list, select your Azure subscription. 56 | 1. In the **Resource group** drop-down list, select the **DocumentIntelligenceResources<xxxx>** created for you. 57 | 1. In the **Document Intelligence or Cognitive Service Resource** drop-down list, select **DocumentIntelligence<xxxx>**. 58 | 1. In the **API version** drop-down list, ensure that **2024-11-30 (4.0 General Availability)** is selected and then select **Continue**. 59 | 1. On the **Connect training data source** page, in the **Subscription** drop-down list, select your Azure subscription. 60 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 61 | 1. In the **Storage account** drop-down list, select the only storage account listed. If you have multiple storage accounts in your subscription, choose the one starting with *docintelstorage* 62 | 1. In the **Blob container** drop-down list, select **1040examples**, and then select **Continue**. 63 | 1. In the **Review and create** page, select **Create project**. 64 | 1. Select **Run now** under *Run layout* in the *Start labeling now* pop up, and wait for the analysis to complete. 65 | 66 | ## Label the 1040 Forms custom model 67 | 68 | Now, let's label the fields in the example forms: 69 | 70 | 1. In the **Label data** page, in the top-right of the page, select **+ Add a field**, and then select **Field**. 71 | 1. Type **FirstName** and then press *Enter*. 72 | 1. Select the document called **f1040_1.pdf** on the left list, select **John** and then select **FirstName**. 73 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 74 | 1. Type **LastName** and then press *Enter*. 75 | 1. In the document, select **Doe** and then select **LastName**. 76 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 77 | 1. Type **City** and then press *Enter*. 78 | 1. In the document, select **Los Angeles** and then select **City**. 79 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 80 | 1. Type **State** and then press *Enter*. 81 | 1. In the document, select **CA** and then select **State**. 82 | 1. Repeat the labeling process for the remaining forms in the list on the left, using the labels you created. Label the same four fields: *FirstName*, *LastName*, *City*, and *State*. Notice that one of the documents does not have city or state data. 83 | 84 | > **IMPORTANT** 85 | > For the purposes of this exercise, we're using only five example forms and labeling only four fields. In your real-world models, you should use as many samples as possible to maximize the accuracy and confidence of your predictions. You should also label all the available fields in the forms, rather than just four fields. 86 | 87 | ## Train the 1040 Forms custom model 88 | 89 | Now that the sample forms are labeled, we can train the first custom model: 90 | 91 | 1. In the Azure AI Document Intelligence Studio, on the top right of the screen, select **Train**. 92 | 1. In the **Train a new model** dialog, in the **Model ID** textbox, type **1040FormsModel**. 93 | 1. In the **Build mode** drop-down list, select **Template**, and then select **Train**. 94 | 1. In the **Training in progress** dialog, select **Go to Models**. 95 | 96 | ## Create the 1099 Forms custom model 97 | 98 | Now, you must create a second model, which you'll train on example 1099 tax forms: 99 | 100 | 1. In Azure AI Document Intelligence Studio, select **Custom extraction model**. 101 | 1. Under **My Projects**, select **+ Create a project**. 102 | 1. In the **Project name** textbox, type **1099 Forms**, and then select **Continue**. 103 | 1. On the **Configure service resource** page, in the **Subscription** drop-down list, select your Azure subscription. 104 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 105 | 1. In the **Document Intelligence or Cognitive Service Resource** drop-down list, select **DocumentIntelligence<xxxx>**. 106 | 1. In the **API version** drop-down list, ensure that **2024-11-30 (4.0 General Availability)** is selected and then select **Continue**. 107 | 1. On the **Connnect training data source** page, in the **Subscription** drop-down list, select your Azure subscription. 108 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 109 | 1. In the **Storage account** drop-down list, select the only storage account listed. 110 | 1. In the **Blob container** drop-down list, select **1099examples**, and then select **Continue**. 111 | 1. In the **Review and create** page, select **Create project**. 112 | 1. Select the drop-down button for **Run layout** and select **Unanalyzed documents**. 113 | 1. Wait for the analysis to complete. 114 | 115 | ## Label the 1099 Forms custom model 116 | 117 | Now, label the example forms with some fields: 118 | 119 | 1. In the **Label data** page, in the top-right of the page, select **+ Add a field**, and then select **Field**. 120 | 1. Type **FirstName** and then press *Enter*. 121 | 1. Select the document called **f1099msc_payer.pdf**, select **John** and then select **FirstName**. 122 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 123 | 1. Type **LastName** and then press *Enter*. 124 | 1. In the document, select **Doe** and then select **LastName**. 125 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 126 | 1. Type **City** and then press *Enter*. 127 | 1. In the document, select **New Haven** and then select **City**. 128 | 1. In the top-right of the page, select **+ Add a field**, and then select **Field**. 129 | 1. Type **State** and then press *Enter*. 130 | 1. In the document, select **CT** and then select **State**. 131 | 1. Repeat the labeling process for the remaining forms in the list on the left. Label the same four fields: *FirstName*, *LastName*, *City*, and *State*. Notice that two of the documents don't have any name data to label. 132 | 133 | ## Train the 1099 Forms custom model 134 | 135 | You can now train the second custom model: 136 | 137 | 1. In the Azure AI Document Intelligence Studio, on the top right, select **Train**. 138 | 1. In the **Train a new model** dialog, in the **Model ID** textbox, type **1099FormsModel**. 139 | 1. In the **Build mode** drop-down list, select **Template**, and then select **Train**. 140 | 1. In the **Training in progress** dialog, select **Go to Models**. 141 | 1. The training process can take a few minutes. Refresh the browser occasionally until both models display the **succeeded** status. 142 | 143 | ## Create a custom classification model 144 | 145 | Before creating the composed model, you need to create a custom classification model. It will be used to first define which type of form a given input file is before extracting data from it. 146 | 147 | 1. In Azure AI Document Intelligence Studio home page, select **Custom classification model**. 148 | 1. Under **My Projects**, select **+ Create a project**. 149 | 1. In the **Project name** textbox, type **Classify Forms**, and then select **Continue**. 150 | 1. On the **Configure service resource** page, in the **Subscription** drop-down list, select your Azure subscription. 151 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 152 | 1. In the **Document Intelligence or Cognitive Service Resource** drop-down list, select **DocumentIntelligence<xxxx>**. 153 | 1. In the **API version** drop-down list, ensure that **2024-11-30 (4.0 General Availability)** is selected and then select **Continue**. 154 | 1. On the **Connnect training data source** page, in the **Subscription** drop-down list, select your Azure subscription. 155 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 156 | 1. In the **Storage account** drop-down list, select the only storage account listed. 157 | 1. In the **Blob container** drop-down list, select **classifyexamples**, and then select **Continue**. 158 | 1. In the **Review and create** page, select **Create project**. 159 | 160 | ## Label the custom classification model 161 | 162 | Now, label the example forms with with each type: 163 | 164 | 1. In the **Label data** page, in the top-right of the page, select **+ Add type**. 165 | 1. Type **1040Form** and then press *Enter*. 166 | 1. Select each **f1040...** document and then select **1040Form**. 167 | 1. In the top-right of the page, select **+ Add type**. 168 | 1. Type **1099Form** and then press *Enter*. 169 | 1. Select each **f1099...** document and then select **1099Form**. 170 | 171 | ## Train the custom classification model 172 | 173 | You can now train the custom classification model: 174 | 175 | 1. In the Azure AI Document Intelligence Studio, on the top right, select **Train**. 176 | 1. In the **Train a new model** dialog, in the **Model ID** textbox, type **ClassifyModel**. 177 | 1. Check the confirmation checkbox and then select **Train**. 178 | 1. In the **Training in progress** dialog, select **Go to Models**. 179 | 1. The training process can take a few minutes. Refresh the browser occasionally until the model displays the **succeeded** status. 180 | 181 | ## Create a composed model 182 | 183 | Now that all required models are trained, let's create the composed model: 184 | 185 | 1. In Azure AI Document Intelligence Studio home page, select **Custom extraction model**. 186 | 1. Under **My Projects**, select **+ Create a project**. 187 | 1. In the **Project name** textbox, type **Compose Model**, and then select **Continue**. 188 | 1. On the **Configure service resource** page, in the **Subscription** drop-down list, select your Azure subscription. 189 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 190 | 1. In the **Document Intelligence or Cognitive Service Resource** drop-down list, select **DocumentIntelligence<xxxx>**. 191 | 1. In the **API version** drop-down list, ensure that **2024-11-30 (4.0 General Availability)** is selected and then select **Continue**. 192 | 1. On the **Connnect training data source** page, in the **Subscription** drop-down list, select your Azure subscription. 193 | 1. In the **Resource group** drop-down list, select **DocumentIntelligenceResources<xxxx>**. 194 | 1. In the **Storage account** drop-down list, select the only storage account listed. 195 | 1. In the **Blob container** drop-down list, select **testdoc**, and then select **Continue**. 196 | 1. In the **Review and create** page, select **Create project**. 197 | 1. In the left pane, select **Models**. 198 | 1. In the Models list, select both **1040Forms** and **1099Forms** and then select **Compose**. 199 | 1. In the **Compose a new model** dialog, in the **Model ID** textbox, type **ComposeModel**. 200 | 1. In the **Classification model** drop-down list, select **ClassifyModel**. 201 | 1. In the **1040Form** drop-down list, select **1040FormsModel**. 202 | 1. In the **1099Form** drop-down list, select **1099FormsModel**. Then select **Compose**. 203 | 204 | ## Use the model 205 | 206 | Now that the composed model is created, let's test it with an example form: 207 | 208 | 1. Download the test form at `https://github.com/MicrosoftLearning/mslearn-ai-document-intelligence/raw/refs/heads/main/Labfiles/03-composed-model/trainingdata/TestDoc/f1040_7.pdf`. 209 | 1. In the Azure AI Document Intelligence Studio, select the **Models** page, and then select the **ComposeModel**. 210 | 1. Select **Test**. 211 | 1. Select **Browse for files** and then browse to the location where you downloaded the form. 212 | 1. Select **f1040_7.pdf**, and then select **Open**. 213 | 1. Select **Run analysis**. Azure AI Document Intelligence analyses the form by using the composed model. 214 | 1. The document you analyzed is an example of the 1040 tax form. Check the **DocType** property to see if the correct custom model has been used. Also check the **FirstName**, **LastName**, **City**, and **State** values identified by the model. 215 | 216 | ## Clean up resources 217 | 218 | Now that you've seen how composed models work, let's remove the resources you created in your Azure subscription. 219 | 220 | 1. In the **Azure portal** at `https://portal.azure.com/`, select **Resource groups**. 221 | 1. In the list of **Resource groups**, select the **DocumentIntelligenceResources<xxxx>** that you created, and then select **Delete resource group**. 222 | 1. In the **TYPE THE RESOURCE GROUP NAME** textbox, type the name of the resource group, and then select **Delete** to delete the Document Intelligence resource and the storage account. 223 | 224 | ## Learn more 225 | 226 | - [Compose custom models](/azure/ai-services/document-intelligence/concept-composed-models) 227 | -------------------------------------------------------------------------------- /Labfiles/04-custom-skill/customskill/Readme.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | languages: 4 | - python 5 | products: 6 | - azure 7 | - azure-cognitive-search 8 | - azure-cognitive-services 9 | - azure-form-recognizer 10 | name: Analyze form sample skill for cognitive search 11 | urlFragment: azure-analyzeform-sample 12 | description: This custom skill extracts specific fields from the results of a trained form recognition. 13 | azureDeploy: https://raw.githubusercontent.com/Azure-Samples/azure-search-power-skills/master/Vision/AnalyzeFormV2/azuredeploy.json 14 | --- 15 | Invoking a Form Recognizer capability within the Cognitive Search pipeline is broken out into three skills 16 | * [Analyze Form](#AnalyzeForm), using a pre trained model 17 | * [Analyze Invoice](#AnalyzeInvoice) using a pretrained model 18 | * [Extract Tables](#ExtractTables) using a pretrained model 19 | 20 | # Deployment 21 | 22 | The analyze form skill requires you to train a custom model to identify and extract key value pairs. Analyze invoice and extract table skills invoke pre trained models. All skills require the `FORMS_RECOGNIZER_ENDPOINT` and `FORMS_RECOGNIZER_KEY` property set in the appsettings, while the analyze form skill additionally requires a `FORMS_RECOGNIZER_MODEL_ID`. 23 | 24 | To deploy the skills: 25 | 1. In the Azure portal, create a Forms Recognizer resource. 26 | 2. Copy the form recognizer URL and key for use in the training and appsettings. 27 | 3. Clone this repository 28 | 4. Upload the training data and test data to an Azure storage account and generate a SAS token 29 | 4. Use the training notebook to train a model (using either the provided training data or your fomrs) 30 | 5. Open the AnalyzeFormV2 folder in VS Code and deploy the function. 31 | 6. Once the function is deployed, set the required appsettings (`FORMS_RECOGNIZER_ENDPOINT`, `FORMS_RECOGNIZER_KEY` and `FORMS_RECOGNIZER_MODEL_ID`) 32 | 33 | # AnalyzeForm 34 | 35 | This custom skill extracts specific fields from the results of a trained form recognition. 36 | 37 | A [full tutorial on this skill is available in the Azure Cognitive Seach documentation](https://docs.microsoft.com/en-us/azure/search/cognitive-search-custom-skill-form). 38 | 39 | 40 | ## Requirements 41 | 42 | In addition to the common requirements described in [the root `README.md` file](../../README.md), this function requires access to an [Azure Form Recognizer](https://azure.microsoft.com/en-us/services/cognitive-services/form-recognizer/) resource. 43 | 44 | [Train a model with your forms](https://docs.microsoft.com/azure/cognitive-services/form-recognizer/quickstarts/client-library) before you can use this skill. As a sample to get started, use the included [sample training forms](Train) and [sample test form](Test) with the [training notebook](FormRecognizerTrainModel.ipynb) to create a model in just a few minutes. 45 | 46 | ## Settings 47 | 48 | This function requires a `FORMS_RECOGNIZER_ENDPOINT` and a `FORMS_RECOGNIZER_KEY` settings set to a valid Azure Forms Recognizer API key and to your custom Form Recognizer 2.1-preview endpoint. 49 | If running locally, this can be set in your project's local environment variables. This ensures your key won't be accidentally checked in with your code. 50 | If running in an Azure function, this can be set in the application settings. 51 | 52 | After training, you will need to set the `FORMS_RECOGNIZER_MODEL_ID` application setting to the model id corresponding to your trained model. 53 | 54 | The list of fields to extract and the fields they get mapped to in the response of the skill need to be configured to reflect your particular scenario. This can be done by editing the [`field-mappings.json`] file(https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Vision/AnalyzeFormV2/AnalyzeForm/field-mappings.json). 55 | 56 | ## Sample Input: 57 | 58 | This sample data is pointing to a file stored in this repository, but when the skill is integrated in a skillset, the URL and token will be provided by cognitive search. 59 | 60 | ```json 61 | { 62 | "values": [ 63 | { 64 | "recordId": "record1", 65 | "data": { 66 | "formUrl": "https://github.com/Azure-Samples/azure-search-power-skills/raw/master/SampleData/Invoice_4.pdf", 67 | "formSasToken": "?st=sasTokenThatWillBeGeneratedByCognitiveSearch" 68 | } 69 | } 70 | ] 71 | } 72 | ``` 73 | 74 | ## Sample Output: 75 | 76 | ```json 77 | { 78 | "values": [ 79 | { 80 | "recordId": "record1", 81 | "data": { 82 | "address": "1111 8th st. Bellevue, WA 99501 ", 83 | "recipient": "Southridge Video 1060 Main St. Atlanta, GA 65024 " 84 | }, 85 | "errors": null, 86 | "warnings": null 87 | } 88 | ] 89 | } 90 | ``` 91 | 92 | ## Sample Skillset Integration 93 | 94 | In order to use this skill in a cognitive search pipeline, you'll need to add a skill definition to your skillset. 95 | Here's a sample skill definition for this example (inputs and outputs should be updated to reflect your particular scenario and skillset environment): 96 | 97 | ```json 98 | { 99 | "@odata.type": "#Microsoft.Skills.Custom.WebApiSkill", 100 | "name": "formrecognizer", 101 | "description": "Extracts fields from a form using a pre-trained form recognition model", 102 | "uri": "[AzureFunctionEndpointUrl]/api/AnalyzeForm?code=[AzureFunctionDefaultHostKey]", 103 | "httpMethod": "POST", 104 | "timeout": "PT1M", 105 | "context": "/document", 106 | "batchSize": 1, 107 | "inputs": [ 108 | { 109 | "name": "formUrl", 110 | "source": "/document/metadata_storage_path" 111 | }, 112 | { 113 | "name": "formSasToken", 114 | "source": "/document/metadata_storage_sas_token" 115 | } 116 | ], 117 | "outputs": [ 118 | { 119 | "name": "address", 120 | "targetName": "address" 121 | }, 122 | { 123 | "name": "recipient", 124 | "targetName": "recipient" 125 | } 126 | ] 127 | } 128 | ``` 129 | 130 | # AnalyzeInvoice 131 | 132 | This custom skill extracts invoice specific fields using a pre trained forms recognizer model. 133 | 134 | 135 | ## Requirements 136 | 137 | In addition to the common requirements described in [the root `README.md` file](../../README.md), this function requires access to an [Azure Forms Recognizer](https://azure.microsoft.com/en-us/services/cognitive-services/form-recognizer/) resource. The [prebuilt invoice model](https://docs.microsoft.com/azure/cognitive-services/form-recognizer/concept-invoices) is available in the 2.1 preview API. 138 | 139 | ## Settings 140 | 141 | This function requires a `FORMS_RECOGNIZER_ENDPOINT` and a `FORMS_RECOGNIZER_KEY` settings set to a valid Azure Forms Recognizer API key and to your custom Form Recognizer 2.1-preview endpoint. 142 | If running locally, this can be set in your project's local environment variables. This ensures your key won't be accidentally checked in with your code. 143 | If running in an Azure function, this can be set in the application settings. 144 | 145 | 146 | ## Sample Input: 147 | 148 | This sample data is pointing to a file stored in this repository, but when the skill is integrated in a skillset, the URL and token will be provided by cognitive search. 149 | 150 | ```json 151 | { 152 | "values": [ 153 | { 154 | "recordId": "record1", 155 | "data": { 156 | "formUrl": "https://github.com/Azure-Samples/azure-search-power-skills/raw/master/SampleData/Invoice_4.pdf", 157 | "formSasToken": "?st=sasTokenThatWillBeGeneratedByCognitiveSearch" 158 | } 159 | } 160 | ] 161 | } 162 | ``` 163 | 164 | ## Sample Output: 165 | 166 | ```json 167 | { 168 | "values": [ 169 | { 170 | "recordId": "0", 171 | "data": { 172 | "invoices": [ 173 | { 174 | "AmountDue": 63.0, 175 | "BillingAddress": "345 North St NY 98052", 176 | "BillingAddressRecipient": "Fabrikam, Inc.", 177 | "DueDate": "2018-05-31", 178 | "InvoiceDate": "2018-05-15", 179 | "InvoiceId": "1785443", 180 | "InvoiceTotal": 56.28, 181 | "VendorAddress": "4567 Main St Buffalo NY 90852", 182 | "SubTotal": 49.3, 183 | "TotalTax": 0.99 184 | } 185 | ] 186 | } 187 | } 188 | ] 189 | } 190 | ``` 191 | 192 | ## Sample Skillset Integration 193 | 194 | In order to use this skill in a cognitive search pipeline, you'll need to add a skill definition to your skillset. 195 | Here's a sample skill definition for this example (inputs and outputs should be updated to reflect your particular scenario and skillset environment): 196 | 197 | ```json 198 | { 199 | "@odata.type": "#Microsoft.Skills.Custom.WebApiSkill", 200 | "name": "formrecognizer", 201 | "description": "Extracts fields from a form using a pre-trained form recognition model", 202 | "uri": "[AzureFunctionEndpointUrl]/api/AnalyzeInvoice?code=[AzureFunctionDefaultHostKey]", 203 | "httpMethod": "POST", 204 | "timeout": "PT1M", 205 | "context": "/document", 206 | "batchSize": 1, 207 | "inputs": [ 208 | { 209 | "name": "formUrl", 210 | "source": "/document/metadata_storage_path" 211 | }, 212 | { 213 | "name": "formSasToken", 214 | "source": "/document/metadata_storage_sas_token" 215 | } 216 | ], 217 | "outputs": [ 218 | { 219 | "name": "invoices", 220 | "targetName": "invoices" 221 | } 222 | ] 223 | } 224 | ``` 225 | 226 | # ExtractTables 227 | 228 | This custom skill extracts tables using a pre trained forms recognizer model. 229 | 230 | 231 | ## Requirements 232 | 233 | In addition to the common requirements described in [the root `README.md` file](../../README.md), this function requires access to an [Azure Forms Recognizer](https://azure.microsoft.com/en-us/services/cognitive-services/form-recognizer/) resource. The [layout service](https://docs.microsoft.com/azure/cognitive-services/form-recognizer/concept-layout) is available in the 2.1 preview API. 234 | 235 | ## Settings 236 | 237 | This function requires a `FORMS_RECOGNIZER_ENDPOINT` and a `FORMS_RECOGNIZER_KEY` settings set to a valid Azure Forms Recognizer API key and to your custom Form Recognizer 2.1-preview endpoint. 238 | If running locally, this can be set in your project's local environment variables. This ensures your key won't be accidentally checked in with your code. 239 | If running in an Azure function, this can be set in the application settings. 240 | 241 | 242 | ## Sample Input: 243 | 244 | This sample data is pointing to a file stored in this repository, but when the skill is integrated in a skillset, the URL and token will be provided by cognitive search. 245 | 246 | ```json 247 | { 248 | "values": [ 249 | { 250 | "recordId": "record1", 251 | "data": { 252 | "formUrl": "https://github.com/Azure-Samples/azure-search-power-skills/raw/master/SampleData/Invoice_4.pdf", 253 | "formSasToken": "?st=sasTokenThatWillBeGeneratedByCognitiveSearch" 254 | } 255 | } 256 | ] 257 | } 258 | ``` 259 | 260 | ## Sample Output: 261 | 262 | ```json 263 | { 264 | "values": [ 265 | { 266 | "recordId": "0", 267 | "data": { 268 | "tables": [ 269 | { 270 | "page_number": 1, 271 | "row_count": 4, 272 | "column_count": 4, 273 | "cells": [ 274 | { 275 | "text": "Item", 276 | "rowIndex": 0, 277 | "colIndex": 0, 278 | "confidence": 1.0, 279 | "is_header": false 280 | }, 281 | { 282 | "text": "Quantity", 283 | "rowIndex": 0, 284 | "colIndex": 1, 285 | "confidence": 1.0, 286 | "is_header": false 287 | }, 288 | { 289 | "text": "Rate", 290 | "rowIndex": 0, 291 | "colIndex": 2, 292 | "confidence": 1.0, 293 | "is_header": false 294 | }, 295 | { 296 | "text": "Amount", 297 | "rowIndex": 0, 298 | "colIndex": 3, 299 | "confidence": 1.0, 300 | "is_header": false 301 | }, 302 | { 303 | "text": "Monthly service fee", 304 | "rowIndex": 1, 305 | "colIndex": 0, 306 | "confidence": 1.0, 307 | "is_header": false 308 | }, 309 | { 310 | "text": "01", 311 | "rowIndex": 1, 312 | "colIndex": 1, 313 | "confidence": 1.0, 314 | "is_header": false 315 | }, 316 | { 317 | "text": "$40.00", 318 | "rowIndex": 1, 319 | "colIndex": 2, 320 | "confidence": 1.0, 321 | "is_header": false 322 | }, 323 | { 324 | "text": "$40.00", 325 | "rowIndex": 1, 326 | "colIndex": 3, 327 | "confidence": 1.0, 328 | "is_header": false 329 | }, 330 | { 331 | "text": "Guarantee monthly fee", 332 | "rowIndex": 2, 333 | "colIndex": 0, 334 | "confidence": 1.0, 335 | "is_header": false 336 | }, 337 | { 338 | "text": "01", 339 | "rowIndex": 2, 340 | "colIndex": 1, 341 | "confidence": 1.0, 342 | "is_header": false 343 | }, 344 | { 345 | "text": "$6.00", 346 | "rowIndex": 2, 347 | "colIndex": 2, 348 | "confidence": 1.0, 349 | "is_header": false 350 | }, 351 | { 352 | "text": "$6.00", 353 | "rowIndex": 2, 354 | "colIndex": 3, 355 | "confidence": 1.0, 356 | "is_header": false 357 | }, 358 | { 359 | "text": "Add-on services", 360 | "rowIndex": 3, 361 | "colIndex": 0, 362 | "confidence": 1.0, 363 | "is_header": false 364 | }, 365 | { 366 | "text": "01", 367 | "rowIndex": 3, 368 | "colIndex": 1, 369 | "confidence": 1.0, 370 | "is_header": false 371 | }, 372 | { 373 | "text": "$3.30", 374 | "rowIndex": 3, 375 | "colIndex": 2, 376 | "confidence": 1.0, 377 | "is_header": false 378 | }, 379 | { 380 | "text": "$3.30", 381 | "rowIndex": 3, 382 | "colIndex": 3, 383 | "confidence": 1.0, 384 | "is_header": false 385 | } 386 | ] 387 | } 388 | ] 389 | } 390 | } 391 | ] 392 | } 393 | ``` 394 | 395 | ## Sample Skillset Integration 396 | 397 | In order to use this skill in a cognitive search pipeline, you'll need to add a skill definition to your skillset. 398 | Here's a sample skill definition for this example (inputs and outputs should be updated to reflect your particular scenario and skillset environment): 399 | 400 | ```json 401 | { 402 | "@odata.type": "#Microsoft.Skills.Custom.WebApiSkill", 403 | "name": "formrecognizer", 404 | "description": "Extracts fields from a form using a pre-trained form recognition model", 405 | "uri": "[AzureFunctionEndpointUrl]/api/ExtractTables?code=[AzureFunctionDefaultHostKey]", 406 | "httpMethod": "POST", 407 | "timeout": "PT1M", 408 | "context": "/document", 409 | "batchSize": 1, 410 | "inputs": [ 411 | { 412 | "name": "formUrl", 413 | "source": "/document/metadata_storage_path" 414 | }, 415 | { 416 | "name": "formSasToken", 417 | "source": "/document/metadata_storage_sas_token" 418 | } 419 | ], 420 | "outputs": [ 421 | { 422 | "name": "tables", 423 | "targetName": "tables" 424 | } 425 | ] 426 | } 427 | ``` 428 | --------------------------------------------------------------------------------