├── .gitattributes ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── docker-image-web-ui.yml ├── .gitignore ├── 00 - Resource Deployment ├── README.md ├── SearchAssets.ps1 ├── deploy.ps1 └── templates │ ├── base-datasource.json │ ├── base-index.json │ ├── base-indexer.json │ └── base-skills.json ├── 01 - Search Index Creation ├── 01.1 - BuiltIn Skills │ └── README.md ├── Cognitive Search Pipeline APIs.postman_collection.json ├── Cognitive Search Pipeline Environment.postman_environment.json ├── Create-Index-AzurePortal.md ├── Create-Index-Postman.md ├── Create-Index-PowerShell.md └── README.md ├── 02 - Web UI Template ├── CognitiveSearch.Template.sln ├── CognitiveSearch.UI │ ├── CognitiveSearch.UI.csproj │ ├── Configuration │ │ ├── ApiConfig.cs │ │ ├── AppConfig.cs │ │ └── OrganizationConfig.cs │ ├── Controllers │ │ ├── AdminController.cs │ │ ├── HomeController.cs │ │ └── StorageController.cs │ ├── Models │ │ ├── ColorSettings.cs │ │ ├── CustomizeViewModel.cs │ │ ├── ErrorViewModel.cs │ │ └── SearchResultViewModel.cs │ ├── Program.cs │ ├── Properties │ │ └── launchSettings.json │ ├── Search │ │ ├── DocumentResult.cs │ │ ├── DocumentSearchClient.cs │ │ ├── FacetGraphGenerator.cs │ │ ├── SearchFacet.cs │ │ ├── SearchModel.cs │ │ └── SearchSchema.cs │ ├── Startup.cs │ ├── Views │ │ ├── Admin │ │ │ ├── Customize.cshtml │ │ │ ├── NotAvailable.cshtml │ │ │ └── UploadData.cshtml │ │ ├── Home │ │ │ ├── Index.cshtml │ │ │ └── Search.cshtml │ │ ├── Shared │ │ │ ├── Error.cshtml │ │ │ ├── _Footer.cshtml │ │ │ ├── _Header.cshtml │ │ │ ├── _Layout.cshtml │ │ │ └── _ValidationScriptsPartial.cshtml │ │ ├── _ViewImports.cshtml │ │ └── _ViewStart.cshtml │ ├── appsettings.Development.json │ ├── appsettings.json │ ├── bundleconfig.json │ └── wwwroot │ │ ├── css │ │ ├── colors.css │ │ ├── navbar.css │ │ ├── site.css │ │ └── site.min.css │ │ ├── favicon.ico │ │ ├── images │ │ ├── ITIcons.png │ │ ├── JumboBackground.jpg │ │ ├── Microsoft-logo_rgb_c-gray.png │ │ ├── ReachNewHeights.png │ │ ├── Video1.PNG │ │ ├── Video2.PNG │ │ ├── collapse.png │ │ ├── expand.png │ │ ├── homepageimage.png │ │ └── logo.png │ │ ├── js │ │ ├── PolygonDrawingTool.js │ │ ├── appInsights.js │ │ ├── details.js │ │ ├── entityMap.js │ │ ├── facets.js │ │ ├── results.js │ │ ├── search.js │ │ ├── site.js │ │ ├── site.min.js │ │ └── tags.js │ │ └── lib │ │ ├── bootstrap │ │ ├── .bower.json │ │ ├── LICENSE │ │ └── dist │ │ │ ├── css │ │ │ ├── bootstrap-grid.css │ │ │ ├── bootstrap-grid.css.map │ │ │ ├── bootstrap-grid.min.css │ │ │ ├── bootstrap-grid.min.css.map │ │ │ ├── bootstrap-reboot.css │ │ │ ├── bootstrap-reboot.css.map │ │ │ ├── bootstrap-reboot.min.css │ │ │ ├── bootstrap-reboot.min.css.map │ │ │ ├── bootstrap-theme.css │ │ │ ├── bootstrap-theme.css.map │ │ │ ├── bootstrap-theme.min.css │ │ │ ├── bootstrap-theme.min.css.map │ │ │ ├── bootstrap.css │ │ │ ├── bootstrap.css.map │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.css.map │ │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ └── glyphicons-halflings-regular.woff2 │ │ │ └── js │ │ │ ├── bootstrap.bundle.js │ │ │ ├── bootstrap.bundle.js.map │ │ │ ├── bootstrap.bundle.min.js │ │ │ ├── bootstrap.bundle.min.js.map │ │ │ ├── bootstrap.js │ │ │ ├── bootstrap.js.map │ │ │ ├── bootstrap.min.js │ │ │ ├── bootstrap.min.js.map │ │ │ └── npm.js │ │ ├── dropzone │ │ └── dist │ │ │ ├── css │ │ │ ├── basic.css │ │ │ ├── basic.min.css │ │ │ ├── dropzone.css │ │ │ └── dropzone.min.css │ │ │ └── js │ │ │ ├── dropzone-amd-module.js │ │ │ ├── dropzone-amd-module.min.js │ │ │ ├── dropzone.js │ │ │ └── dropzone.min.js │ │ ├── fabric-ui │ │ ├── fabric-9.4.0.scoped.css │ │ ├── fabric-9.4.0.scoped.min.css │ │ ├── fabric.css │ │ └── fabric.min.css │ │ ├── jquery-validation-unobtrusive │ │ ├── LICENSE.txt │ │ ├── jquery.validate.unobtrusive.js │ │ └── jquery.validate.unobtrusive.min.js │ │ ├── jquery-validation │ │ ├── LICENSE.md │ │ └── dist │ │ │ ├── additional-methods.js │ │ │ ├── additional-methods.min.js │ │ │ ├── jquery.validate.js │ │ │ └── jquery.validate.min.js │ │ ├── jquery │ │ ├── LICENSE.txt │ │ └── dist │ │ │ ├── jquery.js │ │ │ ├── jquery.min.js │ │ │ └── jquery.min.map │ │ └── tiff │ │ └── tiff.min.js ├── Docker │ ├── .dockerignore │ ├── .env │ └── Dockerfile └── README.md ├── 03 - Data Science and Custom Skills ├── AML Custom Skill Template │ ├── 01_Train_AML_Model.ipynb │ ├── 02_Deploy_AML_Custom_Skill.ipynb │ └── README.md ├── Azure Function Custom Skills │ └── README.md ├── Docker Flask Skill │ ├── Dockerfile │ ├── readme.md │ ├── requirements.txt │ └── ws.py ├── FormRecognizer Skill │ ├── README.md │ ├── SampleInvoices │ │ ├── Invoice_1.pdf │ │ ├── Invoice_2.pdf │ │ ├── Invoice_3.pdf │ │ ├── Invoice_4.pdf │ │ └── Invoice_5.pdf │ └── customskill │ │ ├── .funcignore │ │ ├── .gitignore │ │ ├── .vscode │ │ ├── extensions.json │ │ ├── launch.json │ │ ├── settings.json │ │ └── tasks.json │ │ ├── AnalyzeForm │ │ ├── __init__.py │ │ ├── field_mappings.json │ │ ├── function.json │ │ └── sample.dat │ │ ├── AnalyzeInvoice │ │ ├── __init__.py │ │ ├── function.json │ │ └── sample.dat │ │ ├── ExtractTables │ │ ├── __init__.py │ │ ├── function.json │ │ └── sample.dat │ │ ├── FormRecognizerTrainModel.ipynb │ │ ├── Readme.md │ │ ├── Test │ │ └── Ignite.png │ │ ├── Train │ │ ├── Commercial Motor Vehicle - Adatum.json │ │ ├── Commercial Motor Vehicle - Adatum.pdf │ │ ├── Commercial Motor Vehicle - Fincher.pdf │ │ ├── Commercial Motor Vehicle - Lamna.pdf │ │ ├── Commercial Motor Vehicle - Liberty.pdf │ │ └── Commercial Motor Vehicle - Trey.pdf │ │ ├── azuredeploy.json │ │ ├── host.json │ │ ├── proxies.json │ │ └── requirements.txt ├── README.md └── Useful Notebooks │ ├── query_search_index.ipynb │ └── test_aml_deployment.ipynb ├── 04 - Reporting ├── Cognitive Search.pbit └── README.md ├── CONTRIBUTING.md ├── LICENSE.md ├── PRIVACY.md ├── README.md ├── azuredeploy.json ├── images ├── Config.png ├── SearchModel.png ├── WebUI.jpg ├── appsettings.png ├── architecture.jpg ├── createindex-step0.PNG ├── createindex-step1.PNG ├── createindex-step2.PNG ├── createindex-step3.PNG ├── createindex-step4.PNG ├── createindex-step5.PNG ├── createindex-step6.PNG ├── createindex-step7.PNG ├── createindex-step8.PNG ├── demo.jpg ├── geolocation.png ├── kmheader.png ├── pbi1.png ├── pbi2.png ├── pbi3.JPG ├── postman_edit_environment_variables.png ├── postman_import_files.png ├── translated.png ├── ui.PNG ├── webapp-1.png └── webapp-2.png ├── industry-solutions ├── documents │ ├── KnowledgeMiningWhitePaper.pdf │ └── WhitePaperDocSegmentation.pdf └── journalism │ ├── projectida │ └── readme.md │ └── readme.md ├── sample_documents ├── Azure Search Pricing Table.xlsx ├── Cognitive Services and Bots (spanish).pdf ├── Cognitive Services and Content Intelligence.pdf ├── Deployment strategies and best practices.pdf ├── FAQ - Azure Search.pdf ├── GitHub Facts.pdf ├── LearnAI KnowledgeMiningBootcamp.pdf ├── MSFT Contoso Cloud Architecture.pdf ├── NYSE_LNKD_2015.PDF ├── Pricing details.docx ├── Red Shirts.png ├── What is Azure Search service.pdf ├── create-search-collect-info.png ├── create-search-service.png ├── create-service-full-portal.png ├── geo-search-traffic-mgr.png ├── guthrie.jpg ├── redshirt.jpg ├── satyanadellalinux.jpg ├── satyasletter.html └── scale-indexers.png └── workshops ├── Appendix └── KeyVault.md ├── LICENSE ├── Module 0.md ├── Module 1.md ├── Module 2.md ├── Module 3.md ├── Module 4.md ├── Module 5.md ├── Module 6.md ├── Module 7.md ├── README.md ├── data ├── clinical-trials-small.zip └── words.csv ├── images ├── 4skills.png ├── add_scoring_profile.png ├── addks.png ├── attachenrich.png ├── azurefunction.png ├── chkstatus.png ├── chkstatus2.png ├── chooseconnection.png ├── connectblob.jpg ├── copyblobcontainer.png ├── create-kv-1.png ├── create-kv-2.png ├── customentity.png ├── datasource.png ├── diseaseextractor.png ├── diseases.png ├── editquery.png ├── enrichments.png ├── field_boosting.png ├── freshness_function.png ├── function-url.png ├── get-index-schema.png ├── getdata.png ├── importdata.png ├── indexdef.png ├── indexer.png ├── intresults.png ├── json2.png ├── jsonportal.png ├── kstable.png ├── kstable2.png ├── kv-access-policies.png ├── kv-app-identity.png ├── kv-create-secret.png ├── kv-nuget.png ├── kv-set-secret-blob.png ├── kv-set-secret.png ├── kv-view-list.png ├── location .png ├── lookup2.png ├── mod5 │ ├── ks-pbi-closeandapply.png │ ├── ks-pbi-editquery1.png │ ├── ks-pbi-editquery2.png │ ├── ks-pbi-editquery3.png │ ├── ks-pbi-editquery4.png │ ├── ks-pbi-getdata.png │ ├── ks-pbi-getdata2.png │ ├── ks-pbi-getdata3.png │ ├── ks-pbi-getdata4.png │ ├── ks-pbi-model.png │ ├── ks-pbi-model2.png │ ├── ks-pbi-visual1.png │ ├── ks-pbi-visual2-disease-list.png │ ├── ks-pbi-visual3-lastupdate-date.png │ ├── ks-pbi-visual4-doccount-graph-filtered.png │ ├── ks-pbi-visual4-doccount-graph.png │ ├── ks-pbi-visual5-filledmap-graph.png │ ├── ks-pbi-visual5-filledmap-settings.png │ ├── ks-pbi-visual5-filledmap-settings2.png │ └── ks-pbi-visual6-directed-graph.png ├── new-appservice.png ├── objectmodel.png ├── pasteblobcontainer.png ├── postman.png ├── postman2.png ├── powerbi.png ├── querywithselect.PNG ├── removecolumn.png ├── rerun.png ├── results.png ├── savechanges.png ├── scoring_profile.png ├── select container.png ├── selectcontainer.png ├── setstart.png ├── skillset.png ├── sql-patient-info.png ├── srchexplore.png ├── storeacct.png ├── tablereview.png ├── tablestore.png ├── unstructured.png └── wordscsv.png └── slides └── KM_Workshop.pptx /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | > Please provide us with the following information: 5 | > --------------------------------------------------------------- 6 | 7 | ### This issue is for a: (mark with an `x`) 8 | ``` 9 | - [ ] bug report -> please search issues before submitting 10 | - [ ] feature request 11 | - [ ] documentation issue or request 12 | - [ ] regression (a behavior that used to work and stopped in a new release) 13 | ``` 14 | 15 | ### Minimal steps to reproduce 16 | > 17 | 18 | ### Any log messages given by the failure 19 | > 20 | 21 | ### Expected/desired behavior 22 | > 23 | 24 | ### OS and Version? 25 | > Windows 7, 8 or 10. Linux (which distribution). macOS (Yosemite? El Capitan? Sierra?) 26 | 27 | ### Versions 28 | > 29 | 30 | ### Mention any other details that might be useful 31 | 32 | > --------------------------------------------------------------- 33 | > Thanks! We'll be in touch soon. 34 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | * ... 4 | 5 | ## Does this introduce a breaking change? 6 | 7 | ``` 8 | [ ] Yes 9 | [ ] No 10 | ``` 11 | 12 | ## Pull Request Type 13 | What kind of change does this Pull Request introduce? 14 | 15 | 16 | ``` 17 | [ ] Bugfix 18 | [ ] Feature 19 | [ ] Code style update (formatting, local variables) 20 | [ ] Refactoring (no functional changes, no api changes) 21 | [ ] Documentation content changes 22 | [ ] Other... Please describe: 23 | ``` 24 | 25 | ## How to Test 26 | * Get the code 27 | 28 | ``` 29 | git clone [repo-address] 30 | cd [repo-name] 31 | git checkout [branch-name] 32 | npm install 33 | ``` 34 | 35 | * Test the code 36 | 37 | ``` 38 | ``` 39 | 40 | ## What to Check 41 | Verify that the following are valid 42 | * ... 43 | 44 | ## Other Information 45 | -------------------------------------------------------------------------------- /.github/workflows/docker-image-web-ui.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Azure Container Registry Login 19 | uses: Azure/docker-login@v1 20 | with: 21 | # Container registry username 22 | username: ${{ secrets.ACR_USERNAME }} 23 | # Container registry password 24 | password: ${{ secrets.ACR_PASSWORD }} 25 | # Container registry server url 26 | login-server: ${{ secrets.ACR_LOGIN_SERVER }} 27 | 28 | - uses: actions/checkout@v2 29 | - name: Build the Docker image 30 | run: 31 | cd "02 - Web UI Template"; 32 | docker build . --file Docker/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER }}/web-ui:latest; 33 | docker push ${{ secrets.ACR_LOGIN_SERVER }}/web-ui:latest 34 | 35 | -------------------------------------------------------------------------------- /00 - Resource Deployment/SearchAssets.ps1: -------------------------------------------------------------------------------- 1 | param( 2 | [string] $searchServiceName, 3 | [string] $searchServiceKey, 4 | [string] $storageAccount, 5 | [string] $storageKey, 6 | [string] $storageContainerName, 7 | [string] $cogServicesKey, 8 | [string] $dataSourceName, 9 | [string] $skillsetName, 10 | [string] $indexName, 11 | [string] $indexerName 12 | ) 13 | 14 | $storageConnectionString = "DefaultEndpointsProtocol=https;AccountName=$storageAccount;AccountKey=$storageKey;EndpointSuffix=core.windows.net" 15 | 16 | 17 | function CreateSearchIndex 18 | { 19 | Write-Host "Creating Search Index"; 20 | 21 | function CallSearchAPI 22 | { 23 | param ( 24 | [string]$url, 25 | [string]$body 26 | ) 27 | 28 | $headers = @{ 29 | 'api-key' = $searchServiceKey 30 | 'Content-Type' = 'application/json' 31 | 'Accept' = 'application/json' 32 | } 33 | $baseSearchUrl = "https://"+$searchServiceName+".search.windows.net" 34 | $fullUrl = $baseSearchUrl + $url 35 | 36 | Write-Host "Calling api: '"$fullUrl"'"; 37 | Invoke-RestMethod -Uri $fullUrl -Headers $headers -Method Put -Body $body.TrimStart([regex]::Unescape("\ufeff")) | ConvertTo-Json 38 | }; 39 | 40 | # Create the datasource 41 | $dataSourceBody = (Invoke-WebRequest -Uri "https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/main/00%20-%20Resource%20Deployment/templates/base-datasource.json").Content 42 | $dataSourceBody = $dataSourceBody -replace "{{env_storage_connection_string}}", $storageConnectionString 43 | $dataSourceBody = $dataSourceBody -replace "{{env_storage_container}}", $storageContainerName 44 | CallSearchAPI -url ("/datasources/"+$dataSourceName+"?api-version=2020-06-30") -body $dataSourceBody 45 | 46 | # Create the skillset 47 | $skillBody = (Invoke-WebRequest -Uri "https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/main/00%20-%20Resource%20Deployment/templates/base-skills.json").Content 48 | $skillBody = $skillBody -replace "{{cog_services_key}}", $cogServicesKey 49 | CallSearchAPI -url ("/skillsets/"+$skillsetName+"?api-version=2020-06-30") -body $skillBody 50 | 51 | # Create the index 52 | $indexBody = (Invoke-WebRequest -Uri "https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/main/00%20-%20Resource%20Deployment/templates/base-index.json").Content 53 | CallSearchAPI -url ("/indexes/"+$indexName+"?api-version=2020-06-30") -body $indexBody 54 | 55 | # Create the indexer 56 | $indexerBody = (Invoke-WebRequest -Uri "https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/main/00%20-%20Resource%20Deployment/templates/base-indexer.json").Content 57 | $indexerBody = $indexerBody -replace "{{datasource_name}}", $dataSourceName 58 | $indexerBody = $indexerBody -replace "{{skillset_name}}", $skillsetName 59 | $indexerBody = $indexerBody -replace "{{index_name}}", $indexName 60 | CallSearchAPI -url ("/indexers/"+$indexerName+"?api-version=2020-06-30") -body $indexerBody 61 | } 62 | 63 | CreateSearchIndex; 64 | -------------------------------------------------------------------------------- /00 - Resource Deployment/templates/base-datasource.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Demo files to demonstrate cognitive search capabilities.", 3 | "type": "azureblob", 4 | "credentials": { 5 | "connectionString": "{{env_storage_connection_string}}" 6 | }, 7 | "container": { 8 | "name": "{{env_storage_container}}" 9 | } 10 | } -------------------------------------------------------------------------------- /00 - Resource Deployment/templates/base-index.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "content", 5 | "type": "Edm.String", 6 | "sortable": false, 7 | "searchable": true, 8 | "filterable": false, 9 | "facetable": false 10 | }, 11 | { 12 | "name": "keyPhrases", 13 | "type": "Collection(Edm.String)", 14 | "searchable": true, 15 | "sortable": false, 16 | "filterable": true, 17 | "facetable": true 18 | }, 19 | { 20 | "name": "organizations", 21 | "type": "Collection(Edm.String)", 22 | "searchable": true, 23 | "sortable": false, 24 | "filterable": true, 25 | "facetable": true 26 | }, 27 | { 28 | "name": "persons", 29 | "type": "Collection(Edm.String)", 30 | "searchable": true, 31 | "sortable": false, 32 | "filterable": true, 33 | "facetable": true 34 | }, 35 | { 36 | "name": "locations", 37 | "type": "Collection(Edm.String)", 38 | "searchable": true, 39 | "sortable": false, 40 | "filterable": true, 41 | "facetable": true 42 | }, 43 | { 44 | "name": "metadata_storage_path", 45 | "type": "Edm.String", 46 | "key": true, 47 | "searchable": true, 48 | "sortable": false, 49 | "filterable": false, 50 | "facetable": false 51 | }, 52 | { 53 | "name": "metadata_storage_name", 54 | "type": "Edm.String", 55 | "searchable": true, 56 | "sortable": false, 57 | "filterable": false, 58 | "facetable": false 59 | } 60 | ] 61 | } -------------------------------------------------------------------------------- /00 - Resource Deployment/templates/base-indexer.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataSourceName": "{{datasource_name}}", 3 | "targetIndexName": "{{index_name}}", 4 | "skillsetName": "{{skillset_name}}", 5 | "fieldMappings": [ 6 | { 7 | "sourceFieldName": "metadata_storage_path", 8 | "targetFieldName": "metadata_storage_path", 9 | "mappingFunction": { "name": "base64Encode" } 10 | }, 11 | { 12 | "sourceFieldName": "metadata_storage_name", 13 | "targetFieldName": "metadata_storage_name" 14 | } 15 | ], 16 | "outputFieldMappings": [ 17 | { 18 | "sourceFieldName": "/document/merged_text", 19 | "targetFieldName": "content" 20 | }, 21 | { 22 | "sourceFieldName": "/document/organizations", 23 | "targetFieldName": "organizations" 24 | }, 25 | { 26 | "sourceFieldName": "/document/persons", 27 | "targetFieldName": "persons" 28 | }, 29 | { 30 | "sourceFieldName": "/document/locations", 31 | "targetFieldName": "locations" 32 | }, 33 | { 34 | "sourceFieldName": "/document/pages/*/keyPhrases/*", 35 | "targetFieldName": "keyPhrases" 36 | } 37 | ], 38 | "parameters": { 39 | "batchSize": 1, 40 | "maxFailedItems": -1, 41 | "maxFailedItemsPerBatch": -1, 42 | "configuration": { 43 | "dataToExtract": "contentAndMetadata", 44 | "imageAction": "generateNormalizedImages" 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /00 - Resource Deployment/templates/base-skills.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Extract entities, detect language and extract key-phrases", 3 | "cognitiveServices": { 4 | "@odata.type": "#Microsoft.Azure.Search.CognitiveServicesByKey", 5 | "description": "mycogsvcs", 6 | "key": "{{cog_services_key}}" 7 | }, 8 | "skills": [ 9 | { 10 | "@odata.type": "#Microsoft.Skills.Vision.OcrSkill", 11 | "context": "/document/normalized_images/*", 12 | "defaultLanguageCode": "en", 13 | "detectOrientation": true, 14 | "inputs": [ 15 | { 16 | "name": "image", 17 | "source": "/document/normalized_images/*" 18 | } 19 | ], 20 | "outputs": [ 21 | { 22 | "name": "text" 23 | } 24 | ] 25 | }, 26 | { 27 | "@odata.type": "#Microsoft.Skills.Text.MergeSkill", 28 | "description": "Create merged_text, which includes all the textual representation of each image inserted at the right location in the content field.", 29 | "context": "/document", 30 | "insertPreTag": " ", 31 | "insertPostTag": " ", 32 | "inputs": [ 33 | { 34 | "name": "text", 35 | "source": "/document/content" 36 | }, 37 | { 38 | "name": "itemsToInsert", 39 | "source": "/document/normalized_images/*/text" 40 | }, 41 | { 42 | "name": "offsets", 43 | "source": "/document/normalized_images/*/contentOffset" 44 | } 45 | ], 46 | "outputs": [ 47 | { 48 | "name": "mergedText", 49 | "targetName": "merged_text" 50 | } 51 | ] 52 | }, 53 | { 54 | "@odata.type": "#Microsoft.Skills.Text.SplitSkill", 55 | "textSplitMode": "pages", 56 | "maximumPageLength": 4000, 57 | "defaultLanguageCode": "en", 58 | "context": "/document", 59 | "inputs": [ 60 | { 61 | "name": "text", 62 | "source": "/document/merged_text" 63 | } 64 | ], 65 | "outputs": [ 66 | { 67 | "name": "textItems", 68 | "targetName": "pages" 69 | } 70 | ] 71 | }, 72 | { 73 | "@odata.type": "#Microsoft.Skills.Text.KeyPhraseExtractionSkill", 74 | "context": "/document/pages/*", 75 | "inputs": [ 76 | { 77 | "name": "text", 78 | "source": "/document/pages/*" 79 | } 80 | ], 81 | "outputs": [ 82 | { 83 | "name": "keyPhrases", 84 | "targetName": "keyPhrases" 85 | } 86 | ] 87 | }, 88 | { 89 | "@odata.type": "#Microsoft.Skills.Text.EntityRecognitionSkill", 90 | "categories": [ "Organization" ], 91 | "context": "/document", 92 | "inputs": [ 93 | { 94 | "name": "text", 95 | "source": "/document/merged_text" 96 | } 97 | ], 98 | "outputs": [ 99 | { 100 | "name": "organizations", 101 | "targetName": "organizations" 102 | } 103 | ] 104 | }, 105 | { 106 | "@odata.type": "#Microsoft.Skills.Text.EntityRecognitionSkill", 107 | "categories": [ "Location" ], 108 | "context": "/document", 109 | "inputs": [ 110 | { 111 | "name": "text", 112 | "source": "/document/merged_text" 113 | } 114 | ], 115 | "outputs": [ 116 | { 117 | "name": "locations", 118 | "targetName": "locations" 119 | } 120 | ] 121 | }, 122 | { 123 | "@odata.type": "#Microsoft.Skills.Text.EntityRecognitionSkill", 124 | "categories": [ "Person" ], 125 | "context": "/document", 126 | "inputs": [ 127 | { 128 | "name": "text", 129 | "source": "/document/merged_text" 130 | } 131 | ], 132 | "outputs": [ 133 | { 134 | "name": "persons", 135 | "targetName": "persons" 136 | } 137 | ] 138 | } 139 | ] 140 | } -------------------------------------------------------------------------------- /01 - Search Index Creation/01.1 - BuiltIn Skills/README.md: -------------------------------------------------------------------------------- 1 | # Adding Built In Skill to the Skillset 2 | 3 | Add Sentiment Analysis Skill to the Skillset and verify that sentiment are generated and stored in the index. 4 | 5 | Use https://learn.microsoft.com/en-us/azure/search/cognitive-search-skill-sentiment-v3 as reference for Skill inputs and outputs 6 | 7 | 8 | - Add field `sentiment` to index 9 | ```json 10 | { 11 | "name": "sentiment", 12 | "type": "Edm.String", 13 | "searchable": true, 14 | "sortable": true, 15 | "filterable": true, 16 | "facetable": true 17 | } 18 | ``` 19 | 20 | - Add `"#Microsoft.Skills.Text.V3.SentimentSkill` to skillset 21 | ```json 22 | { 23 | "@odata.type": "#Microsoft.Skills.Text.V3.SentimentSkill", 24 | "name": "sentiment", 25 | "description": "", 26 | "context": "/document", 27 | "defaultLanguageCode": "en", 28 | "modelVersion": "", 29 | "includeOpinionMining": true, 30 | "inputs": [ 31 | { 32 | "name": "text", 33 | "source": "/document/merged_text" 34 | } 35 | ], 36 | "outputs": [ 37 | { 38 | "name": "sentiment", 39 | "targetName": "sentiment" 40 | }, 41 | { 42 | "name": "confidenceScores", 43 | "targetName": "confidenceScores" 44 | }, 45 | { 46 | "name": "sentences", 47 | "targetName": "sentences" 48 | } 49 | ] 50 | } 51 | ``` 52 | 53 | - Update Indexer to add output mappings between skill output and index field 54 | 55 | ```json 56 | { 57 | "sourceFieldName": "/document/sentiment", 58 | "targetFieldName": "sentiment" 59 | } 60 | ``` 61 | 62 | **Refer** to Postman collection for more details 63 | 64 | 65 | # Verify Index data 66 | 67 | - Search for all docments that have 'GitHub` word in them sorting by sentiment 68 | 69 | - Search all document and show sentiment and locations facets 70 | 71 | - Search documents that have location in Europe -------------------------------------------------------------------------------- /01 - Search Index Creation/Cognitive Search Pipeline Environment.postman_environment.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "4868ea28-6e9c-45a5-ac6f-e068348ded70", 3 | "name": "Cognitive Search Env", 4 | "values": [ 5 | { 6 | "key": "search_service", 7 | "value": "", 8 | "enabled": true 9 | }, 10 | { 11 | "key": "index_name", 12 | "value": "", 13 | "enabled": true 14 | }, 15 | { 16 | "key": "env_search_api_key", 17 | "value": "", 18 | "enabled": true 19 | }, 20 | { 21 | "key": "env_storage_connection_string", 22 | "value": "", 23 | "enabled": true 24 | }, 25 | { 26 | "key": "env_storage_container", 27 | "value": "", 28 | "enabled": true 29 | }, 30 | { 31 | "key": "cog_services_key", 32 | "value": "", 33 | "enabled": true 34 | }, 35 | { 36 | "key": "env_function_url", 37 | "value": "", 38 | "enabled": true 39 | } 40 | ], 41 | "_postman_variable_scope": "environment", 42 | "_postman_exported_at": "2019-11-03T20:38:54.049Z", 43 | "_postman_exported_using": "Postman/7.10.0" 44 | } -------------------------------------------------------------------------------- /01 - Search Index Creation/Create-Index-AzurePortal.md: -------------------------------------------------------------------------------- 1 | # Creating a Search Index in the Azure Portal 2 | 3 | The steps below will walk you through creating a search index in the Azure Portal using the [import data wizard](https://docs.microsoft.com/en-us/azure/search/search-import-data-portal). For more detailed information on each of the steps, see the the [Blob Storage Quickstart](https://docs.microsoft.com/en-us/azure/search/cognitive-search-quickstart-blob). 4 | 5 | The web app you'll spin up in the next step expects to receive several fields to work properly. Because of this, be sure to follow the requirements outlined below to avoid any problems during setup. 6 | 7 | ## Requirements 8 | 9 | If you choose to create your index via the Azure Portal, set your search field properties as described in the table below: 10 | 11 | | Field | Notes and Expectations | 12 | |-----------------------|-----------------------------------------------| 13 | |content | Used to show the transcript of the files. Should be **searchable and retrievable** | 14 | |metadata_storage_path | This should be the **key field**. The storage path is used to query the blob indexer for the content so that you can "preview" the file. Should be **retrievable**. If it is **base64 encoded**, make sure to specify it in the [appsettings.json](https://github.com/Azure-Samples/azure-search-knowledge-mining/tree/master/02%20-%20Web%20UI%20Template) file of the front end application. | 15 | |metadata_storage_name | The storage name is used to display the name of the file on the results page. Should be **retrievable**. | 16 | |people | List of strings with the persons identified in the document. Should be **facetable, filterable, searchable and retrievable**. | 17 | |locations | List of strings with the locations identified in the document. Should be **facetable, filterable, searchable and retrievable**. | 18 | |organizations | List of strings with the organizations identified in the document. Should be **facetable, filterable, searchable and retrievable**. | 19 | |keyPhrases | List of strings with the key phrases identified in the document. Should be **facetable, filterable, searchable and retrievable**. | 20 | |language | String containing the main language of the document. Should be **facetable, filterable, searchable and retrievable**. | 21 | 22 | ## Instructions 23 | 24 | Follow the steps and screenshots below to create your index. 25 | 26 | ### 1.0 Navigate to you Search Service 27 | 28 | Start by navigating to your search service in the Azure Portal: 29 | 30 | ![screenshot](../images/createindex-step0.PNG) 31 | 32 | ### 2.0 Select Import data 33 | 34 | ![Navigate to search service](../images/createindex-step1.PNG) 35 | 36 | ### 3.0 Import Data 37 | 38 | #### 3.1 Select Azure Blob Storage 39 | 40 | ![screenshot](../images/createindex-step2.PNG) 41 | 42 | #### 3.2 Follow the wizard to connect to your storage account 43 | 44 | Keep the defaults and use `Choose an existing connection` to connect to your storage container with your data: 45 | 46 | ![screenshot](../images/createindex-step3.PNG) 47 | 48 | ### 4.0 Add Cognitive Skills 49 | 50 | #### 4.1 Attach Cognitive Services 51 | 52 | An Cognitive Services reosurce should have been automatically provisioned, just select it. If it has not been provisioned successfully, use the `Create new Cognitive Services resource` link and create one: 53 | 54 | ![screenshot](../images/createindex-step4.PNG) 55 | 56 | #### 4.2 Add enrichments 57 | 58 | Select the enrichments you want to add. Please note that the `Extract personally identifiable information` skill currently does only support English documents. Some of the sample data is currently in different languages, hence do not enable it when you use the [sample documents](../sample_documents) from this repo. 59 | 60 | ![screenshot](../images/createindex-step5.PNG) 61 | 62 | #### 4.3 Save enrichments to knowledge store 63 | 64 | ![screenshot](../images/createindex-step6.PNG) 65 | 66 | ### 5.0 Customize target index 67 | 68 | > **Note:** This step is essential to properly configuring your index. Make sure your index looks similar to the screenshot before proceeding to the next step. 69 | 70 | ![screenshot](../images/createindex-step7.PNG) 71 | 72 | ### 6.0 Create an indexer 73 | 74 | ![screenshot](../images/createindex-step8.PNG) 75 | 76 | Congratulations! You should be all set to move onto the next folder. 77 | -------------------------------------------------------------------------------- /01 - Search Index Creation/Create-Index-Postman.md: -------------------------------------------------------------------------------- 1 | # Creating a Search Index with Postman 2 | 3 | This folder contains a Postman collection and environment that can be used to create a search index programmatically. Using Postman allows you to parameterize the API calls, edit the API calls to customize your pipeline, and to easily share & preserve your changes. 4 | 5 | This collection is pre-configured to take advantage of a majority of the out of the box Cognitive Search functionality. 6 | 7 | We recommend using this collection to create an initial index and then iterating by editing the postman collection and adding custom skills as needed. This collection assumes that Azure Blob storage is the primary data source. Please see this [documentation](https://docs.microsoft.com/en-us/rest/api/searchservice/create-data-source) to understand the necessary changes if you wish to use a different data source. 8 | 9 | ## Prerequisites 10 | 11 | 1. Postman installed for your operating system 12 | 2. Initial document set uploaded to your Azure Blob storage container 13 | 14 | ## Setup Instructions 15 | 16 | 1. **Install Postman** - navigate to https://www.getpostman.com/apps to download and install postman for your operating system if you haven't already. 17 | 2. **Import Files** - import *Cognitive Search Pipeline APIs.postman_collection.json* and *Cognitive Search Pipeline Environment.postman_environment.json* using the Postman UI. 18 | 19 | ![import files](../images/postman_import_files.png) 20 | 21 | 3. **Update Environment Variables** - Set the environment using the dropdown highlighted below and then update the environment variables by selecting the *eye button* and then selecting edit. These variables are used to parameterize the API calls and save you from having to edit variables for each call. 22 | 23 | ![edit environment variables](../images/postman_edit_environment_variables.png) 24 | 25 | The following variables are required: 26 | * **search_service** - the name of your Azure Search resource 27 | * **index_name** - the name of your search index; this can be whatever you want it to be 28 | * **env_search_api_key** - the API key for your Azure Search service; this can be found in the Azure portal 29 | * **env_storage_connection_string** - the connection string for your blob storage account; this can be found in the Azure portal 30 | * **env_storage_container** - the name of the blob container that your documents are in 31 | * **cog_services_key** - the key for your Cognitive services; this can be found in the Azure portal 32 | 33 | 4. **Run API Calls** 34 | To create a cognitive search index, run the following API calls in order: 35 | 1. Create a datasource 36 | 2. Create index 37 | 3. Create a skillset 38 | 4. Create indexer 39 | 40 | You can then *check the indexer status* to see if documents are processing or if there are any errors. If the indexer does not start running automatically, you can run the indexer manually. 41 | 42 | ## Verify Index 43 | 44 | Use search explorer or postment to search data 45 | 46 | 47 | ## Additional Resources 48 | 49 | For more help working with Postman, see the [documentation](https://learning.postman.com/docs/getting-started/introduction/) on the Postman website. 50 | -------------------------------------------------------------------------------- /01 - Search Index Creation/Create-Index-PowerShell.md: -------------------------------------------------------------------------------- 1 | # Creating a Search Index with PowerShell 2 | 3 | The `deploy.ps1` PowerShell script in the previous step shows how to create a search index 4 | 5 | The following code will create a search index for you based on the json files in the `templates` folder: 6 | 7 | ```powershell 8 | while (!($uniqueName = Read-Host "uniqueName")) { Write-Host "You must provide a uniqueName."; } 9 | 10 | # Generate derivative parameters. 11 | $searchServiceName = $uniqueName + "search"; 12 | $storageAccountName = $uniqueName + "str"; 13 | $storageContainerName = "documents"; 14 | 15 | $dataSourceName = $uniqueName + "-datasource"; 16 | $skillsetName = $uniqueName + "-skillset"; 17 | $indexName = $uniqueName + "-index"; 18 | $indexerName = $uniqueName + "-indexer"; 19 | 20 | $global:storageAccountKey = ""; 21 | $global:searchServiceKey = ""; 22 | $global:storageConnectionString = ""; 23 | $global:cogServicesKey = ""; 24 | 25 | function CreateSearchIndex 26 | { 27 | Write-Host "Creating Search Index"; 28 | 29 | function CallSearchAPI 30 | { 31 | param ( 32 | [string]$url, 33 | [string]$body 34 | ) 35 | 36 | $headers = @{ 37 | 'api-key' = $global:searchServiceKey 38 | 'Content-Type' = 'application/json' 39 | 'Accept' = 'application/json' 40 | } 41 | $baseSearchUrl = "https://"+$searchServiceName+".search.windows.net" 42 | $fullUrl = $baseSearchUrl + $url 43 | 44 | Write-Host "Calling api: '"$fullUrl"'"; 45 | Invoke-RestMethod -Uri $fullUrl -Headers $headers -Method Put -Body $body | ConvertTo-Json 46 | }; 47 | 48 | # Create the datasource 49 | $dataSourceBody = Get-Content -Path .\templates\base-datasource.json 50 | $dataSourceBody = $dataSourceBody -replace "{{env_storage_connection_string}}", $global:storageConnectionString 51 | $dataSourceBody = $dataSourceBody -replace "{{env_storage_container}}", $storageContainerName 52 | CallSearchAPI -url ("/datasources/"+$dataSourceName+"?api-version=2019-05-06") -body $dataSourceBody 53 | 54 | # Create the skillset 55 | $skillBody = Get-Content -Path .\templates\base-skills.json 56 | $skillBody = $skillBody -replace "{{cog_services_key}}", $global:cogServicesKey 57 | CallSearchAPI -url ("/skillsets/"+$skillsetName+"?api-version=2019-05-06") -body $skillBody 58 | 59 | # Create the index 60 | $indexBody = Get-Content -Path .\templates\base-index.json 61 | CallSearchAPI -url ("/indexes/"+$indexName+"?api-version=2019-05-06") -body $indexBody 62 | 63 | # Create the indexer 64 | $indexerBody = Get-Content -Path .\templates\base-indexer.json 65 | $indexerBody = $indexerBody -replace "{{datasource_name}}", $dataSourceName 66 | $indexerBody = $indexerBody -replace "{{skillset_name}}", $skillsetName 67 | $indexerBody = $indexerBody -replace "{{index_name}}", $indexName 68 | CallSearchAPI -url ("/indexers/"+$indexerName+"?api-version=2019-05-06") -body $indexerBody 69 | } 70 | 71 | CreateSearchIndex; 72 | ``` 73 | 74 | ## Prerequisites 75 | 76 | 2. To run the PowerShell script, you'll need to install the [Az PowerShell Module](https://docs.microsoft.com/powershell/azure/install-az-ps) 77 | 78 | ## Running the PowerShell Script 79 | 80 | > If you're new to PowerShell, you can follow the instructions on [How to run PowerShell script file on Windows 10](https://www.windowscentral.com/how-create-and-run-your-first-powershell-script-file-windows-10) to help you get started. 81 | 82 | To run the [PowerShell script](./deploy.ps1): 83 | 84 | 1. Open PowerShell and navigate to this folder. 85 | 86 | ```cmd 87 | cd "00 - Resource Deployment" 88 | ``` 89 | 90 | 2. Run the following command: 91 | 92 | ```cmd 93 | ./deploy.ps1 94 | ``` 95 | 96 | 3. After running the script, you'll be prompted to login and provide additional information. 97 | -------------------------------------------------------------------------------- /01 - Search Index Creation/README.md: -------------------------------------------------------------------------------- 1 | # Search Index Creation 2 | 3 | There are several different ways to create a search index for Azure AI Search. You can choose how to deploy an index based on your skillset, your preferred tools, or your desired level of automation. 4 | 5 | >**Note:** If you ran the `deploy.ps1` script in the previous step, you've already created your search index so you can skip to the next step. 6 | 7 | ## Prerequisites 8 | 9 | At this point, you should have: 10 | 11 | 1. Deployed the necessary resources to Azure using the ARM template or PowerShell script as described in the previous step. 12 | 1. Uploaded your document set to Azure Storage (e.g., a container in Azure Blob Storage). You can also do this later using the Web App. 13 | 14 | ### Uploading sample data 15 | 16 | You can either use your own dataset, or use the [sample documents](../sample_documents/) in this repo. The quickest way to upload the data is to create a new container in the provisioned Storage Account and then use the `Storage Explorer` in the Azure Portal to upload the files. 17 | 18 | ## Options for creating your index 19 | 20 | This folder includes three options for creating an index. Each of these approaches is documented in a separate file: 21 | 22 | 1. [Create a search index using the Azure Portal](./Create-Index-AzurePortal.md) 23 | 2. [Create a search index using PowerShell](./Create-Index-PowerShell.md) 24 | 3. [Create a search index using Postman](./Create-Index-Postman.md) 25 | 26 | 4. Optionally - go thru Sentiment Analysis setup example in [01.1 - BuiltIn Skills](./01.1%20-%20BuiltIn%20Skills/) 27 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.Template.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29911.84 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CognitiveSearch.UI", "CognitiveSearch.UI\CognitiveSearch.UI.csproj", "{7A95287B-A883-45DF-AC29-A821601F9DC0}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {7A95287B-A883-45DF-AC29-A821601F9DC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {7A95287B-A883-45DF-AC29-A821601F9DC0}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {7A95287B-A883-45DF-AC29-A821601F9DC0}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {7A95287B-A883-45DF-AC29-A821601F9DC0}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {EC579B61-33D5-4713-A1E9-FA9AE4A25133} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/CognitiveSearch.UI.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Configuration/ApiConfig.cs: -------------------------------------------------------------------------------- 1 | namespace CognitiveSearch.UI.Configuration 2 | { 3 | public class ApiConfig 4 | { 5 | public string Url { get; set; } 6 | } 7 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Configuration/AppConfig.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace CognitiveSearch.UI.Configuration 3 | { 4 | public class AppConfig 5 | { 6 | public ApiConfig ApiConfig { get; set; } 7 | 8 | public OrganizationConfig Organization { get; set; } 9 | 10 | public bool Customizable { get; set; } 11 | } 12 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Configuration/OrganizationConfig.cs: -------------------------------------------------------------------------------- 1 | namespace CognitiveSearch.UI.Configuration 2 | { 3 | public class OrganizationConfig 4 | { 5 | public string Name { get; set; } 6 | } 7 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Controllers/StorageController.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | using System; 5 | using System.Linq; 6 | using Microsoft.AspNetCore.Mvc; 7 | using Microsoft.Extensions.Configuration; 8 | using System.Threading.Tasks; 9 | using System.IO; 10 | using System.Web; 11 | using Azure.Storage.Blobs; 12 | using Azure.Storage; 13 | using Azure.Storage.Blobs.Models; 14 | 15 | namespace CognitiveSearch.UI.Controllers 16 | { 17 | public class StorageController : Controller 18 | { 19 | private IConfiguration _configuration { get; set; } 20 | private DocumentSearchClient _docSearch { get; set; } 21 | 22 | public StorageController(IConfiguration configuration) 23 | { 24 | _configuration = configuration; 25 | _docSearch = new DocumentSearchClient(configuration); 26 | } 27 | 28 | [HttpPost("upload")] 29 | public async Task Upload() 30 | { 31 | if (Request.Form.Files.Any()) 32 | { 33 | var container = GetStorageContainer(0); 34 | 35 | foreach (var formFile in Request.Form.Files) 36 | { 37 | if (formFile.Length > 0) 38 | { 39 | var blob = container.GetBlobClient(formFile.FileName); 40 | var blobHttpHeader = new BlobHttpHeaders(); 41 | blobHttpHeader.ContentType = formFile.ContentType; 42 | await blob.UploadAsync(formFile.OpenReadStream(), blobHttpHeader); 43 | } 44 | } 45 | } 46 | 47 | await _docSearch.RunIndexer(); 48 | 49 | return new JsonResult("ok"); 50 | } 51 | 52 | /// 53 | /// Returns the requested document with an 'inline' content disposition header. 54 | /// This hints to a browser to show the file instead of downloading it. 55 | /// 56 | /// The storage connection string index. 57 | /// The storage blob filename. 58 | /// The expected mime content type. 59 | /// The file data with inline disposition header. 60 | [HttpGet("preview/{storageIndex}/{fileName}/{mimeType}")] 61 | public async Task GetDocumentInline(int storageIndex, string fileName, string mimeType) 62 | { 63 | var decodedFilename = HttpUtility.UrlDecode(fileName); 64 | var container = GetStorageContainer(storageIndex); 65 | var blob = container.GetBlobClient(decodedFilename); 66 | using (var ms = new MemoryStream()) 67 | { 68 | var downlaodInfo = await blob.DownloadAsync(); 69 | await downlaodInfo.Value.Content.CopyToAsync(ms); 70 | Response.Headers.Add("Content-Disposition", "inline; filename=" + decodedFilename); 71 | return File(ms.ToArray(), HttpUtility.UrlDecode(mimeType)); 72 | } 73 | } 74 | 75 | private BlobContainerClient GetStorageContainer(int storageIndex) 76 | { 77 | string accountName = _configuration.GetSection("StorageAccountName")?.Value; 78 | string accountKey = _configuration.GetSection("StorageAccountKey")?.Value; 79 | 80 | var containerKey = "StorageContainerAddress"; 81 | if (storageIndex > 0) 82 | containerKey += (storageIndex+1).ToString(); 83 | var containerAddress = _configuration.GetSection(containerKey)?.Value.ToLower(); 84 | 85 | var container = new BlobContainerClient(new Uri(containerAddress), new StorageSharedKeyCredential(accountName, accountKey)); 86 | return container; 87 | } 88 | 89 | [HttpGet("download")] 90 | public async Task DownloadBlobAsync(string blobUri="") 91 | { 92 | 93 | Uri blobUriEncoded = new Uri(blobUri); 94 | BlobClient blobClient = new BlobClient(blobUriEncoded); 95 | 96 | var stream = await blobClient.OpenReadAsync(); 97 | return File(stream, "image/tif", "image.tif"); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Models/ColorSettings.cs: -------------------------------------------------------------------------------- 1 | namespace CognitiveSearch.UI.Models 2 | { 3 | public class ColorSettings 4 | { 5 | public string BackgroundColor { get; set; } = "#ffffff"; 6 | public string TextColor { get; set; } = "#4A4A4A"; 7 | } 8 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Models/CustomizeViewModel.cs: -------------------------------------------------------------------------------- 1 | namespace CognitiveSearch.UI.Models 2 | { 3 | public class CustomizeViewModel 4 | { 5 | public ColorSettings NavBar { get; set; } 6 | } 7 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Models/ErrorViewModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | using System; 5 | 6 | namespace CognitiveSearch.UI.Models 7 | { 8 | public class ErrorViewModel 9 | { 10 | public string RequestId { get; set; } 11 | 12 | public bool ShowRequestId => !string.IsNullOrEmpty(RequestId); 13 | } 14 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Models/SearchResultViewModel.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace CognitiveSearch.UI.Models 4 | { 5 | public class SearchResultViewModel 6 | { 7 | public DocumentResult documentResult { get; set; } 8 | 9 | public string query { get; set; } 10 | 11 | public SearchFacet[] selectedFacets { get; set; } 12 | 13 | public int currentPage { get; set; } 14 | 15 | public string searchId { get; set; } 16 | 17 | public string applicationInstrumentationKey { get; set; } 18 | public string searchServiceName { get; set; } 19 | public string indexName { get; set; } 20 | 21 | public string[] facetableFields { get; set; } 22 | 23 | public string answer { get; set; } 24 | 25 | public List captions { get; set; } 26 | 27 | public string queryType { get; set; } 28 | 29 | public bool semanticEnabled { get; set; } 30 | } 31 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using Microsoft.AspNetCore; 7 | using Microsoft.AspNetCore.Hosting; 8 | using Microsoft.Extensions.Configuration; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace CognitiveSearch.UI 12 | { 13 | public class Program 14 | { 15 | public static void Main(string[] args) 16 | { 17 | BuildWebHost(args).Run(); 18 | } 19 | 20 | public static IWebHost BuildWebHost(string[] args) => 21 | WebHost.CreateDefaultBuilder(args) 22 | .UseStartup() 23 | .Build(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "https://localhost:44354/", 7 | "sslPort": 0 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "CognitiveSearch.UI": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "environmentVariables": { 22 | "ASPNETCORE_ENVIRONMENT": "Development" 23 | }, 24 | "applicationUrl": "http://localhost:53096/" 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Search/DocumentResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | using Azure; 5 | using Azure.Search.Documents; 6 | using Azure.Search.Documents.Models; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.Linq; 10 | using System.Threading.Tasks; 11 | 12 | namespace CognitiveSearch.UI 13 | { 14 | public class DocumentResult 15 | { 16 | public List Facets { get; set; } 17 | public SearchDocument Result { get; set; } 18 | public Pageable> Results { get; set; } 19 | public int? Count { get; set; } 20 | public string Token { get; set; } 21 | public int StorageIndex { get; set; } 22 | public string DecodedPath { get; set; } 23 | public List Tags { get; set; } 24 | public string SearchId { get; set; } 25 | public string IdField { get; set; } 26 | public bool IsPathBase64Encoded { get; set; } 27 | 28 | public string Answer { get; set; } 29 | 30 | public List Captions { get; set; } 31 | } 32 | 33 | public class Caption 34 | { 35 | public string metadata_storage_name { get; set; } 36 | public string text { get; set; } 37 | } 38 | 39 | public class Facet 40 | { 41 | public string key { get; set; } 42 | public List value { get; set; } 43 | } 44 | 45 | public class FacetValue 46 | { 47 | public string value { get; set; } 48 | public long? count { get; set; } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Search/SearchFacet.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | namespace CognitiveSearch.UI 5 | { 6 | public class SearchFacet 7 | { 8 | public string Key { get; set; } 9 | public string[] Value { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Search/SearchModel.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Search.Documents.Indexes.Models; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | 8 | namespace CognitiveSearch.UI 9 | { 10 | public class SearchModel 11 | { 12 | private string[] facets = new string[] 13 | { 14 | // Add UI facets here in order 15 | //"people", 16 | //"locations", 17 | //"organizations", 18 | //"keyphrases" 19 | }; 20 | 21 | private string[] tags = new string[] 22 | { 23 | // Add tags fields here in order 24 | //"people", 25 | //"locations", 26 | //"organizations", 27 | //"keyphrases" 28 | }; 29 | 30 | private string[] resultFields = new string[] 31 | { 32 | "metadata_storage_path", 33 | "metadata_storage_name", 34 | "metadata_title", 35 | 36 | // Add fields needed to display results cards 37 | 38 | // NOTE: if you customize the resultFields, be sure to include metadata_storage_name, 39 | // metadata_storage_path as those fields are needed for the UI to work properly 40 | //"people", 41 | //"locations", 42 | //"organizations", 43 | "keyPhrases" 44 | //"geoLocation" 45 | }; 46 | 47 | public List Facets { get; set; } 48 | public List Tags { get; set; } 49 | 50 | public string[] SelectFilter { get; set; } 51 | 52 | public string[] SearchableFields { get; set; } 53 | 54 | public Dictionary SearchFacets { get; set; } 55 | 56 | public SearchModel(SearchSchema schema) 57 | { 58 | Facets = new List(); 59 | Tags = new List(); 60 | 61 | List validatedResultFields = new List(); 62 | foreach (string s in resultFields) 63 | { 64 | if (schema.Fields.ContainsKey(s)) 65 | { 66 | validatedResultFields.Add(s); 67 | } 68 | } 69 | SelectFilter = validatedResultFields.ToArray(); 70 | 71 | if (facets.Count() > 0) 72 | { 73 | // add field to facets if in facets arr 74 | foreach (var field in facets) 75 | { 76 | if (schema.Fields[field] != null && schema.Fields[field].IsFacetable) 77 | { 78 | Facets.Add(schema.Fields[field]); 79 | } 80 | } 81 | } 82 | else 83 | { 84 | foreach (var field in schema.Fields.Where(f => f.Value.IsFacetable)) 85 | { 86 | Facets.Add(field.Value); 87 | } 88 | 89 | foreach (var field in schema.Fields.Where(f => f.Value.IsComplex == true)) 90 | { 91 | foreach(var subField in field.Value.Fields.Where(f => f.Value.IsFacetable)) 92 | { 93 | Facets.Add(subField.Value); 94 | } 95 | } 96 | } 97 | 98 | if (tags.Count() > 0) 99 | { 100 | foreach (var field in tags) 101 | { 102 | if (schema.Fields[field] != null && schema.Fields[field].IsFacetable) 103 | { 104 | Tags.Add(schema.Fields[field]); 105 | } 106 | } 107 | } 108 | else 109 | { 110 | foreach (var field in schema.Fields.Where(f => f.Value.IsFacetable)) 111 | { 112 | Tags.Add(field.Value); 113 | } 114 | } 115 | 116 | SearchableFields = schema.Fields.Where(f => f.Value.IsSearchable).Select(f => f.Key).ToArray(); 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using CognitiveSearch.UI.Configuration; 7 | using Microsoft.AspNetCore.Builder; 8 | using Microsoft.AspNetCore.Hosting; 9 | using Microsoft.AspNetCore.Http; 10 | using Microsoft.AspNetCore.Mvc; 11 | using Microsoft.Extensions.Configuration; 12 | using Microsoft.Extensions.DependencyInjection; 13 | using Microsoft.Extensions.FileProviders; 14 | using Microsoft.Extensions.Hosting; 15 | 16 | namespace CognitiveSearch.UI 17 | { 18 | public class Startup 19 | { 20 | public Startup(IConfiguration configuration) 21 | { 22 | Configuration = configuration; 23 | } 24 | 25 | public IConfiguration Configuration { get; } 26 | 27 | // This method gets called by the runtime. Use this method to add services to the container. 28 | public void ConfigureServices(IServiceCollection services) 29 | { 30 | services.Configure(options => 31 | { 32 | // This lambda determines whether user consent for non-essential cookies is needed for a given request. 33 | options.CheckConsentNeeded = context => true; 34 | options.MinimumSameSitePolicy = SameSiteMode.None; 35 | }); 36 | 37 | var apiConfig = new ApiConfig 38 | { 39 | Url = "/api" 40 | }; 41 | services.AddSingleton(apiConfig); 42 | 43 | var orgConfig = new OrganizationConfig 44 | { 45 | Name = Configuration["OrganizationName"], 46 | }; 47 | services.AddSingleton(orgConfig); 48 | 49 | var appConfig = new AppConfig 50 | { 51 | ApiConfig = apiConfig, 52 | Organization = orgConfig, 53 | Customizable = Configuration["Customizable"].Equals("true", StringComparison.InvariantCultureIgnoreCase) 54 | }; 55 | services.AddSingleton(appConfig); 56 | 57 | services.AddSingleton(new PhysicalFileProvider(Path.Combine(Directory.GetCurrentDirectory(), "wwwroot"))); 58 | 59 | services.AddMvc(options => options.EnableEndpointRouting = false); 60 | } 61 | 62 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 63 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env) 64 | { 65 | if (env.IsDevelopment()) 66 | { 67 | app.UseBrowserLink(); 68 | app.UseDeveloperExceptionPage(); 69 | } 70 | else 71 | { 72 | app.UseExceptionHandler("/Home/Error"); 73 | } 74 | 75 | app.UseHttpsRedirection(); 76 | app.UseStaticFiles(); 77 | app.UseCookiePolicy(); 78 | app.UseMvcWithDefaultRoute(); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Admin/Customize.cshtml: -------------------------------------------------------------------------------- 1 | @model CognitiveSearch.UI.Models.CustomizeViewModel 2 | 3 | @{ 4 | ViewData["Title"] = "Index"; 5 | } 6 | 7 | 8 |
9 |
10 |

Customize the navigation bar

11 |
12 |

13 | Using the options below, you can upload your own logo, as well as set the background and text colors for the navigation bar. 14 |

15 |
17 |
18 | 19 | 20 |
21 | 22 | 23 |
24 | 25 | 26 |
27 |
28 | 29 |
30 |
31 |
32 |
33 | 34 | 45 | 46 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Admin/NotAvailable.cshtml: -------------------------------------------------------------------------------- 1 |  2 | @{ 3 | ViewData["Title"] = "Functionality not available"; 4 | } 5 | 6 |
7 |

The functionality is not available in this deployment

8 | 9 |

10 | This is a shared deployment. Some functionalities including customization and uploading files are not available in this deployment. 11 |

12 |

13 | If you want to use this functionality, you can create it your own deployment. You can find the deployment script and instruction in 14 | the GitHub repo. 15 |

16 |
17 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Admin/UploadData.cshtml: -------------------------------------------------------------------------------- 1 | @model CognitiveSearch.UI.Configuration.AppConfig 2 | 3 | @{ 4 | ViewData["Title"] = "Upload your data"; 5 | } 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 | 15 |
16 |

Upload your files to Blob Storage

17 |
18 |

19 | You can used the area below to upload files into your Azure Blob Storage account. Once uploaded, the files will be indexed by Azure Search, and within a few minutes will be available for searching using this tool. 20 |

21 |
Drag and drop files into this zone to add them to the upload list, or click anywhere in this block to open a file dialog.
22 | 23 |
24 |
25 | 26 | 27 | 28 |
29 | 30 | 65 | 66 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Home/Index.cshtml: -------------------------------------------------------------------------------- 1 |  3 | 4 | @{ 5 | ViewData["Title"] = "Home"; 6 | } 7 | 8 | 9 | 10 |
11 |
12 |
13 |
14 | 15 |
16 | 17 | 18 | 19 | 20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
LEARN MORE
32 | 35 |
36 | Cognitive Services 37 |
38 | 41 |
42 | Knowledge Mining 43 |
44 |
45 |
-------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Shared/Error.cshtml: -------------------------------------------------------------------------------- 1 | @model ErrorViewModel 2 | @{ 3 | ViewData["Title"] = "Error"; 4 | } 5 | 6 |

Error.

7 |

An error occurred while processing your request.

8 | 9 | @if (Model.ShowRequestId) 10 | { 11 |

12 | Request ID: @Model.RequestId 13 |

14 | } 15 | 16 |

Development Mode

17 |

18 | Swapping to Development environment will display more detailed information about the error that occurred. 19 |

20 |

21 | Development environment should not be enabled in deployed applications, as it can result in sensitive information from exceptions being displayed to end users. For local debugging, development environment can be enabled by setting the ASPNETCORE_ENVIRONMENT environment variable to Development, and restarting the application. 22 |

23 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Shared/_Footer.cshtml: -------------------------------------------------------------------------------- 1 | @using CognitiveSearch.UI.Configuration 2 | @inject OrganizationConfig OrgConfig 3 | 4 | 5 |
6 |
7 |
8 | © @DateTime.Now.Year - @OrgConfig.Name 9 |
10 |
11 |
12 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Shared/_Header.cshtml: -------------------------------------------------------------------------------- 1 | @using CognitiveSearch.UI.Configuration 2 | @inject AppConfig AppConfig 3 | 4 | 5 | 33 | 34 | @if (!string.IsNullOrWhiteSpace(ViewBag.Message)) 35 | { 36 | @if (!string.IsNullOrWhiteSpace(ViewBag.Style)) 37 | { 38 |
@ViewBag.Message
39 | } 40 | else 41 | { 42 |
@ViewBag.Message
43 | } 44 | } 45 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Shared/_Layout.cshtml: -------------------------------------------------------------------------------- 1 |  3 | 4 | @using CognitiveSearch.UI.Configuration 5 | @inject AppConfig AppConfig 6 | 7 | 8 | 9 | 10 | 11 | 12 | @ViewData["Title"] 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | @await Html.PartialAsync("_Header") 42 | 43 |
44 | @RenderBody() 45 |
46 | 47 | @await Html.PartialAsync("_Footer") 48 | 49 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 79 | 80 | @RenderSection("Scripts", required: false) 81 | 82 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/Shared/_ValidationScriptsPartial.cshtml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 12 | 18 | 19 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/_ViewImports.cshtml: -------------------------------------------------------------------------------- 1 | @using CognitiveSearch.UI 2 | @using CognitiveSearch.UI.Models 3 | @addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers 4 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/Views/_ViewStart.cshtml: -------------------------------------------------------------------------------- 1 | @{ 2 | Layout = "_Layout"; 3 | } 4 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "IncludeScopes": false, 4 | "LogLevel": { 5 | "Default": "Debug", 6 | "System": "Information", 7 | "Microsoft": "Information" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "IncludeScopes": false, 4 | "LogLevel": { 5 | "Default": "Warning" 6 | } 7 | }, 8 | 9 | // Required fields 10 | "SearchServiceName": "", 11 | "SearchApiKey": "", 12 | "SearchIndexName": "", 13 | "SearchIndexerName": "", 14 | "StorageAccountName": "", 15 | "StorageAccountKey": "", 16 | "StorageContainerAddress": "https://{storage-account-name}.blob.core.windows.net/{container-name}", 17 | "KeyField": "metadata_storage_path", 18 | "IsPathBase64Encoded": true, 19 | 20 | // Optional fields to enable Semantic Search 21 | "QueryLanguage": "en-US", 22 | "SemanticConfiguration": "", 23 | 24 | // Optional instrumentation key 25 | "InstrumentationKey": "", 26 | 27 | // Optional container addresses if using more than one indexer: 28 | "StorageContainerAddress2": "https://{storage-account-name}.blob.core.windows.net/{container-name}", 29 | "StorageContainerAddress3": "https://{storage-account-name}.blob.core.windows.net/{container-name}", 30 | 31 | // Optional key to an Azure Maps account if you would like to display the geoLocation field in a map 32 | "AzureMapsSubscriptionKey": "", 33 | 34 | // Set to the name of a facetable field you would like to represent as a graph. 35 | // You may also set to a comma separated list of the facet names if you would like more than one facet type on the graph. 36 | "GraphFacet": "keyPhrases, locations", 37 | 38 | // Additional Customizations 39 | "Customizable": "true", 40 | "OrganizationName": "Microsoft", 41 | "OrganizationLogo": "~/images/logo.png", 42 | "OrganizationWebSiteUrl": "https://www.microsoft.com" 43 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/bundleconfig.json: -------------------------------------------------------------------------------- 1 | // Configure bundling and minification for the project. 2 | // More info at https://go.microsoft.com/fwlink/?LinkId=808241 3 | [ 4 | { 5 | "outputFileName": "wwwroot/css/site.min.css", 6 | // An array of relative input file paths. Globbing patterns supported 7 | "inputFiles": [ 8 | "wwwroot/css/site.css" 9 | ] 10 | }, 11 | { 12 | "outputFileName": "wwwroot/js/site.min.js", 13 | "inputFiles": [ 14 | "wwwroot/js/site.js" 15 | ], 16 | // Optionally specify minification options 17 | "minify": { 18 | "enabled": true, 19 | "renameLocals": true 20 | }, 21 | // Optionally generate .map file 22 | "sourceMap": false 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/css/colors.css: -------------------------------------------------------------------------------- 1 | /*Change based on customer's styling */ 2 | 3 | /*a { 4 | color: #787878; 5 | } 6 | 7 | a:active, a:hover { 8 | color: #000000; 9 | } 10 | 11 | 12 | .btn-primary { 13 | font-size: 18px; 14 | color: rgb(255, 255, 255); 15 | padding: 10px 20px; 16 | background-color: rgb(228, 30, 40); 17 | margin-top: 20px; 18 | border-radius: 30px; 19 | outline: none; 20 | } 21 | 22 | .btn-primary:active, .btn-primary:focus, .btn-primary:hover, .btn-primary:active:hover, .btn-primary.active:hover, .open > .dropdown-toggle.btn-primary:hover, .btn-primary:active:focus, .btn-primary.active:focus, .open > .dropdown-toggle.btn-primary:focus, .btn-primary:active.focus, .btn-primary.active.focus, .open > .dropdown-toggle.btn-primary.focus { 23 | background-color: #e61d2b; 24 | opacity: 0.7; 25 | border: none; 26 | outline: 0; 27 | } 28 | 29 | .footer { 30 | background-color: rgb(233, 229,223); 31 | } 32 | 33 | .tag { 34 | color: rgb(255, 255, 255); 35 | opacity: 1; 36 | padding: 5px 10px; 37 | background-color: #f73949; 38 | border-radius: 30px; 39 | outline: none; 40 | margin: 3px 3px 3px 0; 41 | } 42 | 43 | .modal-control { 44 | color: #e61d2b; 45 | } 46 | 47 | .ms-CheckBox-field.is-checked::after { 48 | background-color: #e61d2b; 49 | } 50 | 51 | .ms-SearchBox-field { 52 | border: 1px solid darkgrey !important; 53 | } 54 | 55 | .ms-SearchBox-field:focus { 56 | border-color: darkgrey !important; 57 | background-color: transparent !important; 58 | } 59 | 60 | .ms-SearchBox:hover { 61 | background-color: transparent !important; 62 | } 63 | 64 | .ms-SearchBox-clear .ms-CommandButton-button { 65 | background-color: lightgrey !important; 66 | } 67 | 68 | .ms-SearchBox-icon { 69 | color: darkgrey !important; 70 | } 71 | 72 | .ms-Pivot-link.is-selected:before { 73 | background-color: darkgrey !important; 74 | } 75 | 76 | .ms-Pivot-link:focus { 77 | outline: 0; 78 | } 79 | 80 | .search-result { 81 | background-color: rgb(233, 229,223) 82 | } 83 | 84 | .pagination > .active > a, 85 | .pagination > .active > span, 86 | .pagination > .active > a:hover, 87 | .pagination > .active > span:hover, 88 | .pagination > .active > a:focus, 89 | .pagination > .active > span:focus { 90 | background-color: #f73949; 91 | border-color: #f73949; 92 | } 93 | 94 | 95 | ::-webkit-scrollbar-thumb { 96 | background: #787878 97 | }*/ 98 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/css/navbar.css: -------------------------------------------------------------------------------- 1 | .navbar-bg{background-color:#ffffff}.navbar-text-color{color:#000000} -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/css/site.min.css: -------------------------------------------------------------------------------- 1 | body{padding-top:50px;padding-bottom:20px}.body-content{padding-left:15px;padding-right:15px}.carousel-caption p{font-size:20px;line-height:1.4}.carousel-inner .item img[src$=".svg"]{width:100%}#qrCode{margin:15px}@media screen and (max-width:767px){.carousel-caption{display:none}} -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/favicon.ico -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/ITIcons.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/ITIcons.png -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/JumboBackground.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/JumboBackground.jpg -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/Microsoft-logo_rgb_c-gray.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/Microsoft-logo_rgb_c-gray.png -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/ReachNewHeights.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/ReachNewHeights.png -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/Video1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/Video1.PNG -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/Video2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/Video2.PNG -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/collapse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/collapse.png -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/expand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/expand.png -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/homepageimage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/homepageimage.png -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/02 - Web UI Template/CognitiveSearch.UI/wwwroot/images/logo.png -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/js/appInsights.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | var appInsights; 5 | 6 | function AppInsights() { 7 | 8 | appInsights = window.appInsights || 9 | function (config) { 10 | function r(config) { 11 | t[config] = function () { 12 | var i = arguments; t.queue.push(function () { t[config].apply(t, i) }) 13 | } 14 | } 15 | var t = 16 | { 17 | config: config 18 | }, 19 | u = document, e = window, o = "script", s = u.createElement(o), i, f; s.src = config.url || "//az416426.vo.msecnd.net/scripts/a/ai.0.js"; 20 | u.getElementsByTagName(o)[0].parentNode.appendChild(s); 21 | try { 22 | t.cookie = u.cookie 23 | } 24 | catch (h) { } 25 | for (t.queue = [], i = ["Event", "Exception", "Metric", "PageView", "Trace", "Dependency"]; i.length;) 26 | r("track" + i.pop()); 27 | return r("setAuthenticatedUserContext"), 28 | r("clearAuthenticatedUserContext"), 29 | config.disableExceptionTracking || (i = "onerror", r("_" + i), f = e[i], e[i] = function (config, r, u, e, o) { 30 | var s = f && f(config, r, u, e, o); return s !== !0 && t["_" + i](config, r, u, e, o), s 31 | }), t 32 | } 33 | ({ 34 | instrumentationKey: applicationInstrumentationKey 35 | }); 36 | window.appInsights = appInsights; 37 | } 38 | 39 | function LogSearchAnalytics(docCount = 0) { 40 | AppInsights(); 41 | if (docCount !== null) { 42 | var recordedQuery = q; 43 | if (q === undefined || q === null) { 44 | recordedQuery = "*"; 45 | } 46 | 47 | appInsights.trackEvent("Search", { 48 | SearchServiceName: searchServiceName, 49 | SearchId: searchId, 50 | IndexName: indexName, 51 | QueryTerms: recordedQuery, 52 | ResultCount: docCount, 53 | ScoringProfile: scoringProfile 54 | }); 55 | } 56 | } 57 | 58 | function LogClickAnalytics(fileName, index) { 59 | AppInsights(); 60 | appInsights.trackEvent("Click", { 61 | SearchServiceName: searchServiceName, 62 | SearchId: searchId, 63 | ClickedDocId: fileName, 64 | Rank: index 65 | }); 66 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/js/site.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | //Initialize Fabric elements 5 | var SpinnerElements = document.querySelectorAll(".ms-Spinner"); 6 | for (var i = 0; i < SpinnerElements.length; i++) { 7 | new fabric['Spinner'](SpinnerElements[i]); 8 | } 9 | 10 | var SearchBoxElements = document.querySelectorAll(".ms-SearchBox"); 11 | for (var i = 0; i < SearchBoxElements.length; i++) { 12 | new fabric['SearchBox'](SearchBoxElements[i]); 13 | } 14 | 15 | var isGridInitialized = false; 16 | var $grid = $('#doc-details-div'); 17 | 18 | $(document).ready(function () { 19 | if (typeof aspViewModel !== 'undefined' && aspViewModel) 20 | Update(aspViewModel); 21 | }); 22 | 23 | function InitLayout() { 24 | 25 | if (isGridInitialized === true) { 26 | $grid.masonry('destroy'); // destroy 27 | } 28 | 29 | $grid.masonry({ 30 | itemSelector: '.results-div', 31 | columnWidth: '.results-sizer' 32 | }); 33 | 34 | $grid.imagesLoaded().progress(function () { 35 | $grid.masonry('layout'); 36 | }); 37 | 38 | isGridInitialized = true; 39 | } 40 | 41 | function FabricInit() { 42 | var CheckBoxElements = document.querySelectorAll(".ms-CheckBox"); 43 | for (var i = 0; i < CheckBoxElements.length; i++) { 44 | new fabric['CheckBox'](CheckBoxElements[i]); 45 | } 46 | } 47 | 48 | function htmlEncode(value) { 49 | return $('
').text(value).html(); 50 | } 51 | 52 | function htmlDecode(value) { 53 | return $('
').html(value).text(); 54 | } -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/js/site.min.js: -------------------------------------------------------------------------------- 1 | function InitLayout(){isGridInitialized===!0&&$grid.masonry("destroy");$grid.masonry({itemSelector:".results-div",columnWidth:".results-sizer"});$grid.imagesLoaded().progress(function(){$grid.masonry("layout")});isGridInitialized=!0}function FabricInit(){for(var t=document.querySelectorAll(".ms-CheckBox"),n=0;n").text(n).html()}function htmlDecode(n){return $("
").html(n).text()}for(var SearchBoxElements,isGridInitialized,$grid,SpinnerElements=document.querySelectorAll(".ms-Spinner"),i=0;i 4 | */ 5 | .dropzone, .dropzone * { 6 | box-sizing: border-box; } 7 | 8 | .dropzone { 9 | position: relative; } 10 | .dropzone .dz-preview { 11 | position: relative; 12 | display: inline-block; 13 | width: 120px; 14 | margin: 0.5em; } 15 | .dropzone .dz-preview .dz-progress { 16 | display: block; 17 | height: 15px; 18 | border: 1px solid #aaa; } 19 | .dropzone .dz-preview .dz-progress .dz-upload { 20 | display: block; 21 | height: 100%; 22 | width: 0; 23 | background: green; } 24 | .dropzone .dz-preview .dz-error-message { 25 | color: red; 26 | display: none; } 27 | .dropzone .dz-preview.dz-error .dz-error-message, .dropzone .dz-preview.dz-error .dz-error-mark { 28 | display: block; } 29 | .dropzone .dz-preview.dz-success .dz-success-mark { 30 | display: block; } 31 | .dropzone .dz-preview .dz-error-mark, .dropzone .dz-preview .dz-success-mark { 32 | position: absolute; 33 | display: none; 34 | left: 30px; 35 | top: 30px; 36 | width: 54px; 37 | height: 58px; 38 | left: 50%; 39 | margin-left: -27px; } 40 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/lib/dropzone/dist/css/basic.min.css: -------------------------------------------------------------------------------- 1 | .dropzone,.dropzone *{box-sizing:border-box}.dropzone{position:relative}.dropzone .dz-preview{position:relative;display:inline-block;width:120px;margin:0.5em}.dropzone .dz-preview .dz-progress{display:block;height:15px;border:1px solid #aaa}.dropzone .dz-preview .dz-progress .dz-upload{display:block;height:100%;width:0;background:green}.dropzone .dz-preview .dz-error-message{color:red;display:none}.dropzone .dz-preview.dz-error .dz-error-message,.dropzone .dz-preview.dz-error .dz-error-mark{display:block}.dropzone .dz-preview.dz-success .dz-success-mark{display:block}.dropzone .dz-preview .dz-error-mark,.dropzone .dz-preview .dz-success-mark{position:absolute;display:none;left:30px;top:30px;width:54px;height:58px;left:50%;margin-left:-27px} 2 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/lib/jquery-validation-unobtrusive/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) .NET Foundation. All rights reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 4 | these files except in compliance with the License. You may obtain a copy of the 5 | License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software distributed 10 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | specific language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/lib/jquery-validation/LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | ===================== 3 | 4 | Copyright Jörn Zaefferer 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /02 - Web UI Template/CognitiveSearch.UI/wwwroot/lib/jquery/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright JS Foundation and other contributors, https://js.foundation/ 2 | 3 | This software consists of voluntary contributions made by many 4 | individuals. For exact contribution history, see the revision history 5 | available at https://github.com/jquery/jquery 6 | 7 | The following license applies to all parts of this software except as 8 | documented below: 9 | 10 | ==== 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining 13 | a copy of this software and associated documentation files (the 14 | "Software"), to deal in the Software without restriction, including 15 | without limitation the rights to use, copy, modify, merge, publish, 16 | distribute, sublicense, and/or sell copies of the Software, and to 17 | permit persons to whom the Software is furnished to do so, subject to 18 | the following conditions: 19 | 20 | The above copyright notice and this permission notice shall be 21 | included in all copies or substantial portions of the Software. 22 | 23 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 27 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 28 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 29 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 30 | 31 | ==== 32 | 33 | All files located in the node_modules and external directories are 34 | externally maintained libraries used by this software which have their 35 | own licenses; we recommend you read them, as their terms may differ from 36 | the terms above. 37 | -------------------------------------------------------------------------------- /02 - Web UI Template/Docker/.dockerignore: -------------------------------------------------------------------------------- 1 | **/.classpath 2 | **/.dockerignore 3 | **/.env 4 | **/.git 5 | **/.gitignore 6 | **/.project 7 | **/.settings 8 | **/.toolstarget 9 | **/.vs 10 | **/.vscode 11 | **/*.*proj.user 12 | **/*.dbmdl 13 | **/*.jfm 14 | **/azds.yaml 15 | **/bin 16 | **/charts 17 | **/docker-compose* 18 | **/Dockerfile* 19 | **/node_modules 20 | **/npm-debug.log 21 | **/obj 22 | **/secrets.dev.yaml 23 | **/values.dev.yaml 24 | LICENSE 25 | README.md -------------------------------------------------------------------------------- /02 - Web UI Template/Docker/.env: -------------------------------------------------------------------------------- 1 | SearchServiceName= 2 | SearchApiKey= 3 | SearchIndexName= 4 | StorageAccountName= 5 | StorageAccountKey= 6 | StorageContainerAddress=https://{storage-account-name}.blob.core.windows.net/{container-name} 7 | KeyField=metadata_storage_path 8 | IsPathBase64Encoded=true 9 | InstrumentationKey= 10 | StorageContainerAddress2=https://{storage-account-name}.blob.core.windows.net/{container-name} 11 | StorageContainerAddress3=https://{storage-account-name}.blob.core.windows.net/{container-name} 12 | AzureMapsSubscriptionKey= 13 | GraphFacet=keyPhrases, locations 14 | Customizable=true 15 | OrganizationName=Microsoft 16 | OrganizationLogo=~/images/logo.png 17 | OrganizationWebSiteUrl=https://www.microsoft.com -------------------------------------------------------------------------------- /02 - Web UI Template/Docker/Dockerfile: -------------------------------------------------------------------------------- 1 | #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. 2 | 3 | FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-buster-slim AS base 4 | WORKDIR /app 5 | EXPOSE 80 6 | 7 | FROM mcr.microsoft.com/dotnet/core/sdk:3.1-buster AS build 8 | WORKDIR /src 9 | COPY ["CognitiveSearch.UI/CognitiveSearch.UI.csproj", "CognitiveSearch.UI/"] 10 | RUN dotnet restore "CognitiveSearch.UI/CognitiveSearch.UI.csproj" 11 | COPY . . 12 | WORKDIR "/src/CognitiveSearch.UI" 13 | RUN dotnet build "CognitiveSearch.UI.csproj" -c Release -o /app/build 14 | 15 | FROM build AS publish 16 | RUN dotnet publish "CognitiveSearch.UI.csproj" -c Release -o /app/publish 17 | 18 | FROM base AS final 19 | WORKDIR /app 20 | COPY --from=publish /app/publish . 21 | ENTRYPOINT ["dotnet", "CognitiveSearch.UI.dll"] -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/AML Custom Skill Template/README.md: -------------------------------------------------------------------------------- 1 | # AML Custom Skill Template 2 | 3 | Run these two Jupyter notebooks to quickly train and deploy a document classification model as a custom skill for Cognitive Search. 4 | 5 | Walk through the two Jupyter Notebooks for instructions on how to train, deploy, and integrate a custom skill using Azure Machine Learning. 6 | 7 | > This template does not currently use the [AML Custom skill format](https://docs.microsoft.com/azure/search/cognitive-search-aml-skill) that's available in public preview but rather is designed to be deployed as a conventional [Web API skill](https://docs.microsoft.com/azure/search/cognitive-search-custom-skill-web-api). There's a separate tutorial that walks through the process of [creating an AML skill](https://docs.microsoft.com/azure/search/cognitive-search-tutorial-aml-custom-skill) based off of that functionality. 8 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/Azure Function Custom Skills/README.md: -------------------------------------------------------------------------------- 1 | # Custom skills have moved! 2 | 3 | The custom skills and templates that were previously in this directory can now be found in their new home, 4 | [the Power Skills repository](https://github.com/Azure-Samples/azure-search-power-skills). 5 | 6 | [Power Skills](https://github.com/Azure-Samples/azure-search-power-skills) are a collection of useful functions to be deployed as custom skills for Azure Cognitive Search. The skills can be used as [templates](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Template/HelloWorld/README.md) or starting points for your own custom skills, or they can be deployed and used as they are if they happen to meet your requirements. We also invite you to contribute your own work by submitting a [pull request](https://github.com/Azure-Samples/azure-search-power-skills/compare). 7 | 8 | ## Features 9 | 10 | This project provides the following custom skills along with many other options: 11 | 12 | * [**HelloWorld**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Template/HelloWorld/README.md): a minimal skill that can be used as a starting point or template for your own skills. 13 | * [**GeoPointFromName**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Geo/GeoPointFromName/README.md): retrieves coordinates from place names and addresses. 14 | * [**BingEntitySearch**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Text/BingEntitySearch/README.md): finds rich and structured information about public figures, locations, or organizations. 15 | * [**AcronymLinker**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Text/AcronymLinker/README.md): provides definitions for known acronyms. 16 | * [**ImageStore**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Vision/ImageStore/README.md): stores and fetches base64-encoded images to and from blob storage. 17 | * [**HocrGenerator**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Vision/HocrGenerator/README.md): transforms the result of OCR into the hOCR format. 18 | * [**AnalyzeForm**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Vision/AnalyzeForm/README.md): recognizes form fields in a document. 19 | * [**CustomEntityLookup**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Text/CustomEntitySearch/README.md): finds custom entity names in text. 20 | * [**Tokenizer**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Text/Tokenizer/README.md): extracts non-stop words from a text. 21 | * [**Distinct**](https://github.com/Azure-Samples/azure-search-power-skills/blob/master/Text/Distinct/README.md): de-duplicates a list of terms. 22 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/Docker Flask Skill/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.5-slim 2 | RUN apt-get update && apt-get install -y python3-pip 3 | COPY . /storage-emulator 4 | WORKDIR /storage-emulator 5 | 6 | # RUN pip3 install pyopenssl 7 | RUN pip3 install -r requirements.txt 8 | 9 | RUN python -m nltk.downloader stopwords 10 | RUN python -m nltk.downloader universal_tagset 11 | RUN python -m spacy download en 12 | 13 | EXPOSE 80 14 | ENTRYPOINT ["python"] 15 | CMD ["ws.py"] -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/Docker Flask Skill/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.3.2 2 | click==6.7 3 | itsdangerous==0.24 4 | Jinja2==2.11.3 5 | MarkupSafe==1.0 6 | Werkzeug==3.0.1 7 | nltk==3.6.6 8 | spacy==2.1.4 9 | pandas==0.24.2 10 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/Docker Flask Skill/ws.py: -------------------------------------------------------------------------------- 1 | import spacy 2 | import nltk 3 | import pandas as pd 4 | import io 5 | import json 6 | 7 | import flask 8 | from flask import Flask, request, jsonify 9 | app = Flask(__name__) 10 | 11 | @app.route('/process', methods=['POST']) 12 | def process(): 13 | content = request.json 14 | 15 | nlp = spacy.load('en_core_web_sm') 16 | from nltk.corpus import stopwords 17 | stopwords = stopwords.words('english') 18 | 19 | # add some custom stopwords 20 | stopwords.append('part') 21 | stopwords.append('group') 22 | 23 | # return just the key phrases (not counts) 24 | data = {} 25 | data['values'] = [] 26 | 27 | for record in content["values"]: 28 | 29 | recordId = record["recordId"] 30 | metadataStoragePath = record["data"]["metadata_storage_path"] 31 | content = record["data"]["text"] 32 | doc = nlp(content) 33 | 34 | # Get noun phrases (NN*) and include adjectives (JJ*) 35 | cols = ['phrase', 'value'] 36 | lst = [] 37 | for sent in doc.sents: 38 | nounCount = 0 39 | phrase = '' 40 | for token in sent: 41 | if ((token.tag_[0] == 'N') and (token.tag_[1] == 'N')): 42 | if (token.text.lower() not in stopwords): 43 | nounCount += 1 44 | phrase += ' ' + token.text 45 | elif ((token.tag_[0] == 'J') and (token.tag_[1] == 'J')): 46 | if (token.text.lower() not in stopwords): 47 | phrase += ' ' + token.text 48 | else: 49 | if (nounCount > 0): 50 | lst.append([phrase.lower().strip(), 1]) 51 | phrase = '' 52 | nounCount = 0 53 | if (nounCount > 0): 54 | lst.append([phrase.lower().strip(), 1]) 55 | 56 | dfPhraseListGrouped = pd.DataFrame(lst, columns=cols).groupby('phrase').sum() 57 | 58 | for index, row in dfPhraseListGrouped.iterrows(): 59 | print (row.name, row.value) 60 | 61 | document = {} 62 | document['recordId'] = recordId 63 | 64 | keyphrases = [] 65 | for index, row in dfPhraseListGrouped.iterrows(): 66 | keyphrases.append( row.name) 67 | 68 | document['data'] = {"keyphrases": keyphrases} 69 | 70 | data['values'].append(document) 71 | 72 | return jsonify(data) 73 | 74 | if __name__ == '__main__': 75 | app.run(debug=True, host='0.0.0.0', port=80) 76 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Form Recognizer Custom Skill 3 | 4 | Follow MS Learn module [Build a Form Recognizer custom skill for Azure Cognitive Search ](https://learn.microsoft.com/en-us/training/modules/build-form-recognizer-custom-skill-for-azure-cognitive-search/4-exercise-build-deploy) 5 | to create Form Recognizer service and deploy Azure Function using cloud shell. 6 | 7 | Integrate a Form Recognizer Pre-Built Model for Invoices capability within the Cognitive Search pipeline 8 | 9 | # AnalyzeInvoice 10 | 11 | This custom skill extracts invoice specific fields using a pre trained forms recognizer model. 12 | 13 | 14 | ## Settings 15 | 16 | This Azure function requires access to an [Azure Forms Recognizer](https://azure.microsoft.com/en-us/services/cognitive-services/form-recognizer/) resource. The [prebuilt invoice model](https://docs.microsoft.com/azure/cognitive-services/form-recognizer/concept-invoices) is available in the 2.1 preview API. 17 | 18 | 19 | This function requires a `FORMS_RECOGNIZER_ENDPOINT` and a `FORMS_RECOGNIZER_KEY` settings set to a valid Azure Forms Recognizer API key and to your custom Form Recognizer 2.1-preview endpoint. 20 | 21 | 22 | 23 | ## Sample Input: 24 | 25 | This sample data is pointing to a file stored in this repository, but when the skill is integrated in a skillset, the URL and token will be provided by cognitive search. 26 | 27 | ```json 28 | { 29 | "values": [ 30 | { 31 | "recordId": "record1", 32 | "data": { 33 | "formUrl": "https://github.com/Azure-Samples/azure-search-power-skills/raw/master/SampleData/Invoice_4.pdf", 34 | "formSasToken": "?st=sasTokenThatWillBeGeneratedByCognitiveSearch" 35 | } 36 | } 37 | ] 38 | } 39 | ``` 40 | 41 | ## Sample Output: 42 | 43 | ```json 44 | { 45 | "values": [ 46 | { 47 | "recordId": "0", 48 | "data": { 49 | "invoices": [ 50 | { 51 | "AmountDue": 63.0, 52 | "BillingAddress": "345 North St NY 98052", 53 | "BillingAddressRecipient": "Fabrikam, Inc.", 54 | "DueDate": "2018-05-31", 55 | "InvoiceDate": "2018-05-15", 56 | "InvoiceId": "1785443", 57 | "InvoiceTotal": 56.28, 58 | "VendorAddress": "4567 Main St Buffalo NY 90852", 59 | "SubTotal": 49.3, 60 | "TotalTax": 0.99 61 | } 62 | ] 63 | } 64 | } 65 | ] 66 | } 67 | ``` 68 | 69 | ## Sample Skillset Integration 70 | 71 | In order to use this skill in a cognitive search pipeline, you'll need to add a skill definition to your skillset. 72 | Here's a sample skill definition for this example (inputs and outputs should be updated to reflect your particular scenario and skillset environment): 73 | 74 | ```json 75 | { 76 | "@odata.type": "#Microsoft.Skills.Custom.WebApiSkill", 77 | "name": "formrecognizer", 78 | "description": "Extracts fields from a form using a pre-trained form recognition model", 79 | "uri": "[AzureFunctionEndpointUrl]/api/AnalyzeInvoice?code=[AzureFunctionDefaultHostKey]", 80 | "httpMethod": "POST", 81 | "timeout": "PT1M", 82 | "context": "/document", 83 | "batchSize": 1, 84 | "inputs": [ 85 | { 86 | "name": "formUrl", 87 | "source": "/document/metadata_storage_path" 88 | }, 89 | { 90 | "name": "formSasToken", 91 | "source": "/document/metadata_storage_sas_token" 92 | } 93 | ], 94 | "outputs": [ 95 | { 96 | "name": "invoices", 97 | "targetName": "invoices" 98 | } 99 | ] 100 | } 101 | ``` 102 | 103 | Refer to Postman Collection for more details. -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_1.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_2.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_3.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_4.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/SampleInvoices/Invoice_5.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/.funcignore: -------------------------------------------------------------------------------- 1 | .git* 2 | .vscode 3 | local.settings.json 4 | test 5 | .venv -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # pipenv 86 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 87 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 88 | # having no cross-platform support, pipenv may install dependencies that don’t work, or not 89 | # install all needed dependencies. 90 | #Pipfile.lock 91 | 92 | # celery beat schedule file 93 | celerybeat-schedule 94 | 95 | # SageMath parsed files 96 | *.sage.py 97 | 98 | # Environments 99 | .env 100 | .venv 101 | env/ 102 | venv/ 103 | ENV/ 104 | env.bak/ 105 | venv.bak/ 106 | 107 | # Spyder project settings 108 | .spyderproject 109 | .spyproject 110 | 111 | # Rope project settings 112 | .ropeproject 113 | 114 | # mkdocs documentation 115 | /site 116 | 117 | # mypy 118 | .mypy_cache/ 119 | .dmypy.json 120 | dmypy.json 121 | 122 | # Pyre type checker 123 | .pyre/ 124 | 125 | # Azure Functions artifacts 126 | bin 127 | obj 128 | appsettings.json 129 | local.settings.json 130 | .python_packages -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-python.python" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | 5 | { 6 | "name": "Attach to Python Functions", 7 | "type": "python", 8 | "request": "attach", 9 | "port": 9091, 10 | "preLaunchTask": "func: host start" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": ".", 3 | "azureFunctions.scmDoBuildDuringDeployment": true, 4 | "azureFunctions.pythonVenv": ".venv", 5 | "azureFunctions.projectLanguage": "Python", 6 | "azureFunctions.projectRuntime": "~2", 7 | "debug.internalConsoleOptions": "neverOpen" 8 | } -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "func", 6 | "command": "host start", 7 | "problemMatcher": "$func-python-watch", 8 | "isBackground": true, 9 | "dependsOn": "pipInstall" 10 | }, 11 | { 12 | "label": "pipInstall", 13 | "type": "shell", 14 | "osx": { 15 | "command": "${config:azureFunctions.pythonVenv}/bin/python -m pip install -r requirements.txt" 16 | }, 17 | "windows": { 18 | "command": "${config:azureFunctions.pythonVenv}\\Scripts\\python -m pip install -r requirements.txt" 19 | }, 20 | "linux": { 21 | "command": "${config:azureFunctions.pythonVenv}/bin/python -m pip install -r requirements.txt" 22 | }, 23 | "problemMatcher": [] 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/AnalyzeForm/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | import logging 5 | import pathlib 6 | from azure.core.exceptions import ResourceNotFoundError 7 | from azure.ai.formrecognizer import FormRecognizerClient 8 | from azure.ai.formrecognizer import FormTrainingClient 9 | from azure.core.credentials import AzureKeyCredential 10 | import azure.functions as func 11 | 12 | def main(req: func.HttpRequest) -> func.HttpResponse: 13 | logging.info('Invoked AnalyzeForm Skill.') 14 | try: 15 | body = json.dumps(req.get_json()) 16 | if body: 17 | # For testing uncomment the following line to log the incoming request 18 | #logging.info(body) 19 | result = compose_response(body) 20 | return func.HttpResponse(result, mimetype="application/json") 21 | else: 22 | return func.HttpResponse( 23 | "The body of the request could not be parsed", 24 | status_code=400 25 | ) 26 | except ValueError: 27 | return func.HttpResponse( 28 | "The body of the request could not be parsed", 29 | status_code=400 30 | ) 31 | except KeyError: 32 | return func.HttpResponse( 33 | "Skill configuration error. Endpoint, key and model_id required.", 34 | status_code=400 35 | ) 36 | except AssertionError as error: 37 | return func.HttpResponse( 38 | "Request format is not a valid custom skill input", 39 | status_code=400 40 | ) 41 | 42 | def compose_response(json_data): 43 | body = json.loads(json_data) 44 | assert ('values' in body), "request does not implement the custom skill interface" 45 | values = body['values'] 46 | # Prepare the Output before the loop 47 | results = {} 48 | results["values"] = [] 49 | mappings = None 50 | with open(pathlib.Path(__file__).parent / 'field_mappings.json') as file: 51 | mappings = json.loads(file.read()) 52 | endpoint = os.environ["FORMS_RECOGNIZER_ENDPOINT"] 53 | key = os.environ["FORMS_RECOGNIZER_KEY"] 54 | model_id = os.environ["FORMS_RECOGNIZER_MODEL_ID"] 55 | form_recognizer_client = FormRecognizerClient(endpoint, AzureKeyCredential(key)) 56 | for value in values: 57 | output_record = transform_value(value, mappings, form_recognizer_client, model_id) 58 | if output_record != None: 59 | results["values"].append(output_record) 60 | break 61 | return json.dumps(results, ensure_ascii=False) 62 | 63 | ## Perform an operation on a record 64 | def transform_value(value, mappings, form_recognizer_client,model_id): 65 | try: 66 | recordId = value['recordId'] 67 | except AssertionError as error: 68 | return None 69 | try: 70 | assert ('data' in value), "'data' field is required." 71 | data = value['data'] 72 | formUrl = data['formUrl'] 73 | formSasToken = data ['formSasToken'] 74 | formUrl = formUrl + formSasToken 75 | poller = form_recognizer_client.begin_recognize_custom_forms_from_url( 76 | model_id=model_id, form_url=formUrl) 77 | result = poller.result() 78 | recognized = {} 79 | for recognized_form in result: 80 | print("Form type: {}".format(recognized_form.form_type)) 81 | for name, field in recognized_form.fields.items(): 82 | label = field.label_data.text if field.label_data else name 83 | for (k, v) in mappings.items(): 84 | if(label == k): 85 | recognized[v] = field.value 86 | 87 | except AssertionError as error: 88 | return ( 89 | { 90 | "recordId": recordId, 91 | "errors": [ { "message": "Error:" + error.args[0] } ] 92 | }) 93 | except Exception as error: 94 | return ( 95 | { 96 | "recordId": recordId, 97 | "errors": [ { "message": "Error:" + str(error) } ] 98 | }) 99 | return ({ 100 | "recordId": recordId, 101 | "data": { 102 | "recognized": recognized 103 | } 104 | }) 105 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/AnalyzeForm/field_mappings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Vehicle make:": "VehicleMake", 3 | "Vehicle year:": "VehicleYear" 4 | } 5 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/AnalyzeForm/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "__init__.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "function", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "post" 11 | ] 12 | }, 13 | { 14 | "type": "http", 15 | "direction": "out", 16 | "name": "$return" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/AnalyzeForm/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure" 3 | } -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/AnalyzeInvoice/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "__init__.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "function", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "get", 11 | "post" 12 | ] 13 | }, 14 | { 15 | "type": "http", 16 | "direction": "out", 17 | "name": "$return" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/AnalyzeInvoice/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure" 3 | } -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/ExtractTables/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | import logging 5 | import datetime 6 | from json import JSONEncoder 7 | import azure.functions as func 8 | from azure.ai.formrecognizer import FormRecognizerClient 9 | from azure.ai.formrecognizer import FormTrainingClient 10 | from azure.core.credentials import AzureKeyCredential 11 | 12 | class DateTimeEncoder(JSONEncoder): 13 | #Override the default method 14 | def default(self, obj): 15 | if isinstance(obj, (datetime.date, datetime.datetime)): 16 | return obj.isoformat() 17 | 18 | 19 | def main(req: func.HttpRequest) -> func.HttpResponse: 20 | logging.info('Python HTTP trigger function processed a request.') 21 | try: 22 | body = json.dumps(req.get_json()) 23 | if body: 24 | logging.info(body) 25 | result = compose_response(body) 26 | return func.HttpResponse(result, mimetype="application/json") 27 | else: 28 | return func.HttpResponse( 29 | "Invalid body", 30 | status_code=400 31 | ) 32 | except ValueError: 33 | return func.HttpResponse( 34 | "Invalid body", 35 | status_code=400 36 | ) 37 | except KeyError: 38 | return func.HttpResponse( 39 | "Skill configuration error. Endpoint, key and model_id required.", 40 | status_code=400 41 | ) 42 | 43 | 44 | def compose_response(json_data): 45 | values = json.loads(json_data)['values'] 46 | # Prepare the Output before the loop 47 | results = {} 48 | results["values"] = [] 49 | endpoint = os.environ["FORMS_RECOGNIZER_ENDPOINT"] 50 | key = os.environ["FORMS_RECOGNIZER_KEY"] 51 | form_recognizer_client = FormRecognizerClient(endpoint, AzureKeyCredential(key)) 52 | for value in values: 53 | output_record = transform_value(value, form_recognizer_client) 54 | if output_record != None: 55 | results["values"].append(output_record) 56 | 57 | return json.dumps(results, ensure_ascii=False, cls=DateTimeEncoder) 58 | 59 | 60 | ## Perform an operation on a record 61 | def transform_value(value, form_recognizer_client): 62 | try: 63 | recordId = value['recordId'] 64 | except AssertionError as error: 65 | return None 66 | # Validate the inputs 67 | try: 68 | assert ('data' in value), "'data' field is required." 69 | data = value['data'] 70 | print(data) 71 | form_url = data["formUrl"] + data["formSasToken"] 72 | print(form_url) 73 | poller = form_recognizer_client.begin_recognize_content_from_url(form_url) 74 | pages = poller.result() 75 | tables = [] 76 | if not pages: 77 | print("No pages found in doc") 78 | else: 79 | for page in pages: 80 | if not page.tables: 81 | print("No tables on page") 82 | else: 83 | for table in page.tables: 84 | cells = [] 85 | print("Table found on page {}:".format(table.page_number)) 86 | for cell in table.cells: 87 | cells.append( 88 | { 89 | "text": cell.text, 90 | "rowIndex": cell.row_index, 91 | "colIndex": cell.column_index, 92 | "confidence": cell.confidence, 93 | "is_header": cell.is_header 94 | } 95 | ) 96 | tables.append( 97 | { 98 | "page_number": table.page_number, 99 | "row_count": table.row_count, 100 | "column_count": table.column_count, 101 | "cells": cells 102 | } 103 | ) 104 | except AssertionError as error: 105 | return ( 106 | { 107 | "recordId": recordId, 108 | "errors": [ { "message": "Error:" + error.args[0] } ] 109 | }) 110 | except Exception as error: 111 | return ( 112 | { 113 | "recordId": recordId, 114 | "errors": [ { "message": "Error:" + str(error) } ] 115 | }) 116 | return ({ 117 | "recordId": recordId, 118 | "data": { 119 | "tables": tables 120 | } 121 | }) -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/ExtractTables/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "__init__.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "function", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "get", 11 | "post" 12 | ] 13 | }, 14 | { 15 | "type": "http", 16 | "direction": "out", 17 | "name": "$return" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/ExtractTables/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure" 3 | } -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/FormRecognizerTrainModel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "!pip install azure-ai-formrecognizer" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import os\n", 19 | "from azure.core.exceptions import ResourceNotFoundError\n", 20 | "from azure.ai.formrecognizer import FormRecognizerClient\n", 21 | "from azure.ai.formrecognizer import FormTrainingClient\n", 22 | "from azure.core.credentials import AzureKeyCredential" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 3, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "endpoint = \"YourFormsRecognizerEndpoint\"\n", 32 | "key = \"YourFormsRecognizerKey\"\n", 33 | "\n", 34 | "form_recognizer_client = FormRecognizerClient(endpoint, AzureKeyCredential(key))\n", 35 | "form_training_client = FormTrainingClient(endpoint, AzureKeyCredential(key))\n", 36 | "\n" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "# To train a model you need an Azure Storage account.\n", 46 | "# Use the SAS URL to access your training files.\n", 47 | "# Upload the files from the train folder to a storage container and generate a SAS token\n", 48 | "trainingDataUrl = \"\"\n", 49 | "\n", 50 | "poller = form_training_client.begin_training(trainingDataUrl, use_training_labels=False)\n", 51 | "model = poller.result()\n", 52 | "\n", 53 | "print(\"Model ID: {}\".format(model.model_id))\n", 54 | "print(\"Status: {}\".format(model.status))\n", 55 | "print(\"Training started on: {}\".format(model.training_started_on))\n", 56 | "print(\"Training completed on: {}\".format(model.training_completed_on))\n", 57 | "\n", 58 | "print(\"\\nRecognized fields:\")\n", 59 | "for submodel in model.submodels:\n", 60 | " print(\n", 61 | " \"The submodel with form type '{}' has recognized the following fields: {}\".format(\n", 62 | " submodel.form_type,\n", 63 | " \", \".join(\n", 64 | " [\n", 65 | " field.label if field.label else name\n", 66 | " for name, field in submodel.fields.items()\n", 67 | " ]\n", 68 | " ),\n", 69 | " )\n", 70 | " )\n", 71 | "\n", 72 | "# Training result information\n", 73 | "for doc in model.training_documents:\n", 74 | " print(\"Document name: {}\".format(doc.name))\n", 75 | " print(\"Document status: {}\".format(doc.status))\n", 76 | " print(\"Document page count: {}\".format(doc.page_count))\n", 77 | " print(\"Document errors: {}\".format(doc.errors))" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "# Model ID from when you trained your model. Look for the model id in the output from the previous cell\n", 87 | "# Upload the image from the test folder to a storage container and generate a SAS url \n", 88 | "model_id = \"modelid\"\n", 89 | "formUrl = \"test image SAS url\"\n", 90 | "poller = form_recognizer_client.begin_recognize_custom_forms_from_url(\n", 91 | " model_id=model_id, form_url=formUrl)\n", 92 | "result = poller.result()\n", 93 | "\n", 94 | "for recognized_form in result:\n", 95 | " print(\"Form type: {}\".format(recognized_form.form_type))\n", 96 | " for name, field in recognized_form.fields.items():\n", 97 | " print(\"Field '{}' has label '{}' with value '{}' and a confidence score of {}\".format(\n", 98 | " name,\n", 99 | " field.label_data.text if field.label_data else name,\n", 100 | " field.value,\n", 101 | " field.confidence\n", 102 | " ))" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [] 111 | } 112 | ], 113 | "metadata": { 114 | "kernelspec": { 115 | "display_name": "Python 3", 116 | "language": "python", 117 | "name": "python3" 118 | }, 119 | "language_info": { 120 | "codemirror_mode": { 121 | "name": "ipython", 122 | "version": 3 123 | }, 124 | "file_extension": ".py", 125 | "mimetype": "text/x-python", 126 | "name": "python", 127 | "nbconvert_exporter": "python", 128 | "pygments_lexer": "ipython3", 129 | "version": "3.7.4" 130 | } 131 | }, 132 | "nbformat": 4, 133 | "nbformat_minor": 2 134 | } 135 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Test/Ignite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Test/Ignite.png -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Adatum.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Adatum.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Fincher.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Fincher.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Lamna.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Lamna.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Liberty.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Liberty.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Trey.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/Train/Commercial Motor Vehicle - Trey.pdf -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | }, 11 | "extensionBundle": { 12 | "id": "Microsoft.Azure.Functions.ExtensionBundle", 13 | "version": "[2.*, 3.0.0)" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/proxies.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/proxies", 3 | "proxies": {} 4 | } 5 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/FormRecognizer Skill/customskill/requirements.txt: -------------------------------------------------------------------------------- 1 | # DO NOT include azure-functions-worker in this file 2 | # The Python Worker is managed by Azure Functions platform 3 | # Manually managing azure-functions-worker may cause unexpected issues 4 | 5 | azure-functions 6 | azure-ai-formrecognizer 7 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/README.md: -------------------------------------------------------------------------------- 1 | # Data Science & Custom Skills 2 | AI Search allows you to extend the out of the box functionality with custom skills. This extensibility enables a broad range of features such as creating custom filters, classifying documents, extracting custom entities, and more. 3 | 4 | This folder contains some example templates you can leverage to build your own custom skills and some useful Python notebooks you can use if you are building ML models. 5 | 6 | AI Search is agnostic to the tool(s) you use to build your custom skills, which are deployed as restful APIs. The only requirements for building your custom skill(s) are: 7 | 8 | 1. Have a secure, HTTPS, end-point. 9 | 2. Follow the defined input/output schema shown [here](https://docs.microsoft.com/azure/search/cognitive-search-custom-skill-interface). 10 | 11 | The code found in this repo will help you build and deploy these restful APIs in the correct format whether you are building a simple Azure Function or building your own custom ML model. 12 | 13 | ## Prerequisites 14 | 1. If you are using Azure Machine Learning to build custom skills for your solution, you will need to deploy additional resources to Azure subscription. Please see the [AML Quickstart](https://docs.microsoft.com/azure/machine-learning/service/quickstart-get-started) for more information. 15 | 16 | ## Types of Custom Skills 17 | In general, there are two types of custom skills that are most commonly used: 18 | 19 | ### 1.0 Azure Function Custom Skills 20 | Deploying an Azure Function is the quickest and easiest way to create a custom skill. This repo used to have it's own Azure Function templates but now links out to the [PowerSkills repo](https://github.com/Azure-Samples/azure-search-power-skills) that includes an array of custom skill options and fully automates the deployment process. Azure Functions are the recommended approach for deploying skills that do not require an ML model. 21 | 22 | ### 2.0 Azure Machine Learning Custom Skills 23 | ML models can be used to enhance the AI Search pipeline. In this template, Azure Machine Learning is used to build and deploy the model. The AML Custom Skill Template provides the files needed to quickly deploy a model to be used as a custom skill. 24 | 25 | > This template does not currently use the [AML Custom skill format](https://docs.microsoft.com/azure/search/cognitive-search-aml-skill) that's available in public preview but rather is designed to be deployed as a conventional [Web API skill](https://docs.microsoft.com/azure/search/cognitive-search-custom-skill-web-api). There's a separate tutorial that walks through the process of [creating an AML skill](https://docs.microsoft.com/azure/search/cognitive-search-tutorial-aml-custom-skill) based off of that functionality. 26 | 27 | ## Useful Links 28 | See the product documentation for more information on custom skills: 29 | 1. [Define Custom Skill interface](https://docs.microsoft.com/en-us/azure/search/cognitive-search-custom-skill-interface 30 | ) 31 | 2. [Custom Skill example](https://docs.microsoft.com/en-us/azure/search/cognitive-search-create-custom-skill-example 32 | ) 33 | 3. [AML Custom skills](https://docs.microsoft.com/azure/search/cognitive-search-aml-skill) 34 | 4. [AML Custom skill example](https://docs.microsoft.com/azure/search/cognitive-search-tutorial-aml-custom-skill) 35 | 36 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/Useful Notebooks/query_search_index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Query Search Index\n", 8 | "\n", 9 | "This notebook contains the code to pull down the data from the search index. This is particularly useful for building and monitoring custom skills. \n", 10 | "\n", 11 | "Please set the api_key and url here. It should not be checked into source control." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# Copyright (c) Microsoft Corporation. All rights reserved.\n", 21 | "# Licensed under the MIT License.\n", 22 | "\n", 23 | "api_key = ''" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "import requests\n", 33 | "import pandas as pd" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "## 1.0 Define function to pull data down from search index\n", 41 | "#### query_search_index() takes in three parameters:\n", 42 | "\n", 43 | " 1. **url**: The url should be in the format of \"https://{search-service-name}.search.windows.net/indexes/{index-name}/docs?\".\n", 44 | " 2. **api_key**: API key can be found in the Azure Portal. \n", 45 | " 3. **all_rows**: Optional. The default value is False. If False, the function will return the first 50 records. If True, all rows will be returned. Depending on the size of our index this can be time consuming.\n", 46 | " \n", 47 | "Additional customization can be done: for more information please see the Azure Search Rest API Documentation at https://docs.microsoft.com/en-us/rest/api/searchservice/search-documents" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "def query_search_index(url, api_key, all_rows=False):\n", 57 | " headers = {'api-key': api_key,\n", 58 | " 'Content-Type': 'application/json'}\n", 59 | " params = {'api-version': '2017-11-11',\n", 60 | " 'search': '*'}\n", 61 | " r = requests.get(url, params = params, headers = headers)\n", 62 | "\n", 63 | " docs = pd.DataFrame(r.json()['value'])\n", 64 | "\n", 65 | " #Strip whitespace from the column names\n", 66 | " docs.columns = docs.columns.str.strip()\n", 67 | " print(r)\n", 68 | " \n", 69 | " if not all_rows:\n", 70 | " return docs\n", 71 | " else:\n", 72 | " docs_list = [docs]\n", 73 | " while '@odata.nextLink' in r.json():\n", 74 | " r = requests.get(r.json()['@odata.nextLink'], headers = headers)\n", 75 | " docs = pd.DataFrame(r.json()['value'])\n", 76 | " #Strip whitespace from the column names\n", 77 | " docs.columns = docs.columns.str.strip()\n", 78 | " docs_list.append(docs)\n", 79 | "\n", 80 | " df = pd.concat(docs_list)\n", 81 | " return df\n" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "# 2.0 Query Search Index" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "url = \"https://{search-service-name}.search.windows.net/indexes/{index-name}/docs?\"\n", 98 | "df = query_search_index(url, api_key)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "# 3.0 Explore Data" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "len(df)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "metadata": { 121 | "scrolled": true 122 | }, 123 | "outputs": [], 124 | "source": [ 125 | "df.head()" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [] 134 | } 135 | ], 136 | "metadata": { 137 | "kernelspec": { 138 | "display_name": "Python 3", 139 | "language": "python", 140 | "name": "python3" 141 | }, 142 | "language_info": { 143 | "codemirror_mode": { 144 | "name": "ipython", 145 | "version": 3 146 | }, 147 | "file_extension": ".py", 148 | "mimetype": "text/x-python", 149 | "name": "python", 150 | "nbconvert_exporter": "python", 151 | "pygments_lexer": "ipython3", 152 | "version": "3.5.6" 153 | } 154 | }, 155 | "nbformat": 4, 156 | "nbformat_minor": 2 157 | } 158 | -------------------------------------------------------------------------------- /03 - Data Science and Custom Skills/Useful Notebooks/test_aml_deployment.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# Copyright (c) Microsoft Corporation. All rights reserved.\n", 10 | "# Licensed under the MIT License.\n", 11 | "\n", 12 | "import requests\n", 13 | "import json\n", 14 | "import pandas as pd" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "## 1.0 Update Scoring URL and Key" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "url = ''\n", 31 | "key = ''\n", 32 | "\n", 33 | "headers = {\"Authorization\": \"Bearer \" + key, 'Content-Type': 'application/json'}" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "## 2.0 Create or Load Sample Data" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "dict1 = {}\n", 50 | "dict1['recordId'] = \"1\"\n", 51 | "dict1['data'] = {}\n", 52 | "dict1['data']['text'] = \"Hey should I put this data in cosmos db?\"\n", 53 | "\n", 54 | "dict2 = {}\n", 55 | "dict2['recordId'] = \"2\"\n", 56 | "dict2['data'] = {}\n", 57 | "dict2['data']['text'] = \"c# is my favorite programming language\"\n", 58 | "\n", 59 | "dict_bad = {}\n", 60 | "dict_bad['recordId'] = \"3\"" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "## 3.0 Send API Call" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "values = {'values': [dict1]}\n", 77 | "r = requests.post(url, data=json.dumps(values), headers=headers)\n", 78 | "data = json.loads(r.text)\n", 79 | "print(data)" 80 | ] 81 | } 82 | ], 83 | "metadata": { 84 | "kernelspec": { 85 | "display_name": "Python 3", 86 | "language": "python", 87 | "name": "python3" 88 | }, 89 | "language_info": { 90 | "codemirror_mode": { 91 | "name": "ipython", 92 | "version": 3 93 | }, 94 | "file_extension": ".py", 95 | "mimetype": "text/x-python", 96 | "name": "python", 97 | "nbconvert_exporter": "python", 98 | "pygments_lexer": "ipython3", 99 | "version": "3.5.6" 100 | } 101 | }, 102 | "nbformat": 4, 103 | "nbformat_minor": 2 104 | } 105 | -------------------------------------------------------------------------------- /04 - Reporting/Cognitive Search.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/04 - Reporting/Cognitive Search.pbit -------------------------------------------------------------------------------- /04 - Reporting/README.md: -------------------------------------------------------------------------------- 1 | # Reporting 2 | 3 | A PowerBI Template file has been created so you can quickly spin up reports based on the AI Search Template UI. To create the reports, simply open **Cognitive Search.pbit** using PowerBI Desktop. 4 | 5 | ## Prerequisites 6 | 7 | 1. Application Insights is used to capture the telemetry data for these reports. If you chose to not provision Application Insights and update the *InstrumentationKey* in the web app's *appsettings.json* file, you will not have data available to run the reports in this repository. 8 | 1. PowerBI Desktop installed on your computer 9 | 10 | ## Getting Started 11 | 12 | When you open the template, you will be asked for your **Application Insights Application ID**. This Application ID can be found by going to the Azure portal -> navigating to the App Insights resource -> then clicking on API Access: 13 | 14 | After you enter the Application Id, click **Load**. 15 | 16 | Application Id Load 17 | 18 | 19 | 20 | 21 | Next, it will ask for credentials: 22 | 23 | Credentials 24 | 25 | 26 | 27 | 28 | Enter your credentials and you'll have PowerBI reports ready to go like the one seen below: 29 | 30 | PowerBi sample report 31 | 32 | 33 | 34 | ## Troubleshooting 35 | 36 | If you do not see any details in Power BI (even after hitting the Refresh button), you might want to check if you are using a script or adblocker in your browser. For example, uBlock might block the logging requests to Azure, depending on the configuration. 37 | 38 | ## Additional Materials 39 | For more information on PowerBI and developing reports on PowerBI see [What is Power BI](https://docs.microsoft.com/en-us/power-bi/power-bi-overview). 40 | 41 | Alternatively, you can use the new Power BI report that is automatically included with Azure AI Search. In order to access it, goto the Search instance in your subscription, navigate to `Search traffic analytics`, and select `Download Power BI report`. 42 | 43 | Both reports have comparable capabilties and can be used to develop more sophisticated dashboards. 44 | 45 | ## Special Thanks 46 | Special thanks to Emilio D'Angelo for creating this PBI template. 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to azure-search-knowledge-mining 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 6 | 7 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase master -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE -------------------------------------------------------------------------------- /PRIVACY.md: -------------------------------------------------------------------------------- 1 | # Privacy 2 | 3 | When you deploy this template, Microsoft is able to identify the installation of the software with the Azure resources that are deployed. Microsoft is able to correlate the Azure resources that are used to support the software. Microsoft collects this information to provide the best experiences with their products and to operate their business. The data is collected and governed by Microsoft's privacy policies, which can be found at https://www.microsoft.com/trustcenter. 4 | 5 | To disable this, simply remove the following section from [azuredeploy.json](./azuredeploy.json) before deploying the resources to Azure: 6 | 7 | ```json 8 | { 9 | "apiVersion": "2018-02-01", 10 | "name": "pid-205d0e47-324f-5b9e-8dde-f9fc4a92c091", 11 | "type": "Microsoft.Resources/deployments", 12 | "properties": { 13 | "mode": "Incremental", 14 | "template": { 15 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 16 | "contentVersion": "1.0.0.0", 17 | "resources": [] 18 | } 19 | } 20 | } 21 | ``` 22 | 23 | You can see more information on this at https://docs.microsoft.com/en-us/azure/marketplace/azure-partner-customer-usage-attribution. -------------------------------------------------------------------------------- /images/Config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/Config.png -------------------------------------------------------------------------------- /images/SearchModel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/SearchModel.png -------------------------------------------------------------------------------- /images/WebUI.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/WebUI.jpg -------------------------------------------------------------------------------- /images/appsettings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/appsettings.png -------------------------------------------------------------------------------- /images/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/architecture.jpg -------------------------------------------------------------------------------- /images/createindex-step0.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step0.PNG -------------------------------------------------------------------------------- /images/createindex-step1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step1.PNG -------------------------------------------------------------------------------- /images/createindex-step2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step2.PNG -------------------------------------------------------------------------------- /images/createindex-step3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step3.PNG -------------------------------------------------------------------------------- /images/createindex-step4.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step4.PNG -------------------------------------------------------------------------------- /images/createindex-step5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step5.PNG -------------------------------------------------------------------------------- /images/createindex-step6.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step6.PNG -------------------------------------------------------------------------------- /images/createindex-step7.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step7.PNG -------------------------------------------------------------------------------- /images/createindex-step8.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/createindex-step8.PNG -------------------------------------------------------------------------------- /images/demo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/demo.jpg -------------------------------------------------------------------------------- /images/geolocation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/geolocation.png -------------------------------------------------------------------------------- /images/kmheader.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/kmheader.png -------------------------------------------------------------------------------- /images/pbi1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/pbi1.png -------------------------------------------------------------------------------- /images/pbi2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/pbi2.png -------------------------------------------------------------------------------- /images/pbi3.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/pbi3.JPG -------------------------------------------------------------------------------- /images/postman_edit_environment_variables.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/postman_edit_environment_variables.png -------------------------------------------------------------------------------- /images/postman_import_files.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/postman_import_files.png -------------------------------------------------------------------------------- /images/translated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/translated.png -------------------------------------------------------------------------------- /images/ui.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/ui.PNG -------------------------------------------------------------------------------- /images/webapp-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/webapp-1.png -------------------------------------------------------------------------------- /images/webapp-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/images/webapp-2.png -------------------------------------------------------------------------------- /industry-solutions/documents/KnowledgeMiningWhitePaper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/industry-solutions/documents/KnowledgeMiningWhitePaper.pdf -------------------------------------------------------------------------------- /industry-solutions/documents/WhitePaperDocSegmentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/industry-solutions/documents/WhitePaperDocSegmentation.pdf -------------------------------------------------------------------------------- /industry-solutions/journalism/projectida/readme.md: -------------------------------------------------------------------------------- 1 | # Installing Project Ida -------------------------------------------------------------------------------- /industry-solutions/journalism/readme.md: -------------------------------------------------------------------------------- 1 | # Project Ida -------------------------------------------------------------------------------- /sample_documents/Azure Search Pricing Table.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/Azure Search Pricing Table.xlsx -------------------------------------------------------------------------------- /sample_documents/Cognitive Services and Bots (spanish).pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/Cognitive Services and Bots (spanish).pdf -------------------------------------------------------------------------------- /sample_documents/Cognitive Services and Content Intelligence.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/Cognitive Services and Content Intelligence.pdf -------------------------------------------------------------------------------- /sample_documents/Deployment strategies and best practices.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/Deployment strategies and best practices.pdf -------------------------------------------------------------------------------- /sample_documents/FAQ - Azure Search.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/FAQ - Azure Search.pdf -------------------------------------------------------------------------------- /sample_documents/GitHub Facts.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/GitHub Facts.pdf -------------------------------------------------------------------------------- /sample_documents/LearnAI KnowledgeMiningBootcamp.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/LearnAI KnowledgeMiningBootcamp.pdf -------------------------------------------------------------------------------- /sample_documents/MSFT Contoso Cloud Architecture.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/MSFT Contoso Cloud Architecture.pdf -------------------------------------------------------------------------------- /sample_documents/NYSE_LNKD_2015.PDF: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/NYSE_LNKD_2015.PDF -------------------------------------------------------------------------------- /sample_documents/Pricing details.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/Pricing details.docx -------------------------------------------------------------------------------- /sample_documents/Red Shirts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/Red Shirts.png -------------------------------------------------------------------------------- /sample_documents/What is Azure Search service.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/What is Azure Search service.pdf -------------------------------------------------------------------------------- /sample_documents/create-search-collect-info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/create-search-collect-info.png -------------------------------------------------------------------------------- /sample_documents/create-search-service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/create-search-service.png -------------------------------------------------------------------------------- /sample_documents/create-service-full-portal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/create-service-full-portal.png -------------------------------------------------------------------------------- /sample_documents/geo-search-traffic-mgr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/geo-search-traffic-mgr.png -------------------------------------------------------------------------------- /sample_documents/guthrie.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/guthrie.jpg -------------------------------------------------------------------------------- /sample_documents/redshirt.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/redshirt.jpg -------------------------------------------------------------------------------- /sample_documents/satyanadellalinux.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/satyanadellalinux.jpg -------------------------------------------------------------------------------- /sample_documents/satyasletter.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/satyasletter.html -------------------------------------------------------------------------------- /sample_documents/scale-indexers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/sample_documents/scale-indexers.png -------------------------------------------------------------------------------- /workshops/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 cynotebo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /workshops/Module 0.md: -------------------------------------------------------------------------------- 1 | # Pre-Requisites for Knowledge Mining Workshop 2 | 3 | Please make sure you fulfill the following pre-requisites before starting the workshop. 4 | 5 | 1. Have your own Azure account 6 | 1. Be familiar [Azure Portal](https://portal.azure.com) 7 | 1. Make sure you can create Azure resources in your subscription (including paid resources). 8 | 9 | *Note, if your organizations policy prohibits you from creating resources in the subscription, you can use a [free subscription](https://signup.azure.com) for the purposes of this lab.* 10 | 11 | 1. Create a resource group for this workshop where you will add each of the resources you will create in the next steps. 12 | 1. **Create** an [Azure Storage Account](https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=azure-portal). 13 | * Select Performance: *Standard* tier, not Premium 14 | * Select Account kind: *StorageV2 (general purpose v2)* 15 | 1. **Install** [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) 16 | 1. **Copy** the storage container that holds clinical trials from a read only location to your Storage Account. 17 | 1. **Open** Azure Storage Explorer and select *Manage Accounts*, *Add and Account...*, *Use a shared access signature (SAS) URI*. Select *Next* 18 | 1. **Add** the following *URI*, then select *Next*, then select *Connect* 19 | ``` 20 | https://kmworkshop.blob.core.windows.net/clinical-trials-small?sv=2020-10-02&st=2022-05-06T17%3A02%3A00Z&se=2024-06-07T17%3A02%3A00Z&sr=c&sp=rl&sig=EkgZHm2A5l509O%2FBZf7j2B60QsIlHOQuXJpvX46%2BO3o%3D 21 | ``` 22 | 1. **Select** *Toggle Explorer* to view the Explorer. Right click on the *clinical-trials-small* Blob Container that you just connected to and select *Copy Blob Container* 23 | ![](images/copyblobcontainer.png) 24 | 1. **Find** your Storage Account in the Explorer. Right click on its *Blob Containers* and select *Paste Blob Container*. 25 | ![](images/pasteblobcontainer.png) 26 | 1. Confirm that the container copied successfully by checking the Activities at the bottom of the Azure Storage Explorer. 27 | 1. **Create** an [Azure Search](https://docs.microsoft.com/en-us/azure/search/search-create-service-portal) resource. (A Free Tier should be sufficient for this workshop). 28 | [Learn more](https://docs.microsoft.com/en-us/azure/search/search-sku-tier) 29 | 30 | 1. **Create** a [AI Services resource](https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Cwindows). 31 | 32 | *Note - You need to create the AI Services resource in the same region as you Azure Search resource.* 33 | 34 | 1. **Install** [Visual Studio 2019](https://visualstudio.microsoft.com/downloads/). Make sure you can create ASP.Net websites with it. We'll be using .NET Core 3.1 35 | 1. **Install** [Postman](https://www.getpostman.com/) 36 | 1. **Install** [PowerBI desktop](https://powerbi.microsoft.com/en-us/desktop/). 37 | 38 | ### Next: [Module 1: Using Azure Portal to Create Your Index - No Code Required](Module 1.md) 39 | -------------------------------------------------------------------------------- /workshops/Module 2.md: -------------------------------------------------------------------------------- 1 | # Module 2: Visualizing the Results with a Demo FrontEnd 2 | Now that you've built your Search index, we'll take a moment to build and deploy a simple web-page that will allow you to visualize your initial search results. We'll come back to this view throughout the lab and take special note of how our search results change as we add additional features and capabilities. 3 | 4 | To get started with this exercise, we will be using and getting familiar with the [Knowledge Mining solution accelerator](https://github.com/Azure-Samples/azure-search-knowledge-mining) to create our front-end experience. This accelerator was published to provide developers with all of the steps and tools required to build a working minimally viable knowledge mining solution. Take a few moments to note that it contains modules to: deploy the required Azure resources; build custom skills; and present the results in a simple, but elegant front-end. At the end of this lab, your results will look similar to this: 5 | 6 | ![](images/intresults.png) 7 | 8 | ## 1. Clone the repository 9 | ``` 10 | git clone https://github.com/Azure-Samples/azure-search-knowledge-mining.git 11 | ``` 12 | 13 | ## 2. Start the project 14 | 15 | Open **CognitiveSearch.UI.sln** (02-Web UI Template\CognitiveSearch.UI) in Visual Studio 16 | 17 | ## 3. Update appsettings.json 18 | 19 | Update the following fields in the *appsettings.json* file to connect the web app to your storage account, search index, and app insights account: 20 | 21 | ```json 22 | "SearchServiceName": "Your Search Service Name", 23 | "SearchApiKey": "Your Search Service key", 24 | "SearchIndexName": "clinical-trials-small", 25 | "InstrumentationKey": "", 26 | "StorageAccountName": "Your storage Account Name", 27 | "StorageAccountKey": "Your Storage Account Key", 28 | "StorageContainerAddress": "Your Storage Container Address", 29 | "KeyField": "metadata_storage_path", 30 | "IsPathBase64Encoded": true, 31 | "GraphFacet": "diseases" 32 | ``` 33 | 34 | ### Notes 35 | 1. **SearchServiceName** should be set to the name of the search service. (i.e. "myservice") 36 | 1. **SearchApiKey** should be to the name of the search service. (i.e. "B8365AC95521089B7E3FA4CC98435") 37 | 1. **SearchIndexName** should be set to the name of the index (i.e. "clinical-trials-small") 38 | 1. **StorageAccountName** should be set to the name of the storage account (i.e. "mystorageaccount") 39 | 1. **StorageContainerAddress** should be in the following format: *"https://*storageaccountname*.blob.core.windows.net/*containername*"* 40 | 1. **InstrumentationKey** is an optional field. The instrumentation key connects the web app to Application Insights in order to populate the Power BI reports. 41 | 1. **KeyField** should be set to the field specified as a key document Id in the index. (i.e. "metadata_storage_path") 42 | 1. Sometimes metadata_storage_path is the key, and it gets base64 encoded. In that case set **IsPathBase64Encoded** to true. 43 | 1. The **GraphFacet** is used for generating the relationship graph, set it to the name of the facet that you would like to use (i.e. "diseases"). Or leave blank if you won't use the node graph. 44 | 45 | 46 | ### 47 | *Important:* 48 | While this tutorial is optimizing for efficiency of allowing you to see results, and investigate the code, please note that entering your credentials into code is not a good practice to follow. We recommend you use a service like [Azure Key Vault](https://docs.microsoft.com/en-us/azure/key-vault/key-vault-overview) to do this. 49 | 50 | ## 3. Set the **Startup Project** 51 | 52 | ![](images/setstart.png) 53 | 54 | ## 4. Run the project and see the results 55 | 56 | ![](images/intresults.png) 57 | 58 | ## 5. Inspect the code 59 | 60 | Much of the UI is rendered dynamically by javascript. Some important files to know when making changes to the UI are: 61 | 62 | 1. **wwroot/js/results.js** - contains the code used to render search results on the UI 63 | 64 | 2. **wwroot/js/details.js** - contains the code for rending the detail view once a result is selected 65 | 66 | ### Next: [Module 3: Introduction to Azure Functions and Custom Skills](Module 3.md) 67 | 68 | -------------------------------------------------------------------------------- /workshops/README.md: -------------------------------------------------------------------------------- 1 | # Knowledge Mining Workshop 2 | 3 | **Welcome to the Knowledge Mining workshop!** 4 | 5 | During this lab, we will be exploring a data set that is comprised of clinical trials in PDF format. This demo data set contains over 100 records which contain complex medical terms and disease information in an unstructured format - meaning that there is no easy way to search the data set for information or records that pertain to a specific disease, like cirrhosis. And worse yet, if (like the authors of this lab) you don't always accurately spell cirrhosis, you may have no way of ever finding the information you're looking for - structured or otherwise. 6 | 7 | The challenge is, how can you take a vast amount of unstructured content and all of the latent data that it contains and provide an easy and effective way for a human to find the information that is most meaningful and relevant to them in the most efficient means possible? Metaphorically, we are going to be finding meaning in information that starts out looking like this: 8 | 9 | ![](images/unstructured.png) 10 | 11 | This is where the power of Knowledge Mining with Azure AI Search comes to the rescue. Using the skills and techniques we'll be teaching you today, you will be able to quickly and easily ingest this content, build custom skills to identify and extract specific medical terms related to disease conditions and then either search for information through a web front end 12 | 13 | ![](images/results.png) 14 | 15 | or project the data into powerful visuals you create in PowerBI. 16 | 17 | ![](images/mod5/ks-pbi-visual5-filledmap-graph.png) 18 | 19 | Along the way, we will also show you how easy and powerful it is to introduce more advanced search topics like phonetic search to your index (so that your users can find the information they are looking for regardless of their spelling skills) and boost relevancy of results. The end result of today's workshop will be a fully searchable medical repository that will allow users to find and extract really powerful information, even if they are not trained medical professionals. 20 | 21 | More concretely, you'll learn how to build an Azure AI Search Index and Knowledge Store repository through the Azure Portal. From there, we'll explore more advanced concepts in Azure AI Search, build custom skills to extend the solution and create a basic web page front end to visualize your search results. Finally, you will build a PowerBI dashboard on top of your Knowledge Store to demonstrate how you can use your data store for data visualization and advanced analytics. We have also included an Optional Module 7 which is a stand alone module that will walk you through indexing data from an AzureSQL data source. We encourage you to walk through this module on your own time and review how this further extends the data available in your solution. 22 | 23 | + *Note: Please make sure to complete all of pre-requisites listed in Module 0 before moving on, as you will not be able to complete the exercises in this lab without them*. 24 | 25 | ## Agenda 26 | 27 | + [Module 0 - Pre-Requisites](./Module%200.md) (*you must complete prior to moving on!*) 28 | + [Module 1 - Using Azure Portal to Build a Search Index and Knowledge Store](./Module%201.md) 29 | + [Module 2 - Visualizing the Results with a Demo FrontEnd](./Module%202.md) 30 | + [Module 3 - Introduction to Custom Skills and Azure Functions](./Module%203.md) 31 | + [Module 4 - Learning the Object Model](./Module%204.md) 32 | + [Module 5 - Advanced Azure AI Search: Analyzers and Scoring Profiles](./Module%205.md) 33 | + [Module 6 - Analyzing Your Data with PowerBI](./Module%206.md) 34 | + [Module 7 - Using Azure AI Search to index structured data](./Module%207.md) (Optional) 35 | -------------------------------------------------------------------------------- /workshops/data/clinical-trials-small.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/data/clinical-trials-small.zip -------------------------------------------------------------------------------- /workshops/images/4skills.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/4skills.png -------------------------------------------------------------------------------- /workshops/images/add_scoring_profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/add_scoring_profile.png -------------------------------------------------------------------------------- /workshops/images/addks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/addks.png -------------------------------------------------------------------------------- /workshops/images/attachenrich.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/attachenrich.png -------------------------------------------------------------------------------- /workshops/images/azurefunction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/azurefunction.png -------------------------------------------------------------------------------- /workshops/images/chkstatus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/chkstatus.png -------------------------------------------------------------------------------- /workshops/images/chkstatus2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/chkstatus2.png -------------------------------------------------------------------------------- /workshops/images/chooseconnection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/chooseconnection.png -------------------------------------------------------------------------------- /workshops/images/connectblob.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/connectblob.jpg -------------------------------------------------------------------------------- /workshops/images/copyblobcontainer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/copyblobcontainer.png -------------------------------------------------------------------------------- /workshops/images/create-kv-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/create-kv-1.png -------------------------------------------------------------------------------- /workshops/images/create-kv-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/create-kv-2.png -------------------------------------------------------------------------------- /workshops/images/customentity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/customentity.png -------------------------------------------------------------------------------- /workshops/images/datasource.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/datasource.png -------------------------------------------------------------------------------- /workshops/images/diseaseextractor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/diseaseextractor.png -------------------------------------------------------------------------------- /workshops/images/diseases.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/diseases.png -------------------------------------------------------------------------------- /workshops/images/editquery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/editquery.png -------------------------------------------------------------------------------- /workshops/images/enrichments.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/enrichments.png -------------------------------------------------------------------------------- /workshops/images/field_boosting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/field_boosting.png -------------------------------------------------------------------------------- /workshops/images/freshness_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/freshness_function.png -------------------------------------------------------------------------------- /workshops/images/function-url.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/function-url.png -------------------------------------------------------------------------------- /workshops/images/get-index-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/get-index-schema.png -------------------------------------------------------------------------------- /workshops/images/getdata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/getdata.png -------------------------------------------------------------------------------- /workshops/images/importdata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/importdata.png -------------------------------------------------------------------------------- /workshops/images/indexdef.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/indexdef.png -------------------------------------------------------------------------------- /workshops/images/indexer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/indexer.png -------------------------------------------------------------------------------- /workshops/images/intresults.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/intresults.png -------------------------------------------------------------------------------- /workshops/images/json2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/json2.png -------------------------------------------------------------------------------- /workshops/images/jsonportal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/jsonportal.png -------------------------------------------------------------------------------- /workshops/images/kstable.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kstable.png -------------------------------------------------------------------------------- /workshops/images/kstable2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kstable2.png -------------------------------------------------------------------------------- /workshops/images/kv-access-policies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kv-access-policies.png -------------------------------------------------------------------------------- /workshops/images/kv-app-identity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kv-app-identity.png -------------------------------------------------------------------------------- /workshops/images/kv-create-secret.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kv-create-secret.png -------------------------------------------------------------------------------- /workshops/images/kv-nuget.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kv-nuget.png -------------------------------------------------------------------------------- /workshops/images/kv-set-secret-blob.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kv-set-secret-blob.png -------------------------------------------------------------------------------- /workshops/images/kv-set-secret.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kv-set-secret.png -------------------------------------------------------------------------------- /workshops/images/kv-view-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/kv-view-list.png -------------------------------------------------------------------------------- /workshops/images/location .png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/location .png -------------------------------------------------------------------------------- /workshops/images/lookup2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/lookup2.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-closeandapply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-closeandapply.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-editquery1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-editquery1.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-editquery2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-editquery2.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-editquery3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-editquery3.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-editquery4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-editquery4.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-getdata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-getdata.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-getdata2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-getdata2.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-getdata3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-getdata3.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-getdata4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-getdata4.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-model.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-model2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-model2.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual1.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual2-disease-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual2-disease-list.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual3-lastupdate-date.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual3-lastupdate-date.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual4-doccount-graph-filtered.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual4-doccount-graph-filtered.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual4-doccount-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual4-doccount-graph.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual5-filledmap-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual5-filledmap-graph.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual5-filledmap-settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual5-filledmap-settings.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual5-filledmap-settings2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual5-filledmap-settings2.png -------------------------------------------------------------------------------- /workshops/images/mod5/ks-pbi-visual6-directed-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/mod5/ks-pbi-visual6-directed-graph.png -------------------------------------------------------------------------------- /workshops/images/new-appservice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/new-appservice.png -------------------------------------------------------------------------------- /workshops/images/objectmodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/objectmodel.png -------------------------------------------------------------------------------- /workshops/images/pasteblobcontainer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/pasteblobcontainer.png -------------------------------------------------------------------------------- /workshops/images/postman.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/postman.png -------------------------------------------------------------------------------- /workshops/images/postman2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/postman2.png -------------------------------------------------------------------------------- /workshops/images/powerbi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/powerbi.png -------------------------------------------------------------------------------- /workshops/images/querywithselect.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/querywithselect.PNG -------------------------------------------------------------------------------- /workshops/images/removecolumn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/removecolumn.png -------------------------------------------------------------------------------- /workshops/images/rerun.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/rerun.png -------------------------------------------------------------------------------- /workshops/images/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/results.png -------------------------------------------------------------------------------- /workshops/images/savechanges.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/savechanges.png -------------------------------------------------------------------------------- /workshops/images/scoring_profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/scoring_profile.png -------------------------------------------------------------------------------- /workshops/images/select container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/select container.png -------------------------------------------------------------------------------- /workshops/images/selectcontainer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/selectcontainer.png -------------------------------------------------------------------------------- /workshops/images/setstart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/setstart.png -------------------------------------------------------------------------------- /workshops/images/skillset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/skillset.png -------------------------------------------------------------------------------- /workshops/images/sql-patient-info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/sql-patient-info.png -------------------------------------------------------------------------------- /workshops/images/srchexplore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/srchexplore.png -------------------------------------------------------------------------------- /workshops/images/storeacct.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/storeacct.png -------------------------------------------------------------------------------- /workshops/images/tablereview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/tablereview.png -------------------------------------------------------------------------------- /workshops/images/tablestore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/tablestore.png -------------------------------------------------------------------------------- /workshops/images/unstructured.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/unstructured.png -------------------------------------------------------------------------------- /workshops/images/wordscsv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/images/wordscsv.png -------------------------------------------------------------------------------- /workshops/slides/KM_Workshop.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-search-knowledge-mining/3ac6db28739c58e0036a43c318c69d4e68617150/workshops/slides/KM_Workshop.pptx --------------------------------------------------------------------------------