├── .gitattributes ├── .gitignore ├── LICENSE ├── PBTextAnalyticsExamples.pbix ├── README.md ├── TextAnalytics.sln └── TextAnalytics ├── TextAnalytics.mproj ├── TextAnalytics.pq ├── TextAnalytics.query.pq ├── TextAnalytics16.png ├── TextAnalytics20.png ├── TextAnalytics24.png ├── TextAnalytics32.png ├── TextAnalytics40.png ├── TextAnalytics48.png ├── TextAnalytics64.png ├── TextAnalytics80.png └── resources.resx /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | [Xx]64/ 19 | [Xx]86/ 20 | [Bb]uild/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | artifacts/ 46 | 47 | *_i.c 48 | *_p.c 49 | *_i.h 50 | *.ilk 51 | *.meta 52 | *.obj 53 | *.pch 54 | *.pdb 55 | *.pgc 56 | *.pgd 57 | *.rsp 58 | *.sbr 59 | *.tlb 60 | *.tli 61 | *.tlh 62 | *.tmp 63 | *.tmp_proj 64 | *.log 65 | *.vspscc 66 | *.vssscc 67 | .builds 68 | *.pidb 69 | *.svclog 70 | *.scc 71 | 72 | # Chutzpah Test files 73 | _Chutzpah* 74 | 75 | # Visual C++ cache files 76 | ipch/ 77 | *.aps 78 | *.ncb 79 | *.opendb 80 | *.opensdf 81 | *.sdf 82 | *.cachefile 83 | *.VC.db 84 | 85 | # Visual Studio profiler 86 | *.psess 87 | *.vsp 88 | *.vspx 89 | *.sap 90 | 91 | # TFS 2012 Local Workspace 92 | $tf/ 93 | 94 | # Guidance Automation Toolkit 95 | *.gpState 96 | 97 | # ReSharper is a .NET coding add-in 98 | _ReSharper*/ 99 | *.[Rr]e[Ss]harper 100 | *.DotSettings.user 101 | 102 | # JustCode is a .NET coding add-in 103 | .JustCode 104 | 105 | # TeamCity is a build add-in 106 | _TeamCity* 107 | 108 | # DotCover is a Code Coverage Tool 109 | *.dotCover 110 | 111 | # NCrunch 112 | _NCrunch_* 113 | .*crunch*.local.xml 114 | nCrunchTemp_* 115 | 116 | # MightyMoose 117 | *.mm.* 118 | AutoTest.Net/ 119 | 120 | # Web workbench (sass) 121 | .sass-cache/ 122 | 123 | # Installshield output folder 124 | [Ee]xpress/ 125 | 126 | # DocProject is a documentation generator add-in 127 | DocProject/buildhelp/ 128 | DocProject/Help/*.HxT 129 | DocProject/Help/*.HxC 130 | DocProject/Help/*.hhc 131 | DocProject/Help/*.hhk 132 | DocProject/Help/*.hhp 133 | DocProject/Help/Html2 134 | DocProject/Help/html 135 | 136 | # Click-Once directory 137 | publish/ 138 | 139 | # Publish Web Output 140 | *.[Pp]ublish.xml 141 | *.azurePubxml 142 | 143 | # TODO: Un-comment the next line if you do not want to checkin 144 | # your web deploy settings because they may include unencrypted 145 | # passwords 146 | #*.pubxml 147 | *.publishproj 148 | 149 | # NuGet Packages 150 | *.nupkg 151 | # The packages folder can be ignored because of Package Restore 152 | **/packages/* 153 | # except build/, which is used as an MSBuild target. 154 | !**/packages/build/ 155 | # Uncomment if necessary however generally it will be regenerated when needed 156 | #!**/packages/repositories.config 157 | # NuGet v3's project.json files produces more ignoreable files 158 | *.nuget.props 159 | *.nuget.targets 160 | 161 | # Microsoft Azure Build Output 162 | csx/ 163 | *.build.csdef 164 | 165 | # Microsoft Azure Emulator 166 | ecf/ 167 | rcf/ 168 | 169 | # Microsoft Azure ApplicationInsights config file 170 | ApplicationInsights.config 171 | 172 | # Windows Store app package directory 173 | AppPackages/ 174 | BundleArtifacts/ 175 | 176 | # Visual Studio cache files 177 | # files ending in .cache can be ignored 178 | *.[Cc]ache 179 | # but keep track of directories ending in .cache 180 | !*.[Cc]ache/ 181 | 182 | # Others 183 | ClientBin/ 184 | [Ss]tyle[Cc]op.* 185 | ~$* 186 | *~ 187 | *.dbmdl 188 | *.dbproj.schemaview 189 | *.pfx 190 | *.publishsettings 191 | node_modules/ 192 | orleans.codegen.cs 193 | 194 | # RIA/Silverlight projects 195 | Generated_Code/ 196 | 197 | # Backup & report files from converting an old project file 198 | # to a newer Visual Studio version. Backup files are not needed, 199 | # because we have git ;-) 200 | _UpgradeReport_Files/ 201 | Backup*/ 202 | UpgradeLog*.XML 203 | UpgradeLog*.htm 204 | 205 | # SQL Server files 206 | *.mdf 207 | *.ldf 208 | 209 | # Business Intelligence projects 210 | *.rdl.data 211 | *.bim.layout 212 | *.bim_*.settings 213 | 214 | # Microsoft Fakes 215 | FakesAssemblies/ 216 | 217 | # GhostDoc plugin setting file 218 | *.GhostDoc.xml 219 | 220 | # Node.js Tools for Visual Studio 221 | .ntvs_analysis.dat 222 | 223 | # Visual Studio 6 build log 224 | *.plg 225 | 226 | # Visual Studio 6 workspace options file 227 | *.opt 228 | 229 | # Visual Studio LightSwitch build output 230 | **/*.HTMLClient/GeneratedArtifacts 231 | **/*.DesktopClient/GeneratedArtifacts 232 | **/*.DesktopClient/ModelManifest.xml 233 | **/*.Server/GeneratedArtifacts 234 | **/*.Server/ModelManifest.xml 235 | _Pvt_Extensions 236 | 237 | # LightSwitch generated files 238 | GeneratedArtifacts/ 239 | ModelManifest.xml 240 | 241 | # Paket dependency manager 242 | .paket/paket.exe 243 | 244 | # FAKE - F# Make 245 | .fake/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Christopher Webb 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PBTextAnalyticsExamples.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cwebbbi/PowerBITextAnalytics/785319cec916ce997bdc04216044121c6ccb630d/PBTextAnalyticsExamples.pbix -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PowerBITextAnalytics 2 | A Power BI custom data connector for the Microsoft Cognitive Services Text Analytics API 3 | 4 | Full documentation for the Text Analytics API can be found [here](https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/overview) and there is more detailed documentation available for the [Detect Language](https://westus.dev.cognitive.microsoft.com/docs/services/TextAnalytics.V2.0/operations/56f30ceeeda5650db055a3c7), [Key Phrases](https://westus.dev.cognitive.microsoft.com/docs/services/TextAnalytics.V2.0/operations/56f30ceeeda5650db055a3c6) and [Sentiment](https://westus.dev.cognitive.microsoft.com/docs/services/TextAnalytics.V2.0/operations/56f30ceeeda5650db055a3c9) APIs. 5 | 6 | Note: you will need to [sign up for the Text Analytics API](https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/text-analytics-howto-signup) and [obtain an access key](https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/text-analytics-howto-accesskey) before you use this custom data connector. A number of [pricing tiers](https://azure.microsoft.com/en-us/pricing/details/cognitive-services/text-analytics/) are available, including a free tier that allows for 5000 calls per month. 7 | 8 | # Functionality 9 | This custom data connector exposes three M functions: 10 | - TextAnalytics.DetectLanguage(inputtext as list, optional numberoflanguages as number) as table 11 | This function takes a list of text values and returns a table containing the input text and the language detected in each piece of text 12 | - TextAnalytics.KeyPhrases(inputtext as list, optional languages as list) as table 13 | This function takes a list of text values (and an optional list of language identifiers for each piece of text) and returns a table containing the input text and key phrases detected in each piece of text. More than one key phrase may be returned for each piece of text. 14 | - TextAnalytics.Sentiment(inputtext as list, optional languages as list) as table 15 | This function takes a list of text values (and an optional list of language identifiers for each piece of text) and returns a table containing the input text and a score representing the sentiment detected for each piece of text. 16 | 17 | There are a number of examples of how these functions can be used in the TextAnalytics.query.pq file included in the project, in the [example Power BI .pbix file](https://github.com/cwebbbi/PowerBITextAnalytics/blob/master/PBTextAnalyticsExamples.pbix), as well as [this blog post](https://blog.crossjoin.co.uk/2017/09/04/power-bi-custom-data-connector-for-language-detection-key-phrase-extraction-and-sentiment-analysis/) on Chris Webb's BI Blog. 18 | -------------------------------------------------------------------------------- /TextAnalytics.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.25420.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{4DF76451-A46A-4C0B-BE03-459FAAFA07E6}") = "TextAnalytics", "TextAnalytics\TextAnalytics.mproj", "{FAE68D50-EBFE-461F-A40C-2E47F099E053}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|x86 = Debug|x86 11 | Release|x86 = Release|x86 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {FAE68D50-EBFE-461F-A40C-2E47F099E053}.Debug|x86.ActiveCfg = Debug|x86 15 | {FAE68D50-EBFE-461F-A40C-2E47F099E053}.Debug|x86.Build.0 = Debug|x86 16 | {FAE68D50-EBFE-461F-A40C-2E47F099E053}.Release|x86.ActiveCfg = Release|x86 17 | {FAE68D50-EBFE-461F-A40C-2E47F099E053}.Release|x86.Build.0 = Release|x86 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | EndGlobal 23 | -------------------------------------------------------------------------------- /TextAnalytics/TextAnalytics.mproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | 2.0 6 | 7 | 8 | Exe 9 | MyRootNamespace 10 | MyAssemblyName 11 | False 12 | False 13 | False 14 | False 15 | False 16 | False 17 | False 18 | False 19 | False 20 | False 21 | 1000 22 | Yes 23 | TextAnalytics 24 | 25 | 26 | false 27 | 28 | bin\Debug\ 29 | 30 | 31 | false 32 | bin\Release\ 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | Code 43 | 44 | 45 | Code 46 | 47 | 48 | Code 49 | 50 | 51 | Code 52 | 53 | 54 | Code 55 | 56 | 57 | Code 58 | 59 | 60 | Code 61 | 62 | 63 | Code 64 | 65 | 66 | Code 67 | 68 | 69 | Code 70 | 71 | 72 | Code 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /TextAnalytics/TextAnalytics.pq: -------------------------------------------------------------------------------- 1 | section TextAnalytics; 2 | 3 | //Function parameters with documentation 4 | 5 | //Function to expose the Key Phrases web service 6 | [DataSource.Kind="TextAnalytics"] 7 | shared TextAnalytics.KeyPhrases = Value.ReplaceType(KeyPhrasesImpl, KeyPhrasesParameters); 8 | 9 | KeyPhrasesParameters = 10 | type function ( 11 | optional inputtext as ( type list meta[ 12 | Documentation.FieldCaption = "Input Text" 13 | ]), 14 | optional languages as ( type list meta[ 15 | Documentation.FieldCaption = "Languages" 16 | ]) 17 | ) 18 | as table meta[ 19 | Documentation.Name = "KeyPhrases", 20 | Documentation.LongDescription = "Calls the Microsoft Cognitive Services Text Analytics Key Phrases API", 21 | Documentation.Examples = {[ 22 | Description = "Returns a table with the key phrases identified from each piece of text", 23 | Code = "TextAnalytics.KeyPhrases({""blue is my favourite colour"", ""what time it is please?"", ""twinkle, twinkle little star, how I wonder what you are""})", 24 | Result = "#table({""id"", ""text"", ""call bucket"", ""keyPhrases""}, {{0, ""blue is my favourite colour"", 1, ""blue""}, {1, ""Hello world"", 1, ""time""}, {2, ""twinkle, twinkle little star, how I wonder what you are"", 1, ""twinkle""}, {2, ""twinkle, twinkle little star, how I wonder what you are"", 1, ""little star""}})" 25 | ]} 26 | ]; 27 | 28 | KeyPhrasesImpl = (optional inputtext as list, optional languages as list) as table => 29 | let 30 | //if inputtext parameter is null, use an empty list 31 | CheckForInputText = if inputtext = null then {} else List.Buffer(inputtext), 32 | BufferLanguages = if languages = null then null else List.Buffer(languages), 33 | PrepareData = if languages = null then PrepareInputText(CheckForInputText) else PrepareInputText(CheckForInputText, BufferLanguages), 34 | //Call the web service once for each partition 35 | CallWebService = List.Transform(PrepareData, each CallTextAnalyticsWebService("keyPhrases", _)), 36 | //Combine list of tables into one table 37 | OneTable = Table.Combine(CallWebService), 38 | #"Expanded Column1" = Table.ExpandRecordColumn(OneTable, "Column1", {"id", "keyPhrases"}, {"id", "keyPhrases"}), 39 | #"Expanded Column2" = Table.ExpandListColumn(#"Expanded Column1", "keyPhrases"), 40 | JoinWithOriginalTable = Table.Join(Table.Combine(PrepareData), "id", #"Expanded Column2", "id"), 41 | SetDataTypes = Table.TransformColumnTypes(JoinWithOriginalTable,{{"id", Int64.Type}, {"text", type text}, {"call bucket", Int64.Type}, {"keyPhrases", type text}}), 42 | SetLanguageDataType = if languages = null then SetDataTypes else Table.TransformColumnTypes(SetDataTypes,{{"language", type text}}) 43 | in 44 | SetLanguageDataType; 45 | 46 | 47 | //Function to expose the Detect Language web service 48 | [DataSource.Kind="TextAnalytics"] 49 | shared TextAnalytics.DetectLanguage = Value.ReplaceType(DetectLanguageImpl, DetectLanguageParameters); 50 | 51 | DetectLanguageParameters = 52 | type function ( 53 | optional inputtext as ( type list meta[ 54 | Documentation.FieldCaption = "Input Text" 55 | ]), 56 | optional numberoflanguages as ( type number meta[ 57 | Documentation.FieldCaption = "Number Of Languages" 58 | ]) 59 | ) 60 | as table meta[ 61 | Documentation.Name = "DetectLanguages", 62 | Documentation.LongDescription = "Calls the Microsoft Cognitive Services Text Analytics Detect Language API", 63 | Documentation.Examples = {[ 64 | Description = "Returns a table with the languages identified from each piece of text", 65 | Code = "TextAnalytics.DetectLanguage({""Ich spreche nicht viel Deutsch"", ""Bonjour"", ""¿Dónde hay un restaurante?""})", 66 | Result = "#table({""id"", ""text"", ""call bucket"", ""name"", ""iso6391Name"", ""score""}, {{0, ""Ich spreche nicht viel Deutsch"", 1, ""German"", ""de"", 1}, {1, ""Bonjour"", 1, ""French"", ""fr"", 1}, {2, ""¿Dónde hay un restaurante?"", 1, ""Spanish"", ""es"", 1}})" 67 | ]} 68 | ]; 69 | 70 | DetectLanguageImpl = (optional inputtext as list, optional numberoflanguages as number) as table => 71 | let 72 | //if inputtext parameter is null, use an empty list 73 | CheckForInputText = if inputtext = null then {} else List.Buffer(inputtext), 74 | PrepareData = PrepareInputText(CheckForInputText), 75 | //Call the web service once for each partition 76 | CallWebService = List.Transform(PrepareData, each if numberoflanguages=null then CallTextAnalyticsWebService("languages", _) else CallTextAnalyticsWebService("languages", _, numberoflanguages)), 77 | //Combine list of tables into one table 78 | OneTable = Table.Combine(CallWebService), 79 | #"Expanded Column1" = Table.ExpandRecordColumn(OneTable, "Column1", {"id", "detectedLanguages"}, {"id", "detectedLanguages"}), 80 | #"Expanded Column2" = Table.ExpandListColumn(#"Expanded Column1", "detectedLanguages"), 81 | #"Expanded Column3" = Table.ExpandRecordColumn(#"Expanded Column2", "detectedLanguages", {"name", "iso6391Name", "score"}, {"name", "iso6391Name", "score"}), 82 | JoinWithOriginalTable = Table.Join(Table.Combine(PrepareData), "id", #"Expanded Column3", "id"), 83 | SetDataTypes = Table.TransformColumnTypes(JoinWithOriginalTable,{{"id", Int64.Type}, {"text", type text}, {"call bucket", Int64.Type}, {"name", type text}, {"iso6391Name", type text}, {"score", type number}}) 84 | in 85 | SetDataTypes; 86 | 87 | //Function to expose the Sentiment web service 88 | [DataSource.Kind="TextAnalytics"] 89 | shared TextAnalytics.Sentiment = Value.ReplaceType(SentimentImpl, SentimentParameters); 90 | 91 | SentimentParameters = 92 | type function ( 93 | optional inputtext as ( type list meta[ 94 | Documentation.FieldCaption = "Input Text" 95 | ]), 96 | optional languages as ( type list meta[ 97 | Documentation.FieldCaption = "Languages" 98 | ]) 99 | ) 100 | as table meta[ 101 | Documentation.Name = "Sentiment", 102 | Documentation.LongDescription = "Calls the Microsoft Cognitive Services Text Analytics Sentiment API", 103 | Documentation.Examples = {[ 104 | Description = "Returns a table with the sentiment identified from each piece of text", 105 | Code = "TextAnalytics.Sentiment({""this is great"", ""this is terrible""})", 106 | Result = "#table({""id"", ""text"", ""call bucket"", ""score""}, {{0, ""this is great"", 0.99797004461288452}, {1, ""this is terrible"", 1, 0.00062239170074462891})" 107 | ]} 108 | ]; 109 | 110 | SentimentImpl = (optional inputtext as list, optional languages as list) as table => 111 | let 112 | //if inputtext parameter is null, use an empty list 113 | CheckForInputText = if inputtext = null then {} else List.Buffer(inputtext), 114 | BufferLanguages = if languages = null then null else List.Buffer(languages), 115 | PrepareData = if languages = null then PrepareInputText(CheckForInputText) else PrepareInputText(CheckForInputText, BufferLanguages), 116 | //Call the web service once for each partition 117 | CallWebService = List.Transform(PrepareData, each CallTextAnalyticsWebService("sentiment", _)), 118 | //Combine list of tables into one table 119 | OneTable = Table.Combine(CallWebService), 120 | #"Expanded Column1" = Table.ExpandRecordColumn(OneTable, "Column1", {"id", "score"}, {"id", "score"}), 121 | JoinWithOriginalTable = Table.Join(Table.Combine(PrepareData), "id", #"Expanded Column1", "id"), 122 | SetDataTypes = Table.TransformColumnTypes(JoinWithOriginalTable,{{"id", Int64.Type}, {"text", type text}, {"call bucket", Int64.Type}, {"score", type number}}), 123 | SetLanguageDataType = if languages = null then SetDataTypes else Table.TransformColumnTypes(SetDataTypes,{{"language", type text}}) 124 | in 125 | SetLanguageDataType; 126 | 127 | 128 | //Helper functions 129 | 130 | //Make a single call to one of the Text Analytics web services 131 | CallTextAnalyticsWebService = (webservicename as text, inputtable as table, optional numberoflanguages as number) as table => 132 | let 133 | //Get API key 134 | apiKey = Extension.CurrentCredential()[Key], 135 | //Prepare call 136 | TableForJSON = [documents=inputtable], 137 | OptionsRecord = [ 138 | Content=Json.FromValue(TableForJSON), 139 | Headers = [#"Ocp-Apim-Subscription-Key"=apiKey], 140 | ManualCredentials = true 141 | ], 142 | AddNumberOfLanguagesToDetect = if numberoflanguages=null then OptionsRecord else Record.AddField(OptionsRecord, "Query", [numberOfLanguagesToDetect=Text.From(numberoflanguages)]), 143 | CallService = ()=> 144 | Web.Contents( 145 | "https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/" & webservicename, 146 | AddNumberOfLanguagesToDetect), 147 | //Call the service with a delay of 1 second between each call, to avoid hitting the rate limit 148 | CallServiceWithDelay = Function.InvokeAfter(CallService, #duration(0,0,0,1)), 149 | ResultsList = Json.Document(CallServiceWithDelay)[documents], 150 | #"Converted to Table" = Table.FromList(ResultsList, Splitter.SplitByNothing(), null, null, ExtraValues.Error) 151 | in 152 | #"Converted to Table"; 153 | 154 | PrepareInputText = (inputtext as list, optional languages as list) => 155 | //Prepare input data for all web services 156 | let 157 | //Truncate each document to 5120 characters, the maximum the web services will accept for a single document (ie 10KB) 158 | TruncateText = List.Transform(inputtext, each Text.Start(_, 5120)), 159 | //The web services only allow requests of 1MB at a time, and no more than 1000 documents at a time 160 | //so we need to create a list of tables with 1000 or less rows and where the size of the combined text is less than 1MB 161 | DocumentCount = List.Count(TruncateText), 162 | FindBuckets = List.Generate( 163 | ()=>[rowcounter=0, bucketrowcounter=0, runningsum= Text.Length(TruncateText{0}), bucketid=1], 164 | each [rowcounter] 2 | 3 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | text/microsoft-resx 110 | 111 | 112 | 2.0 113 | 114 | 115 | System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 116 | 117 | 118 | System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 119 | 120 | 121 | Connect to TextAnalytics 122 | 123 | 124 | TextAnalytics 125 | 126 | 127 | TextAnalytics 128 | 129 | --------------------------------------------------------------------------------