├── .gitignore ├── GetDataType.ps1 ├── data ├── fixedData.txt ├── test.xls ├── fixedDataSchema.csv ├── testData.xlsx ├── datatypes.csv ├── places.csv ├── targetData.csv ├── air_quality_parameters.csv ├── mindex_ex.csv ├── testdata.json ├── air_quality_stations.csv ├── baseball.csv ├── tips.csv └── countries.csv ├── write-ups ├── Lookup.png ├── stats.png ├── SQLSelect.png ├── fixedwidth.png ├── FindMatching.png ├── sqlquery.md ├── fixeddata.md ├── stats.md ├── find.md └── lookupTable.md ├── lib └── MathNet.Numerics.dll ├── sampleCsv ├── new.csv ├── areas.csv ├── output.csv ├── stocks.csv ├── wmt.csv ├── weather.csv ├── test_scores.csv ├── customers.csv ├── scores.csv ├── stock_data.csv ├── weather3.csv ├── weather_data.csv ├── fb.csv ├── hiring.csv ├── msft.csv ├── weather2.csv ├── insurance_data.csv ├── prediction.csv ├── carprices.csv ├── homeprices.csv ├── weather_by_cities.csv ├── income.csv ├── aapl_no_dates.csv ├── salaries.csv ├── aapl.csv ├── canada_per_capita_income.csv └── nyc_weather.csv ├── PublishToGallery.ps1 ├── src ├── Microsoft.ML.DotNet.Interactive │ ├── obj │ │ └── Debug │ │ │ └── netstandard2.0 │ │ │ ├── Microsoft.ML.DotNet.Interactive.AssemblyInfoInputs.cache │ │ │ └── Microsoft.ML.DotNet.Interactive.AssemblyInfo.cs │ ├── DecisionTreeData.cs │ ├── Microsoft.ML.DotNet.Interactive.csproj │ ├── NodeData.cs │ └── DecisionTreeDataExtensions.cs ├── Microsoft.Data.Analysis.Interactive │ ├── obj │ │ └── Debug │ │ │ └── netcoreapp3.1 │ │ │ ├── Microsoft.Data.Analysis.Interactive.AssemblyInfoInputs.cache │ │ │ └── Microsoft.Data.Analysis.Interactive.AssemblyInfo.cs │ ├── Microsoft.Data.Analysis.Interactive.csproj │ └── DataFrameKernelExtension.cs ├── Microsoft.ML.DotNet.Interactive.Extensions │ ├── obj │ │ └── Debug │ │ │ └── netcoreapp3.1 │ │ │ ├── Microsoft.ML.DotNet.Interactive.Extensions.AssemblyInfoInputs.cache │ │ │ ├── Microsoft.ML.DotNet.Interactive.Extensions.csprojAssemblyReference.cache │ │ │ └── Microsoft.ML.DotNet.Interactive.Extensions.AssemblyInfo.cs │ ├── KernelExtension.cs │ ├── Microsoft.ML.DotNet.Interactive.Extensions.csproj │ ├── DecisionTreeDataFormatting.cs │ └── RegressionTree.js └── NuGet.config ├── DumpFunctionsUtil.ps1 ├── InstallModule.ps1 ├── DoTests.ps1 ├── GetDataTypePrecedence.ps1 ├── demo.txt ├── ScanProperties.ps1 ├── __tests__ ├── PSKit.MathNetNumerics.tests.ps1 ├── PSKit.ScanProperties.tests.ps1 ├── PSKit.ConvertFromFixedData.tests.ps1 ├── PSKit.ConvertIntoCsv.tests.ps1 ├── PSKit.SQLQuery.tests.ps1 ├── PSKit.Dataframe.tests.ps1 ├── PSKit.ReadCsv.tests.ps1 ├── PSKit.NewLookupTable.tests.ps1 ├── PSKit.GetPropertyName.tests.ps1 ├── PSKit.GetDataInfo.tests.ps1 ├── PSKit.GetPropertyStats.tests.ps1 ├── PSKit.TypeData.tests.ps1 ├── PSKit.GetDateRange.tests.ps1 ├── PSKit.GroupByAndMeasure.tests.ps1 ├── ConvertFromSSV.tests.ps1 └── PSKit.InvokeTranspileSQL.tests.ps1 ├── ConvertIntoCSV.ps1 ├── NuGet.config ├── ConvertFromFixedData.ps1 ├── GetPropertyName.ps1 ├── LICENSE ├── GetPropertyStats.ps1 ├── ReadCsv.ps1 ├── azure-pipelines.yml ├── NewLookupTable.ps1 ├── NotebookExamples ├── DataAnalysis.ipynb └── TryPSKit.ipynb ├── GenerateStats.ps1 ├── NewDataframe.ps1 ├── GetDateRange.ps1 ├── InferData.ps1 ├── PSKit.psm1 ├── CustomTypeData.ps1 ├── GroupByAndMeasure.ps1 ├── ConvertFromSSV.ps1 ├── GetDataInfo.ps1 ├── Dockerfile ├── InvokeTranspileSQL.ps1 ├── PSKit.psd1 ├── PSKitAndMore.ipynb ├── README.md └── start-demo.ps1 /.gitignore: -------------------------------------------------------------------------------- 1 | src/* -------------------------------------------------------------------------------- /GetDataType.ps1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixedData.txt: -------------------------------------------------------------------------------- 1 | Chris44 72 2 | Brian26110 3 | Ryan 18145 4 | Joe 34 83 -------------------------------------------------------------------------------- /data/test.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/data/test.xls -------------------------------------------------------------------------------- /data/fixedDataSchema.csv: -------------------------------------------------------------------------------- 1 | column,start,length 2 | name,1,5 3 | age,6,2 4 | cash,8,3 -------------------------------------------------------------------------------- /data/testData.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/data/testData.xlsx -------------------------------------------------------------------------------- /write-ups/Lookup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/write-ups/Lookup.png -------------------------------------------------------------------------------- /write-ups/stats.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/write-ups/stats.png -------------------------------------------------------------------------------- /lib/MathNet.Numerics.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/lib/MathNet.Numerics.dll -------------------------------------------------------------------------------- /write-ups/SQLSelect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/write-ups/SQLSelect.png -------------------------------------------------------------------------------- /write-ups/fixedwidth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/write-ups/fixedwidth.png -------------------------------------------------------------------------------- /sampleCsv/new.csv: -------------------------------------------------------------------------------- 1 | tickers,price 2 | GOOGL,845 3 | WMT,65 4 | MSFT,64 5 | RIL ,1023 6 | TATA,n.a. 7 | -------------------------------------------------------------------------------- /write-ups/FindMatching.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/write-ups/FindMatching.png -------------------------------------------------------------------------------- /PublishToGallery.ps1: -------------------------------------------------------------------------------- 1 | $p = @{ 2 | Name = "PSKit" 3 | NuGetApiKey = $NuGetApiKey 4 | } 5 | 6 | Publish-Module @p -------------------------------------------------------------------------------- /sampleCsv/areas.csv: -------------------------------------------------------------------------------- 1 | area 2 | 1000 3 | 1500 4 | 2300 5 | 3540 6 | 4120 7 | 4560 8 | 5490 9 | 3460 10 | 4750 11 | 2300 12 | 9000 13 | 8600 14 | 7100 15 | -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive/obj/Debug/netstandard2.0/Microsoft.ML.DotNet.Interactive.AssemblyInfoInputs.cache: -------------------------------------------------------------------------------- 1 | b1648eedee313466ac0f253106d5009d5886e8b4 2 | -------------------------------------------------------------------------------- /DumpFunctionsUtil.ps1: -------------------------------------------------------------------------------- 1 | "|Function|" 2 | "|---|" 3 | (Get-Command -Module pskit).name -notmatch 'Add*|Test*' | Sort-Object | ForEach-Object { 4 | "|{0}" -f $_ 5 | } -------------------------------------------------------------------------------- /src/Microsoft.Data.Analysis.Interactive/obj/Debug/netcoreapp3.1/Microsoft.Data.Analysis.Interactive.AssemblyInfoInputs.cache: -------------------------------------------------------------------------------- 1 | cb246eb79f0637c522846bf762267ae164cd5ac2 2 | -------------------------------------------------------------------------------- /data/datatypes.csv: -------------------------------------------------------------------------------- 1 | a,b,c,d,e,f,g 2 | 2,0.0,FALSE,"""Yes!""",2011-11-11 11:00,2012-09-08,12:34 3 | 42,3.1415,TRUE,"Oh, good",2014-09-15,12/6/70,0:07 PM 4 | 66,,False,2198,,, 5 | -------------------------------------------------------------------------------- /sampleCsv/output.csv: -------------------------------------------------------------------------------- 1 | Company Name,PE Ratio, PB Ratio 2 | Reliance,22.23,2.25 3 | Tata Steel,4.39,0.68 4 | Infosys,18.57,4.42 5 | Axis Bank,38.89,2.81 6 | Bajaj Finance,58.61,11.86 7 | -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive.Extensions/obj/Debug/netcoreapp3.1/Microsoft.ML.DotNet.Interactive.Extensions.AssemblyInfoInputs.cache: -------------------------------------------------------------------------------- 1 | 475d3fe9554f518b20f34e22da5de8c1bc095451 2 | -------------------------------------------------------------------------------- /sampleCsv/stocks.csv: -------------------------------------------------------------------------------- 1 | Company Name,Price,Earnings Per Share, Book Value 2 | Reliance,1467,66,653 3 | Tata Steel,391,89,572 4 | Infosys,650,35,147 5 | Axis Bank,739,19,263 6 | Bajaj Finance,4044,69,341 -------------------------------------------------------------------------------- /sampleCsv/wmt.csv: -------------------------------------------------------------------------------- 1 | Line Item,2017Q1,2017Q2,2017Q3,2017Q4,2018Q1 2 | Revenue,115904,120854,118179,130936,117542 3 | Expenses,86544,89485,87484,97743,87688 4 | Profit,29360,31369,30695,33193,29854 5 | -------------------------------------------------------------------------------- /sampleCsv/weather.csv: -------------------------------------------------------------------------------- 1 | day,chicago,chennai,berlin 2 | Monday,32,75,41 3 | Tuesday,30,77,43 4 | Wednesday,28,75,45 5 | Thursday,22,82,38 6 | Friday,30,83,30 7 | Saturday,20,81,45 8 | Sunday,25,77,47 9 | -------------------------------------------------------------------------------- /InstallModule.ps1: -------------------------------------------------------------------------------- 1 | $fullPath = 'C:\Program Files\WindowsPowerShell\Modules\PSKit' 2 | 3 | Robocopy . $fullPath /mir /XD .vscode .git examples data /XF appveyor.yml azure-pipelines.yml .gitattributes .gitignore -------------------------------------------------------------------------------- /sampleCsv/test_scores.csv: -------------------------------------------------------------------------------- 1 | name,math,cs 2 | david,92,98 3 | laura,56,68 4 | sanjay,88,81 5 | wei,70,80 6 | jeff,80,83 7 | aamir,49,52 8 | venkat,65,66 9 | virat,35,30 10 | arthur,66,68 11 | paul,67,73 -------------------------------------------------------------------------------- /data/places.csv: -------------------------------------------------------------------------------- 1 | slug,place,latitude,longitude 2 | dcl,Downtown Coffee Lounge,32.35066,-95.30181 3 | tyler-museum,Tyler Museum of Art,32.33396,-95.28174 4 | genecov,Genecov Sculpture,32.299076986939205,-95.31571447849274 -------------------------------------------------------------------------------- /sampleCsv/customers.csv: -------------------------------------------------------------------------------- 1 | Customer Name,Customer Phone 2 | rafael nadal,4567895647 3 | maria sharapova,434534545 4 | vladimir putin,89345345 5 | kim un jong,123434456 6 | jeff bezos,934534543 7 | rahul gandhi,44324222 8 | -------------------------------------------------------------------------------- /sampleCsv/scores.csv: -------------------------------------------------------------------------------- 1 | rohit,9 2 | shakib,56 3 | babar,56 4 | rohit,120 5 | rohit,105 6 | shakib,78 7 | rohit,140 8 | babar,45 9 | rohit,130 10 | shakib,102 11 | babar,120 12 | babar,5 13 | shakib,72 14 | babar,67 -------------------------------------------------------------------------------- /sampleCsv/stock_data.csv: -------------------------------------------------------------------------------- 1 | tickers,eps,revenue,price,people 2 | GOOGL,27.82,87,845,larry page 3 | WMT,4.61,484,65,n.a. 4 | MSFT,-1,85,64,bill gates 5 | RIL ,not available,50,1023,mukesh ambani 6 | TATA,5.6,-1,n.a.,ratan tata 7 | -------------------------------------------------------------------------------- /sampleCsv/weather3.csv: -------------------------------------------------------------------------------- 1 | date,city,temperature,humidity 2 | 5/1/2017,new york,65,56 3 | 5/2/2017,new york,61,54 4 | 5/3/2017,new york,70,60 5 | 12/1/2017,new york,30,50 6 | 12/2/2017,new york,28,52 7 | 12/3/2017,new york,25,51 8 | -------------------------------------------------------------------------------- /data/targetData.csv: -------------------------------------------------------------------------------- 1 | "Cost","Date","Name" 2 | "1.1","1/1/2015","John" 3 | "2.1","1/2/2015","Tom" 4 | "5.1","1/2/2015","Dick" 5 | "11.1","1/2/2015","Harry" 6 | "7.1","1/2/2015","Jane" 7 | "22.1","1/2/2015","Mary" 8 | "32.1","1/2/2015","Liz" -------------------------------------------------------------------------------- /sampleCsv/weather_data.csv: -------------------------------------------------------------------------------- 1 | day,temperature,windspeed,event 2 | 1/1/2017,32,6,Rain 3 | 1/2/2017,-99999,7,Sunny 4 | 1/3/2017,28,-99999,Snow 5 | 1/4/2017,-99999,7,0 6 | 1/5/2017,32,-99999,Rain 7 | 1/6/2017,31,2,Sunny 8 | 1/6/2017,34,5,0 9 | -------------------------------------------------------------------------------- /sampleCsv/fb.csv: -------------------------------------------------------------------------------- 1 | Date,Price 2 | 15-Aug-17,171 3 | 16-Aug-17,170 4 | 17-Aug-17,166.91 5 | 18-Aug-17,167.41 6 | 21-Aug-17,167.78 7 | 22-Aug-17,169.64 8 | 23-Aug-17,168.71 9 | 24-Aug-17,167.74 10 | 25-Aug-17,166.32 11 | 28-Aug-17,167.24 12 | -------------------------------------------------------------------------------- /sampleCsv/hiring.csv: -------------------------------------------------------------------------------- 1 | experience,test_score(out of 10),interview_score(out of 10),salary($) 2 | ,8,9,50000 3 | ,8,6,45000 4 | five,6,7,60000 5 | two,10,10,65000 6 | seven,9,6,70000 7 | three,7,10,62000 8 | ten,,7,72000 9 | eleven,7,8,80000 10 | -------------------------------------------------------------------------------- /sampleCsv/msft.csv: -------------------------------------------------------------------------------- 1 | "Microsoft Stock Price: 17 August, 2017", 2 | Date Time,Price 3 | 8/17/2017 9:00:00 AM,72.38 4 | 8/17/2017 9:15:00 AM,71 5 | 8/17/2017 9:30:00 AM,71.67 6 | 8/17/2017 10:00:00 AM,72.8 7 | 8/17/2017 10:30:00 AM,73 8 | 8/17/2017 11:00:00 AM,72.5 9 | -------------------------------------------------------------------------------- /sampleCsv/weather2.csv: -------------------------------------------------------------------------------- 1 | date,city,temperature,humidity 2 | 5/1/2017,new york,65,56 3 | 5/1/2017,new york,61,54 4 | 5/2/2017,new york,70,60 5 | 5/2/2017,new york,72,62 6 | 5/1/2017,mumbai,75,80 7 | 5/1/2017,mumbai,78,83 8 | 5/2/2017,mumbai,82,85 9 | 5/2/2017,mumbai,80,26 10 | -------------------------------------------------------------------------------- /data/air_quality_parameters.csv: -------------------------------------------------------------------------------- 1 | id,description,name 2 | bc,Black Carbon,BC 3 | co,Carbon Monoxide,CO 4 | no2,Nitrogen Dioxide,NO2 5 | o3,Ozone,O3 6 | pm10,Particulate matter less than 10 micrometers in diameter,PM10 7 | pm25,Particulate matter less than 2.5 micrometers in diameter,PM2.5 8 | so2,Sulfur Dioxide,SO2 9 | -------------------------------------------------------------------------------- /sampleCsv/insurance_data.csv: -------------------------------------------------------------------------------- 1 | age,bought_insurance 2 | 22,0 3 | 25,0 4 | 47,1 5 | 52,0 6 | 46,1 7 | 56,1 8 | 55,0 9 | 60,1 10 | 62,1 11 | 61,1 12 | 18,0 13 | 28,0 14 | 27,0 15 | 29,0 16 | 49,1 17 | 55,1 18 | 25,1 19 | 58,1 20 | 19,0 21 | 18,0 22 | 21,0 23 | 26,0 24 | 40,1 25 | 45,1 26 | 50,1 27 | 54,1 28 | 23,0 -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive.Extensions/obj/Debug/netcoreapp3.1/Microsoft.ML.DotNet.Interactive.Extensions.csprojAssemblyReference.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dfinke/PSKit/HEAD/src/Microsoft.ML.DotNet.Interactive.Extensions/obj/Debug/netcoreapp3.1/Microsoft.ML.DotNet.Interactive.Extensions.csprojAssemblyReference.cache -------------------------------------------------------------------------------- /data/mindex_ex.csv: -------------------------------------------------------------------------------- 1 | year,indiv,zit,xit 2 | 1977,"A",1.2,.6 3 | 1977,"B",1.5,.5 4 | 1977,"C",1.7,.8 5 | 1978,"A",.2,.06 6 | 1978,"B",.7,.2 7 | 1978,"C",.8,.3 8 | 1978,"D",.9,.5 9 | 1978,"E",1.4,.9 10 | 1979,"C",.2,.15 11 | 1979,"D",.14,.05 12 | 1979,"E",.5,.15 13 | 1979,"F",1.2,.5 14 | 1979,"G",3.4,1.9 15 | 1979,"H",5.4,2.7 16 | 1979,"I",6.4,1.2 17 | -------------------------------------------------------------------------------- /DoTests.ps1: -------------------------------------------------------------------------------- 1 | $PSVersionTable.PSVersion 2 | 3 | $psModules = 'Pester', 'PSStringScanner' 4 | 5 | foreach ($module in $psModules) { 6 | Install-Module -Name $module -Repository PSGallery -Force -Scope CurrentUser 7 | } 8 | 9 | $result = Invoke-Pester -Script $PSScriptRoot\__tests__ -Verbose -PassThru 10 | 11 | if ($result.FailedCount -gt 0) { 12 | throw "$($result.FailedCount) tests failed." 13 | } -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive/DecisionTreeData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation and contributors. All rights reserved. 2 | // Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | namespace Microsoft.ML.DotNet.Interactive 5 | { 6 | public class DecisionTreeData 7 | { 8 | public NodeData Root { get; set; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /sampleCsv/prediction.csv: -------------------------------------------------------------------------------- 1 | ,area,prices 2 | 0,1000,316404.1095890411 3 | 1,1500,384297.9452054794 4 | 2,2300,492928.0821917808 5 | 3,3540,661304.794520548 6 | 4,4120,740061.6438356165 7 | 5,4560,799808.2191780822 8 | 6,5490,926090.7534246575 9 | 7,3460,650441.7808219178 10 | 8,4750,825607.8767123288 11 | 9,2300,492928.0821917808 12 | 10,9000,1402705.479452055 13 | 11,8600,1348390.4109589043 14 | 12,7100,1144708.904109589 15 | -------------------------------------------------------------------------------- /sampleCsv/carprices.csv: -------------------------------------------------------------------------------- 1 | Mileage,Age(yrs),Sell Price($) 2 | 69000,6,18000 3 | 35000,3,34000 4 | 57000,5,26100 5 | 22500,2,40000 6 | 46000,4,31500 7 | 59000,5,26750 8 | 52000,5,32000 9 | 72000,6,19300 10 | 91000,8,12000 11 | 67000,6,22000 12 | 83000,7,18700 13 | 79000,7,19500 14 | 59000,5,26000 15 | 58780,4,27500 16 | 82450,7,19400 17 | 25400,3,35000 18 | 28000,2,35500 19 | 69000,5,19700 20 | 87600,8,12800 21 | 52000,5,28200 22 | -------------------------------------------------------------------------------- /sampleCsv/homeprices.csv: -------------------------------------------------------------------------------- 1 | town,area,price 2 | monroe township,2600,550000 3 | monroe township,3000,565000 4 | monroe township,3200,610000 5 | monroe township,3600,680000 6 | monroe township,4000,725000 7 | west windsor,2600,585000 8 | west windsor,2800,615000 9 | west windsor,3300,650000 10 | west windsor,3600,710000 11 | robinsville,2600,575000 12 | robinsville,2900,600000 13 | robinsville,3100,620000 14 | robinsville,3600,695000 15 | -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive/Microsoft.ML.DotNet.Interactive.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | Microsoft.ML.DotNet.Interactive 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /sampleCsv/weather_by_cities.csv: -------------------------------------------------------------------------------- 1 | day,city,temperature,windspeed,event 2 | 1/1/2017,new york,32,6,Rain 3 | 1/2/2017,new york,36,7,Sunny 4 | 1/3/2017,new york,28,12,Snow 5 | 1/4/2017,new york,33,7,Sunny 6 | 1/1/2017,mumbai,90,5,Sunny 7 | 1/2/2017,mumbai,85,12,Fog 8 | 1/3/2017,mumbai,87,15,Fog 9 | 1/4/2017,mumbai,92,5,Rain 10 | 1/1/2017,paris,45,20,Sunny 11 | 1/2/2017,paris,50,13,Cloudy 12 | 1/3/2017,paris,54,8,Cloudy 13 | 1/4/2017,paris,42,10,Cloudy 14 | -------------------------------------------------------------------------------- /GetDataTypePrecedence.ps1: -------------------------------------------------------------------------------- 1 | function GetDataTypePrecedence { 2 | param($list) 3 | 4 | $precedence = @{ 5 | 'String' = 1 6 | 'Double' = 2 7 | 'Int' = 3 8 | 'DateTime' = 4 9 | 'bool' = 5 10 | 'null' = 6 11 | } 12 | 13 | ($(foreach ($item in $list) { 14 | "$($precedence.$item)" + $item 15 | }) | Sort-Object | Select-Object -First 1) -replace "^\d", "" 16 | } 17 | -------------------------------------------------------------------------------- /demo.txt: -------------------------------------------------------------------------------- 1 | # Import the module 2 | Import-Module .\PSKit.psm1 -Force 3 | 4 | # Check out the csv 5 | Get-Content .\data\places.csv 6 | 7 | # Import the csv 8 | $data = Import-Csv .\data\places.csv 9 | 10 | # Convert it to a hash table, with a key from the `slug` property 11 | New-LookupTable -InputObject $data -key slug 12 | 13 | # Convert it to a hash table, with a key from the `slug` property and return it as JSON 14 | New-LookupTable -InputObject $data -key slug -AsJSON -------------------------------------------------------------------------------- /ScanProperties.ps1: -------------------------------------------------------------------------------- 1 | function Invoke-ScanProperties { 2 | param( 3 | $InputObject, 4 | $Pattern 5 | ) 6 | 7 | $regex = New-Object regex $pattern, 'Compiled, IgnoreCase' 8 | 9 | $propertyNames = $InputObject[0].psobject.Properties.name 10 | foreach ($record in $InputObject) { 11 | foreach ($pn in $propertyNames) { 12 | if ($regex.IsMatch($record.$pn)) { 13 | $record 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /write-ups/sqlquery.md: -------------------------------------------------------------------------------- 1 | # Execute a SQL SELECT query directly on PowerShell Arrays 2 | 3 | ```powershell 4 | class Person { 5 | $Name 6 | [int]$Age 7 | [int]$Cash 8 | } 9 | 10 | [Person[]]$data = ConvertFrom-Csv @" 11 | name,age,cash 12 | Chris,44,72 13 | Brian,26,110 14 | Ryan,18,145 15 | Joe,34,83 16 | "@ 17 | 18 | $data.query("SELECT cash, name FROM data where name like '*i*' and cash > 100") 19 | ``` 20 | 21 | ```powershell 22 | cash name 23 | ---- ---- 24 | 110 Brian 25 | ``` -------------------------------------------------------------------------------- /sampleCsv/income.csv: -------------------------------------------------------------------------------- 1 | Name,Age,Income($) 2 | Rob,27,70000 3 | Michael,29,90000 4 | Mohan,29,61000 5 | Ismail,28,60000 6 | Kory,42,150000 7 | Gautam,39,155000 8 | David,41,160000 9 | Andrea,38,162000 10 | Brad,36,156000 11 | Angelina,35,130000 12 | Donald,37,137000 13 | Tom,26,45000 14 | Arnold,27,48000 15 | Jared,28,51000 16 | Stark,29,49500 17 | Ranbir,32,53000 18 | Dipika,40,65000 19 | Priyanka,41,63000 20 | Nick,43,64000 21 | Alia,39,80000 22 | Sid,41,82000 23 | Abdul,39,58000 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive/NodeData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation and contributors. All rights reserved. 2 | // Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | using System.Collections.Generic; 5 | 6 | namespace Microsoft.ML.DotNet.Interactive 7 | { 8 | public class NodeData 9 | { 10 | public string Label { get; set; } 11 | public float Data { get; set; } 12 | public float Value { get; set; } 13 | public List Children { get; } = new List(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /sampleCsv/aapl_no_dates.csv: -------------------------------------------------------------------------------- 1 | Open,High,Low,Close,Volume 2 | 144.88,145.3,143.1,143.5,14277848 3 | 143.69,144.79,142.72,144.09,21569557 4 | 143.02,143.5,142.41,142.73,24128782 5 | 142.9,144.75,142.9,144.18,19201712 6 | 144.11,145.95,143.37,145.06,21090636 7 | 144.73,145.85,144.38,145.53,19781836 8 | 145.87,146.18,144.82,145.74,24884478 9 | 145.5,148.49,145.44,147.77,25199373 10 | 147.97,149.33,147.33,149.04,20132061 11 | 148.82,150.9,148.57,149.56,23793456 12 | 149.2,150.13,148.67,150.08,17868792 13 | 150.48,151.42,149.95,151.02,20922969 14 | 151.5,151.74,150.19,150.34,17243748 15 | 149.99,150.44,148.88,150.27,26252630 16 | -------------------------------------------------------------------------------- /__tests__/PSKit.MathNetNumerics.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Math.Net Numerics" { 4 | BeforeAll { 5 | [double[]]$script:data = 1..20 6 | } 7 | 8 | It "Mean shoud be correct" { 9 | $Mean = [MathNet.Numerics.Statistics.Statistics]::Mean($data) 10 | $Mean | Should Be 10.5 11 | } 12 | 13 | It "StandardDeviation shoud be correct" { 14 | $StdDev = [MathNet.Numerics.Statistics.Statistics]::StandardDeviation($data) 15 | $StdDev = [math]::Round($StdDev, 2) 16 | $StdDev | Should Be 5.92 17 | } 18 | } -------------------------------------------------------------------------------- /__tests__/PSKit.ScanProperties.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Scan Properties" { 4 | BeforeAll { 5 | $script:data = ConvertFrom-Csv @" 6 | name,phoneNumber 7 | Chris,555-999-1111 8 | Brian,555-123-4567 9 | Ryan,555-123-8901 10 | Joe,555-777-1111 11 | Jane,555-777-2222 12 | "@ 13 | } 14 | 15 | It "Data should have 2 records searching the phone number" { 16 | $actual = $data.ScanProperties("\d{3}-123-\d{4}") 17 | @($actual).Count | Should Be 2 18 | } 19 | 20 | It "Data should have 2 records search for text" { 21 | $actual = $data.ScanProperties("an$") 22 | @($actual).Count | Should Be 2 23 | } 24 | } -------------------------------------------------------------------------------- /sampleCsv/salaries.csv: -------------------------------------------------------------------------------- 1 | company,job,degree,salary_more_then_100k 2 | google,sales executive,bachelors,0 3 | google,sales executive,masters,0 4 | google,business manager,bachelors,1 5 | google,business manager,masters,1 6 | google,computer programmer,bachelors,0 7 | google,computer programmer,masters,1 8 | abc pharma,sales executive,masters,0 9 | abc pharma,computer programmer,bachelors,0 10 | abc pharma,business manager,bachelors,0 11 | abc pharma,business manager,masters,1 12 | facebook,sales executive,bachelors,1 13 | facebook,sales executive,masters,1 14 | facebook,business manager,bachelors,1 15 | facebook,business manager,masters,1 16 | facebook,computer programmer,bachelors,1 17 | facebook,computer programmer,masters,1 -------------------------------------------------------------------------------- /data/testdata.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Name": "Sally Whittaker", 4 | "Class": 2018, 5 | "Dorm": "McCarren House", 6 | "Room": 312, 7 | "GPA": 3.75 8 | }, 9 | { 10 | "Name": "Belinda Jameson", 11 | "Class": 2017, 12 | "Dorm": "Cushing House", 13 | "Room": 148, 14 | "GPA": 3.52 15 | }, 16 | { 17 | "Name": "Jeff Smith", 18 | "Class": 2018, 19 | "Dorm": "Prescott House", 20 | "Room": "17-D", 21 | "GPA": 3.2 22 | }, 23 | { 24 | "Name": "Sandy Allen", 25 | "Class": 2019, 26 | "Dorm": "Oliver House", 27 | "Room": 108, 28 | "GPA": 3.48 29 | } 30 | ] 31 | -------------------------------------------------------------------------------- /sampleCsv/aapl.csv: -------------------------------------------------------------------------------- 1 | Date,Open,High,Low,Close,Volume 2 | 3-Jul-17,144.88,145.3,143.1,143.5,14277848 3 | 5-Jul-17,143.69,144.79,142.72,144.09,21569557 4 | 6-Jul-17,143.02,143.5,142.41,142.73,24128782 5 | 7-Jul-17,142.9,144.75,142.9,144.18,19201712 6 | 10-Jul-17,144.11,145.95,143.37,145.06,21090636 7 | 11-Jul-17,144.73,145.85,144.38,145.53,19781836 8 | 12-Jul-17,145.87,146.18,144.82,145.74,24884478 9 | 13-Jul-17,145.5,148.49,145.44,147.77,25199373 10 | 14-Jul-17,147.97,149.33,147.33,149.04,20132061 11 | 17-Jul-17,148.82,150.9,148.57,149.56,23793456 12 | 18-Jul-17,149.2,150.13,148.67,150.08,17868792 13 | 19-Jul-17,150.48,151.42,149.95,151.02,20922969 14 | 20-Jul-17,151.5,151.74,150.19,150.34,17243748 15 | 21-Jul-17,149.99,150.44,148.88,150.27,26252630 16 | -------------------------------------------------------------------------------- /write-ups/fixeddata.md: -------------------------------------------------------------------------------- 1 | # Conquer fixed-width formats 2 | 3 | Fixed-width files are particularly challenging to parse. Save yourself some frustration by using a CSV-formatted schema to convert your fixed-width file into PowerShell objects. 4 | 5 | ## Fixed data 6 | 7 | ``` 8 | Chris44 72 9 | Brian26110 10 | Ryan 18145 11 | Joe 34 83 12 | ``` 13 | 14 | ## Schema CSV 15 | 16 | ``` 17 | column,start,length 18 | name,1,5 19 | age,6,2 20 | cash,8,3 21 | ``` 22 | 23 | ## Parse the Data 24 | 25 | ```powershell 26 | ConvertFrom-FixedData -fixedData (Get-Content .\fixedData.txt) -schema (Import-Csv .\fixedDataSchema.csv) 27 | 28 | name age cash 29 | ---- --- ---- 30 | Chris 44 72 31 | Brian 26 110 32 | Ryan 18 145 33 | Joe 34 83 34 | ``` -------------------------------------------------------------------------------- /__tests__/PSKit.ConvertFromFixedData.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Convert-IntoCsv" { 4 | BeforeAll { 5 | # schema.csv 6 | $script:schema = ConvertFrom-Csv @" 7 | column,start,length 8 | name,1,5 9 | age,6,2 10 | cash,8,3 11 | "@ 12 | 13 | # data.fixed 14 | $script:fixedData = @" 15 | Chris44 72 16 | Brian26110 17 | Ryan 18145 18 | Joe 34 83 19 | "@ -split "\r?\n" 20 | } 21 | 22 | It "Data should convert from fixed Data" { 23 | $actual = ConvertFrom-FixedData $script:fixedData $script:schema 24 | 25 | $actual.Count | Should Be 4 26 | 27 | $actual[1].name | Should BeExactly 'Brian' 28 | $actual[1].age | Should Be 26 29 | $actual[1].cash | Should Be 110 30 | } 31 | } -------------------------------------------------------------------------------- /src/NuGet.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive.Extensions/KernelExtension.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation and contributors. All rights reserved. 2 | // Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | using System.Threading.Tasks; 5 | using Microsoft.DotNet.Interactive; 6 | using Microsoft.DotNet.Interactive.Formatting; 7 | 8 | namespace Microsoft.ML.DotNet.Interactive 9 | { 10 | public class MlKernelExtension : IKernelExtension 11 | { 12 | public Task OnLoadAsync(IKernel kernel) 13 | { 14 | Formatter.Register((tree, writer) => 15 | { 16 | writer.Write(DecisionTreeDataFormatting.GenerateTreeView(tree)); 17 | }, "text/html"); 18 | 19 | return Task.CompletedTask; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ConvertIntoCSV.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .SYNOPSIS 3 | Converts files to CSV, from either JSON or Excel 4 | 5 | .DESCRIPTION 6 | Target either a JSON or Excel file and it will be converted to the CSV format. Note: The Excel file works only if there is a sheet named `Sheet1` 7 | 8 | .EXAMPLE 9 | Convert-IntoCSV C:\Temp\testData.xlsx 10 | 11 | .EXAMPLE 12 | Convert-IntoCSV C:\Temp\testData.json 13 | #> 14 | function Convert-IntoCSV { 15 | param( 16 | $Path 17 | ) 18 | 19 | # Needs test-path 20 | # Needs requires for ImportExcel 21 | $ext = [System.IO.Path]::GetExtension($path) 22 | 23 | switch ($ext) { 24 | '.json' {$fileContents = Get-Content $Path | ConvertFrom-Json} 25 | '.xlsx' {$fileContents = Import-Excel -Path $Path} 26 | } 27 | 28 | $fileContents | ConvertTo-Csv -NoTypeInformation 29 | } -------------------------------------------------------------------------------- /write-ups/stats.md: -------------------------------------------------------------------------------- 1 | # Generate summary statistics for PowerShell Arrays 2 | 3 | Calculates different statistics based on the type of each column 4 | 5 | ```powershell 6 | PS C:\> $data = ConvertFrom-Csv @" 7 | name,age,cash,dateAdded 8 | Chris,44,72,1/1 9 | Brian,26,110,3/2 10 | Ryan,18,145 11 | Joe,34,83,4/7 12 | Ryan,27,300 13 | ,98,1,6/9 14 | "@ 15 | 16 | PS C:\ $data.Stats() 17 | ```` 18 | 19 | ```powershell 20 | ColumnName DataType HasNulls Min Max Range Median StandardDeviation Variance Sum 21 | ---------- -------- -------- --- --- ----- ------ ----------------- -------- --- 22 | name string True 0 23 | age int False 18 98 80 30.5 29.1781882005492 851.366666666667 247 24 | cash int False 1 300 299 96.5 100.941071918224 10189.1 711 25 | dateAdded datetime True 0 26 | ``` -------------------------------------------------------------------------------- /NuGet.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /write-ups/find.md: -------------------------------------------------------------------------------- 1 | # Find properties matching a regular expression 2 | 3 | ```powershell 4 | $data = ConvertFrom-Csv @" 5 | name,phone_number,location 6 | Chris,555-999-1111,Cushing House 7 | Brian,555-123-4567,Prescott House 8 | Ryan,555-123-8901,Oliver House 9 | Joe,555-777-1111,Oliver House 10 | "@ 11 | ``` 12 | 13 | Find phone numbers following the pattern "ddd–123-dddd": 14 | 15 | ```powershell 16 | PS C:\> $data.ScanProperties("\d{3}-123-\d{4}") 17 | 18 | name phone_number location 19 | ---- ------------ -------- 20 | Brian 555-123-4567 Prescott House 21 | Ryan 555-123-8901 Oliver House 22 | ``` 23 | 24 | Find any record whose properties match "ryan" or "prescott" 25 | 26 | ```powershell 27 | PS C:\> $data.ScanProperties("ryan|prescott") 28 | 29 | name phone_number location 30 | ---- ------------ -------- 31 | Brian 555-123-4567 Prescott House 32 | Ryan 555-123-8901 Oliver House 33 | ``` -------------------------------------------------------------------------------- /__tests__/PSKit.ConvertIntoCsv.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | # Describe "PSKit tests - Convert-IntoCsv" { 4 | # BeforeAll { 5 | # $script:expectedResult = @" 6 | # "Name","Class","Dorm","Room","GPA" 7 | # "Sally Whittaker","2018","McCarren House","312","3.75" 8 | # "Belinda Jameson","2017","Cushing House","148","3.52" 9 | # "Jeff Smith","2018","Prescott House","17-D","3.2" 10 | # "Sandy Allen","2019","Oliver House","108","3.48" 11 | 12 | # "@ 13 | # } 14 | 15 | # It "Data should convert from Excel" { 16 | # $actual = Convert-IntoCsv -Path $PSScriptRoot/../data/testData.xlsx | Out-String 17 | # $actual | Should BeExactly $expectedResult 18 | # } 19 | 20 | # It "Data should convert from JSON" { 21 | # $actual = Convert-IntoCsv -Path $PSScriptRoot/../data/testData.json | Out-String 22 | # $actual | Should BeExactly $expectedResult 23 | # } 24 | 25 | # } -------------------------------------------------------------------------------- /ConvertFromFixedData.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .SYNOPSIS 3 | Convert your fixed-width file into a PowerShell Array using a CSV-formatted schema 4 | 5 | .EXAMPLE 6 | ConvertFrom-FixedData -fixedData (Get-Content .\data\fixedData.txt) -schema (Import-Csv .\data\fixedDataSchema.csv) 7 | 8 | name age cash 9 | ---- --- ---- 10 | Chris 44 72 11 | Brian 26 110 12 | Ryan 18 145 13 | Joe 34 83 14 | #> 15 | 16 | function ConvertFrom-FixedData { 17 | param( 18 | $fixedData, 19 | $schema 20 | ) 21 | 22 | foreach ($record in $fixedData) { 23 | $h = [ordered]@{ } 24 | foreach ($schemaRecord in $schema) { 25 | if ($schemaRecord.length -eq '$') { 26 | $h.($schemaRecord.column) = $record.SubString($schemaRecord.start - 1) 27 | } 28 | else { 29 | $h.($schemaRecord.column) = $record.SubString($schemaRecord.start - 1, $schemaRecord.length) 30 | } 31 | } 32 | 33 | [PSCustomObject]$h 34 | } 35 | } -------------------------------------------------------------------------------- /write-ups/lookupTable.md: -------------------------------------------------------------------------------- 1 | # Create a Lookup Table 2 | 3 | Have data that with a unique id column? Want to use it as a lookup table? Here you go: 4 | 5 | ```powershell 6 | PS C:\> $data = ConvertFrom-Csv @" 7 | slug,place,latitude,longitude 8 | dcl,Downtown Coffee Lounge,32.35066,-95.30181 9 | tyler-museum,Tyler Museum of Art,32.33396,-95.28174 10 | genecov,Genecov Sculpture,32.299076986939205,-95.31571447849274 11 | "@ 12 | ``` 13 | 14 | Similar to `Group-Object` in PowerShell. New-LookupTable handles missing data. 15 | 16 | ``` 17 | PS C:\> New-LookupTable $data slug 18 | 19 | Name Value 20 | ---- ----- 21 | dcl @{slug=dcl; place=Downtown Coffee Lounge; latitude=32.35066; longitude=-95.30181} 22 | tyler-museum @{slug=tyler-museum; place=Tyler Museum of Art; latitude=32.33396; longitude=-95.28174} 23 | genecov @{slug=genecov; place=Genecov Sculpture; latitude=32.299076986939205; longitude=-95.3157144... 24 | 25 | ``` -------------------------------------------------------------------------------- /sampleCsv/canada_per_capita_income.csv: -------------------------------------------------------------------------------- 1 | year,per capita income (US$) 2 | 1970,3399.299037 3 | 1971,3768.297935 4 | 1972,4251.175484 5 | 1973,4804.463248 6 | 1974,5576.514583 7 | 1975,5998.144346 8 | 1976,7062.131392 9 | 1977,7100.12617 10 | 1978,7247.967035 11 | 1979,7602.912681 12 | 1980,8355.96812 13 | 1981,9434.390652 14 | 1982,9619.438377 15 | 1983,10416.53659 16 | 1984,10790.32872 17 | 1985,11018.95585 18 | 1986,11482.89153 19 | 1987,12974.80662 20 | 1988,15080.28345 21 | 1989,16426.72548 22 | 1990,16838.6732 23 | 1991,17266.09769 24 | 1992,16412.08309 25 | 1993,15875.58673 26 | 1994,15755.82027 27 | 1995,16369.31725 28 | 1996,16699.82668 29 | 1997,17310.75775 30 | 1998,16622.67187 31 | 1999,17581.02414 32 | 2000,18987.38241 33 | 2001,18601.39724 34 | 2002,19232.17556 35 | 2003,22739.42628 36 | 2004,25719.14715 37 | 2005,29198.05569 38 | 2006,32738.2629 39 | 2007,36144.48122 40 | 2008,37446.48609 41 | 2009,32755.17682 42 | 2010,38420.52289 43 | 2011,42334.71121 44 | 2012,42665.25597 45 | 2013,42676.46837 46 | 2014,41039.8936 47 | 2015,35175.18898 48 | 2016,34229.19363 49 | -------------------------------------------------------------------------------- /GetPropertyName.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .SYNOPSIS 3 | Show only the property names of an object 4 | 5 | .EXAMPLE 6 | Get-Service | Get-PropertyName 7 | 8 | Name 9 | RequiredServices 10 | CanPauseAndContinue 11 | CanShutdown 12 | CanStop 13 | DisplayName 14 | DependentServices 15 | MachineName 16 | ServiceName 17 | ServicesDependedOn 18 | ServiceHandle 19 | Status 20 | ServiceType 21 | StartType 22 | Site 23 | Container 24 | #> 25 | function Get-PropertyName { 26 | param( 27 | $Name, 28 | [Parameter(ValueFromPipeline)] 29 | $Data, 30 | $InputObject 31 | ) 32 | 33 | Begin { 34 | if (!$InputObject) { $list = @() } 35 | } 36 | 37 | Process { 38 | if (!$InputObject) { $list += $Data } 39 | } 40 | 41 | End { 42 | if (!$InputObject) { 43 | $names = $List[0].psobject.properties.name 44 | } 45 | else { 46 | $names = $InputObject[0].psobject.properties.name 47 | } 48 | 49 | if (!$name) { $name = "*" } 50 | 51 | $names.Where( { $_ -like $name } ) 52 | } 53 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Doug Finke 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | true 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | all 17 | runtime; build; native; contentfiles; analyzers; buildtransitive 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive/obj/Debug/netstandard2.0/Microsoft.ML.DotNet.Interactive.AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // This code was generated by a tool. 4 | // Runtime Version:4.0.30319.42000 5 | // 6 | // Changes to this file may cause incorrect behavior and will be lost if 7 | // the code is regenerated. 8 | // 9 | //------------------------------------------------------------------------------ 10 | 11 | using System; 12 | using System.Reflection; 13 | 14 | [assembly: System.Reflection.AssemblyCompanyAttribute("Microsoft.ML.DotNet.Interactive")] 15 | [assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")] 16 | [assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")] 17 | [assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")] 18 | [assembly: System.Reflection.AssemblyProductAttribute("Microsoft.ML.DotNet.Interactive")] 19 | [assembly: System.Reflection.AssemblyTitleAttribute("Microsoft.ML.DotNet.Interactive")] 20 | [assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")] 21 | 22 | // Generated by the MSBuild WriteCodeFragment class. 23 | 24 | -------------------------------------------------------------------------------- /src/Microsoft.Data.Analysis.Interactive/obj/Debug/netcoreapp3.1/Microsoft.Data.Analysis.Interactive.AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // This code was generated by a tool. 4 | // Runtime Version:4.0.30319.42000 5 | // 6 | // Changes to this file may cause incorrect behavior and will be lost if 7 | // the code is regenerated. 8 | // 9 | //------------------------------------------------------------------------------ 10 | 11 | using System; 12 | using System.Reflection; 13 | 14 | [assembly: System.Reflection.AssemblyCompanyAttribute("Microsoft.Data.Analysis.Interactive")] 15 | [assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")] 16 | [assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")] 17 | [assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")] 18 | [assembly: System.Reflection.AssemblyProductAttribute("Microsoft.Data.Analysis.Interactive")] 19 | [assembly: System.Reflection.AssemblyTitleAttribute("Microsoft.Data.Analysis.Interactive")] 20 | [assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")] 21 | 22 | // Generated by the MSBuild WriteCodeFragment class. 23 | 24 | -------------------------------------------------------------------------------- /GetPropertyStats.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .SYNOPSIS 3 | Generate summary statistics for any PowerShell array 4 | 5 | .DESCRIPTION 6 | Prints descriptive statistics for all columns in a #PowerShell array. Will intelligently determine the type of each column and then print analysis relevant to that type, for example, min, max, avg, sum for numbers. Also determines of there are nulls in the data for that column. 7 | 8 | .EXAMPLE 9 | ConvertFrom-Csv "Name,Age`r`nJane,10`r`nJohn,15" | Get-PropertyStats | Format-Table 10 | 11 | ColumnName DataType HasNulls Min Max Avg Sum 12 | ---------- -------- -------- --- --- --- --- 13 | Name string False 14 | Age int False 10 15 12.5 25 15 | 16 | #> 17 | function Get-PropertyStats { 18 | param( 19 | [Parameter(ValueFromPipeline)] 20 | $Data, 21 | $InputObject, 22 | $NumberOfRowsToCheck = 0 23 | ) 24 | 25 | Begin { 26 | if (!$InputObject) { $list = @() } 27 | } 28 | 29 | Process { 30 | if (!$InputObject) { $list += $Data } 31 | } 32 | 33 | End { 34 | if (!$InputObject) { 35 | GenerateStats $list 36 | } 37 | else { 38 | GenerateStats $InputObject 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive.Extensions/obj/Debug/netcoreapp3.1/Microsoft.ML.DotNet.Interactive.Extensions.AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // This code was generated by a tool. 4 | // Runtime Version:4.0.30319.42000 5 | // 6 | // Changes to this file may cause incorrect behavior and will be lost if 7 | // the code is regenerated. 8 | // 9 | //------------------------------------------------------------------------------ 10 | 11 | using System; 12 | using System.Reflection; 13 | 14 | [assembly: System.Reflection.AssemblyCompanyAttribute("Microsoft.ML.DotNet.Interactive.Extensions")] 15 | [assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")] 16 | [assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")] 17 | [assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")] 18 | [assembly: System.Reflection.AssemblyProductAttribute("Microsoft.ML.DotNet.Interactive.Extensions")] 19 | [assembly: System.Reflection.AssemblyTitleAttribute("Microsoft.ML.DotNet.Interactive.Extensions")] 20 | [assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")] 21 | 22 | // Generated by the MSBuild WriteCodeFragment class. 23 | 24 | -------------------------------------------------------------------------------- /__tests__/PSKit.SQLQuery.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | Describe "PSKit tests - Query Method" { 3 | 4 | BeforeAll { 5 | class Person { 6 | $name 7 | [int]$age 8 | [double]$cash 9 | } 10 | 11 | [Person[]]$script:data = ConvertFrom-Csv @" 12 | name,age,cash 13 | Chris,44,72 14 | Brian,26,110 15 | Ryan,18,145 16 | Joe,34,83 17 | "@ 18 | } 19 | 20 | It "Should query values" { 21 | $actual = $data.query("SELECT * FROM data") 22 | 23 | $actual.Count | Should Be 4 24 | $actual[2].Age | Should Be 18 25 | } 26 | 27 | It "Should return only one property" { 28 | $actual = $data.query("SELECT cash FROM data") 29 | 30 | $actual.Count | Should Be 4 31 | $actual[0].psobject.properties.Count | Should Be 1 32 | $actual[0].psobject.properties.Name | Should BeExactly 'cash' 33 | } 34 | 35 | It "Should return many" { 36 | $actual = $data.query("SELECT cash, name FROM data where name like '*i*' and cash > 100") 37 | 38 | @($actual).Count | Should Be 1 39 | @($actual.psobject.properties).Count | Should Be 2 40 | $actual.psobject.properties.Name[0] | Should BeExactly 'cash' 41 | $actual.psobject.properties.Name[1] | Should BeExactly 'name' 42 | $actual.cash | Should Be 110 43 | $actual.name | Should BeExactly "Brian" 44 | } 45 | } -------------------------------------------------------------------------------- /sampleCsv/nyc_weather.csv: -------------------------------------------------------------------------------- 1 | EST,Temperature,DewPoint,Humidity,Sea Level PressureIn,VisibilityMiles,WindSpeedMPH,PrecipitationIn,CloudCover,Events,WindDirDegrees 2 | 1/1/2016,38,23,52,30.03,10,8,0,5,,281 3 | 1/2/2016,36,18,46,30.02,10,7,0,3,,275 4 | 1/3/2016,40,21,47,29.86,10,8,0,1,,277 5 | 1/4/2016,25,9,44,30.05,10,9,0,3,,345 6 | 1/5/2016,20,-3,41,30.57,10,5,0,0,,333 7 | 1/6/2016,33,4,35,30.5,10,4,0,0,,259 8 | 1/7/2016,39,11,33,30.28,10,2,0,3,,293 9 | 1/8/2016,39,29,64,30.2,10,4,0,8,,79 10 | 1/9/2016,44,38,77,30.16,9,8,T,8,Rain,76 11 | 1/10/2016,50,46,71,29.59,4,,1.8,7,Rain,109 12 | 1/11/2016,33,8,37,29.92,10,,0,1,,289 13 | 1/12/2016,35,15,53,29.85,10,6,T,4,,235 14 | 1/13/2016,26,4,42,29.94,10,10,0,0,,284 15 | 1/14/2016,30,12,47,29.95,10,5,T,7,,266 16 | 1/15/2016,43,31,62,29.82,9,5,T,2,,101 17 | 1/16/2016,47,37,70,29.52,8,7,0.24,7,Rain,340 18 | 1/17/2016,36,23,66,29.78,8,6,0.05,6,Fog-Snow,345 19 | 1/18/2016,25,6,53,29.83,9,12,T,2,Snow,293 20 | 1/19/2016,22,3,42,30.03,10,11,0,1,,293 21 | 1/20/2016,32,15,49,30.13,10,6,0,2,,302 22 | 1/21/2016,31,11,45,30.15,10,6,0,1,,312 23 | 1/22/2016,26,6,41,30.21,9,,0.01,3,Snow,34 24 | 1/23/2016,26,21,78,29.77,1,16,2.31,8,Fog-Snow,42 25 | 1/24/2016,28,11,53,29.92,8,6,T,3,Snow,327 26 | 1/25/2016,34,18,54,30.25,10,3,0,2,,286 27 | 1/26/2016,43,29,56,30.03,10,7,0,2,,244 28 | 1/27/2016,41,22,45,30.03,10,7,T,3,Rain,311 29 | 1/28/2016,37,20,51,29.9,10,5,0,1,,234 30 | 1/29/2016,36,21,50,29.58,10,8,0,4,,298 31 | 1/30/2016,34,16,46,30.01,10,7,0,0,,257 32 | 1/31/2016,46,28,52,29.9,10,5,0,0,,241 33 | -------------------------------------------------------------------------------- /ReadCsv.ps1: -------------------------------------------------------------------------------- 1 | function Read-Csv { 2 | <# 3 | .Synopsis 4 | Read comma-separated values (csv). $target can be a URL, a file, or a string 5 | 6 | .Example 7 | #$file = "targetData.csv" 8 | $url = 'https://raw.githubusercontent.com/dfinke/ImportExcel/master/Examples/JustCharts/TargetData.csv' 9 | $str = @" 10 | "Cost","Date","Name" 11 | "1.1","1/1/2015","John" 12 | "2.1","1/2/2015","Tom" 13 | "5.1","1/2/2015","Dick" 14 | "11.1","1/2/2015","Harry" 15 | "7.1","1/2/2015","Jane" 16 | "22.1","1/2/2015","Mary" 17 | "32.1","1/2/2015","Liz" 18 | "@ 19 | 20 | $str, $url | Read-Csv 21 | 22 | Cost Date Name 23 | ---- ---- ---- 24 | 1.1 1/1/2015 John 25 | 2.1 1/2/2015 Tom 26 | 5.1 1/2/2015 Dick 27 | 11.1 1/2/2015 Harry 28 | 7.1 1/2/2015 Jane 29 | 22.1 1/2/2015 Mary 30 | 32.1 1/2/2015 Liz 31 | 1.1 1/1/2015 John 32 | 2.1 1/2/2015 Tom 33 | 5.1 1/2/2015 Dick 34 | 11.1 1/2/2015 Harry 35 | 7.1 1/2/2015 Jane 36 | 22.1 1/2/2015 Mary 37 | 32.1 1/2/2015 Liz 38 | 39 | #> 40 | param( 41 | [Parameter(ValueFromPipeline)] 42 | $target, 43 | $Delimiter = "," 44 | ) 45 | 46 | Process { 47 | if ([System.Uri]::IsWellFormedUriString($target, [System.UriKind]::Absolute)) { 48 | ConvertFrom-Csv (Invoke-RestMethod $target) -Delimiter $Delimiter 49 | } 50 | elseif (Test-Path $target -ErrorAction SilentlyContinue) { 51 | Import-Csv $target -Delimiter $Delimiter 52 | } 53 | else { 54 | ConvertFrom-Csv $target -Delimiter $Delimiter 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - job: Build_PS_Win2016 3 | pool: 4 | vmImage: vs2017-win2016 5 | steps: 6 | - powershell: | 7 | .\DoTests.ps1 8 | displayName: 'Run Tests on Windows' 9 | 10 | - job: Build_PSCore_Ubuntu1604 11 | 12 | pool: 13 | vmImage: ubuntu-16.04 14 | 15 | steps: 16 | - script: | 17 | curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - 18 | curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list | sudo tee /etc/apt/sources.list.d/microsoft.list 19 | sudo apt-get update 20 | sudo apt-get install -y powershell 21 | displayName: 'Install PowerShell Core' 22 | 23 | - script: | 24 | pwsh -c '.\DoTests.ps1' 25 | displayName: 'Run Tests on Linux' 26 | 27 | - job: Build_PSCore_MacIS1014 28 | pool: 29 | vmImage: macOS-10.14 30 | steps: 31 | - script: | 32 | brew update 33 | brew tap caskroom/cask 34 | brew cask install powershell 35 | displayName: 'Install PowerShell Core' 36 | 37 | - script: | 38 | pwsh -c '.\DoTests.ps1' 39 | displayName: 'Run Tests on macOS' 40 | 41 | trigger: 42 | paths: 43 | exclude: 44 | - README.md 45 | 46 | # resources: 47 | # - repo: self 48 | # queue: 49 | # name: Hosted VS2017 50 | # steps: 51 | # - powershell: .\DoTests.ps1 52 | # displayName: 'PowerShell Script' 53 | 54 | # - task: ArchiveFiles@2 55 | # displayName: 'Archive $(Build.BinariesDirectory)' 56 | 57 | # trigger: 58 | # paths: 59 | # exclude: 60 | # - README.md -------------------------------------------------------------------------------- /NewLookupTable.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .SYNOPSIS 3 | Turn your data into a lookup table 4 | 5 | .DESCRIPTION 6 | Do you have data that with a unique id column? Would you like to be able to load that data into your browser keyed by its unique id so that you can use it as a lookup table? Well then. 7 | 8 | .EXAMPLE 9 | $data = ConvertFrom-Csv @" 10 | slug,place,latitude,longitude 11 | dcl,Downtown Coffee Lounge,32.35066,-95.30181 12 | tyler-museum,Tyler Museum of Art,32.33396,-95.28174 13 | genecov,Genecov Sculpture,32.299076986939205,-95.31571447849274 14 | "@ 15 | 16 | New-LookupTable $data slug 17 | 18 | Name Value 19 | ---- ----- 20 | dcl @{slug=dcl; place=Downtown Coffee Lounge; latitude=32.35066; longitude=-95.30181} 21 | tyler-museum @{slug=tyler-museum; place=Tyler Museum of Art; latitude=32.33396; longitude=-95.28174} 22 | genecov @{slug=genecov; place=Genecov Sculpture; latitude=32.299076986939205; longitude=-95.3157144... 23 | #> 24 | function New-LookupTable { 25 | param( 26 | $InputObject, 27 | $key 28 | ) 29 | 30 | $h = [ordered]@{ } 31 | foreach ($record in $InputObject) { 32 | $theKey = $record.$key 33 | if (![string]::IsNullOrEmpty($thekey)) { 34 | if ($h.contains($theKey)) { 35 | if (!$h.$theKey -isnot [array]) { 36 | $h.$theKey = @($h.$theKey) 37 | } 38 | 39 | $h.$theKey += $record 40 | } 41 | else { 42 | $h.$theKey = $record 43 | } 44 | } 45 | } 46 | 47 | $h 48 | } -------------------------------------------------------------------------------- /NotebookExamples/DataAnalysis.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "Import-Module PSKit -Force\n", 10 | "\n", 11 | "$url = \"https://raw.githubusercontent.com/jennybc/gapminder/master/inst/extdata/gapminder.tsv\"\n", 12 | "\n", 13 | "$data = Read-Csv $url \"`t\"\n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "$data.head() | ctmt\n" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "$data.shape() | ctmt\n" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "$data.columns() #| ctmt\n" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "$data.dTypes() | ctmt\n" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "$data.info() " 59 | ] 60 | } 61 | ], 62 | "metadata": { 63 | "kernelspec": { 64 | "display_name": ".NET (PowerShell)", 65 | "language": "PowerShell", 66 | "name": ".net-powershell" 67 | }, 68 | "language_info": { 69 | "file_extension": ".ps1", 70 | "mimetype": "text/x-powershell", 71 | "name": "PowerShell", 72 | "pygments_lexer": "powershell", 73 | "version": "7.0" 74 | } 75 | }, 76 | "nbformat": 4, 77 | "nbformat_minor": 2 78 | } 79 | -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive.Extensions/Microsoft.ML.DotNet.Interactive.Extensions.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | Microsoft.ML.DotNet.Interactive 6 | preview 7 | false 8 | True 9 | 10 | 11 | 12 | 13 | Always 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | all 26 | runtime; build; native; contentfiles; analyzers; buildtransitive 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /GenerateStats.ps1: -------------------------------------------------------------------------------- 1 | function GenerateStats { 2 | param( 3 | [Parameter(Mandatory)] 4 | $TargetData 5 | ) 6 | 7 | $names = $TargetData[0].psobject.properties.name 8 | 9 | $NumberOfRowsToCheck = 2 10 | foreach ($name in $names) { 11 | $h = [Ordered]@{ } 12 | $h.ColumnName = $name 13 | 14 | $dt = for ($idx = 0; $idx -lt $NumberOfRowsToCheck; $idx++) { 15 | if ([string]::IsNullOrEmpty($TargetData[$idx].$name)) { 16 | "null" 17 | } 18 | else { 19 | (Invoke-AllTests $TargetData[$idx].$name -OnlyPassing -FirstOne).datatype 20 | } 21 | } 22 | 23 | $DataType = GetDataTypePrecedence @($dt) 24 | 25 | $h.DataType = $DataType 26 | $h.HasNulls = if ($DataType) { @($TargetData.$name -match '^$').count -gt 0 } 27 | $h.Min = $null 28 | $h.Max = $null 29 | $h.Range = $null 30 | $h.Median = $null 31 | $h.StandardDeviation = $null 32 | $h.Variance = $null 33 | $h.Sum = $null 34 | 35 | $validDataTypes = 'int|double|float|decimal' 36 | 37 | $stats = [MathNet.Numerics.Statistics.Statistics] 38 | switch -Regex ($DataType) { 39 | $validDataTypes { 40 | $h.Min = $stats::Minimum([double[]]$TargetData.$name) 41 | $h.Max = $stats::Maximum([double[]]$TargetData.$name) 42 | $h.Median = $stats::Median([double[]]$TargetData.$name) 43 | $h.StandardDeviation = $stats::StandardDeviation([double[]]$TargetData.$name) 44 | $h.Variance = $stats::Variance([double[]]$TargetData.$name) 45 | $h.Sum = ($TargetData.$name | Measure-Object -Sum).Sum 46 | } 47 | } 48 | 49 | $h.Range = $h.Max - $h.Min 50 | [PSCustomObject]$h 51 | } 52 | } -------------------------------------------------------------------------------- /__tests__/PSKit.Dataframe.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | Describe "PSKit tests - New-Dataframe" { 3 | 4 | BeforeAll { 5 | $propertyNames = 'a', 'b', 'c' 6 | } 7 | 8 | It "Should return 1 row" { 9 | $actual = @(New-DataFrame 1 $propertyNames) 10 | 11 | $actual.Count | Should be 1 12 | 13 | $actual.a | should be '[missing]' 14 | $actual.b | should be '[missing]' 15 | $actual.c | should be '[missing]' 16 | } 17 | 18 | It "Should return 1 row and properties set to 1" { 19 | $actual = @(New-DataFrame 1 $propertyNames { 1 }) 20 | 21 | $actual.Count | Should be 1 22 | 23 | $actual.a | should be 1 24 | $actual.b | should be 1 25 | $actual.c | should be 1 26 | } 27 | 28 | It "Should return 3 rows and properties set to 'a'" { 29 | $actual = New-DataFrame (1..3) $propertyNames { 'a' } 30 | 31 | $actual.Count | Should be 3 32 | 33 | $actual[0].Index | should be 1 34 | $actual[0].a | should be 'a' 35 | $actual[0].b | should be 'a' 36 | $actual[0].c | should be 'a' 37 | 38 | $actual[1].Index | should be 2 39 | $actual[1].a | should be 'a' 40 | $actual[1].b | should be 'a' 41 | $actual[1].c | should be 'a' 42 | 43 | $actual[2].Index | should be 3 44 | $actual[2].a | should be 'a' 45 | $actual[2].b | should be 'a' 46 | $actual[2].c | should be 'a' 47 | } 48 | 49 | It "Should return 3 rows Index set to correct date" { 50 | $actual = New-DataFrame (Get-DateRange 1/1 -periods 3) $propertyNames 51 | 52 | $actual.Count | Should be 3 53 | 54 | $actual[0].Index | should be '2020-01-01' 55 | $actual[1].Index | should be '2020-01-02' 56 | $actual[2].Index | should be '2020-01-03' 57 | } 58 | } -------------------------------------------------------------------------------- /__tests__/PSKit.ReadCsv.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Read-Csv" { 4 | 5 | BeforeAll { 6 | $script:url = 'https://raw.githubusercontent.com/dfinke/ImportExcel/master/Examples/JustCharts/TargetData.csv' 7 | 8 | $script:file = "$PSScriptRoot\..\data\targetData.csv" 9 | 10 | $script:str = @" 11 | "Cost","Date","Name" 12 | "1.1","1/1/2015","John" 13 | "2.1","1/2/2015","Tom" 14 | "5.1","1/2/2015","Dick" 15 | "11.1","1/2/2015","Harry" 16 | "7.1","1/2/2015","Jane" 17 | "22.1","1/2/2015","Mary" 18 | "32.1","1/2/2015","Liz" 19 | "@ 20 | } 21 | 22 | It "Shoud Read from a url" { 23 | $actual = Read-Csv $url 24 | 25 | $actual.Count | Should -Be 7 26 | 27 | $actual = $url | Read-Csv 28 | $actual.Count | Should -Be 7 29 | } 30 | 31 | It "Shoud Read from a file" { 32 | $actual = Read-Csv $file 33 | $actual.Count | Should -Be 7 34 | 35 | $actual = $file | Read-Csv 36 | $actual.Count | Should -Be 7 37 | } 38 | 39 | It "Shoud Read from a string" { 40 | 41 | $actual = Read-Csv $str 42 | $actual.Count | Should -Be 7 43 | 44 | $actual = $str | Read-Csv 45 | $actual.Count | Should -Be 7 46 | } 47 | 48 | It "Should do all" { 49 | $actual = $url, $str, $file | Read-Csv 50 | 51 | $actual.Count | Should -Be 21 52 | } 53 | 54 | It "Should Read Tab Separated Values from a string" { 55 | $tsvData = @" 56 | Region`tItemName`tTotalSold 57 | South`torange`t31 58 | West`tmelon`t91 59 | South`tpear`t40 60 | North`tdrill`t43 61 | South`torange`t77 62 | South`tpeach`t67 63 | West`tscrews`t48 64 | North`tavocado`t52 65 | North`tpeach`t63 66 | East`tavocado1`t62 67 | "@ 68 | $actual = Read-Csv -target $tsvData -Delimiter "`t" 69 | 70 | $actual.Count | Should -Be 10 71 | } 72 | } -------------------------------------------------------------------------------- /__tests__/PSKit.NewLookupTable.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - New-LookupTable" { 4 | BeforeAll { 5 | $script:data = ConvertFrom-Csv @" 6 | slug,place,latitude,longitude 7 | dcl,Downtown Coffee Lounge,32.35066,-95.30181 8 | tyler-museum,Tyler Museum of Art,32.33396,-95.28174 9 | genecov,Genecov Sculpture,32.299076986939205,-95.31571447849274 10 | "@ 11 | } 12 | 13 | It "Data should have 3 records" { 14 | $data.count | Should Be 3 15 | } 16 | 17 | It "New-LookupTable should return a hashtable" { 18 | $actual = New-LookupTable $data slug 19 | {$actual -is [hashtable]} | Should Be $true 20 | } 21 | 22 | It "New-LookupTable should have 3 keys" { 23 | $actual = New-LookupTable $data slug 24 | 25 | $actual.Keys.Count | Should Be 3 26 | $actual.Contains('dcl') | Should Be $true 27 | $actual.Contains('tyler-museum') | Should Be $true 28 | $actual.Contains('genecov') | Should Be $true 29 | } 30 | 31 | It "New-LookupTable key should have correct property names" { 32 | $actual = New-LookupTable $data slug 33 | 34 | $actual.dcl | Should Not Be $null 35 | $names = $actual.dcl.psobject.Properties.Name 36 | 37 | $names.Count | Should Be 4 38 | 39 | $names[0] | Should BeExactly 'slug' 40 | $names[1] | Should BeExactly 'place' 41 | $names[2] | Should BeExactly 'latitude' 42 | $names[3] | Should BeExactly 'longitude' 43 | } 44 | 45 | It "New-LookupTable key should have correct data" { 46 | $actual = New-LookupTable $data slug 47 | 48 | $actual.dcl | Should Not Be $null 49 | 50 | $actual.dcl.slug | Should BeExactly 'dcl' 51 | $actual.dcl.place | Should BeExactly 'Downtown Coffee Lounge' 52 | $actual.dcl.latitude | Should Be 32.35066 53 | $actual.dcl.longitude | Should Be -95.30181 54 | } 55 | } -------------------------------------------------------------------------------- /src/Microsoft.Data.Analysis.Interactive/DataFrameKernelExtension.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation and contributors. All rights reserved. 2 | // Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | using System.CommandLine; 5 | using System.CommandLine.Invocation; 6 | using System.IO; 7 | using System.Threading.Tasks; 8 | using Microsoft.DotNet.Interactive; 9 | using Microsoft.DotNet.Interactive.Commands; 10 | using Microsoft.DotNet.Interactive.Events; 11 | using Microsoft.DotNet.Interactive.Formatting; 12 | 13 | namespace Microsoft.Data.Analysis.Interactive 14 | { 15 | public class DataFrameKernelExtension : IKernelExtension 16 | { 17 | public Task OnLoadAsync(IKernel kernel) 18 | { 19 | //Formatter.Register((tree, writer) => 20 | //{ 21 | // writer.Write(""); 22 | //}, "text/html"); 23 | 24 | var kernelBase = kernel as KernelBase; 25 | var directive = new Command("#!doit") 26 | { 27 | Handler = CommandHandler.Create(async (FileInfo csv, string typeName, KernelInvocationContext context) => 28 | { 29 | // do the job 30 | var command = new SubmitCode(@$"public class {typeName}{{}}"); 31 | context.Publish(new DisplayedValueProduced($"emitting {typeName} from {csv.FullName}", context.Command)); 32 | await context.HandlingKernel.SendAsync(command); 33 | }) 34 | }; 35 | 36 | directive.AddOption(new Option( 37 | "csv").ExistingOnly()); 38 | 39 | directive.AddOption(new Option( 40 | "typeName", 41 | getDefaultValue:() => "Foo")); 42 | 43 | kernelBase.AddDirective(directive); 44 | 45 | return Task.CompletedTask; 46 | 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /__tests__/PSKit.GetPropertyName.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Get-PropertyName" { 4 | BeforeAll { 5 | $script:data = ConvertFrom-Csv @" 6 | Name,Class,Dorm,Room,GPA 7 | Sally Whittaker,2018,McCarren House,312,3.75 8 | Belinda Jameson,2017,Cushing House,148,3.52 9 | Jeff Smith,2018,Prescott House,17-D,3.2 10 | Sandy Allen,2019,Oliver House,108,3.48 11 | "@ 12 | } 13 | 14 | It "Should have five properties" { 15 | $actual = Get-PropertyName -InputObject $data 16 | $actual.Count | Should Be 5 17 | } 18 | 19 | It "Should have five properties when piped" { 20 | $actual = $data| Get-PropertyName 21 | $actual.Count | Should Be 5 22 | } 23 | 24 | It "Should have these names" { 25 | $actual = Get-PropertyName -InputObject $data 26 | 27 | $actual[0] | Should BeExactly 'Name' 28 | $actual[1] | Should BeExactly 'Class' 29 | $actual[2] | Should BeExactly 'Dorm' 30 | $actual[3] | Should BeExactly 'Room' 31 | $actual[4] | Should BeExactly 'GPA' 32 | } 33 | 34 | It "Should have these names when piped" { 35 | $actual = $data | Get-PropertyName 36 | 37 | $actual[0] | Should BeExactly 'Name' 38 | $actual[1] | Should BeExactly 'Class' 39 | $actual[2] | Should BeExactly 'Dorm' 40 | $actual[3] | Should BeExactly 'Room' 41 | $actual[4] | Should BeExactly 'GPA' 42 | } 43 | 44 | It "Should find one name" { 45 | $actual = $data | Get-PropertyName '*e' 46 | 47 | $actual.Count | Should Be 1 48 | $actual | Should BeExactly 'Name' 49 | } 50 | 51 | It "Should find these wildcard names" { 52 | $actual = $data | Get-PropertyName '*m*' 53 | 54 | $actual.Count | Should Be 3 55 | $actual[0] | Should BeExactly 'Name' 56 | $actual[1] | Should BeExactly 'Dorm' 57 | $actual[2] | Should BeExactly 'Room' 58 | } 59 | } -------------------------------------------------------------------------------- /NotebookExamples/TryPSKit.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "ipmo pskit -force\n", 10 | "\n", 11 | "new-dataframe (get-daterange 1/1 -p 5) a,b,c {get-random} | ctmt" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "Read-Csv https://raw.githubusercontent.com/codebasics/py/master/pandas/10_pivot/weather.csv | ctmt" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "Read-Csv https://raw.githubusercontent.com/codebasics/py/master/ML/14_naive_bayes/titanic.csv | ctmt" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "Read-Csv https://raw.githubusercontent.com/codebasics/py/master/ML/14_naive_bayes/titanic.csv | Export-Excel ./titanic.xlsx\n", 39 | "Import-Excel ./titanic.xlsx | ctmt" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": null, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "Read-Csv https://raw.githubusercontent.com/codebasics/py/master/pandas/4_read_write_to_excel/new.csv | ctmt" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "Read-Csv https://raw.githubusercontent.com/codebasics/py/master/pandas/4_read_write_to_excel/stock_data.csv | ctmt" 58 | ] 59 | } 60 | ], 61 | "metadata": { 62 | "kernelspec": { 63 | "display_name": ".NET (PowerShell)", 64 | "language": "PowerShell", 65 | "name": ".net-powershell" 66 | }, 67 | "language_info": { 68 | "file_extension": ".ps1", 69 | "mimetype": "text/x-powershell", 70 | "name": "PowerShell", 71 | "pygments_lexer": "powershell", 72 | "version": "7.0" 73 | } 74 | }, 75 | "nbformat": 4, 76 | "nbformat_minor": 4 77 | } 78 | -------------------------------------------------------------------------------- /data/air_quality_stations.csv: -------------------------------------------------------------------------------- 1 | location,coordinates.latitude,coordinates.longitude 2 | BELAL01,51.23619,4.38522 3 | BELHB23,51.1703,4.341 4 | BELLD01,51.10998,5.00486 5 | BELLD02,51.12038,5.02155 6 | BELR833,51.32766,4.36226 7 | BELSA04,51.31393,4.40387 8 | BELWZ02,51.1928,5.22153 9 | BETM802,51.26099,4.4244 10 | BETN016,51.23365,5.16398 11 | BETR801,51.20966,4.43182 12 | BETR802,51.20952,4.43179 13 | BETR803,51.22863,4.42845 14 | BETR805,51.20823,4.42156 15 | BETR811,51.2521,4.49136 16 | BETR815,51.2147,4.33221 17 | BETR817,51.17713,4.41795 18 | BETR820,51.32042,4.44481 19 | BETR822,51.26429,4.34128 20 | BETR831,51.3488,4.33971 21 | BETR834,51.092,4.3801 22 | BETR891,51.25581,4.38536 23 | BETR893,51.28138,4.38577 24 | BETR894,51.2835,4.3495 25 | BETR897,51.25011,4.3421 26 | FR04004,48.89167,2.34667 27 | FR04012,48.82778,2.3275 28 | FR04014,48.83724,2.3939 29 | FR04014,48.83722,2.3939 30 | FR04031,48.86887,2.31194 31 | FR04031,48.86889,2.31194 32 | FR04037,48.82861,2.36028 33 | FR04060,48.8572,2.2933 34 | FR04071,48.8564,2.33528 35 | FR04071,48.85639,2.33528 36 | FR04118,48.87027,2.3325 37 | FR04118,48.87029,2.3325 38 | FR04131,48.87333,2.33028 39 | FR04135,48.83795,2.40806 40 | FR04135,48.83796,2.40806 41 | FR04141,48.85278,2.36056 42 | FR04141,48.85279,2.36056 43 | FR04143,48.859,2.351 44 | FR04143,48.85944,2.35111 45 | FR04179,48.83038,2.26989 46 | FR04329,48.8386,2.41279 47 | FR04329,48.83862,2.41278 48 | Camden Kerbside,51.54421,-0.17527 49 | Ealing Horn Lane,51.51895,-0.26562 50 | Haringey Roadside,51.5993,-0.06822 51 | London Bexley,51.46603,0.18481 52 | London Bloomsbury,51.52229,-0.12589 53 | London Eltham,51.45258,0.07077 54 | London Haringey Priory Park South,51.58413,-0.12525 55 | London Harlington,51.48879,-0.44161 56 | London Harrow Stanmore,51.61733,-0.29878 57 | London Hillingdon,51.49633,-0.46086 58 | London Marylebone Road,51.52253,-0.15461 59 | London N. Kensington,51.52105,-0.21349 60 | London Teddington,51.42099,-0.33965 61 | London Teddington Bushy Park,51.42529,-0.34561 62 | London Westminster,51.49467,-0.13193 63 | Southend-on-Sea,51.5442,0.67841 64 | Southwark A2 Old Kent Road,51.4805,-0.05955 65 | Thurrock,51.47707,0.31797 66 | Tower Hamlets Roadside,51.52253,-0.04216 67 | Groton Fort Griswold,41.3536,-72.0789 68 | -------------------------------------------------------------------------------- /NewDataframe.ps1: -------------------------------------------------------------------------------- 1 | function New-DataFrame { 2 | <# 3 | .Synopsis 4 | Creates an array of objects, size-mutable, can be heterogeneous, tabular data 5 | 6 | .Example 7 | New-DataFrame (Get-DateRange 1/1 -periods 5) p1,p2,3 {Get-Random} 8 | Index p1 p2 3 9 | ----- -- -- - 10 | 2020-01-01 708420917 1112428663 523426202 11 | 2020-01-02 1643869654 2086787197 1127195815 12 | 2020-01-03 1095068483 2006354687 1612194161 13 | 2020-01-04 1561123134 1004618008 1431794170 14 | 2020-01-05 851611997 189055864 871342612 15 | 16 | .Example 17 | New-DataFrame (Get-DateRange 1/1 -periods 5) p1,p2,3 18 | Index p1 p2 3 19 | ----- -- -- - 20 | 2020-01-01 [missing] [missing] [missing] 21 | 2020-01-02 [missing] [missing] [missing] 22 | 2020-01-03 [missing] [missing] [missing] 23 | 2020-01-04 [missing] [missing] [missing] 24 | 2020-01-05 [missing] [missing] [missing] 25 | 26 | .Example 27 | New-DataFrame (Get-DateRange 1/1 -periods 5 -freq M) person,state,region {Invoke-Generate "$args"} 28 | Index person state region 29 | ----- ------ ----- ------ 30 | 2020-01-01 Jalen Austin Oklahoma East 31 | 2020-02-01 Jaydon Pratt Nevada North 32 | 2020-03-01 Jazlynn Zuniga Florida West 33 | 2020-04-01 Dylan Nash Nebraska South 34 | 2020-05-01 Jabari Dodson Massachusetts West 35 | 36 | .Example 37 | New-DataFrame -data (1..3) -propertyNames a,b,c,d -getRandomData {2*[math]::pi} 38 | Index a b c d 39 | ----- - - - - 40 | 1 6.28318530717959 6.28318530717959 6.28318530717959 6.28318530717959 41 | 2 6.28318530717959 6.28318530717959 6.28318530717959 6.28318530717959 42 | 3 6.28318530717959 6.28318530717959 6.28318530717959 6.28318530717959 43 | #> 44 | param($data, $propertyNames, [scriptblock]$getRandomData = { '[missing]' }) 45 | 46 | foreach ($row in $data) { 47 | $obj = [Ordered]@{Index = $row } 48 | foreach ($propertyName in $propertyNames) { 49 | $obj.$propertyName = &$getRandomData $propertyName 50 | } 51 | 52 | [PSCustomObject]$obj 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /GetDateRange.ps1: -------------------------------------------------------------------------------- 1 | function Get-DateRange { 2 | <# 3 | .Synopsis 4 | Return a fixed frequency Datetime Index 5 | 6 | .Example 7 | Get-DateRange 1/1/2020 -periods 6 8 | 9 | 2020-01-01 10 | 2020-01-02 11 | 2020-01-03 12 | 2020-01-04 13 | 2020-01-05 14 | 2020-01-06 15 | 16 | .Example 17 | New-DataFrame (Get-DateRange 1/1/2020 -periods 3 -freq M) a,b,c 18 | 19 | Index a b c 20 | ----- - - - 21 | 2020-01-01 [missing] [missing] [missing] 22 | 2020-02-01 [missing] [missing] [missing] 23 | 2020-03-01 [missing] [missing] [missing] 24 | 25 | .Example 26 | Get-DateRange 1/1/2020 -periods 6 -freq M 27 | 28 | 2020-01-01 29 | 2020-02-01 30 | 2020-03-01 31 | 2020-04-01 32 | 2020-05-01 33 | 2020-06-01 34 | 35 | .Example 36 | Get-DateRange 1/1/2020 -periods 6 -freq Y 37 | 38 | 2020-01-01 39 | 2021-01-01 40 | 2022-01-01 41 | 2023-01-01 42 | 2024-01-01 43 | 2025-01-01 44 | 45 | .Example 46 | Get-DateRange 1/1/2020 1/5/2020 -periods 3 47 | 48 | 2020-01-01 49 | 2020-01-02 50 | 2020-01-05 51 | 52 | #> 53 | param( 54 | [datetime]$start = (Get-Date), 55 | [datetime]$end, 56 | $periods, 57 | [ValidateSet('D', 'M', 'Y')] 58 | $freq = 'D' 59 | ) 60 | 61 | switch ($freq) { 62 | 'D' { $targetMethod = 'AddDays' } 63 | 'M' { $targetMethod = 'AddMonths' } 64 | 'Y' { $targetMethod = 'AddYears' } 65 | } 66 | 67 | $fmt = 'yyyy-MM-dd' 68 | 69 | if ((@{ } + $PSBoundParameters).count -eq 0) { 70 | $n = 0 71 | } 72 | elseif ($start -and !$end -and !$periods) { 73 | $n = 0 74 | } 75 | else { 76 | $n = $periods - 1 77 | } 78 | 79 | if ($end) { 80 | $totalNumberOfDates = ($end - $start).totaldays 81 | if (!$periods) { 82 | $n = $totalNumberOfDates 83 | } 84 | } 85 | 86 | $r = 0..$n | ForEach-Object { $start.$targetMethod($_).ToString($fmt) } 87 | 88 | # set the last element to the end date 89 | if ($totalNumberOfDates -gt $periods) { 90 | $r[-1] = $start.$targetMethod($totalNumberOfDates).ToString($fmt) 91 | } 92 | 93 | $r 94 | } -------------------------------------------------------------------------------- /InferData.ps1: -------------------------------------------------------------------------------- 1 | function Test-String { 2 | param($p) 3 | 4 | [PSCustomObject]@{ 5 | Test = ($p -is [string] -or $p.GetType().BaseType.Name -eq 'Enum') 6 | DataType = "string" 7 | } 8 | } 9 | 10 | function Test-Date { 11 | param($p) 12 | 13 | [datetime]$result = [datetime]::MinValue 14 | 15 | [PSCustomObject]@{ 16 | Test = [datetime]::TryParse($p, [ref]$result) 17 | DataType = "datetime" 18 | } 19 | } 20 | 21 | function Test-Boolean { 22 | param($p) 23 | 24 | #[bool]$result = [bool]::FalseString 25 | [bool]$result = $false 26 | 27 | [PSCustomObject]@{ 28 | Test = [bool]::TryParse($p, [ref]$result) 29 | DataType = "bool" 30 | } 31 | } 32 | 33 | function Test-Number { 34 | param($p) 35 | 36 | [double]$result = [double]::MinValue 37 | 38 | [PSCustomObject]@{ 39 | Test = [double]::TryParse($p, [ref]$result) 40 | DataType = "double" 41 | } 42 | } 43 | 44 | function Test-Integer { 45 | param($p) 46 | 47 | [int]$result = [int]::MinValue 48 | 49 | [PSCustomObject]@{ 50 | Test = [int]::TryParse($p, [ref]$result) 51 | DataType = "int" 52 | } 53 | } 54 | 55 | $tests = [ordered]@{ 56 | TestBoolean = Get-Command Test-Boolean 57 | TestInteger = Get-Command Test-Integer 58 | TestNumber = Get-Command Test-Number 59 | TestDate = Get-Command Test-Date 60 | TestString = Get-Command Test-String 61 | } 62 | 63 | function Invoke-AllTests { 64 | param( 65 | $target, 66 | [Switch]$OnlyPassing, 67 | [Switch]$FirstOne 68 | ) 69 | 70 | $resultCount = 0 71 | $tests.GetEnumerator() | ForEach-Object { 72 | 73 | $result = & $_.Value $target 74 | 75 | $testResult = [PSCustomObject]@{ 76 | Test = $_.Key 77 | Target = $target 78 | Result = $result.Test 79 | DataType = $result.DataType 80 | } 81 | 82 | if (!$OnlyPassing) { 83 | $testResult 84 | } 85 | elseif ($result.Test -eq $true) { 86 | if ($FirstOne) { 87 | if ($resultCount -ne 1) { 88 | $testResult 89 | $resultCount += 1 90 | } 91 | } 92 | else { 93 | $testResult 94 | } 95 | } 96 | } 97 | } -------------------------------------------------------------------------------- /PSKit.psm1: -------------------------------------------------------------------------------- 1 | Add-Type -Path "$PSScriptRoot\lib\MathNet.Numerics.dll" 2 | 3 | . $PSScriptRoot/ConvertFromFixedData.ps1 4 | . $PSScriptRoot/ConvertFromSSV.ps1 5 | . $PSScriptRoot/ConvertIntoCSV.ps1 6 | . $PSScriptRoot/CustomTypeData.ps1 7 | . $PSScriptRoot/GenerateStats.ps1 8 | . $PSScriptRoot/GetDataInfo.ps1 9 | . $PSScriptRoot/GetDataTypePrecedence.ps1 10 | . $PSScriptRoot/GetDateRange.ps1 11 | . $PSScriptRoot/GetPropertyName.ps1 12 | . $PSScriptRoot/GetPropertyStats.ps1 13 | . $PSScriptRoot/GroupByAndMeasure.ps1 14 | . $PSScriptRoot/InferData.ps1 15 | . $PSScriptRoot/InvokeTranspileSQL.ps1 16 | . $PSScriptRoot/NewDataframe.ps1 17 | . $PSScriptRoot/NewLookupTable.ps1 18 | . $PSScriptRoot/ReadCsv.ps1 19 | . $PSScriptRoot/ScanProperties.ps1 20 | 21 | filter ConvertTo-Property { 22 | $i = $_ 23 | "" | Select-Object @{n = "P1"; e = { $i } } 24 | } 25 | 26 | function Script:Test-JupyterNotebook { 27 | if (Get-Command Out-Display -ErrorAction SilentlyContinue) { 28 | $true 29 | } 30 | else { 31 | $false 32 | } 33 | } 34 | 35 | # Added here, just for PS Jupyter Notebooks. May move elsewhere. 36 | if (Test-JupyterNotebook) { 37 | function ConvertTo-MarkdownTable { 38 | param( 39 | [parameter(ValueFromPipeline)] 40 | $targetData 41 | ) 42 | 43 | Begin { $allData = @() } 44 | 45 | Process { 46 | # $allData += $targetData 47 | if ($targetData.gettype().name -match 'HashTable|OrderedDictionary') { 48 | $allData = [PSCustomObject]$targetData 49 | } 50 | else { 51 | $allData += $targetData 52 | } 53 | } 54 | 55 | End { 56 | $names = $allData[0].psobject.Properties.name 57 | 58 | $result = foreach ($record in $allData) { 59 | $inner = @() 60 | foreach ($name in $names) { 61 | $inner += $record.$name 62 | } 63 | '|' + ($inner -join '|') + '|' + "`n" 64 | } 65 | 66 | $html = (@" 67 | $('|' + ($names -join '|') + '|') 68 | $(('|---' * ($names.Count - 1)) + '|') 69 | $($result) 70 | "@ | ConvertFrom-Markdown).html 71 | 72 | if (Get-Command Get-HtmlContent -ErrorAction SilentlyContinue) { 73 | $html | Get-HtmlContent | Out-Display 74 | } 75 | else { 76 | $html | Out-Display 77 | } 78 | } 79 | } 80 | 81 | Set-Alias ctmt ConvertTo-MarkdownTable 82 | } -------------------------------------------------------------------------------- /CustomTypeData.ps1: -------------------------------------------------------------------------------- 1 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName info -Value { 2 | Get-DataInfo -TargetData $this 3 | } 4 | 5 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName stats -Value { 6 | Get-PropertyStats -InputObject $this 7 | } 8 | 9 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName GroupAndMeasure -Value { 10 | param( 11 | $GroupBy, 12 | $MeasureProperty, 13 | [ValidateSet('Average', 'Maximum', 'Minimum', 'Sum', 'Count')] 14 | $MeasureOperation 15 | ) 16 | 17 | Group-ByAndMeasure -targetData $this -GroupBy $GroupBy -MeasureProperty $MeasureProperty -MeasureOperation $MeasureOperation 18 | } 19 | 20 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName query -Value { 21 | param($q) 22 | 23 | $psquery = Invoke-TranspileSQL $q | ConvertFrom-TranspileSQL 24 | 25 | Invoke-Expression "`$this $psquery" 26 | } 27 | 28 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName SetIndex -Value { 29 | param($key) 30 | 31 | New-LookupTable -InputObject $this -key $key 32 | } 33 | 34 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName ScanProperties -Value { 35 | param($pattern) 36 | 37 | Invoke-ScanProperties -InputObject ($this) -Pattern $pattern 38 | } 39 | 40 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName Head -Value { 41 | <# 42 | .Synopsis 43 | This function returns the first n rows for the object based on position. It is useful for quickly testing if your object has the right type of data in it. 44 | #> 45 | param($numberOfRows = 5) 46 | 47 | $this[0..($numberOfRows - 1)] 48 | } 49 | 50 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName Tail -Value { 51 | <# 52 | .Synopsis 53 | This function returns last n rows from the object based on position. It is useful for quickly verifying data, for example, after sorting or appending rows. 54 | #> 55 | param($numberOfRows = 5) 56 | 57 | $this[ (-$numberOfRows)..-1] 58 | } 59 | 60 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName Shape -Value { 61 | 62 | [PSCustomObject][Ordered]@{ 63 | Rows = $this.Count 64 | Columns = $this[0].psobject.Properties.name.count 65 | } 66 | } 67 | 68 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName Columns -Value { 69 | #$this[0].psobject.properties.name 70 | $this.DTypes().ColumnName 71 | } 72 | 73 | Update-TypeData -Force -TypeName Array -MemberType ScriptMethod -MemberName DTypes -Value { 74 | Get-DataType $this 75 | } -------------------------------------------------------------------------------- /GroupByAndMeasure.ps1: -------------------------------------------------------------------------------- 1 | function Group-ByAndMeasure { 2 | <# 3 | .Synopsis 4 | Groups data and can either get the Count, Average, Sum, Maximum or Minimum 5 | 6 | .Example 7 | $str = @" 8 | Region,Item,TotalSold 9 | West,apple,2 10 | South,lemon,4 11 | East,avocado,12 12 | South,screwdriver,70 13 | North,avocado,59 14 | North,hammer,33 15 | North,screws,69 16 | East,apple,21 17 | West,lemon,67 18 | South,drill,52 19 | "@ 20 | 21 | Group-ByAndMeasure (Read-Csv $str) Region TotalSold Sum 22 | 23 | Region Sum 24 | ------ --- 25 | West 69 26 | South 126 27 | East 33 28 | North 161 29 | 30 | .Example 31 | Group-ByAndMeasure (Read-Csv $str) Region,Item TotalSold Sum 32 | 33 | Region, Item Sum 34 | ------------ --- 35 | West, apple 2 36 | South, lemon 4 37 | East, avocado 12 38 | South, screwdriver 70 39 | North, avocado 59 40 | North, hammer 33 41 | North, screws 69 42 | East, apple 21 43 | West, lemon 67 44 | South, drill 52 45 | 46 | .Example 47 | Group-ByAndMeasure (Get-Process) -GroupBy Company -MeasureProperty Handles -MeasureOperation Sum 48 | 49 | Company Sum 50 | ------- --- 51 | Microsoft Corporation 81410 52 | Citrix Systems, Inc. 3302 53 | Google LLC 16804 54 | Zoom Video Communications, Inc. 160 55 | The CefSharp Authors 1016 56 | Node.js 649 57 | Realtek Semiconductor 1218 58 | Helios Software Solutions 303 59 | TechSmith Corporation 208 60 | #> 61 | param( 62 | [Parameter(Mandatory)] 63 | $targetData, 64 | [Parameter(Mandatory)] 65 | $GroupBy, 66 | [Parameter(Mandatory)] 67 | $MeasureProperty, 68 | [Parameter(Mandatory)] 69 | [ValidateSet('Average', 'Maximum', 'Minimum', 'Sum', 'Count')] 70 | $MeasureOperation 71 | ) 72 | 73 | $params = @{ 74 | Property = $MeasureProperty 75 | Average = if ($MeasureOperation -eq 'Average') { $true } else { $false } 76 | Maximum = if ($MeasureOperation -eq 'Maximum') { $true } else { $false } 77 | Minimum = if ($MeasureOperation -eq 'Minimum') { $true } else { $false } 78 | Sum = if ($MeasureOperation -eq 'Sum') { $true } else { $false } 79 | } 80 | 81 | $groubyName = $GroupBy -join ', ' 82 | foreach ($group in ($targetData | Group-Object $GroupBy)) { 83 | [PSCustomObject][ordered]@{ 84 | $groubyName = $group.name 85 | $MeasureOperation = ($group.group | Measure-Object @params).$MeasureOperation 86 | } 87 | } 88 | 89 | } -------------------------------------------------------------------------------- /__tests__/PSKit.GetDataInfo.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Get-DataInfo" { 4 | BeforeAll { 5 | $data = ConvertFrom-Csv @" 6 | Region,ItemName,TotalSold 7 | South,screws,3 8 | North,drill,58 9 | East,drill,4 10 | North,apple,67 11 | North,nail,45 12 | East,orange,40 13 | South,apple,2 14 | West,hammer,55 15 | North,screws,49 16 | West,peach,67 17 | "@ 18 | } 19 | 20 | It "Should have the correct summary info" { 21 | $actual = Get-DataInfo $data -Raw 22 | 23 | $actual.Entries | Should -Be 10 24 | $actual.Columns | Should -Be 3 25 | $actual.Result.Count | Should -Be 3 26 | $actual.DataTypeSummary.Count | Should -Be 2 27 | } 28 | 29 | It "Should have the correct detailed Result info" { 30 | <# 31 | ColumnName NonNull DataType 32 | ---------- ------- -------- 33 | Region 10 string 34 | ItemName 10 string 35 | TotalSold 10 int 36 | #> 37 | 38 | $actual = Get-DataInfo $data -Raw 39 | 40 | $actual.Result[0].ColumnName | Should -BeExactly 'Region' 41 | $actual.Result[0].NonNull | Should -Be 10 42 | $actual.Result[0].DataType | Should -BeExactly 'string' 43 | 44 | $actual.Result[1].ColumnName | Should -BeExactly 'ItemName' 45 | $actual.Result[1].NonNull | Should -Be 10 46 | $actual.Result[1].DataType | Should -BeExactly 'string' 47 | 48 | $actual.Result[2].ColumnName | Should -BeExactly 'TotalSold' 49 | $actual.Result[2].NonNull | Should -Be 10 50 | $actual.Result[2].DataType | Should -BeExactly 'int' 51 | } 52 | 53 | It "Should have the correct detailed DataTypeSummary info" { 54 | $actual = Get-DataInfo $data -Raw 55 | 56 | $actual.DataTypeSummary[0] | Should -BeExactly 'int(1)' 57 | $actual.DataTypeSummary[1] | Should -BeExactly 'string(2)' 58 | } 59 | 60 | It "Should have the correct overall summary" -Skip { 61 | 62 | $actual = Get-DataInfo $data 63 | $expected = @" 64 | Entries: 10 65 | Columns: 3 66 | 67 | 68 | ColumnName NonNull DataType 69 | ---------- ------- -------- 70 | Region 10 string 71 | ItemName 10 string 72 | TotalSold 10 int 73 | 74 | 75 | 76 | string(2) int(1) 77 | 78 | "@ 79 | $nl = [System.Environment]::NewLine 80 | 81 | $records = $actual.split($nl) 82 | $expectedRecords = $expected.split($nl) 83 | 84 | $records.Count | Should -Be 25 85 | 86 | foreach ($item in 0..25) { 87 | if ($records[$item].length -gt 0 -and $expectedRecords[$item].length -gt 0) { 88 | $records[$item] | Should -BeExactly $expectedRecords[$item] 89 | } 90 | } 91 | } 92 | } -------------------------------------------------------------------------------- /ConvertFromSSV.ps1: -------------------------------------------------------------------------------- 1 | #Requires -Modules PSStringScanner 2 | 3 | function ConvertFrom-SSV { 4 | <# 5 | .SYNOPSIS 6 | Parse text as space-separated values and create objects in PowerShell 7 | 8 | .EXAMPLE 9 | Sample `ps` for Linux and uses `-MinimumWhiteSpaceLength 1` in order to parse the `TIME` and `CMD` 10 | @" 11 | PID TTY TIME CMD 12 | 103 pts/0 00:00:00 bash 13 | 136 pts/0 00:00:13 pwsh 14 | 305 pts/0 00:00:00 ps 15 | 16 | "@ -split "`n" | ConvertFrom-SSV -MinimumWhiteSpaceLength 1 | Sort pid -desc 17 | 18 | PID TTY TIME CMD 19 | --- --- ---- --- 20 | 305 pts/0 00:00:00 ps 21 | 136 pts/0 00:00:13 pwsh 22 | 103 pts/0 00:00:00 bash 23 | 24 | #> 25 | 26 | param( 27 | $MinimumWhiteSpaceLength = 2, 28 | [Parameter(ValueFromPipeline)] 29 | $data 30 | ) 31 | 32 | begin { 33 | $pattern = "\s{$($MinimumWhiteSpaceLength),}" 34 | $firstTimeThru = $true 35 | } 36 | process { 37 | if ($firstTimeThru) { 38 | $ss = New-PSStringScanner $data 39 | $h = @() 40 | while ($ss.Check($pattern)) { 41 | $h += $ss.ScanUntil($pattern).trim() 42 | } 43 | $h += $ss.Scan(".*").trim() 44 | 45 | $firstTimeThru = $false 46 | 47 | } 48 | else { 49 | foreach ($item in $data) { 50 | $ss = New-PSStringScanner $item 51 | 52 | $index = 0 53 | $d = [ordered]@{ } 54 | while ($ss.Check($pattern)) { 55 | $s = $ss.ScanUntil($pattern).trim() 56 | 57 | # blank header 58 | if ($h[$index].Length -eq 0) { 59 | $index++ 60 | continue 61 | } 62 | else { 63 | if ($s -match "-{$($s.length)}") { } 64 | else { 65 | $d.($h[$index]) = $s 66 | $index++ 67 | } 68 | } 69 | } 70 | 71 | $s = $ss.Scan(".*").trim() 72 | # blank header 73 | if ($h[$index].Length -eq 0) { 74 | $index++ 75 | } 76 | else { 77 | if ($s -match "-{$($s.length)}") { } 78 | else { 79 | $d.($h[$index]) = $s 80 | } 81 | } 82 | 83 | if ($d.keys.count -gt 0) { 84 | [PSCustomObject]$d 85 | } 86 | } 87 | } 88 | } 89 | } 90 | 91 | function Import-SSV { 92 | param( 93 | $FileName, 94 | $MinimumWhiteSpaceLength = 2 95 | ) 96 | 97 | [System.IO.File]::ReadLines($FileName) | ConvertFrom-SSV $MinimumWhiteSpaceLength 98 | } -------------------------------------------------------------------------------- /GetDataInfo.ps1: -------------------------------------------------------------------------------- 1 | function Get-DataInfo { 2 | <# 3 | 4 | .Example 5 | (ConvertFrom-Csv @" 6 | Region,ItemName,Units,TotalSold 7 | ,screws,5.3,3 8 | North,,5.7,58 9 | East,drill,6.3 10 | "@).info() 11 | 12 | Entries: 3 13 | Columns: 4 14 | 15 | 16 | ColumnName NonNull DataType 17 | ---------- ------- -------- 18 | Region 2 string 19 | ItemName 2 string 20 | Units 3 double 21 | TotalSold 2 int 22 | 23 | 24 | 25 | string(2) double(1) int(1) 26 | 27 | #> 28 | param( 29 | [Parameter(Mandatory)] 30 | $TargetData, 31 | [Switch]$Raw 32 | ) 33 | 34 | $totalRecords = $TargetData.Count 35 | $names = $TargetData[0].psobject.properties.name 36 | 37 | $NumberOfRowsToCheck = 2 38 | 39 | $result = foreach ($name in $names) { 40 | $h = [Ordered]@{ } 41 | $h.ColumnName = $name 42 | 43 | $dt = for ($idx = 0; $idx -lt $NumberOfRowsToCheck; $idx++) { 44 | if ([string]::IsNullOrEmpty($TargetData[$idx].$name)) { 45 | "null" 46 | } 47 | else { 48 | (Invoke-AllTests $TargetData[$idx].$name -OnlyPassing -FirstOne).datatype 49 | } 50 | } 51 | 52 | $h.NonNull = $totalRecords - @($TargetData.$name -match '^$').count 53 | $h.DataType = GetDataTypePrecedence @($dt) 54 | 55 | [pscustomobject]$h 56 | } 57 | 58 | $rawData = [PSCustomObject][Ordered]@{ 59 | Result = $result 60 | Entries = $totalRecords 61 | Columns = $names.count 62 | DataTypeSummary = foreach ($record in $result | Group-Object -NoElement DataType | Sort-Object Name) { 63 | "{0}({1})" -f $record.Name, $record.Count 64 | } 65 | } 66 | 67 | if ($Raw) { 68 | $rawData 69 | } 70 | else { 71 | @" 72 | Entries: $($rawData.Entries) 73 | Columns: $($rawData.Columns) 74 | 75 | $($rawData.Result | Out-String) 76 | $($rawData.DataTypeSummary) 77 | "@ 78 | 79 | } 80 | } 81 | 82 | ## TODO: Refactor `Get-DataInfo` to use this 83 | function Get-DataType { 84 | param( 85 | [Parameter(Mandatory)] 86 | $TargetData 87 | ) 88 | 89 | $names = $TargetData[0].psobject.properties.name 90 | 91 | $NumberOfRowsToCheck = 2 92 | foreach ($name in $names) { 93 | $h = [Ordered]@{ } 94 | $h.ColumnName = $name 95 | 96 | $dt = for ($idx = 0; $idx -lt $NumberOfRowsToCheck; $idx++) { 97 | if ([string]::IsNullOrEmpty($TargetData[$idx].$name)) { 98 | "null" 99 | } 100 | else { 101 | (Invoke-AllTests $TargetData[$idx].$name -OnlyPassing -FirstOne).datatype 102 | } 103 | } 104 | 105 | $h.DataType = GetDataTypePrecedence @($dt) 106 | 107 | [pscustomobject]$h 108 | } 109 | } -------------------------------------------------------------------------------- /__tests__/PSKit.GetPropertyStats.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Get-PropertyStat -InputObject" { 4 | BeforeAll { 5 | $script:data = ConvertFrom-Csv @" 6 | Name,Age 7 | Jane,10 8 | John,5 9 | ,15 10 | "@ 11 | <# 12 | ColumnName DataType HasNulls Min Max Avg Sum 13 | ---------- -------- -------- --- --- --- --- 14 | Name string False 15 | Age int False 5 15 10 30 16 | #> 17 | 18 | } 19 | 20 | It "Should calculate property stats when piped" { 21 | $actual = $data | Get-PropertyStats 22 | 23 | $actual.Count | Should Be 2 24 | 25 | $actual[0].ColumnName | Should BeExactly 'Name' 26 | $actual[0].DataType | Should BeExactly 'string' 27 | $actual[0].HasNulls | Should Be $true 28 | 29 | $actual[0].Min | Should Be $null 30 | $actual[0].Max | Should Be $null 31 | $actual[0].Median | Should Be $null 32 | $actual[0].Sum | Should Be $null 33 | $actual[0].StandardDeviation | Should Be $null 34 | $actual[0].Variance | Should Be $null 35 | 36 | $actual[1].ColumnName | Should BeExactly 'Age' 37 | $actual[1].DataType | Should BeExactly 'int' 38 | $actual[1].HasNulls | Should Be $false 39 | $actual[1].Min | Should Be 5 40 | $actual[1].Max | Should Be 15 41 | $actual[1].Median | Should Be 10 42 | $actual[1].Sum | Should Be 30 43 | $actual[1].StandardDeviation | Should Be 5 44 | $actual[1].Variance | Should Be 25 45 | } 46 | 47 | It "Should calculate property stats" { 48 | 49 | $actual = Get-PropertyStats -InputObject $data 50 | $actual.Count | Should Be 2 51 | 52 | $actual[0].ColumnName | Should BeExactly 'Name' 53 | $actual[0].DataType | Should BeExactly 'string' 54 | $actual[0].HasNulls | Should Be $true 55 | $actual[0].Min | Should Be $null 56 | $actual[0].Max | Should Be $null 57 | $actual[0].Median | Should Be $null 58 | $actual[0].StandardDeviation | Should Be $null 59 | $actual[0].Variance | Should Be $null 60 | $actual[0].Sum | Should Be $null 61 | 62 | $actual[1].ColumnName | Should BeExactly 'Age' 63 | $actual[1].DataType | Should BeExactly 'int' 64 | $actual[1].HasNulls | Should Be $false 65 | $actual[1].Min | Should Be 5 66 | $actual[1].Max | Should Be 15 67 | $actual[1].Median | Should Be 10 68 | $actual[1].StandardDeviation | Should Be 5 69 | $actual[1].Variance | Should Be 25 70 | $actual[1].Sum | Should Be 30 71 | } 72 | 73 | It "Should calculate Range" { 74 | # $actual = (30..50) | ForEach-Object { $i = $_; "" | Select-Object @{name = "P1"; e = { $i } } } | Get-PropertyStats 75 | $actual = (30..50) | ConvertTo-Property | Get-PropertyStats 76 | 77 | $actual.Min | Should Be 30 78 | $actual.Range | Should Be 20 79 | } 80 | } -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive/DecisionTreeDataExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation and contributors. All rights reserved. 2 | // Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | using System; 5 | using System.Linq; 6 | using System.Text; 7 | using Microsoft.ML.Data; 8 | using Microsoft.ML.Trainers.FastTree; 9 | 10 | namespace Microsoft.ML.DotNet.Interactive 11 | { 12 | public static class DecisionTreeDataExtensions 13 | { 14 | public static DecisionTreeData ToDecisionTreeData(this RegressionTreeEnsemble ensemble,in VBuffer> featureNames = default) 15 | { 16 | // just get the first tree, for now 17 | return ensemble.Trees.FirstOrDefault().ToDecisionTreeData(featureNames); 18 | } 19 | 20 | public static DecisionTreeData ToDecisionTreeData(this RegressionTree tree, in VBuffer> featureNames = default) 21 | { 22 | DecisionTreeData treeData = new DecisionTreeData(); 23 | 24 | if (tree == null) 25 | { 26 | return treeData; 27 | } 28 | 29 | var nodes = new NodeData[tree.NumberOfNodes]; 30 | var labelBuilder = new StringBuilder(); 31 | for (int node = 0; node < tree.NumberOfNodes; node++) 32 | { 33 | nodes[node] = new NodeData(); 34 | int featureIndex = tree.NumericalSplitFeatureIndexes[node]; 35 | float splitThreshold = tree.NumericalSplitThresholds[node]; 36 | 37 | ReadOnlyMemory featureName = featureNames.GetItemOrDefault(featureIndex); 38 | if (!featureName.IsEmpty) 39 | { 40 | labelBuilder.Append(featureName); 41 | } 42 | else 43 | { 44 | labelBuilder.Append('f'); 45 | labelBuilder.Append(featureIndex); 46 | } 47 | labelBuilder.Append($" > "); 48 | labelBuilder.Append(splitThreshold.ToString("n2")); 49 | 50 | nodes[node].Label = labelBuilder.ToString(); 51 | labelBuilder.Clear(); 52 | } 53 | 54 | NodeData[] leaves = new NodeData[tree.NumberOfLeaves]; 55 | for (int leaf = 0; leaf < tree.NumberOfLeaves; leaf++) 56 | { 57 | leaves[leaf] = new NodeData {Label = tree.LeafValues[leaf].ToString("n2")}; 58 | } 59 | 60 | NodeData GetNodeData(int child) 61 | { 62 | return child >= 0 63 | ? nodes[child] 64 | : leaves[~child]; 65 | } 66 | 67 | // hook the nodes up 68 | for (int node = 0; node < tree.NumberOfNodes; node++) 69 | { 70 | // the RightChild is the 'greater than' path, so put that first 71 | nodes[node].Children.Add(GetNodeData(tree.RightChild[node])); 72 | nodes[node].Children.Add(GetNodeData(tree.LeftChild[node])); 73 | } 74 | 75 | if (nodes.Length > 0) 76 | { 77 | treeData.Root = nodes[0]; 78 | } 79 | 80 | return treeData; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/scipy-notebook:latest 2 | 3 | # Install .NET CLI dependencies 4 | 5 | ARG NB_USER=jovyan 6 | ARG NB_UID=1000 7 | ENV USER ${NB_USER} 8 | ENV NB_UID ${NB_UID} 9 | ENV HOME /home/${NB_USER} 10 | 11 | WORKDIR ${HOME} 12 | 13 | USER root 14 | 15 | RUN apt-get update \ 16 | && apt-get install -y --no-install-recommends \ 17 | curl \ 18 | # Install .NET CLI dependencies 19 | libc6 \ 20 | libgcc1 \ 21 | libgssapi-krb5-2 \ 22 | libicu60 \ 23 | libssl1.1 \ 24 | libstdc++6 \ 25 | zlib1g \ 26 | # For ImportExcel autosize 27 | libgdiplus libc6-dev \ 28 | && rm -rf /var/lib/apt/lists/* 29 | 30 | # Install .NET Core SDK 31 | ENV DOTNET_SDK_VERSION 3.1.101 32 | 33 | RUN curl -SL --output dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-x64.tar.gz \ 34 | && dotnet_sha512='eeee75323be762c329176d5856ec2ecfd16f06607965614df006730ed648a5b5d12ac7fd1942fe37cfc97e3013e796ef278e7c7bc4f32b8680585c4884a8a6a1' \ 35 | && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ 36 | && mkdir -p /usr/share/dotnet \ 37 | && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \ 38 | && rm dotnet.tar.gz \ 39 | && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet 40 | 41 | # Enable detection of running in a container 42 | ENV DOTNET_RUNNING_IN_CONTAINER=true \ 43 | # Enable correct mode for dotnet watch (only mode supported in a container) 44 | DOTNET_USE_POLLING_FILE_WATCHER=true \ 45 | # Skip extraction of XML docs - generally not useful within an image/container - helps performance 46 | NUGET_XMLDOC_MODE=skip \ 47 | # Opt out of telemetry until after we install jupyter when building the image, this prevents caching of machine id 48 | DOTNET_TRY_CLI_TELEMETRY_OPTOUT=true 49 | 50 | # Trigger first run experience by running arbitrary cmd 51 | RUN dotnet help 52 | 53 | # Copy notebooks, package sources, and source code 54 | # NOTE: Do this before installing dotnet-try so we get the 55 | # latest dotnet-try everytime we change sources. 56 | COPY ./NotebookExamples/ ${HOME}/Notebooks/ 57 | COPY ./NuGet.config ${HOME}/nuget.config 58 | COPY ./src/ ${HOME}/src/ 59 | 60 | RUN mkdir ${HOME}/packages/ ${HOME}/localNuget/ 61 | 62 | RUN chown -R ${NB_UID} ${HOME} 63 | USER ${USER} 64 | 65 | # Install Microsoft.DotNet.Interactive 66 | RUN dotnet tool install -g Microsoft.dotnet-interactive --add-source "https://dotnet.myget.org/F/dotnet-try/api/v3/index.json" 67 | 68 | ENV PATH="${PATH}:${HOME}/.dotnet/tools" 69 | RUN echo "$PATH" 70 | 71 | # Install kernel specs 72 | RUN dotnet interactive jupyter install 73 | 74 | # Build extensions 75 | RUN dotnet build ${HOME}/src/Microsoft.ML.DotNet.Interactive.Extensions -c Release 76 | RUN dotnet pack ${HOME}/src/Microsoft.ML.DotNet.Interactive.Extensions -c Release 77 | 78 | RUN dotnet build ${HOME}/src/Microsoft.Data.Analysis.Interactive -c Release 79 | RUN dotnet pack ${HOME}/src/Microsoft.Data.Analysis.Interactive -c Release 80 | 81 | # Publish nuget if there is any 82 | WORKDIR ${HOME}/src/ 83 | RUN dotnet nuget push **/*.nupkg -s ${HOME}/localNuget/ 84 | 85 | RUN rm -fr ${HOME}/src/ 86 | 87 | # install powershell modules 88 | RUN dotnet tool install --global PowerShell 89 | RUN pwsh -c "install-module pskit -force" 90 | RUN pwsh -c "install-module nameit -force" 91 | RUN pwsh -c "install-module psstringscanner -force" 92 | RUN pwsh -c "install-module importexcel -force" 93 | 94 | # Enable telemetry once we install jupyter for the image 95 | ENV DOTNET_TRY_CLI_TELEMETRY_OPTOUT=false 96 | 97 | # Set root to Notebooks 98 | WORKDIR ${HOME}/Notebooks/ 99 | -------------------------------------------------------------------------------- /__tests__/PSKit.TypeData.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Custom Datatypes" { 4 | 5 | BeforeAll { 6 | $data = ConvertFrom-Csv @" 7 | Region,ItemName,TotalSold 8 | South,avocado,5 9 | East,nail,13 10 | South,melon,34 11 | West,drill,5 12 | North,kiwi,48 13 | North,nail,2 14 | North,melon,74 15 | West,hammer,37 16 | East,pear,34 17 | South,screws,71 18 | "@ 19 | } 20 | 21 | It "Should have these data types added" { 22 | $actual = (Get-TypeData -TypeName Array).Members 23 | 24 | 25 | $actual.ContainsKey('DTypes') | Should Be $true 26 | $actual.ContainsKey('columns') | Should Be $true 27 | $actual.ContainsKey('head') | Should Be $true 28 | $actual.ContainsKey('tail') | Should Be $true 29 | $actual.ContainsKey('info') | Should Be $true 30 | $actual.ContainsKey('stats') | Should Be $true 31 | $actual.ContainsKey('query') | Should Be $true 32 | $actual.ContainsKey('GroupAndMeasure') | Should Be $true 33 | $actual.ContainsKey('SetIndex') | Should Be $true 34 | $actual.ContainsKey('ScanProperties') | Should Be $true 35 | } 36 | 37 | It "Should return the correct # of rows from the top" { 38 | $actual = $data.Head(3) 39 | $actual.Count | Should -Be 3 40 | 41 | $actual[0].Region | Should -BeExactly 'South' 42 | $actual[0].ItemName | Should -BeExactly 'avocado' 43 | $actual[0].TotalSold | Should -BeExactly 5 44 | 45 | $actual[1].Region | Should -BeExactly 'East' 46 | $actual[1].ItemName | Should -BeExactly 'nail' 47 | $actual[1].TotalSold | Should -BeExactly 13 48 | 49 | $actual[2].Region | Should -BeExactly 'South' 50 | $actual[2].ItemName | Should -BeExactly 'melon' 51 | $actual[2].TotalSold | Should -BeExactly 34 52 | } 53 | 54 | It "Should return the correct # of rows from the bottom" { 55 | $actual = $data.Tail(3) 56 | $actual.Count | Should -Be 3 57 | 58 | # West, hammer, 37 59 | # East, pear, 34 60 | # South, screws, 71 61 | 62 | $actual[0].Region | Should -BeExactly 'West' 63 | $actual[0].ItemName | Should -BeExactly 'hammer' 64 | $actual[0].TotalSold | Should -BeExactly 37 65 | 66 | $actual[1].Region | Should -BeExactly 'East' 67 | $actual[1].ItemName | Should -BeExactly 'pear' 68 | $actual[1].TotalSold | Should -BeExactly 34 69 | 70 | $actual[2].Region | Should -BeExactly 'South' 71 | $actual[2].ItemName | Should -BeExactly 'screws' 72 | $actual[2].TotalSold | Should -BeExactly 71 73 | } 74 | 75 | It "Should return the correct # of rows and columns from the bottom" { 76 | $actual = $data.Shape() 77 | 78 | $actual[0].Rows | Should -Be 10 79 | $actual[0].Columns | Should -Be 3 80 | } 81 | 82 | It "Should return the correct column names" { 83 | $actual = $data.Columns() 84 | 85 | $actual.Count | Should -Be 3 86 | $actual[0] | Should -BeExactly "Region" 87 | $actual[1] | Should -BeExactly "ItemName" 88 | $actual[2] | Should -BeExactly "TotalSold" 89 | } 90 | 91 | It "Should return the correct datatypes names" { 92 | $actual = $data.DTypes() 93 | 94 | $actual.Count | Should -Be 3 95 | 96 | $actual[0].ColumnName | Should -BeExactly "Region" 97 | $actual[0].DataType | Should -BeExactly "string" 98 | $actual[1].ColumnName | Should -BeExactly "ItemName" 99 | $actual[1].DataType | Should -BeExactly "string" 100 | $actual[2].ColumnName | Should -BeExactly "TotalSold" 101 | $actual[2].DataType | Should -BeExactly "int" 102 | } 103 | } -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive.Extensions/DecisionTreeDataFormatting.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation and contributors. All rights reserved. 2 | // Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | using System; 5 | using System.IO; 6 | using System.Linq; 7 | using System.Text; 8 | using HtmlAgilityPack; 9 | using System.Text.Json; 10 | 11 | namespace Microsoft.ML.DotNet.Interactive 12 | { 13 | public static class DecisionTreeDataFormatting 14 | { 15 | private static readonly JsonSerializerOptions JsonSerializerOptions = new JsonSerializerOptions 16 | { 17 | PropertyNamingPolicy = JsonNamingPolicy.CamelCase, 18 | WriteIndented = true 19 | }; 20 | 21 | internal static string GenerateTreeView(DecisionTreeData tree) 22 | { 23 | var newHtmlDocument = new HtmlDocument(); 24 | 25 | var renderingId = $"a{Guid.NewGuid()}"; 26 | 27 | newHtmlDocument.DocumentNode.ChildNodes.Add(HtmlNode.CreateNode($"")); 28 | newHtmlDocument.DocumentNode.ChildNodes.Add(GetRenderingScript()); 29 | newHtmlDocument.DocumentNode.ChildNodes.Add(GetScriptNodeWithRequire(renderingId, tree)); 30 | 31 | return newHtmlDocument.DocumentNode.WriteContentTo(); 32 | } 33 | 34 | private static HtmlNode GetRenderingScript() 35 | { 36 | var newScript = new StringBuilder(); 37 | newScript.AppendLine(""); 48 | return HtmlNode.CreateNode(newScript.ToString()); 49 | } 50 | 51 | private static HtmlNode GetScriptNodeWithRequire(string renderingId, DecisionTreeData tree) 52 | { 53 | var newScript = new StringBuilder(); 54 | newScript.AppendLine(""); 78 | return HtmlNode.CreateNode(newScript.ToString()); 79 | } 80 | 81 | private static string GenerateData(DecisionTreeData tree) 82 | { 83 | return JsonSerializer.Serialize(tree.Root, options: JsonSerializerOptions); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /InvokeTranspileSQL.ps1: -------------------------------------------------------------------------------- 1 | #requires -Modules PSStringScanner 2 | 3 | function ConvertFrom-SQLToPS { 4 | param( 5 | [Parameter(Mandatory)] 6 | $SQL 7 | ) 8 | 9 | Invoke-TranspileSQL $SQL | ConvertFrom-TranspileSQL 10 | } 11 | 12 | function Add-PSOp { 13 | param($target) 14 | 15 | $target | Add-Member -PassThru -MemberType ScriptProperty -Name PSOp -Value { 16 | switch ($this.operation) { 17 | "<>" { "-ne" } 18 | ">=" { "-ge" } 19 | "<=" { "-le" } 20 | "=" { "-eq" } 21 | ">" { "-gt" } 22 | "<" { "-lt" } 23 | "like" { "-like" } 24 | "match" { "-match" } 25 | default { $_ } 26 | } 27 | } 28 | } 29 | 30 | function Add-PSLogicOp { 31 | param($target) 32 | 33 | $target | Add-Member -PassThru -MemberType ScriptProperty -Name PSLogicOp -Value { 34 | if ($this.LogicOp) { 35 | "-" + $this.LogicOp 36 | } 37 | } 38 | } 39 | 40 | function Invoke-TranspileSQL { 41 | param( 42 | [Parameter(Mandatory)] 43 | $SQL 44 | ) 45 | 46 | $ss = New-PSStringScanner $sql 47 | 48 | $SELECT_KW = "^[Ss][Ee][Ll][Ee][Cc][Tt]\s+" 49 | $FROM_KW = "[Ff][Rr][Oo][Mm]" 50 | $WHERE_KW = "[Ww][Hh][Ee][Rr][Ee]" 51 | $OPERATIONS = "<>|<=|>=|>|<|=|like|match" 52 | $LOGICAL = "[Oo][rR]|[Aa][Nn][Dd]" 53 | $WHITESPACE = "\s+" 54 | 55 | $h = [Ordered]@{ } 56 | 57 | if ($ss.Check($SELECT_KW)) { 58 | $null = $ss.Scan($SELECT_KW) 59 | 60 | $h.SelectPropertyNames = ($ss.ScanUntil("(?=$FROM_KW)")).trim() 61 | 62 | if ($h.SelectPropertyNames.Contains(',')) { 63 | $h.SelectPropertyNames = $h.SelectPropertyNames.Split(',').foreach( { $_.trim() }) 64 | } 65 | 66 | $null = $ss.Skip($FROM_KW) 67 | 68 | if ($ss.Check($WHERE_KW)) { 69 | $h.DataSetName = $ss.ScanUntil("(?=$WHERE_KW)").trim() 70 | $null = $ss.Skip("$WHERE_KW") 71 | 72 | $ssWhere = New-PSStringScanner $ss.Scan(".*") 73 | 74 | $whereResults = @() 75 | 76 | while (!$ssWhere.EoS()) { 77 | $currentResult = [Ordered]@{ } 78 | $currentResult.propertyName = $ssWhere.ScanUntil("(?=$OPERATIONS)").trim() 79 | $currentResult.operation = $ssWhere.Scan($OPERATIONS) 80 | 81 | if ($ssWhere.Check("$($WHITESPACE)$($LOGICAL)")) { 82 | $currentResult.value = $ssWhere.ScanUntil("(?=$($WHITESPACE)$($LOGICAL))") 83 | $currentResult.logicOp = $ssWhere.Scan($LOGICAL) 84 | } 85 | else { 86 | $currentResult.value = $ssWhere.Scan('.*').Trim() 87 | } 88 | 89 | $obj = Add-PSOp ([PSCustomObject]$currentResult) 90 | $obj = Add-PSLogicOp $obj 91 | 92 | $whereResults += [PSCustomObject]$obj 93 | } 94 | } 95 | else { 96 | $h.DataSetName = $ss.Scan(".*").trim() 97 | } 98 | } 99 | 100 | if ($whereResults) { 101 | $h.where = [PSCustomObject[]]$whereResults 102 | } 103 | $h 104 | } 105 | 106 | function ConvertFrom-TranspileSQL { 107 | param( 108 | [Parameter(ValueFromPipeline)] 109 | [System.Collections.Specialized.OrderedDictionary] 110 | $map 111 | ) 112 | 113 | $SelectPropertyNames = $map.SelectPropertyNames 114 | 115 | if ($SelectPropertyNames -ne '*') { 116 | $SelectPropertyNames = $SelectPropertyNames -join '","' 117 | $SelectPropertyNames = '"' + $SelectPropertyNames + '"' 118 | } 119 | 120 | if ($map.Contains("where")) { 121 | $sqlResult += "| Where-Object {" 122 | foreach ($whereRecord in $map.Where) { 123 | $sqlResult += '$_.{0} {1} {2} {3} ' -f $whereRecord.propertyName, $whereRecord.PSOp, $whereRecord.value.trim(), $whereRecord.PSLogicOp 124 | if ($null -eq $whereRecord.PSLogicOp) { $sqlResult = $sqlResult.Trim() } 125 | } 126 | $sqlResult += "}" 127 | } 128 | 129 | $sqlResult += " | Select-Object -Property $($SelectPropertyNames)" 130 | 131 | $sqlResult 132 | } -------------------------------------------------------------------------------- /__tests__/PSKit.GetDateRange.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | Describe "PSKit tests - Get-DateRange" { 3 | 4 | BeforeAll { 5 | $script:fmt = 'yyyy-MM-dd' 6 | } 7 | 8 | It "Should return one date" { 9 | $actual = Get-DateRange 10 | 11 | $actual.count | should be 1 12 | } 13 | 14 | It "Should return date run on" { 15 | $actual = Get-DateRange -periods 1 16 | 17 | $actual.Count | Should be 1 18 | $actual | should be (Get-Date).ToString($fmt) 19 | } 20 | 21 | It "Should return one date" { 22 | $date = '1/1/2020' 23 | $actual = Get-DateRange -start $date -periods 1 24 | 25 | $actual.Count | Should be 1 26 | $actual | should be (Get-Date $date).ToString($fmt) 27 | } 28 | 29 | It "Should return multiple dates" { 30 | $date = '1/1/2020' 31 | $actual = Get-DateRange $date -periods 6 32 | 33 | $actual.Count | Should be 6 34 | $actual[0] | should be (Get-Date $date).AddDays(0).ToString($fmt) 35 | $actual[1] | should be (Get-Date $date).AddDays(1).ToString($fmt) 36 | $actual[2] | should be (Get-Date $date).AddDays(2).ToString($fmt) 37 | $actual[3] | should be (Get-Date $date).AddDays(3).ToString($fmt) 38 | $actual[4] | should be (Get-Date $date).AddDays(4).ToString($fmt) 39 | $actual[5] | should be (Get-Date $date).AddDays(5).ToString($fmt) 40 | } 41 | 42 | It "Should return multiple dates, offset my month" { 43 | $date = '1/1/2020' 44 | $actual = Get-DateRange $date -periods 6 -freq M 45 | 46 | $actual.Count | Should be 6 47 | $actual[0] | should be (Get-Date $date).AddMonths(0).ToString($fmt) 48 | $actual[1] | should be (Get-Date $date).AddMonths(1).ToString($fmt) 49 | $actual[2] | should be (Get-Date $date).AddMonths(2).ToString($fmt) 50 | $actual[3] | should be (Get-Date $date).AddMonths(3).ToString($fmt) 51 | $actual[4] | should be (Get-Date $date).AddMonths(4).ToString($fmt) 52 | $actual[5] | should be (Get-Date $date).AddMonths(5).ToString($fmt) 53 | } 54 | 55 | It "Should return multiple dates, offset my year" { 56 | $date = '1/1/2020' 57 | $actual = Get-DateRange $date -periods 6 -freq Y 58 | 59 | $actual.Count | Should be 6 60 | $actual[0] | should be (Get-Date $date).AddYears(0).ToString($fmt) 61 | $actual[1] | should be (Get-Date $date).AddYears(1).ToString($fmt) 62 | $actual[2] | should be (Get-Date $date).AddYears(2).ToString($fmt) 63 | $actual[3] | should be (Get-Date $date).AddYears(3).ToString($fmt) 64 | $actual[4] | should be (Get-Date $date).AddYears(4).ToString($fmt) 65 | $actual[5] | should be (Get-Date $date).AddYears(5).ToString($fmt) 66 | } 67 | 68 | It "Should return multiple dates, based on end and freq is D" { 69 | $date = '1/1/2020' 70 | $end = '1/5/2020' 71 | 72 | $actual = Get-DateRange $date $end 73 | 74 | $actual.Count | should be 5 75 | 76 | $actual[0] | should be (Get-Date $date).AddDays(0).ToString($fmt) 77 | $actual[1] | should be (Get-Date $date).AddDays(1).ToString($fmt) 78 | $actual[2] | should be (Get-Date $date).AddDays(2).ToString($fmt) 79 | $actual[3] | should be (Get-Date $date).AddDays(3).ToString($fmt) 80 | $actual[4] | should be (Get-Date $date).AddDays(4).ToString($fmt) 81 | } 82 | 83 | It "Should return multiple dates, based on end, respects period and freq is D" { 84 | $date = '1/1/2020' 85 | $end = '1/5/2020' 86 | 87 | $actual = Get-DateRange $date $end -periods 3 88 | 89 | $actual.Count | should be 3 90 | 91 | $actual[0] | should be (Get-Date $date).AddDays(0).ToString($fmt) 92 | $actual[1] | should be (Get-Date $date).AddDays(1).ToString($fmt) 93 | $actual[2] | should be (Get-Date $end).ToString($fmt) 94 | # $actual[3] | should be (Get-Date $date).AddDays(3).ToString($fmt) 95 | # $actual[4] | should be (Get-Date $date).AddDays(4).ToString($fmt) 96 | } 97 | 98 | It "Should return month intervals" { 99 | $date = '1/1/2020' 100 | $end = '1/5/2020' 101 | $periods = 3 102 | 103 | $actual = Get-DateRange $date $end -periods $periods -freq M 104 | 105 | $actual.Count | should be 3 106 | 107 | $actual[0] | should be (Get-Date $date).AddMonths(0).ToString($fmt) 108 | $actual[1] | should be (Get-Date $date).AddMonths(1).ToString($fmt) 109 | $actual[2] | should be '2020-05-01' 110 | } 111 | 112 | It "Should return one date" { 113 | $actual = @(Get-DateRange 1/1) 114 | 115 | $actual.Count | should be 1 116 | $actual[0] | should be '2020-01-01' 117 | } 118 | } -------------------------------------------------------------------------------- /PSKit.psd1: -------------------------------------------------------------------------------- 1 | @{ 2 | 3 | # Script module or binary module file associated with this manifest. 4 | RootModule = 'PSKit.psm1' 5 | 6 | # Version number of this module. 7 | ModuleVersion = '1.4.0' 8 | 9 | # ID used to uniquely identify this module 10 | GUID = '92cd0d76-f455-4049-b129-bede0e1041af' 11 | 12 | # Author of this module 13 | Author = 'Doug Finke' 14 | 15 | # Company or vendor of this module 16 | CompanyName = 'Doug Finke' 17 | 18 | # Copyright statement for this module 19 | Copyright = 'c 2019 All rights reserved.' 20 | 21 | # Description of the functionality provided by this module 22 | Description = @' 23 | A suite of command-line tools for working with PowerShell Arrays. From querying to doing array statistics. 24 | '@ 25 | 26 | # Minimum version of the Windows PowerShell engine required by this module 27 | # PowerShellVersion = '' 28 | 29 | # Name of the Windows PowerShell host required by this module 30 | # PowerShellHostName = '' 31 | 32 | # Minimum version of the Windows PowerShell host required by this module 33 | # PowerShellHostVersion = '' 34 | 35 | # Minimum version of Microsoft .NET Framework required by this module 36 | # DotNetFrameworkVersion = '' 37 | 38 | # Minimum version of the common language runtime (CLR) required by this module 39 | # CLRVersion = '' 40 | 41 | # Processor architecture (None, X86, Amd64) required by this module 42 | # ProcessorArchitecture = '' 43 | 44 | # Modules that must be imported into the global environment prior to importing this module 45 | RequiredModules = @("PSStringScanner") 46 | 47 | # Assemblies that must be loaded prior to importing this module 48 | # RequiredAssemblies = @() 49 | 50 | # Script files (.ps1) that are run in the caller's environment prior to importing this module. 51 | # ScriptsToProcess = @() 52 | 53 | # Type files (.ps1xml) to be loaded when importing this module 54 | # TypesToProcess = @() 55 | 56 | # Format files (.ps1xml) to be loaded when importing this module 57 | # FormatsToProcess = @() 58 | 59 | # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess 60 | # NestedModules = @() 61 | 62 | # Functions to export from this module 63 | FunctionsToExport = "*" 64 | # Cmdlets to export from this module 65 | #CmdletsToExport = '*' 66 | 67 | # Variables to export from this module 68 | #VariablesToExport = '*' 69 | 70 | # Aliases to export from this module 71 | AliasesToExport = "*" 72 | 73 | # List of all modules packaged with this module 74 | # ModuleList = @() 75 | 76 | # List of all files packaged with this module 77 | # FileList = @() 78 | 79 | # Private data to pass to the module specified in RootModule/ModuleToProcess 80 | PrivateData = @{ 81 | # PSData is module packaging and gallery metadata embedded in PrivateData 82 | # It's for rebuilding PowerShellGet (and PoshCode) NuGet-style packages 83 | # We had to do this because it's the only place we're allowed to extend the manifest 84 | # https://connect.microsoft.com/PowerShell/feedback/details/421837 85 | PSData = @{ 86 | # The primary categorization of this module (from the TechNet Gallery tech tree). 87 | Category = "Scripting" 88 | 89 | # Keyword tags to help users find this module via navigations and search. 90 | Tags = @("PowerShell") 91 | 92 | # The web address of an icon which can be used in galleries to represent this module 93 | #IconUri = "http://pesterbdd.com/images/Pester.png" 94 | 95 | # The web address of this module's project or support homepage. 96 | ProjectUri = "https://github.com/dfinke/PSKit" 97 | 98 | # The web address of this module's license. Points to a page that's embeddable and linkable. 99 | LicenseUri = "https://github.com/dfinke/PSKit/blob/master/LICENSE" 100 | 101 | # Release notes for this particular version of the module 102 | #ReleaseNotes = $True 103 | 104 | # If true, the LicenseUrl points to an end-user license (not just a source license) which requires the user agreement before use. 105 | # RequireLicenseAcceptance = "" 106 | 107 | # Indicates this is a pre-release/testing version of the module. 108 | IsPrerelease = 'False' 109 | } 110 | } 111 | 112 | # HelpInfo URI of this module 113 | # HelpInfoURI = '' 114 | 115 | # Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. 116 | # DefaultCommandPrefix = '' 117 | 118 | } -------------------------------------------------------------------------------- /__tests__/PSKit.GroupByAndMeasure.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Group-ByAndMeasure" { 4 | BeforeAll { 5 | $script:data = @" 6 | Region,Item,TotalSold 7 | West,peach,10 8 | West,peach,15 9 | North,orange,20 10 | North,orange,25 11 | South,nail,30 12 | South,nail,35 13 | East,saw,40 14 | East,saw,45 15 | "@ 16 | } 17 | 18 | It "Should calc count" { 19 | $actual = Group-ByAndMeasure -targetData (Read-Csv $data) -GroupBy Region -MeasureProperty TotalSold -MeasureOperation Count 20 | 21 | $actual.Count | should be 4 22 | 23 | $r = $actual | Where-Object region -CContains 'West' 24 | $r.Region | should beexactly 'West' 25 | $r.Count | should be 2 26 | 27 | $r = $actual | Where-Object region -CContains 'North' 28 | $r.Region | should beexactly 'North' 29 | $r.Count | should be 2 30 | 31 | $r = $actual | Where-Object region -CContains 'South' 32 | $r.Region | should beexactly 'South' 33 | $r.Count | should be 2 34 | 35 | $r = $actual | Where-Object region -CContains 'East' 36 | $r.Region | should beexactly 'East' 37 | $r.Count | should be 2 38 | 39 | $names = $actual[0].psobject.properties.name 40 | $names.count | should be 2 41 | $names[0] | should beexactly 'Region' 42 | $names[1] | should beexactly 'Count' 43 | } 44 | 45 | It "Should calc Average" { 46 | $actual = Group-ByAndMeasure -targetData (Read-Csv $data) -GroupBy Region -MeasureProperty TotalSold -MeasureOperation Average 47 | 48 | $actual.Count | should be 4 49 | 50 | $r = $actual | Where-Object region -CContains 'West' 51 | $r.Region | should beexactly 'West' 52 | $r.Average | should be 12.5 53 | 54 | $r = $actual | Where-Object region -CContains 'North' 55 | $r.Region | should beexactly 'North' 56 | $r.Average | should be 22.5 57 | 58 | $r = $actual | Where-Object region -CContains 'South' 59 | $r.Region | should beexactly 'South' 60 | $r.Average | should be 32.5 61 | 62 | $r = $actual | Where-Object region -CContains 'East' 63 | $r.Region | should beexactly 'East' 64 | $r.Average | should be 42.5 65 | 66 | $names = $actual[0].psobject.properties.name 67 | $names.count | should be 2 68 | $names[0] | should beexactly 'Region' 69 | $names[1] | should beexactly 'Average' 70 | } 71 | 72 | It "Should calc Sum" { 73 | $actual = Group-ByAndMeasure -targetData (Read-Csv $data) -GroupBy Region -MeasureProperty TotalSold -MeasureOperation Sum 74 | 75 | $actual.Count | should be 4 76 | 77 | $r = $actual | Where-Object region -CContains 'West' 78 | $r.Region | should beexactly 'West' 79 | $r.Sum | should be 25 80 | 81 | $r = $actual | Where-Object region -CContains 'North' 82 | $r.Region | should beexactly 'North' 83 | $r.Sum | should be 45 84 | 85 | $r = $actual | Where-Object region -CContains 'South' 86 | $r.Region | should beexactly 'South' 87 | $r.Sum | should be 65 88 | 89 | $r = $actual | Where-Object region -CContains 'East' 90 | $r.Region | should beexactly 'East' 91 | $r.Sum | should be 85 92 | 93 | $names = $actual[0].psobject.properties.name 94 | $names.count | should be 2 95 | $names[0] | should beexactly 'Region' 96 | $names[1] | should beexactly 'Sum' 97 | } 98 | 99 | It "Should calc Minimum" { 100 | $actual = Group-ByAndMeasure -targetData (Read-Csv $data) -GroupBy Region -MeasureProperty TotalSold -MeasureOperation Minimum 101 | 102 | $actual.Count | should be 4 103 | 104 | $r = $actual | Where-Object region -CContains 'West' 105 | $r.Region | should beexactly 'West' 106 | $r.Minimum | should be 10 107 | 108 | $r = $actual | Where-Object region -CContains 'North' 109 | $r.Region | should beexactly 'North' 110 | $r.Minimum | should be 20 111 | 112 | $r = $actual | Where-Object region -CContains 'South' 113 | $r.Region | should beexactly 'South' 114 | $r.Minimum | should be 30 115 | 116 | $r = $actual | Where-Object region -CContains 'East' 117 | $r.Region | should beexactly 'East' 118 | $r.Minimum | should be 40 119 | 120 | $names = $actual[0].psobject.properties.name 121 | $names.count | should be 2 122 | $names[0] | should beexactly 'Region' 123 | $names[1] | should beexactly 'Minimum' 124 | } 125 | 126 | It "Should calc Maximum" { 127 | $actual = Group-ByAndMeasure -targetData (Read-Csv $data) -GroupBy Region -MeasureProperty TotalSold -MeasureOperation Maximum 128 | 129 | $actual.Count | should be 4 130 | 131 | $r = $actual | Where-Object region -CContains 'West' 132 | $r.Region | should beexactly 'West' 133 | $r.Maximum | should be 15 134 | 135 | $r = $actual | Where-Object region -CContains 'North' 136 | $r.Region | should beexactly 'North' 137 | $r.Maximum | should be 25 138 | 139 | $r = $actual | Where-Object region -CContains 'South' 140 | $r.Region | should beexactly 'South' 141 | $r.Maximum | should be 35 142 | 143 | $r = $actual | Where-Object region -CContains 'East' 144 | $r.Region | should beexactly 'East' 145 | $r.Maximum | should be 45 146 | 147 | $names = $actual[0].psobject.properties.name 148 | $names.count | should be 2 149 | $names[0] | should beexactly 'Region' 150 | $names[1] | should beexactly 'Maximum' 151 | } 152 | } -------------------------------------------------------------------------------- /__tests__/ConvertFromSSV.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | 3 | Describe "PSKit tests - Test ConverFrom-SSV" { 4 | It "Should be docker data" { 5 | $data = @" 6 | NAME LABELS SELECTOR IP PORT(S) 7 | docker-registry docker-registry=default docker-registry=default 172.30.78.158 5000/TCP 8 | kubernetes component=apiserver,provider=kubernetes 172.30.0.2 443/TCP 9 | kubernetes-ro component=apiserver,provider=kubernetes 172.30.0.1 80/TCP 10 | "@ 11 | 12 | # $actual = $data -split "`n" | ConvertFrom-SSV | Select-Object -Skip 1 -First 1 13 | $actual = $data -split "`n" | ConvertFrom-SSV 14 | $expected = "172.30.0.2" 15 | 16 | $actual[1].IP | should be $expected 17 | } 18 | 19 | It "Should parse simple data with blank lines" { 20 | $data = @" 21 | a b 22 | 23 | 1 2 24 | 25 | 3 4 26 | "@ 27 | 28 | $actual = $data -split "`n" | ConvertFrom-SSV 29 | 30 | $actual[0].a | should be 1 31 | $actual[1].b | should be 4 32 | } 33 | 34 | It "Should parse" { 35 | $data = @" 36 | a 37 | 1 38 | 2 39 | "@ 40 | $actual = $data -split "`n" | ConvertFrom-SSV 41 | 42 | $actual[0].a | should be 1 43 | $actual[1].a | should be 2 44 | } 45 | 46 | It "Should parse pulumi data" { 47 | $data = @" 48 | NAME LAST UPDATE RESOURCE COUNT URL 49 | dfinke/azure-functions-raw/dev 4 months ago 0 https://app.pulumi.com/dfinke/azure-functions-raw/dev 50 | dfinke/azure-py-webserver-component/pulumicomponent 2 months ago 0 https://app.pulumi.com/dfinke/azure-py-webserver-component/pulumicomponent 51 | dfinke/foo/dev 3 months ago 0 https://app.pulumi.com/dfinke/foo/dev 52 | dfinke/guestbook-csharp/kubernetes-cs n/a n/a https://app.pulumi.com/dfinke/guestbook-csharp/kubernetes-cs 53 | dfinke/kata/dev 4 months ago 0 https://app.pulumi.com/dfinke/kata/dev 54 | "@ 55 | $actual = $data -split "`n" | ConvertFrom-SSV 56 | $expected = "4 months ago" 57 | 58 | $actual[4].'LAST UPDATE' | should be $expected 59 | } 60 | 61 | It "Should parse data with whitespace in the begining" { 62 | $data = @" 63 | PID TTY TIME CMD 64 | 103 pts/0 00:00:00 bash 65 | 136 pts/0 00:00:13 pwsh 66 | 305 pts/0 00:00:00 ps 67 | 68 | "@ 69 | 70 | $actual = $data -split "`n" | ConvertFrom-SSV -min 1 71 | $names = $actual[0].psobject.properties.name 72 | 73 | $names.Count | Should Be 4 74 | $names[0] | Should BeExactly "PID" 75 | $names[1] | Should BeExactly "TTY" 76 | $names[2] | Should BeExactly "TIME" 77 | $names[3] | Should BeExactly "CMD" 78 | 79 | $actual[0].PID | Should Be 103 80 | $actual[1].PID | Should Be 136 81 | $actual[2].PID | Should Be 305 82 | 83 | $actual[0].TTY | Should Be "pts/0" 84 | $actual[1].TTY | Should Be "pts/0" 85 | $actual[2].TTY | Should Be "pts/0" 86 | 87 | $actual[0].TIME | Should Be "00:00:00" 88 | $actual[1].TIME | Should Be "00:00:13" 89 | $actual[2].TIME | Should Be "00:00:00" 90 | 91 | $actual[0].CMD | Should Be "bash" 92 | $actual[1].CMD | Should Be "pwsh" 93 | $actual[2].CMD | Should Be "ps" 94 | } 95 | 96 | It "Should multi whitespace" { 97 | $data = @" 98 | column a column b 99 | entry 1 entry number 2 100 | 3 four 101 | "@ 102 | 103 | $actual = $data -split "`n" | ConvertFrom-SSV -MinimumWhiteSpaceLength 3 104 | 105 | $names = $actual[0].psobject.properties.name 106 | $actual.count | should be 2 107 | 108 | $names[0] | should beexactly 'column a' 109 | $names[1] | should beexactly 'column b' 110 | 111 | $actual[0].'column a' | should beexactly 'entry 1' 112 | $actual[1].'column a' | should beexactly 3 113 | 114 | $actual[0].'column b' | should beexactly 'entry number 2' 115 | $actual[1].'column b' | should beexactly 'four' 116 | } 117 | 118 | It "Should multi whitespace and single row" { 119 | $data = @" 120 | colA colB colC 121 | val1 val2 val3 122 | "@ 123 | $actual = @($data -split "`n" | ConvertFrom-SSV -MinimumWhiteSpaceLength 3) 124 | $names = $actual[0].psobject.properties.name 125 | 126 | $actual.Count | Should be 1 127 | $names.Count | Should be 3 128 | 129 | $actual[0].colA | Should beexactly 'val1' 130 | $actual[0].colB | Should beexactly 'val2' 131 | $actual[0].colC | Should beexactly 'val3' 132 | } 133 | 134 | It "Should multi whitespace and single row" -Skip { 135 | $data = @" 136 | colA col B col C 137 | val2 val3 138 | val4 val 5 val 6 139 | val7 val8 140 | 141 | "@ 142 | $actual = @($data -split "`n" | ConvertFrom-SSV -MinimumWhiteSpaceLength 3) 143 | $names = $actual[0].psobject.properties.name 144 | 145 | $names.Count | Should be 3 146 | 147 | } 148 | 149 | It "Should handle trailing values" -Skip { 150 | $data = @" 151 | colA col B 152 | val1 val2 trailing value that should be included 153 | "@ 154 | $actual = @($data -split "`n" | ConvertFrom-SSV ) 155 | $names = $actual[0].psobject.properties.name 156 | 157 | $actual.Count | Should be 1 158 | $names.Count | Should be 2 159 | 160 | $actual[0].colA | Should beexactly 'val1' 161 | #$actual[0].colB | Should beexactly 'val2' 162 | } 163 | 164 | } -------------------------------------------------------------------------------- /PSKitAndMore.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/html": [ 11 | "
PROGRESS: Completed.
\n" 12 | ] 13 | }, 14 | "metadata": {}, 15 | "output_type": "display_data" 16 | }, 17 | { 18 | "data": { 19 | "text/html": [ 20 | "
PROGRESS: Completed.
\n" 21 | ] 22 | }, 23 | "metadata": {}, 24 | "output_type": "display_data" 25 | }, 26 | { 27 | "data": { 28 | "text/html": [ 29 | "
PROGRESS: Completed.
\n" 30 | ] 31 | }, 32 | "metadata": {}, 33 | "output_type": "display_data" 34 | } 35 | ], 36 | "source": [ 37 | "Install-Module pskit -force\n", 38 | "Install-Module nameit -force\n", 39 | "\n", 40 | "Import-Module nameit\n" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "function ConvertTo-MarkdownTable {\n", 50 | " param($targetData)\n", 51 | "\n", 52 | " $names = $targetData[0].psobject.Properties.name \n", 53 | "\n", 54 | " $all = @()\n", 55 | " 1..$names.count | foreach {\n", 56 | " if($_ -eq $names.count) {\n", 57 | " $all += '|'\n", 58 | " } else {\n", 59 | " $all += '|---'\n", 60 | " }\n", 61 | " }\n", 62 | "\n", 63 | " $result = foreach($record in $targetData) {\n", 64 | " $inner=@()\n", 65 | " foreach($name in $names) { \n", 66 | " $inner+=$record.$name\n", 67 | " } \n", 68 | " '|' + ($inner -join '|') + '|' + \"`n\"\n", 69 | " }\n", 70 | " \n", 71 | "@\"\n", 72 | "$('|' + ($names -join '|') + '|')\n", 73 | "$($all)\n", 74 | "$($result)\n", 75 | "\"@ | ConvertFrom-Markdown | % html | Get-HtmlContent | Out-Display\n", 76 | "}" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 3, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "data": { 86 | "text/html": [ 87 | "\n", 88 | "\n", 89 | "\n", 90 | "\n", 91 | "\n", 92 | "\n", 93 | "\n", 94 | "\n", 95 | "\n", 96 | "\n", 97 | "\n", 98 | "\n", 99 | "\n", 100 | "\n", 101 | "\n", 102 | "\n", 103 | "\n", 104 | "\n", 105 | "\n", 106 | "\n", 107 | "\n", 108 | "\n", 109 | "\n", 110 | "\n", 111 | "\n", 112 | "\n", 113 | "\n", 114 | "\n", 115 | "\n", 116 | "\n", 117 | "\n", 118 | "\n", 119 | "\n", 120 | "\n", 121 | "\n", 122 | "\n", 123 | "\n", 124 | "\n", 125 | "\n", 126 | "\n", 127 | "\n", 128 | "\n", 129 | "\n", 130 | "\n", 131 | "\n", 132 | "
CostDateName
1.11/1/2015John
2.11/2/2015Tom
5.11/2/2015Dick
11.11/2/2015Harry
7.11/2/2015Jane
22.11/2/2015Mary
32.11/2/2015Liz
\n" 133 | ] 134 | }, 135 | "metadata": {}, 136 | "output_type": "display_data" 137 | } 138 | ], 139 | "source": [ 140 | "ConvertTo-MarkdownTable (Read-Csv https://raw.githubusercontent.com/dfinke/ImportExcel/master/Examples/JustCharts/TargetData.csv)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 6, 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "data": { 150 | "text/html": [ 151 | "\n", 152 | "\n", 153 | "\n", 154 | "\n", 155 | "\n", 156 | "\n", 157 | "\n", 158 | "\n", 159 | "\n", 160 | "\n", 161 | "\n", 162 | "\n", 163 | "\n", 164 | "\n", 165 | "\n", 166 | "\n", 167 | "\n", 168 | "\n", 169 | "\n", 170 | "\n", 171 | "\n", 172 | "\n", 173 | "\n", 174 | "\n", 175 | "\n", 176 | "\n", 177 | "\n", 178 | "\n", 179 | "\n", 180 | "\n", 181 | "\n", 182 | "\n", 183 | "\n", 184 | "\n", 185 | "\n", 186 | "\n", 187 | "\n", 188 | "\n", 189 | "\n", 190 | "\n", 191 | "\n", 192 | "\n", 193 | "\n", 194 | "\n", 195 | "\n", 196 | "\n", 197 | "\n", 198 | "
Indexpersonstateverbnoun
2020-01-01London OconnorKentuckyIndicateZebra
2020-02-01Kamari AcevedoIowaFlowFoundation
2020-03-01Martha BarnesIowaDetailedComparison
2020-04-01Joselyn LucasLouisianaEngageWill
2020-05-01Zechariah BarronNorth DakotaSupposeBoss
\n" 199 | ] 200 | }, 201 | "metadata": {}, 202 | "output_type": "display_data" 203 | } 204 | ], 205 | "source": [ 206 | "ConvertTo-MarkdownTable (New-DataFrame (Get-DateRange 1/1 -periods 5 -freq M) person,state,verb,noun {invoke-generate \"$args\"})" 207 | ] 208 | } 209 | ], 210 | "metadata": { 211 | "kernelspec": { 212 | "display_name": ".NET (PowerShell)", 213 | "language": "PowerShell", 214 | "name": ".net-powershell" 215 | }, 216 | "language_info": { 217 | "file_extension": ".ps1", 218 | "mimetype": "text/x-powershell", 219 | "name": "PowerShell", 220 | "pygments_lexer": "powershell", 221 | "version": "7.0" 222 | } 223 | }, 224 | "nbformat": 4, 225 | "nbformat_minor": 4 226 | } 227 | -------------------------------------------------------------------------------- /__tests__/PSKit.InvokeTranspileSQL.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module $PSScriptRoot/../PSKit.psd1 -Force 2 | Describe "PSKit tests - Invoke-TranspileSQL" { 3 | It "Should have a select and from value" { 4 | $actual = Invoke-TranspileSQL "Select * FROM X" 5 | 6 | @($actual.SelectPropertyNames).Count | Should Be 1 7 | $actual.SelectPropertyNames | Should BeExactly '*' 8 | $actual.DataSetName | Should BeExactly 'X' 9 | } 10 | 11 | It "Should have 2 select values and a from value" { 12 | $actual = Invoke-TranspileSQL "Select cash, name FROM X" 13 | 14 | @($actual.SelectPropertyNames).Count | Should Be 2 15 | $actual.SelectPropertyNames | Should BeExactly 'cash', 'name' 16 | $actual.DataSetName | Should BeExactly 'X' 17 | } 18 | 19 | It "Should have 2 select values and a from value even with empty Where clause" { 20 | $actual = Invoke-TranspileSQL "Select cash, name FROM X" 21 | 22 | @($actual.SelectPropertyNames).Count | Should Be 2 23 | $actual.SelectPropertyNames | Should BeExactly 'cash', 'name' 24 | $actual.DataSetName | Should BeExactly 'X' 25 | } 26 | 27 | It "Should have 2 select values and a from value and a Where clause" { 28 | $actual = Invoke-TranspileSQL "Select cash, name FROM X Where age >= 44 and age <= 50" 29 | 30 | @($actual.SelectPropertyNames).Count | Should Be 2 31 | $actual.SelectPropertyNames | Should BeExactly 'cash', 'name' 32 | $actual.DataSetName | Should BeExactly 'X' 33 | 34 | $actual.where.Count | Should Be 2 35 | 36 | $actual.where[0].propertyName | Should BeExactly 'age' 37 | $actual.where[0].operation | Should Be '>=' 38 | $actual.where[0].value | Should BeExactly 44 39 | $actual.where[0].logicOp | Should BeExactly 'and' 40 | $actual.where[0].PSOp | Should BeExactly '-ge' 41 | $actual.where[0].PSLogicOp | Should BeExactly '-and' 42 | 43 | $actual.where[1].propertyName | Should BeExactly 'age' 44 | $actual.where[1].operation | Should Be '<=' 45 | $actual.where[1].value | Should BeExactly 50 46 | $actual.where[1].logicOp | Should Be $null 47 | $actual.where[1].PSOp | Should BeExactly '-le' 48 | $actual.where[1].PSLogicOp | Should Be $null 49 | } 50 | } 51 | 52 | Describe "PSKit tests - ConvertFrom-TranspileSQL - Select" { 53 | It "Should translate *" { 54 | $actual = Invoke-TranspileSQL "Select * FROM X" | ConvertFrom-TranspileSQL 55 | $actual | Should BeExactly ' | Select-Object -Property *' 56 | } 57 | 58 | It "Should translate a select value" { 59 | $actual = Invoke-TranspileSQL "Select cash FROM X" | ConvertFrom-TranspileSQL 60 | 61 | $actual | Should BeExactly ' | Select-Object -Property "cash"' 62 | } 63 | 64 | It "Should translate many select values" { 65 | $actual = Invoke-TranspileSQL "Select cash, name FROM X" | ConvertFrom-TranspileSQL 66 | 67 | $actual | Should BeExactly ' | Select-Object -Property "cash","name"' 68 | } 69 | } 70 | 71 | Describe "PSKit tests - ConvertFrom-TranspileSQL - Where" { 72 | 73 | It "Should translate select and where =" { 74 | $actual = Invoke-TranspileSQL "Select * FROM X where age = 34" | ConvertFrom-TranspileSQL 75 | $actual | Should BeExactly '| Where-Object {$_.age -eq 34} | Select-Object -Property *' 76 | } 77 | 78 | It "Should translate select and where >" { 79 | $actual = Invoke-TranspileSQL "Select * FROM X where age > 34" | ConvertFrom-TranspileSQL 80 | $actual | Should BeExactly '| Where-Object {$_.age -gt 34} | Select-Object -Property *' 81 | } 82 | 83 | It "Should translate select and where <" { 84 | $actual = Invoke-TranspileSQL "Select * FROM X where age < 34" | ConvertFrom-TranspileSQL 85 | $actual | Should BeExactly '| Where-Object {$_.age -lt 34} | Select-Object -Property *' 86 | } 87 | 88 | It "Should translate select and where >=" { 89 | $actual = Invoke-TranspileSQL "Select * FROM X where age >= 34" | ConvertFrom-TranspileSQL 90 | $actual | Should BeExactly '| Where-Object {$_.age -ge 34} | Select-Object -Property *' 91 | } 92 | 93 | It "Should translate select and where <=" { 94 | $actual = Invoke-TranspileSQL "Select * FROM X where age <= 34" | ConvertFrom-TranspileSQL 95 | $actual | Should BeExactly '| Where-Object {$_.age -le 34} | Select-Object -Property *' 96 | } 97 | 98 | It "Should translate select and where <>" { 99 | $actual = Invoke-TranspileSQL "Select * FROM X where age <> 34" | ConvertFrom-TranspileSQL 100 | $actual | Should BeExactly "| Where-Object {`$_.age -ne 34} | Select-Object -Property *" 101 | } 102 | 103 | It "Should translate select and where <> 'abc" { 104 | $actual = Invoke-TranspileSQL "Select * FROM X where age <> 'abc'" | ConvertFrom-TranspileSQL 105 | $actual | Should BeExactly "| Where-Object {`$_.age -ne 'abc'} | Select-Object -Property *" 106 | } 107 | 108 | It "Should translate select and where like 'abc" { 109 | $actual = Invoke-TranspileSQL "Select * FROM X where name like 'chris'" | ConvertFrom-TranspileSQL 110 | $actual | Should BeExactly "| Where-Object {`$_.name -like 'chris'} | Select-Object -Property *" 111 | } 112 | 113 | It "Should translate select and where match '^chris$" { 114 | $actual = Invoke-TranspileSQL "Select * FROM X where name match '^chris$'" | ConvertFrom-TranspileSQL 115 | $actual | Should BeExactly "| Where-Object {`$_.name -match '^chris$'} | Select-Object -Property *" 116 | } 117 | } 118 | 119 | Describe "PSKit tests - ConvertFrom-TranspileSQL - Multiple items in where clause" { 120 | 121 | It "Should translate select and where > and <" { 122 | $actual = Invoke-TranspileSQL "Select * FROM X where age > 34 and age < 70" | ConvertFrom-TranspileSQL 123 | $actual | Should BeExactly '| Where-Object {$_.age -gt 34 -and $_.age -lt 70} | Select-Object -Property *' 124 | } 125 | 126 | It "Should translate multi select and where > and <" { 127 | $actual = Invoke-TranspileSQL "Select cash, name FROM X where age > 34 and age < 70" | ConvertFrom-TranspileSQL 128 | $actual | Should BeExactly '| Where-Object {$_.age -gt 34 -and $_.age -lt 70} | Select-Object -Property "cash","name"' 129 | } 130 | 131 | } -------------------------------------------------------------------------------- /src/Microsoft.ML.DotNet.Interactive.Extensions/RegressionTree.js: -------------------------------------------------------------------------------- 1 | var dnRegressionTree = (function () { 2 | const blockHeight = 60; 3 | const blockWidth = 250; 4 | const dotSize = 10; 5 | 6 | function renderRegressionTree(renderTarget, regressionTree, d3) { 7 | 8 | 9 | let root = d3.hierarchy(regressionTree); 10 | let treeSize = getTreeBoundaries(root); 11 | 12 | let margin = { top: 20, right: 120, bottom: 20, left: 180 }; 13 | let width = treeSize[1] - margin.right - margin.left; 14 | let height = treeSize[0] - margin.top - margin.bottom; 15 | 16 | let viewBox = [0, 0, getDepth(root) * width / 8 + margin.right + margin.left, height + margin.top + margin.bottom]; 17 | renderTarget 18 | .attr("viewBox", `${viewBox[0]} ${viewBox[1]} ${viewBox[2]} ${viewBox[3]}`) 19 | .append("g") 20 | .attr("class", "rootTransform") 21 | .attr("transform", `translate(${0},${0} )`); 22 | 23 | let rootTransform = renderTarget.select("g"); 24 | 25 | let zoom = d3.zoom().on("zoom", () => { 26 | rootTransform.attr("transform", d3.event.transform); 27 | }); 28 | 29 | renderTarget.call(zoom); 30 | 31 | let toolTip = createToolTip(renderTarget); 32 | 33 | 34 | let treeLayout = d3 35 | .tree() 36 | .size(treeSize); 37 | 38 | 39 | root.dx = blockHeight / 2; 40 | root.dy = blockWidth * 1.5; 41 | 42 | let id = 0; 43 | root.eachBefore(c => c.id = id++); 44 | 45 | rootTransform 46 | .append("g") 47 | .attr("class", "linkLayer"); 48 | 49 | rootTransform.append("g") 50 | .attr("class", "nodeLayer") 51 | .attr("stroke-linejoin", "round") 52 | .attr("stroke-width", 3); 53 | 54 | root.children.forEach(collapse); 55 | update(root, rootTransform, d3); 56 | let currentTreeSize = getTreeBoundaries(root); 57 | let initialTranform = d3.zoomIdentity.translate(currentTreeSize[0] / 2, currentTreeSize[1] / 2).scale((treeSize[0] / currentTreeSize[0]) * 0.15); 58 | 59 | zoom.transform(rootTransform, initialTranform); 60 | renderTarget.property("__zoom", initialTranform); 61 | } 62 | 63 | function collapse(d) { 64 | if (d.children) { 65 | d._children = d.children; 66 | d._children.forEach(collapse); 67 | d.children = null; 68 | } 69 | } 70 | 71 | function expand(d) { 72 | if (d._children) { 73 | d.children = d._children; 74 | d.children.forEach(expand); 75 | d._children = null; 76 | } 77 | } 78 | 79 | 80 | function toggleChildren(d) { 81 | if (d.children) { 82 | d._children = d.children; 83 | d.children = null; 84 | } else if (d._children) { 85 | d.children = d._children; 86 | d._children = null; 87 | } 88 | return d; 89 | } 90 | 91 | 92 | function getDepth(treeNode) { 93 | let depth = 0; 94 | if (treeNode.children) { 95 | treeNode.children.forEach((d) => { 96 | var tmpDepth = getDepth(d); 97 | if (tmpDepth > depth) { 98 | depth = tmpDepth; 99 | } 100 | }); 101 | } 102 | return 1 + depth; 103 | } 104 | 105 | function getTreeBoundaries(treeNode) { 106 | return [count_leaves(treeNode) * blockHeight * 1.7, getDepth(treeNode) * blockWidth * 1.3]; 107 | } 108 | 109 | function count_leaves(treeNode) { 110 | let count = 0; 111 | function count_leaves_r(node) { 112 | if (node.children) { 113 | //go through all its children 114 | for (var i = 0; i < node.children.length; i++) { 115 | //if the current child in the for loop has children of its own 116 | //call recurse again on it to decend the whole tree 117 | if (node.children[i].children) { 118 | count_leaves_r(node.children[i]); 119 | } 120 | //if not then it is a leaf so we count it 121 | else { 122 | count++; 123 | } 124 | } 125 | } 126 | } 127 | 128 | count_leaves_r(treeNode); 129 | 130 | return count; 131 | 132 | } 133 | 134 | function createToolTip(renderTarget) { 135 | 136 | } 137 | 138 | function rightRoundedRect(x, y, width, height, radius) { 139 | return "M" + x + "," + y 140 | + "h" + (width - radius) 141 | + "a" + radius + "," + radius + " 0 0 1 " + radius + "," + radius 142 | + "v" + (height - 2 * radius) 143 | + "a" + radius + "," + radius + " 0 0 1 " + -radius + "," + radius 144 | + "h" + (radius - width) 145 | + "z"; 146 | } 147 | 148 | function updateLinks(root, renderTarget) { 149 | let offset = blockWidth; 150 | let internalOffset = root.dy; 151 | 152 | let link = renderTarget 153 | .select("g.linkLayer") 154 | .attr("fill", "none") 155 | .attr("stroke", "#555") 156 | .attr("stroke-opacity", 0.4) 157 | .attr("stroke-width", 1.5) 158 | .selectAll("path") 159 | .data(root.links(), d => `${d.source.id}_${d.target.id}`) 160 | .join("path") 161 | .attr("d", d => { 162 | return ` 163 | M${d.target.y},${d.target.x} 164 | C${d.source.y + internalOffset},${d.target.x} 165 | ${d.source.y + internalOffset},${d.source.x} 166 | ${d.source.y + offset},${d.source.x} 167 | `; 168 | }); 169 | } 170 | 171 | function updateNodes(root, renderTarget, d3) { 172 | let node = renderTarget 173 | .select("g.nodeLayer") 174 | .selectAll("g.nodeRootTransform") 175 | .data(root.descendants(), d => d.id) 176 | .join("g") 177 | .attr("class", "nodeRootTransform") 178 | .attr("transform", d => `translate(${d.y + blockWidth / 2},${d.x})`); 179 | 180 | // root 181 | node 182 | .append("g") 183 | .attr("class", "nodeExpander") 184 | .attr("transform", d => `translate(${blockWidth / 2},0)`) 185 | .append("circle") 186 | .attr("strong", "black") 187 | .attr("fill", d => d.children ? "#555" : "#999") 188 | .attr("r", d => (d.children || d._children) ? dotSize : 0) 189 | .on("click", d => { 190 | toggleChildren(d); 191 | update(root, renderTarget, d3); 192 | }); 193 | 194 | 195 | let strokeSize = 1; 196 | let boxX = -(blockWidth / 2); 197 | let boxy = -(blockHeight / 2); 198 | let boxW = blockWidth; 199 | let boxH = blockHeight; 200 | let boxR = (blockHeight / 4); 201 | 202 | // node block 203 | node 204 | .append("path") 205 | .attr("class", "nodeBlock") 206 | .attr("d", rightRoundedRect(boxX, boxy, boxW, boxH, boxR)) 207 | .attr("fill", "white") 208 | .attr("stroke", "black").on("click", d => { 209 | toggleChildren(d); 210 | update(root, renderTarget, d3); 211 | }); 212 | 213 | 214 | // data flow part 215 | node 216 | .append("path") 217 | .attr("class", "dataFlow") 218 | .attr("d", rightRoundedRect(boxX + strokeSize, boxy + strokeSize, boxW - (2 * strokeSize), boxH - (2 * strokeSize), boxR)) 219 | .attr("fill", "teal") 220 | .attr("style", d => `clip-path: inset( 0% 0% 0% ${(1 - d.data.data) * 100}% );`); 221 | 222 | node 223 | .append("text") 224 | .attr("class", "nodeText") 225 | .attr("user-select", "none") 226 | .attr("dy", "0.31em") 227 | .attr("dx", blockWidth * 0.4) 228 | .text(d => d.data.label ? d.data.label : d.data.value) 229 | .attr("text-anchor", "end") 230 | .attr("stroke-width", "1px") 231 | .attr("stroke", "white") 232 | .clone(true) 233 | .attr("stroke-width", "none") 234 | .attr("stroke", "none"); 235 | 236 | 237 | } 238 | function update(root, renderTarget, d3) { 239 | let treeSize = getTreeBoundaries(root); 240 | let treeLayout = d3.tree().size(treeSize); 241 | treeLayout(root); 242 | updateLinks(root, renderTarget); 243 | updateNodes(root, renderTarget, d3); 244 | } 245 | 246 | return { 247 | render: renderRegressionTree 248 | }; 249 | })(); 250 | -------------------------------------------------------------------------------- /data/baseball.csv: -------------------------------------------------------------------------------- 1 | id,player,year,stint,team,lg,g,ab,r,h,X2b,X3b,hr,rbi,sb,cs,bb,so,ibb,hbp,sh,sf,gidp 2 | 88641,womacto01,2006,2,CHN,NL,19,50,6,14,1,0,1,2.0,1.0,1.0,4,4.0,0.0,0.0,3.0,0.0,0.0 3 | 88643,schilcu01,2006,1,BOS,AL,31,2,0,1,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 4 | 88645,myersmi01,2006,1,NYA,AL,62,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 5 | 88649,helliri01,2006,1,MIL,NL,20,3,0,0,0,0,0,0.0,0.0,0.0,0,2.0,0.0,0.0,0.0,0.0,0.0 6 | 88650,johnsra05,2006,1,NYA,AL,33,6,0,1,0,0,0,0.0,0.0,0.0,0,4.0,0.0,0.0,0.0,0.0,0.0 7 | 88652,finlest01,2006,1,SFN,NL,139,426,66,105,21,12,6,40.0,7.0,0.0,46,55.0,2.0,2.0,3.0,4.0,6.0 8 | 88653,gonzalu01,2006,1,ARI,NL,153,586,93,159,52,2,15,73.0,0.0,1.0,69,58.0,10.0,7.0,0.0,6.0,14.0 9 | 88662,seleaa01,2006,1,LAN,NL,28,26,2,5,1,0,0,0.0,0.0,0.0,1,7.0,0.0,0.0,6.0,0.0,1.0 10 | 89177,francju01,2007,2,ATL,NL,15,40,1,10,3,0,0,8.0,0.0,0.0,4,10.0,1.0,0.0,0.0,1.0,1.0 11 | 89178,francju01,2007,1,NYN,NL,40,50,7,10,0,0,1,8.0,2.0,1.0,10,13.0,0.0,0.0,0.0,1.0,1.0 12 | 89330,zaungr01,2007,1,TOR,AL,110,331,43,80,24,1,10,52.0,0.0,0.0,51,55.0,8.0,2.0,1.0,6.0,9.0 13 | 89333,witasja01,2007,1,TBA,AL,3,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 14 | 89334,williwo02,2007,1,HOU,NL,33,59,3,6,0,0,1,2.0,0.0,0.0,0,25.0,0.0,0.0,5.0,0.0,1.0 15 | 89335,wickmbo01,2007,2,ARI,NL,8,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 16 | 89336,wickmbo01,2007,1,ATL,NL,47,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 17 | 89337,whitero02,2007,1,MIN,AL,38,109,8,19,4,0,4,20.0,0.0,0.0,6,19.0,0.0,3.0,0.0,1.0,2.0 18 | 89338,whiteri01,2007,1,HOU,NL,20,1,0,0,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 19 | 89339,wellsda01,2007,2,LAN,NL,7,15,2,4,1,0,0,1.0,0.0,0.0,0,6.0,0.0,0.0,0.0,0.0,0.0 20 | 89340,wellsda01,2007,1,SDN,NL,22,38,1,4,0,0,0,0.0,0.0,0.0,0,12.0,0.0,0.0,4.0,0.0,0.0 21 | 89341,weathda01,2007,1,CIN,NL,67,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 22 | 89343,walketo04,2007,1,OAK,AL,18,48,5,13,1,0,0,4.0,0.0,0.0,2,4.0,0.0,0.0,0.0,2.0,2.0 23 | 89345,wakefti01,2007,1,BOS,AL,1,2,0,0,0,0,0,0.0,0.0,0.0,0,2.0,0.0,0.0,0.0,0.0,0.0 24 | 89347,vizquom01,2007,1,SFN,NL,145,513,54,126,18,3,4,51.0,14.0,6.0,44,48.0,6.0,1.0,14.0,3.0,14.0 25 | 89348,villoro01,2007,1,NYA,AL,6,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 26 | 89352,valenjo03,2007,1,NYN,NL,51,166,18,40,11,1,3,18.0,2.0,1.0,15,28.0,4.0,0.0,1.0,1.0,5.0 27 | 89354,trachst01,2007,2,CHN,NL,4,7,0,1,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 28 | 89355,trachst01,2007,1,BAL,AL,3,5,0,0,0,0,0,0.0,0.0,0.0,0,3.0,0.0,0.0,0.0,0.0,0.0 29 | 89359,timlimi01,2007,1,BOS,AL,4,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 30 | 89360,thomeji01,2007,1,CHA,AL,130,432,79,119,19,0,35,96.0,0.0,1.0,95,134.0,11.0,6.0,0.0,3.0,10.0 31 | 89361,thomafr04,2007,1,TOR,AL,155,531,63,147,30,0,26,95.0,0.0,0.0,81,94.0,3.0,7.0,0.0,5.0,14.0 32 | 89363,tavarju01,2007,1,BOS,AL,2,4,0,1,0,0,0,0.0,0.0,0.0,1,3.0,0.0,0.0,0.0,0.0,0.0 33 | 89365,sweenma01,2007,2,LAN,NL,30,33,2,9,1,0,0,3.0,0.0,0.0,1,11.0,0.0,0.0,0.0,0.0,0.0 34 | 89366,sweenma01,2007,1,SFN,NL,76,90,18,23,8,0,2,10.0,2.0,0.0,13,18.0,0.0,3.0,1.0,0.0,0.0 35 | 89367,suppaje01,2007,1,MIL,NL,33,61,4,8,0,0,0,2.0,0.0,0.0,3,16.0,0.0,0.0,11.0,0.0,2.0 36 | 89368,stinnke01,2007,1,SLN,NL,26,82,7,13,3,0,1,5.0,0.0,0.0,5,22.0,2.0,0.0,0.0,0.0,2.0 37 | 89370,stantmi02,2007,1,CIN,NL,67,2,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 38 | 89371,stairma01,2007,1,TOR,AL,125,357,58,103,28,1,21,64.0,2.0,1.0,44,66.0,5.0,2.0,0.0,2.0,7.0 39 | 89372,sprinru01,2007,1,SLN,NL,72,1,0,0,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 40 | 89374,sosasa01,2007,1,TEX,AL,114,412,53,104,24,1,21,92.0,0.0,0.0,34,112.0,3.0,3.0,0.0,5.0,11.0 41 | 89375,smoltjo01,2007,1,ATL,NL,30,54,1,5,1,0,0,2.0,0.0,0.0,1,19.0,0.0,0.0,13.0,0.0,0.0 42 | 89378,sheffga01,2007,1,DET,AL,133,494,107,131,20,1,25,75.0,22.0,5.0,84,71.0,2.0,9.0,0.0,6.0,10.0 43 | 89381,seleaa01,2007,1,NYN,NL,31,4,0,0,0,0,0,0.0,0.0,0.0,1,1.0,0.0,0.0,1.0,0.0,0.0 44 | 89382,seaneru01,2007,1,LAN,NL,68,1,0,0,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 45 | 89383,schmija01,2007,1,LAN,NL,6,7,1,1,0,0,1,1.0,0.0,0.0,0,4.0,0.0,0.0,1.0,0.0,0.0 46 | 89384,schilcu01,2007,1,BOS,AL,1,2,0,1,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 47 | 89385,sandere02,2007,1,KCA,AL,24,73,12,23,7,0,2,11.0,0.0,1.0,11,15.0,0.0,1.0,0.0,0.0,2.0 48 | 89388,rogerke01,2007,1,DET,AL,1,2,0,0,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 49 | 89389,rodriiv01,2007,1,DET,AL,129,502,50,141,31,3,11,63.0,2.0,2.0,9,96.0,1.0,1.0,1.0,2.0,16.0 50 | 89396,ramirma02,2007,1,BOS,AL,133,483,84,143,33,1,20,88.0,0.0,0.0,71,92.0,13.0,7.0,0.0,8.0,21.0 51 | 89398,piazzmi01,2007,1,OAK,AL,83,309,33,85,17,1,8,44.0,0.0,0.0,18,61.0,0.0,0.0,0.0,2.0,9.0 52 | 89400,perezne01,2007,1,DET,AL,33,64,5,11,3,0,1,6.0,0.0,0.0,4,8.0,0.0,0.0,3.0,0.0,2.0 53 | 89402,parkch01,2007,1,NYN,NL,1,1,0,0,0,0,0,0.0,0.0,0.0,0,1.0,0.0,0.0,0.0,0.0,0.0 54 | 89406,oliveda02,2007,1,LAA,AL,5,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 55 | 89410,myersmi01,2007,1,NYA,AL,6,1,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 56 | 89411,mussimi01,2007,1,NYA,AL,2,2,0,0,0,0,0,0.0,0.0,0.0,1,0.0,0.0,0.0,0.0,0.0,0.0 57 | 89412,moyerja01,2007,1,PHI,NL,33,73,4,9,2,0,0,2.0,0.0,0.0,2,26.0,0.0,0.0,8.0,0.0,1.0 58 | 89420,mesajo01,2007,1,PHI,NL,38,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 59 | 89421,martipe02,2007,1,NYN,NL,5,9,1,1,1,0,0,0.0,0.0,0.0,0,6.0,0.0,0.0,2.0,0.0,0.0 60 | 89425,maddugr01,2007,1,SDN,NL,33,62,2,9,2,0,0,0.0,1.0,0.0,1,19.0,0.0,0.0,9.0,0.0,2.0 61 | 89426,mabryjo01,2007,1,COL,NL,28,34,4,4,1,0,1,5.0,0.0,0.0,5,10.0,0.0,0.0,0.0,0.0,1.0 62 | 89429,loftoke01,2007,2,CLE,AL,52,173,24,49,9,3,0,15.0,2.0,3.0,17,23.0,0.0,0.0,4.0,2.0,1.0 63 | 89430,loftoke01,2007,1,TEX,AL,84,317,62,96,16,3,7,23.0,21.0,4.0,39,28.0,1.0,2.0,2.0,3.0,5.0 64 | 89431,loaizes01,2007,1,LAN,NL,5,7,0,1,0,0,0,2.0,0.0,0.0,0,2.0,0.0,0.0,2.0,0.0,1.0 65 | 89438,kleskry01,2007,1,SFN,NL,116,362,51,94,27,3,6,44.0,5.0,1.0,46,68.0,2.0,1.0,1.0,1.0,14.0 66 | 89439,kentje01,2007,1,LAN,NL,136,494,78,149,36,1,20,79.0,1.0,3.0,57,61.0,4.0,5.0,0.0,6.0,17.0 67 | 89442,jonesto02,2007,1,DET,AL,5,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 68 | 89445,johnsra05,2007,1,ARI,NL,10,15,0,1,0,0,0,0.0,0.0,0.0,1,7.0,0.0,0.0,2.0,0.0,0.0 69 | 89450,hoffmtr01,2007,1,SDN,NL,60,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 70 | 89451,hernaro01,2007,2,LAN,NL,22,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 71 | 89452,hernaro01,2007,1,CLE,AL,2,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 72 | 89460,guarded01,2007,1,CIN,NL,15,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 73 | 89462,griffke02,2007,1,CIN,NL,144,528,78,146,24,1,30,93.0,6.0,1.0,85,99.0,14.0,1.0,0.0,9.0,14.0 74 | 89463,greensh01,2007,1,NYN,NL,130,446,62,130,30,1,10,46.0,11.0,1.0,37,62.0,4.0,5.0,1.0,1.0,14.0 75 | 89464,graffto01,2007,1,MIL,NL,86,231,34,55,8,0,9,30.0,0.0,1.0,24,44.0,6.0,3.0,0.0,2.0,7.0 76 | 89465,gordoto01,2007,1,PHI,NL,44,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 77 | 89466,gonzalu01,2007,1,LAN,NL,139,464,70,129,23,2,15,68.0,6.0,2.0,56,56.0,4.0,4.0,0.0,2.0,11.0 78 | 89467,gomezch02,2007,2,CLE,AL,19,53,4,15,2,0,0,5.0,0.0,0.0,0,6.0,0.0,0.0,1.0,1.0,1.0 79 | 89468,gomezch02,2007,1,BAL,AL,73,169,17,51,10,1,1,16.0,1.0,2.0,10,20.0,1.0,0.0,5.0,1.0,5.0 80 | 89469,glavito02,2007,1,NYN,NL,33,56,3,12,1,0,0,4.0,0.0,0.0,6,5.0,0.0,0.0,12.0,1.0,0.0 81 | 89473,floydcl01,2007,1,CHN,NL,108,282,40,80,10,1,9,45.0,0.0,0.0,35,47.0,5.0,5.0,0.0,0.0,6.0 82 | 89474,finlest01,2007,1,COL,NL,43,94,9,17,3,0,1,2.0,0.0,0.0,8,4.0,1.0,0.0,0.0,0.0,2.0 83 | 89480,embreal01,2007,1,OAK,AL,4,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 84 | 89481,edmonji01,2007,1,SLN,NL,117,365,39,92,15,2,12,53.0,0.0,2.0,41,75.0,2.0,0.0,2.0,3.0,9.0 85 | 89482,easleda01,2007,1,NYN,NL,76,193,24,54,6,0,10,26.0,0.0,1.0,19,35.0,1.0,5.0,0.0,1.0,2.0 86 | 89489,delgaca01,2007,1,NYN,NL,139,538,71,139,30,0,24,87.0,4.0,0.0,52,118.0,8.0,11.0,0.0,6.0,12.0 87 | 89493,cormirh01,2007,1,CIN,NL,6,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 88 | 89494,coninje01,2007,2,NYN,NL,21,41,2,8,2,0,0,5.0,0.0,0.0,7,8.0,2.0,0.0,1.0,1.0,1.0 89 | 89495,coninje01,2007,1,CIN,NL,80,215,23,57,11,1,6,32.0,4.0,0.0,20,28.0,0.0,0.0,1.0,6.0,4.0 90 | 89497,clemero02,2007,1,NYA,AL,2,2,0,1,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 91 | 89498,claytro01,2007,2,BOS,AL,8,6,1,0,0,0,0,0.0,0.0,0.0,0,3.0,0.0,0.0,0.0,0.0,2.0 92 | 89499,claytro01,2007,1,TOR,AL,69,189,23,48,14,0,1,12.0,2.0,1.0,14,50.0,0.0,1.0,3.0,3.0,8.0 93 | 89501,cirilje01,2007,2,ARI,NL,28,40,6,8,4,0,0,6.0,0.0,0.0,4,6.0,0.0,0.0,0.0,0.0,1.0 94 | 89502,cirilje01,2007,1,MIN,AL,50,153,18,40,9,2,2,21.0,2.0,0.0,15,13.0,0.0,1.0,3.0,2.0,9.0 95 | 89521,bondsba01,2007,1,SFN,NL,126,340,75,94,14,0,28,66.0,5.0,0.0,132,54.0,43.0,3.0,0.0,2.0,13.0 96 | 89523,biggicr01,2007,1,HOU,NL,141,517,68,130,31,3,10,50.0,4.0,3.0,23,112.0,0.0,3.0,7.0,5.0,5.0 97 | 89525,benitar01,2007,2,FLO,NL,34,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 98 | 89526,benitar01,2007,1,SFN,NL,19,0,0,0,0,0,0,0.0,0.0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0 99 | 89530,ausmubr01,2007,1,HOU,NL,117,349,38,82,16,3,3,25.0,6.0,1.0,37,74.0,3.0,6.0,4.0,1.0,11.0 100 | 89533,aloumo01,2007,1,NYN,NL,87,328,51,112,19,1,13,49.0,3.0,0.0,27,30.0,5.0,2.0,0.0,3.0,13.0 101 | 89534,alomasa02,2007,1,NYN,NL,8,22,1,3,1,0,0,0.0,0.0,0.0,0,3.0,0.0,0.0,0.0,0.0,0.0 102 | -------------------------------------------------------------------------------- /data/tips.csv: -------------------------------------------------------------------------------- 1 | total_bill,tip,sex,smoker,day,time,size 2 | 16.99,1.01,Female,No,Sun,Dinner,2 3 | 10.34,1.66,Male,No,Sun,Dinner,3 4 | 21.01,3.5,Male,No,Sun,Dinner,3 5 | 23.68,3.31,Male,No,Sun,Dinner,2 6 | 24.59,3.61,Female,No,Sun,Dinner,4 7 | 25.29,4.71,Male,No,Sun,Dinner,4 8 | 8.77,2.0,Male,No,Sun,Dinner,2 9 | 26.88,3.12,Male,No,Sun,Dinner,4 10 | 15.04,1.96,Male,No,Sun,Dinner,2 11 | 14.78,3.23,Male,No,Sun,Dinner,2 12 | 10.27,1.71,Male,No,Sun,Dinner,2 13 | 35.26,5.0,Female,No,Sun,Dinner,4 14 | 15.42,1.57,Male,No,Sun,Dinner,2 15 | 18.43,3.0,Male,No,Sun,Dinner,4 16 | 14.83,3.02,Female,No,Sun,Dinner,2 17 | 21.58,3.92,Male,No,Sun,Dinner,2 18 | 10.33,1.67,Female,No,Sun,Dinner,3 19 | 16.29,3.71,Male,No,Sun,Dinner,3 20 | 16.97,3.5,Female,No,Sun,Dinner,3 21 | 20.65,3.35,Male,No,Sat,Dinner,3 22 | 17.92,4.08,Male,No,Sat,Dinner,2 23 | 20.29,2.75,Female,No,Sat,Dinner,2 24 | 15.77,2.23,Female,No,Sat,Dinner,2 25 | 39.42,7.58,Male,No,Sat,Dinner,4 26 | 19.82,3.18,Male,No,Sat,Dinner,2 27 | 17.81,2.34,Male,No,Sat,Dinner,4 28 | 13.37,2.0,Male,No,Sat,Dinner,2 29 | 12.69,2.0,Male,No,Sat,Dinner,2 30 | 21.7,4.3,Male,No,Sat,Dinner,2 31 | 19.65,3.0,Female,No,Sat,Dinner,2 32 | 9.55,1.45,Male,No,Sat,Dinner,2 33 | 18.35,2.5,Male,No,Sat,Dinner,4 34 | 15.06,3.0,Female,No,Sat,Dinner,2 35 | 20.69,2.45,Female,No,Sat,Dinner,4 36 | 17.78,3.27,Male,No,Sat,Dinner,2 37 | 24.06,3.6,Male,No,Sat,Dinner,3 38 | 16.31,2.0,Male,No,Sat,Dinner,3 39 | 16.93,3.07,Female,No,Sat,Dinner,3 40 | 18.69,2.31,Male,No,Sat,Dinner,3 41 | 31.27,5.0,Male,No,Sat,Dinner,3 42 | 16.04,2.24,Male,No,Sat,Dinner,3 43 | 17.46,2.54,Male,No,Sun,Dinner,2 44 | 13.94,3.06,Male,No,Sun,Dinner,2 45 | 9.68,1.32,Male,No,Sun,Dinner,2 46 | 30.4,5.6,Male,No,Sun,Dinner,4 47 | 18.29,3.0,Male,No,Sun,Dinner,2 48 | 22.23,5.0,Male,No,Sun,Dinner,2 49 | 32.4,6.0,Male,No,Sun,Dinner,4 50 | 28.55,2.05,Male,No,Sun,Dinner,3 51 | 18.04,3.0,Male,No,Sun,Dinner,2 52 | 12.54,2.5,Male,No,Sun,Dinner,2 53 | 10.29,2.6,Female,No,Sun,Dinner,2 54 | 34.81,5.2,Female,No,Sun,Dinner,4 55 | 9.94,1.56,Male,No,Sun,Dinner,2 56 | 25.56,4.34,Male,No,Sun,Dinner,4 57 | 19.49,3.51,Male,No,Sun,Dinner,2 58 | 38.01,3.0,Male,Yes,Sat,Dinner,4 59 | 26.41,1.5,Female,No,Sat,Dinner,2 60 | 11.24,1.76,Male,Yes,Sat,Dinner,2 61 | 48.27,6.73,Male,No,Sat,Dinner,4 62 | 20.29,3.21,Male,Yes,Sat,Dinner,2 63 | 13.81,2.0,Male,Yes,Sat,Dinner,2 64 | 11.02,1.98,Male,Yes,Sat,Dinner,2 65 | 18.29,3.76,Male,Yes,Sat,Dinner,4 66 | 17.59,2.64,Male,No,Sat,Dinner,3 67 | 20.08,3.15,Male,No,Sat,Dinner,3 68 | 16.45,2.47,Female,No,Sat,Dinner,2 69 | 3.07,1.0,Female,Yes,Sat,Dinner,1 70 | 20.23,2.01,Male,No,Sat,Dinner,2 71 | 15.01,2.09,Male,Yes,Sat,Dinner,2 72 | 12.02,1.97,Male,No,Sat,Dinner,2 73 | 17.07,3.0,Female,No,Sat,Dinner,3 74 | 26.86,3.14,Female,Yes,Sat,Dinner,2 75 | 25.28,5.0,Female,Yes,Sat,Dinner,2 76 | 14.73,2.2,Female,No,Sat,Dinner,2 77 | 10.51,1.25,Male,No,Sat,Dinner,2 78 | 17.92,3.08,Male,Yes,Sat,Dinner,2 79 | 27.2,4.0,Male,No,Thur,Lunch,4 80 | 22.76,3.0,Male,No,Thur,Lunch,2 81 | 17.29,2.71,Male,No,Thur,Lunch,2 82 | 19.44,3.0,Male,Yes,Thur,Lunch,2 83 | 16.66,3.4,Male,No,Thur,Lunch,2 84 | 10.07,1.83,Female,No,Thur,Lunch,1 85 | 32.68,5.0,Male,Yes,Thur,Lunch,2 86 | 15.98,2.03,Male,No,Thur,Lunch,2 87 | 34.83,5.17,Female,No,Thur,Lunch,4 88 | 13.03,2.0,Male,No,Thur,Lunch,2 89 | 18.28,4.0,Male,No,Thur,Lunch,2 90 | 24.71,5.85,Male,No,Thur,Lunch,2 91 | 21.16,3.0,Male,No,Thur,Lunch,2 92 | 28.97,3.0,Male,Yes,Fri,Dinner,2 93 | 22.49,3.5,Male,No,Fri,Dinner,2 94 | 5.75,1.0,Female,Yes,Fri,Dinner,2 95 | 16.32,4.3,Female,Yes,Fri,Dinner,2 96 | 22.75,3.25,Female,No,Fri,Dinner,2 97 | 40.17,4.73,Male,Yes,Fri,Dinner,4 98 | 27.28,4.0,Male,Yes,Fri,Dinner,2 99 | 12.03,1.5,Male,Yes,Fri,Dinner,2 100 | 21.01,3.0,Male,Yes,Fri,Dinner,2 101 | 12.46,1.5,Male,No,Fri,Dinner,2 102 | 11.35,2.5,Female,Yes,Fri,Dinner,2 103 | 15.38,3.0,Female,Yes,Fri,Dinner,2 104 | 44.3,2.5,Female,Yes,Sat,Dinner,3 105 | 22.42,3.48,Female,Yes,Sat,Dinner,2 106 | 20.92,4.08,Female,No,Sat,Dinner,2 107 | 15.36,1.64,Male,Yes,Sat,Dinner,2 108 | 20.49,4.06,Male,Yes,Sat,Dinner,2 109 | 25.21,4.29,Male,Yes,Sat,Dinner,2 110 | 18.24,3.76,Male,No,Sat,Dinner,2 111 | 14.31,4.0,Female,Yes,Sat,Dinner,2 112 | 14.0,3.0,Male,No,Sat,Dinner,2 113 | 7.25,1.0,Female,No,Sat,Dinner,1 114 | 38.07,4.0,Male,No,Sun,Dinner,3 115 | 23.95,2.55,Male,No,Sun,Dinner,2 116 | 25.71,4.0,Female,No,Sun,Dinner,3 117 | 17.31,3.5,Female,No,Sun,Dinner,2 118 | 29.93,5.07,Male,No,Sun,Dinner,4 119 | 10.65,1.5,Female,No,Thur,Lunch,2 120 | 12.43,1.8,Female,No,Thur,Lunch,2 121 | 24.08,2.92,Female,No,Thur,Lunch,4 122 | 11.69,2.31,Male,No,Thur,Lunch,2 123 | 13.42,1.68,Female,No,Thur,Lunch,2 124 | 14.26,2.5,Male,No,Thur,Lunch,2 125 | 15.95,2.0,Male,No,Thur,Lunch,2 126 | 12.48,2.52,Female,No,Thur,Lunch,2 127 | 29.8,4.2,Female,No,Thur,Lunch,6 128 | 8.52,1.48,Male,No,Thur,Lunch,2 129 | 14.52,2.0,Female,No,Thur,Lunch,2 130 | 11.38,2.0,Female,No,Thur,Lunch,2 131 | 22.82,2.18,Male,No,Thur,Lunch,3 132 | 19.08,1.5,Male,No,Thur,Lunch,2 133 | 20.27,2.83,Female,No,Thur,Lunch,2 134 | 11.17,1.5,Female,No,Thur,Lunch,2 135 | 12.26,2.0,Female,No,Thur,Lunch,2 136 | 18.26,3.25,Female,No,Thur,Lunch,2 137 | 8.51,1.25,Female,No,Thur,Lunch,2 138 | 10.33,2.0,Female,No,Thur,Lunch,2 139 | 14.15,2.0,Female,No,Thur,Lunch,2 140 | 16.0,2.0,Male,Yes,Thur,Lunch,2 141 | 13.16,2.75,Female,No,Thur,Lunch,2 142 | 17.47,3.5,Female,No,Thur,Lunch,2 143 | 34.3,6.7,Male,No,Thur,Lunch,6 144 | 41.19,5.0,Male,No,Thur,Lunch,5 145 | 27.05,5.0,Female,No,Thur,Lunch,6 146 | 16.43,2.3,Female,No,Thur,Lunch,2 147 | 8.35,1.5,Female,No,Thur,Lunch,2 148 | 18.64,1.36,Female,No,Thur,Lunch,3 149 | 11.87,1.63,Female,No,Thur,Lunch,2 150 | 9.78,1.73,Male,No,Thur,Lunch,2 151 | 7.51,2.0,Male,No,Thur,Lunch,2 152 | 14.07,2.5,Male,No,Sun,Dinner,2 153 | 13.13,2.0,Male,No,Sun,Dinner,2 154 | 17.26,2.74,Male,No,Sun,Dinner,3 155 | 24.55,2.0,Male,No,Sun,Dinner,4 156 | 19.77,2.0,Male,No,Sun,Dinner,4 157 | 29.85,5.14,Female,No,Sun,Dinner,5 158 | 48.17,5.0,Male,No,Sun,Dinner,6 159 | 25.0,3.75,Female,No,Sun,Dinner,4 160 | 13.39,2.61,Female,No,Sun,Dinner,2 161 | 16.49,2.0,Male,No,Sun,Dinner,4 162 | 21.5,3.5,Male,No,Sun,Dinner,4 163 | 12.66,2.5,Male,No,Sun,Dinner,2 164 | 16.21,2.0,Female,No,Sun,Dinner,3 165 | 13.81,2.0,Male,No,Sun,Dinner,2 166 | 17.51,3.0,Female,Yes,Sun,Dinner,2 167 | 24.52,3.48,Male,No,Sun,Dinner,3 168 | 20.76,2.24,Male,No,Sun,Dinner,2 169 | 31.71,4.5,Male,No,Sun,Dinner,4 170 | 10.59,1.61,Female,Yes,Sat,Dinner,2 171 | 10.63,2.0,Female,Yes,Sat,Dinner,2 172 | 50.81,10.0,Male,Yes,Sat,Dinner,3 173 | 15.81,3.16,Male,Yes,Sat,Dinner,2 174 | 7.25,5.15,Male,Yes,Sun,Dinner,2 175 | 31.85,3.18,Male,Yes,Sun,Dinner,2 176 | 16.82,4.0,Male,Yes,Sun,Dinner,2 177 | 32.9,3.11,Male,Yes,Sun,Dinner,2 178 | 17.89,2.0,Male,Yes,Sun,Dinner,2 179 | 14.48,2.0,Male,Yes,Sun,Dinner,2 180 | 9.6,4.0,Female,Yes,Sun,Dinner,2 181 | 34.63,3.55,Male,Yes,Sun,Dinner,2 182 | 34.65,3.68,Male,Yes,Sun,Dinner,4 183 | 23.33,5.65,Male,Yes,Sun,Dinner,2 184 | 45.35,3.5,Male,Yes,Sun,Dinner,3 185 | 23.17,6.5,Male,Yes,Sun,Dinner,4 186 | 40.55,3.0,Male,Yes,Sun,Dinner,2 187 | 20.69,5.0,Male,No,Sun,Dinner,5 188 | 20.9,3.5,Female,Yes,Sun,Dinner,3 189 | 30.46,2.0,Male,Yes,Sun,Dinner,5 190 | 18.15,3.5,Female,Yes,Sun,Dinner,3 191 | 23.1,4.0,Male,Yes,Sun,Dinner,3 192 | 15.69,1.5,Male,Yes,Sun,Dinner,2 193 | 19.81,4.19,Female,Yes,Thur,Lunch,2 194 | 28.44,2.56,Male,Yes,Thur,Lunch,2 195 | 15.48,2.02,Male,Yes,Thur,Lunch,2 196 | 16.58,4.0,Male,Yes,Thur,Lunch,2 197 | 7.56,1.44,Male,No,Thur,Lunch,2 198 | 10.34,2.0,Male,Yes,Thur,Lunch,2 199 | 43.11,5.0,Female,Yes,Thur,Lunch,4 200 | 13.0,2.0,Female,Yes,Thur,Lunch,2 201 | 13.51,2.0,Male,Yes,Thur,Lunch,2 202 | 18.71,4.0,Male,Yes,Thur,Lunch,3 203 | 12.74,2.01,Female,Yes,Thur,Lunch,2 204 | 13.0,2.0,Female,Yes,Thur,Lunch,2 205 | 16.4,2.5,Female,Yes,Thur,Lunch,2 206 | 20.53,4.0,Male,Yes,Thur,Lunch,4 207 | 16.47,3.23,Female,Yes,Thur,Lunch,3 208 | 26.59,3.41,Male,Yes,Sat,Dinner,3 209 | 38.73,3.0,Male,Yes,Sat,Dinner,4 210 | 24.27,2.03,Male,Yes,Sat,Dinner,2 211 | 12.76,2.23,Female,Yes,Sat,Dinner,2 212 | 30.06,2.0,Male,Yes,Sat,Dinner,3 213 | 25.89,5.16,Male,Yes,Sat,Dinner,4 214 | 48.33,9.0,Male,No,Sat,Dinner,4 215 | 13.27,2.5,Female,Yes,Sat,Dinner,2 216 | 28.17,6.5,Female,Yes,Sat,Dinner,3 217 | 12.9,1.1,Female,Yes,Sat,Dinner,2 218 | 28.15,3.0,Male,Yes,Sat,Dinner,5 219 | 11.59,1.5,Male,Yes,Sat,Dinner,2 220 | 7.74,1.44,Male,Yes,Sat,Dinner,2 221 | 30.14,3.09,Female,Yes,Sat,Dinner,4 222 | 12.16,2.2,Male,Yes,Fri,Lunch,2 223 | 13.42,3.48,Female,Yes,Fri,Lunch,2 224 | 8.58,1.92,Male,Yes,Fri,Lunch,1 225 | 15.98,3.0,Female,No,Fri,Lunch,3 226 | 13.42,1.58,Male,Yes,Fri,Lunch,2 227 | 16.27,2.5,Female,Yes,Fri,Lunch,2 228 | 10.09,2.0,Female,Yes,Fri,Lunch,2 229 | 20.45,3.0,Male,No,Sat,Dinner,4 230 | 13.28,2.72,Male,No,Sat,Dinner,2 231 | 22.12,2.88,Female,Yes,Sat,Dinner,2 232 | 24.01,2.0,Male,Yes,Sat,Dinner,4 233 | 15.69,3.0,Male,Yes,Sat,Dinner,3 234 | 11.61,3.39,Male,No,Sat,Dinner,2 235 | 10.77,1.47,Male,No,Sat,Dinner,2 236 | 15.53,3.0,Male,Yes,Sat,Dinner,2 237 | 10.07,1.25,Male,No,Sat,Dinner,2 238 | 12.6,1.0,Male,Yes,Sat,Dinner,2 239 | 32.83,1.17,Male,Yes,Sat,Dinner,2 240 | 35.83,4.67,Female,No,Sat,Dinner,3 241 | 29.03,5.92,Male,No,Sat,Dinner,3 242 | 27.18,2.0,Female,Yes,Sat,Dinner,2 243 | 22.67,2.0,Male,Yes,Sat,Dinner,2 244 | 17.82,1.75,Male,No,Sat,Dinner,2 245 | 18.78,3.0,Female,No,Thur,Dinner,2 246 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://dougfinke.visualstudio.com/PSKit/_apis/build/status/dfinke.PSKit?branchName=master)](https://dougfinke.visualstudio.com/PSKit/_build/latest?definitionId=18&branchName=master) 2 | 3 | # PSKit - PowerShell Kit 4 | 5 | Try it out in a PowerShell Jupyter Notebook here: 6 | 7 | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/dfinke/PSKit/master) 8 | 9 | A suite of command-line tools for working with PowerShell arrays. 10 | 11 | |Function| 12 | |---| 13 | |ConvertFrom-FixedData 14 | |ConvertFrom-SQLToPS 15 | |ConvertFrom-SSV 16 | |ConvertFrom-TranspileSQL 17 | |Convert-IntoCSV 18 | |ConvertTo-Property 19 | |Get-DataInfo 20 | |GetDataTypePrecedence 21 | |Get-DateRange 22 | |Get-PropertyName 23 | |Get-PropertyStats 24 | |Group-ByAndMeasure 25 | |Import-SSV 26 | |Invoke-ScanProperties 27 | |Invoke-TranspileSQL 28 | |New-DataFrame 29 | |New-LookupTable 30 | 31 | ## Get-DataInfo 32 | 33 | This function prints information about a PowerShell object array including the column name, column data type, non-null values. 34 | 35 | ```powershell 36 | $data = ConvertFrom-Csv @" 37 | Region,ItemName,Units,TotalSold 38 | ,screws,5.3,3 39 | North,,5.7,58 40 | East,drill,6.3 41 | "@ 42 | 43 | $data.info() 44 | # Get-DataInfo $data 45 | ``` 46 | 47 | ``` 48 | Entries: 3 49 | Columns: 4 50 | 51 | 52 | ColumnName NonNull DataType 53 | ---------- ------- -------- 54 | Region 2 string 55 | ItemName 2 string 56 | Units 3 double 57 | TotalSold 2 int 58 | 59 | 60 | 61 | string(2) double(1) int(1) 62 | 63 | 64 | ``` 65 | 66 | 67 | ## Read-Csv 68 | 69 | Read comma-separated values (csv). $target can be a URL, a file, or a string 70 | 71 | ```powershell 72 | #$file = "targetData.csv" 73 | $url = 'https://raw.githubusercontent.com/dfinke/ImportExcel/master/Examples/JustCharts/TargetData.csv' 74 | $str = @" 75 | "Cost","Date","Name" 76 | "1.1","1/1/2015","John" 77 | "2.1","1/2/2015","Tom" 78 | "5.1","1/2/2015","Dick" 79 | "11.1","1/2/2015","Harry" 80 | "7.1","1/2/2015","Jane" 81 | "22.1","1/2/2015","Mary" 82 | "32.1","1/2/2015","Liz" 83 | "@ 84 | 85 | $str, $url | Read-Csv 86 | ``` 87 | 88 | ``` 89 | Cost Date Name 90 | ---- ---- ---- 91 | 1.1 1/1/2015 John 92 | 2.1 1/2/2015 Tom 93 | 5.1 1/2/2015 Dick 94 | 11.1 1/2/2015 Harry 95 | 7.1 1/2/2015 Jane 96 | 22.1 1/2/2015 Mary 97 | 32.1 1/2/2015 Liz 98 | 1.1 1/1/2015 John 99 | 2.1 1/2/2015 Tom 100 | 5.1 1/2/2015 Dick 101 | 11.1 1/2/2015 Harry 102 | 7.1 1/2/2015 Jane 103 | 22.1 1/2/2015 Mary 104 | 32.1 1/2/2015 Liz 105 | ``` 106 | 107 | ## New-DataFrame 108 | 109 | Creates an array of objects, size-mutable, can be heterogeneous, tabular data 110 | 111 | ```powershell 112 | New-DataFrame (Get-DateRange 1/1 -periods 5) p1,p2,3 {Get-Random} 113 | ``` 114 | 115 | ``` 116 | Index p1 p2 3 117 | ----- -- -- - 118 | 2020-01-01 708420917 1112428663 523426202 119 | 2020-01-02 1643869654 2086787197 1127195815 120 | 2020-01-03 1095068483 2006354687 1612194161 121 | 2020-01-04 1561123134 1004618008 1431794170 122 | 2020-01-05 851611997 189055864 871342612 123 | ``` 124 | ## Group-ByAndMeasure 125 | 126 | Groups data and can either get the Count, Average, Sum, Maximum or Minimum 127 | 128 | ```powershell 129 | $str = @" 130 | Region,Item,TotalSold 131 | West,apple,2 132 | South,lemon,4 133 | East,avocado,12 134 | South,screwdriver,70 135 | North,avocado,59 136 | North,hammer,33 137 | North,screws,69 138 | East,apple,21 139 | West,lemon,67 140 | South,drill,52 141 | "@ 142 | 143 | Group-ByAndMeasure (Read-Csv $str) Region TotalSold Sum 144 | ``` 145 | 146 | ``` 147 | Region Sum 148 | ------ --- 149 | West 69 150 | South 126 151 | East 33 152 | North 161 153 | ``` 154 | 155 | ## Get-DateRange 156 | 157 | Return a fixed frequency Datetime Index 158 | 159 | ```powershell 160 | Get-DateRange 1/1/2020 -periods 6 161 | 162 | 2020-01-01 163 | 2020-01-02 164 | 2020-01-03 165 | 2020-01-04 166 | 2020-01-05 167 | 2020-01-06 168 | 169 | New-DataFrame (Get-DateRange 1/1/2020 -periods 3 -freq M) a,b,c 170 | ``` 171 | 172 | ``` 173 | Index a b c 174 | ----- - - - 175 | 2020-01-01 [missing] [missing] [missing] 176 | 2020-02-01 [missing] [missing] [missing] 177 | 2020-03-01 [missing] [missing] [missing] 178 | ``` 179 | # SQL Query 180 | 181 | The `PSKit` module adds a method `Query()` to lists of objects. You pass a SQL statement to it to work on that set of data. Currently, the SQL syntax is limited to Select the properties you want to see and a Where clause with value type comparison and logical operators. It does not support multiple arrays, joins or aliasing etc. 182 | 183 | ```powershell 184 | $data = ConvertFrom-Csv @" 185 | name,age,cash 186 | Chris,44,72 187 | Brian,26,110 188 | Ryan,18,145 189 | Joe,34,83 190 | "@ 191 | ``` 192 | 193 | ## Scan All The Properties 194 | 195 | ```powershell 196 | $data = ConvertFrom-Csv @" 197 | name,phoneNumber 198 | Chris,555-999-1111 199 | Brian,555-123-4567 200 | Ryan,555-123-8901 201 | Joe,555-777-1111 202 | Jane,555-777-2222 203 | "@ 204 | ``` 205 | 206 | Find phone numbers matching the pattern "ddd–123-dddd". 207 | 208 | ```powershell 209 | PS C:\ $data.ScanProperties("\d{3}-123-\d{4}") 210 | 211 | name phoneNumber 212 | ---- ----------- 213 | Brian 555-123-4567 214 | Ryan 555-123-8901 215 | ``` 216 | 217 | Find records that end in "an$". 218 | 219 | ```powershell 220 | PS C:\ $data.ScanProperties("an$") 221 | 222 | name phoneNumber 223 | ---- ----------- 224 | Brian 555-123-4567 225 | Ryan 555-123-8901 226 | ``` 227 | 228 | ### Works with built in PowerShell Functions 229 | 230 | ```powershell 231 | PS C:\ (Get-Service).ScanProperties("^mssql") 232 | 233 | Status Name DisplayName ServicesDependedOn 234 | ------ ---- ----------- ------------------ 235 | Stopped MSSQLSERVER SQL Server (MSSQLSERVER) {} 236 | Stopped MSSQLSERVER SQL Server (MSSQLSERVER) {} 237 | Stopped SQLSERVERAGENT SQL Server Agent (MSSQLSERVER) {MSSQLSERVER} 238 | Stopped SQLSERVERAGENT SQL Server Agent (MSSQLSERVER) {MSSQLSERVER} 239 | ``` 240 | 241 | ## Use SQL like query 242 | 243 | **Note**: This feature requires the PowerShell module PSStringScanner. 244 | 245 | `Install-Module PSStringScanner` 246 | 247 | Supports a simple SQL Select statement syntax for querying arrays of data. 248 | 249 | ```powershell 250 | $actual = $data.query("SELECT cash, name FROM data where name like '*i*' and cash > 100") 251 | 252 | cash name 253 | ---- ---- 254 | 72 Chris 255 | 110 Brian 256 | ``` 257 | 258 | # Generate summary statistics 259 | 260 | `Get-PropertyStats` will calculate different statistics based on the type of each column. 261 | 262 | **Note**: There are two other ways to get the same results `$data | Get-PropertyStats` or `$data.stats()` 263 | 264 | ```powershell 265 | $data = ConvertFrom-Csv @" 266 | a,b,c,d,e,f,g 267 | 2,0.0,FALSE,"""Yes!""",2011-11-11 11:00,2012-09-08,12:34 268 | 42,3.1415,TRUE,"Oh, good",2014-09-15,12/6/70,0:07 PM 269 | 66,,False,2198,,, 270 | "@ 271 | 272 | Get-PropertyStats $data 273 | ``` 274 | 275 | ``` 276 | ColumnName DataType HasNulls Min Max Median StandardDeviation Variance Sum 277 | ---------- -------- -------- --- --- ------ ----------------- -------- --- 278 | a int False 2 66 42 32.331615074619 1045.33333333333 110 279 | b double True 0 3.1415 0 1.81374587065921 3.28967408333333 3.1415 280 | c bool False 281 | d string False 282 | e datetime True 283 | f datetime True 284 | g datetime True 285 | ``` 286 | 287 | # Create a Lookup Table 288 | 289 | Have data that with a unique id column? Want to use it as a lookup table? Here you go: 290 | 291 | ```powershell 292 | $data = ConvertFrom-Csv @" 293 | slug,place,latitude,longitude 294 | dcl,Downtown Coffee Lounge,32.35066,-95.30181 295 | tyler-museum,Tyler Museum of Art,32.33396,-95.28174 296 | genecov,Genecov Sculpture,32.299076986939205,-95.31571447849274 297 | "@ 298 | ``` 299 | 300 | This is similar to `Group-Object` built into `PowerShell`. New-LookupTable also handles missing data. 301 | 302 | ```powershell 303 | PS C:\> New-LookupTable $data slug 304 | 305 | Name Value 306 | ---- ----- 307 | dcl @{slug=dcl; place=Downtown Coffee Lounge; latitude=32.35066; longitude=-95.30181} 308 | tyler-museum @{slug=tyler-museum; place=Tyler Museum of Art; latitude=32.33396; longitude=-95.28174} 309 | genecov @{slug=genecov; place=Genecov Sculpture; latitude=32.299076986939205; longitude=-95.3157144... 310 | ``` 311 | 312 | # Convert Fixed Data Based on a Schema 313 | 314 | Fixed-width files are particularly challenging to parse. Save yourself some frustration by using a CSV-formatted schema to convert your fixed-width file into PowerShell objects. 315 | 316 | ## Fixed data 317 | 318 | ``` 319 | Chris44 72 320 | Brian26110 321 | Ryan 18145 322 | Joe 34 83 323 | ``` 324 | 325 | ## Schema CSV 326 | 327 | ``` 328 | column,start,length 329 | name,1,5 330 | age,6,2 331 | cash,8,3 332 | ``` 333 | 334 | The function that parses the data. 335 | 336 | ```powershell 337 | ConvertFrom-FixedData -fixedData (Get-Content .\fixedData.txt) -schema (Import-Csv .\fixedDataSchema.csv) 338 | 339 | name age cash 340 | ---- --- ---- 341 | Chris 44 72 342 | Brian 26 110 343 | Ryan 18 145 344 | Joe 34 83 345 | ``` -------------------------------------------------------------------------------- /start-demo.ps1: -------------------------------------------------------------------------------- 1 | ## Start-Demo.ps1 2 | ################################################################################################## 3 | ## This is an overhaul of Jeffrey Snover's original Start-Demo script by Joel "Jaykul" Bennett 4 | ## 5 | ## I've switched it to using ReadKey instead of ReadLine (you don't have to hit Enter each time) 6 | ## As a result, I've changed the names and keys for a lot of the operations, so that they make 7 | ## sense with only a single letter to tell them apart (sorry if you had them memorized). 8 | ## 9 | ## I've also been adding features as I come across needs for them, and you'll contribute your 10 | ## improvements back to the PowerShell Script repository as well. 11 | ################################################################################################## 12 | ## Revision History (version 3.3) 13 | ## 3.3.3 Fixed: Script no longer says "unrecognized key" when you hit shift or ctrl, etc. 14 | ## Fixed: Blank lines in script were showing as errors (now printed like comments) 15 | ## 3.3.2 Fixed: Changed the "x" to match the "a" in the help text 16 | ## 3.3.1 Fixed: Added a missing bracket in the script 17 | ## 3.3 - Added: Added a "Clear Screen" option 18 | ## - Added: Added a "Rewind" function (which I'm not using much) 19 | ## 3.2 - Fixed: Put back the trap { continue; } 20 | ## 3.1 - Fixed: No Output when invoking Get-Member (and other cmdlets like it???) 21 | ## 3.0 - Fixed: Commands which set a variable, like: $files = ls 22 | ## - Fixed: Default action doesn't continue 23 | ## - Changed: Use ReadKey instead of ReadLine 24 | ## - Changed: Modified the option prompts (sorry if you had them memorized) 25 | ## - Changed: Various time and duration strings have better formatting 26 | ## - Enhance: Colors are settable: prompt, command, comment 27 | ## - Added: NoPauseAfterExecute switch removes the extra pause 28 | ## If you set this, the next command will be displayed immediately 29 | ## - Added: Auto Execute mode (FullAuto switch) runs the rest of the script 30 | ## at an automatic speed set by the AutoSpeed parameter (or manually) 31 | ## - Added: Automatically append an empty line to the end of the demo script 32 | ## so you have a chance to "go back" after the last line of you demo 33 | ################################################################################################## 34 | ## 35 | param( 36 | $file = ".\demo.txt", 37 | [int]$command = 0, 38 | [System.ConsoleColor]$promptColor = "Yellow", 39 | [System.ConsoleColor]$commandColor = "White", 40 | [System.ConsoleColor]$commentColor = "Green", 41 | [switch]$FullAuto, 42 | [int]$AutoSpeed = 3, 43 | [switch]$NoPauseAfterExecute 44 | ) 45 | 46 | $RawUI = $Host.UI.RawUI 47 | $hostWidth = $RawUI.BufferSize.Width 48 | 49 | # A function for reading in a character 50 | function Read-Char() { 51 | $_OldColor = $RawUI.ForeGroundColor 52 | $RawUI.ForeGroundColor = "Red" 53 | $inChar = $RawUI.ReadKey("IncludeKeyUp") 54 | # loop until they press a character, so Shift or Ctrl, etc don't terminate us 55 | while ($inChar.Character -eq 0) { 56 | $inChar = $RawUI.ReadKey("IncludeKeyUp") 57 | } 58 | $RawUI.ForeGroundColor = $_OldColor 59 | return $inChar.Character 60 | } 61 | 62 | function Rewind($lines, $index, $steps = 1) { 63 | $started = $index; 64 | $index -= $steps; 65 | while (($index -ge 0) -and ($lines[$index].Trim(" `t").StartsWith("#"))) { 66 | $index-- 67 | } 68 | if ( $index -lt 0 ) { $index = $started } 69 | return $index 70 | } 71 | 72 | $file = Resolve-Path $file 73 | while (-not(Test-Path $file)) { 74 | $file = Read-Host "Please enter the path of your demo script (Crtl+C to cancel)" 75 | $file = Resolve-Path $file 76 | } 77 | 78 | Clear-Host 79 | 80 | $_lines = Get-Content $file 81 | # Append an extra (do nothing) line on the end so we can still go back after the last line. 82 | $_lines += "Write-Host 'The End'" 83 | $_starttime = [DateTime]::now 84 | #$FullAuto = $false 85 | 86 | 87 | Write-Host -nonew -back black -fore $promptColor $(" " * $hostWidth) 88 | Write-Host -nonew -back black -fore $promptColor @" 89 | $(' ' * ($hostWidth -(18 + $(split-path $file -leaf).Length))) 90 | "@ 91 | Write-Host -nonew -back black -fore $promptColor "Press" 92 | Write-Host -nonew -back black -fore Red " ? " 93 | Write-Host -nonew -back black -fore $promptColor "for help.$(' ' * ($hostWidth -17))" 94 | Write-Host -nonew -back black -fore $promptColor $(" " * $hostWidth) 95 | 96 | # We use a FOR and an INDEX ($_i) instead of a FOREACH because 97 | # it is possible to start at a different location and/or jump 98 | # around in the order. 99 | for ($_i = $Command; $_i -lt $_lines.count; $_i++) { 100 | # Put the current command in the Window Title along with the demo duration 101 | $Dur = [DateTime]::Now - $_StartTime 102 | $RawUI.WindowTitle = "$(if($dur.Hours -gt 0){'{0}h '})$(if($dur.Minutes -gt 0){'{1}m '}){2}s {3}" -f 103 | $dur.Hours, $dur.Minutes, $dur.Seconds, $($_Lines[$_i]) 104 | 105 | # Echo out the commmand to the console with a prompt as though it were real 106 | Write-Host -nonew -fore $promptColor "[$_i]$([char]0x2265) " 107 | if ($_lines[$_i].Trim(" ").StartsWith("#") -or $_lines[$_i].Trim(" ").Length -le 0) { 108 | Write-Host -fore $commentColor "$($_Lines[$_i]) " 109 | continue 110 | } 111 | else { 112 | Write-Host -nonew -fore $commandColor "$($_Lines[$_i]) " 113 | } 114 | 115 | if ( $FullAuto ) { Start-Sleep $autoSpeed; $ch = [char]13 } else { $ch = Read-Char } 116 | switch ($ch) { 117 | "?" { 118 | Write-Host -Fore $promptColor @" 119 | 120 | Running demo: $file 121 | (n) Next (p) Previous 122 | (q) Quit (s) Suspend 123 | (t) Timecheck (v) View $(split-path $file -leaf) 124 | (g) Go to line by number 125 | (f) Find lines by string 126 | (a) Auto Execute mode 127 | (c) Clear Screen 128 | "@ 129 | $_i-- # back a line, we're gonna step forward when we loop 130 | } 131 | "n" { 132 | # Next (do nothing) 133 | Write-Host -Fore $promptColor "" 134 | } 135 | "p" { 136 | # Previous 137 | Write-Host -Fore $promptColor "" 138 | while ($_lines[--$_i].Trim(" ").StartsWith("#")) {} 139 | $_i-- # back a line, we're gonna step forward when we loop 140 | } 141 | "a" { 142 | # EXECUTE (Go Faster) 143 | $AutoSpeed = [int](Read-Host "Pause (seconds)") 144 | $FullAuto = $true; 145 | Write-Host -Fore $promptColor "" 146 | $_i-- # Repeat this line, and then just blow through the rest 147 | } 148 | "q" { 149 | # Quit 150 | Write-Host -Fore $promptColor "" 151 | $_i = $_lines.count; 152 | break; 153 | } 154 | "v" { 155 | # View Source 156 | $lines[0..($_i - 1)] | Write-Host -Fore Yellow 157 | $lines[$_i] | Write-Host -Fore Green 158 | $lines[($_i + 1)..$lines.Count] | Write-Host -Fore Yellow 159 | $_i-- # back a line, we're gonna step forward when we loop 160 | } 161 | "t" { 162 | # Time Check 163 | $dur = [DateTime]::Now - $_StartTime 164 | Write-Host -Fore $promptColor $( 165 | "{3} -- $(if($dur.Hours -gt 0){'{0}h '})$(if($dur.Minutes -gt 0){'{1}m '}){2}s" -f 166 | $dur.Hours, $dur.Minutes, $dur.Seconds, ([DateTime]::Now.ToShortTimeString())) 167 | $_i-- # back a line, we're gonna step forward when we loop 168 | } 169 | "s" { 170 | # Suspend (Enter Nested Prompt) 171 | Write-Host -Fore $promptColor "" 172 | $Host.EnterNestedPrompt() 173 | $_i-- # back a line, we're gonna step forward when we loop 174 | } 175 | "g" { 176 | # GoTo Line Number 177 | $i = [int](Read-Host "line number") 178 | if ($i -le $_lines.Count) { 179 | if ($i -gt 0) { 180 | # extra line back because we're gonna step forward when we loop 181 | $_i = Rewind $_lines $_i (($_i - $i) + 1) 182 | } 183 | else { 184 | $_i = -1 # Start negative, because we step forward when we loop 185 | } 186 | } 187 | } 188 | "f" { 189 | # Find by pattern 190 | $match = $_lines | Select-String (Read-Host "search string") 191 | if ($match -eq $null) { 192 | Write-Host -Fore Red "Can't find a matching line" 193 | } 194 | else { 195 | $match | % { Write-Host -Fore $promptColor $("[{0,2}] {1}" -f ($_.LineNumber - 1), $_.Line) } 196 | if ($match.Count -lt 1) { 197 | $_i = $match.lineNumber - 2 # back a line, we're gonna step forward when we loop 198 | } 199 | else { 200 | $_i-- # back a line, we're gonna step forward when we loop 201 | } 202 | } 203 | } 204 | "c" { 205 | Clear-Host 206 | $_i-- # back a line, we're gonna step forward when we loop 207 | } 208 | "$([char]13)" { 209 | # on enter 210 | Write-Host 211 | trap [System.Exception] {Write-Error $_; continue; } 212 | Invoke-Expression ($_lines[$_i]) | out-default 213 | if (-not $NoPauseAfterExecute -and -not $FullAuto) { 214 | $null = $RawUI.ReadKey("NoEcho,IncludeKeyUp") # Pause after output for no apparent reason... ;) 215 | } 216 | } 217 | default { 218 | Write-Host -Fore Green "`nKey not recognized. Press ? for help, or ENTER to execute the command." 219 | $_i-- # back a line, we're gonna step forward when we loop 220 | } 221 | } 222 | } 223 | $dur = [DateTime]::Now - $_StartTime 224 | Write-Host -Fore $promptColor $( 225 | "" -f 226 | $dur.Hours, $dur.Minutes, $dur.Seconds, [DateTime]::Now.ToLongTimeString()) 227 | Write-Host -Fore $promptColor $([DateTime]::now) 228 | Write-Host -------------------------------------------------------------------------------- /data/countries.csv: -------------------------------------------------------------------------------- 1 | "Rank","Country or territory","Total length of land borders (km)","Total surface area (km?)","Border/area ratio (km/km?)" 2 | "1","Vatican City","3.2","0.44","7.2727273" 3 | "2","Monaco","4.4","2","2.2000000" 4 | "3","San Marino","39","61","0.6393443" 5 | "4","Liechtenstein","76","160","0.4750000" 6 | "5","Sint Maarten (Netherlands)","10.2","34","0.3000000" 7 | "6","Andorra","120.3","468","0.2570513" 8 | "7","Gibraltar (United Kingdom)","1.2","6","0.2000000" 9 | "8","Saint Martin (France)","10.2","54","0.1888889" 10 | "9","Luxembourg","359","2586","0.1388244" 11 | "10","Palestinian territories","466","6220","0.0749196" 12 | "11","Brunei","381","5765","0.0660885" 13 | "12","Slovenia","1334","20273","0.0658018" 14 | "13","The Gambia","740","11295","0.0655157" 15 | "14","Kosovo","701","10887","0.0643887" 16 | "15","Israel","1017","20770","0.0489649" 17 | "16","Belgium","1385","30528","0.0453682" 18 | "17","Montenegro","625","13812","0.0452505" 19 | "18","Switzerland","1852","41284","0.0448600" 20 | "19","Lebanon","454","10452","0.0434367" 21 | "20","Armenia","1254","29743","0.0421612" 22 | "21","Moldova","1389","33846","0.0410388" 23 | "22","Croatia","2197","56594","0.0388204" 24 | "23","Burundi","974","27834","0.0349932" 25 | "24","Rwanda","893","26338","0.0339054" 26 | "25","Slovakia","1524","49037","0.0310786" 27 | "26","Swaziland","535","17364","0.0308109" 28 | "27","Austria","2562","83871","0.0305469" 29 | "28","Lesotho","909","30355","0.0299456" 30 | "29","Macedonia","766","25713","0.0297904" 31 | "30","Bangladesh","4246","143998","0.0294865" 32 | "31","Togo","1647","56785","0.0290041" 33 | "32","Bosnia and Herzegovina","1459","51197","0.0284978" 34 | "33","Bhutan","1075","38394","0.0279992" 35 | "34","Hong Kong (People's Republic of China)","30","1104","0.0271739" 36 | "35","Kuwait","462","17818","0.0259288" 37 | "36","El Salvador","545","21041","0.0259018" 38 | "37","Tajikistan","3651","143100","0.0255136" 39 | "38","Albania","720","28748","0.0250452" 40 | "39","Netherlands","1027","41528","0.0247303" 41 | "40","Malawi","2881","118484","0.0243155" 42 | "41","Czech Republic","1881","78865","0.0238509" 43 | "42","Hungary","2171","93028","0.0233371" 44 | "43","Azerbaijan","2013","86600","0.0232448" 45 | "44","Serbia","2027","88361","0.0229400" 46 | "45","Belize","516","22966","0.0224680" 47 | "46","Djibouti","516","23200","0.0222414" 48 | "47","Laos","5083","236800","0.0214654" 49 | "48","Georgia","1461","69700","0.0209613" 50 | "49","Guinea-Bissau","724","36125","0.0200415" 51 | "50","Nepal","2926","147181","0.0198803" 52 | "51","Lithuania","1273","65300","0.0194946" 53 | "52","Kyrgyzstan","3878","199951","0.0193948" 54 | "53","Equatorial Guinea","539","28051","0.0192150" 55 | "54","Jordan","1635","89342","0.0183005" 56 | "55","Latvia","1150","64559","0.0178132" 57 | "56","Benin","1989","112622","0.0176608" 58 | "57","Cyprus","152","9251","0.0164307" 59 | "58","Bulgaria","1808","110879","0.0163061" 60 | "59","Republic of the Congo","5504","342000","0.0160936" 61 | "60","Guatemala","1687","108889","0.0154928" 62 | "61","East Timor","228","14874","0.0153288" 63 | "62","Liberia","1585","111369","0.0142320" 64 | "63","Cambodia","2572","181035","0.0142072" 65 | "64","French Guiana (France)","1183","83534","0.0141619" 66 | "65","Vietnam","4639","331212","0.0140061" 67 | "66","Estonia","633","45227","0.0139961" 68 | "67","Belarus","2900","207600","0.0139692" 69 | "68","Uzbekistan","6221","447400","0.0139048" 70 | "69","North Korea","1673","120538","0.0138794" 71 | "70","Eritrea","1626","117600","0.0138265" 72 | "71","Guinea","3399","245857","0.0138251" 73 | "72","Honduras","1520","112492","0.0135121" 74 | "73","Senegal","2640","196722","0.0134200" 75 | "74","Sierra Leone","958","71740","0.0133538" 76 | "75","Portugal","1214","92090","0.0131828" 77 | "76","Haiti","360","27750","0.0129730" 78 | "77","Costa Rica","639","51100","0.0125049" 79 | "78","Syria","2253","185180","0.0121665" 80 | "79","Burkina Faso","3193","272967","0.0116974" 81 | "80","Guyana","2462","214969","0.0114528" 82 | "81","Macau (People's Republic of China)","0.34","30","0.0113333" 83 | "82","Uganda","2698","241550","0.0111695" 84 | "83","Romania","2508","238391","0.0105205" 85 | "84","Suriname","1707","163820","0.0104200" 86 | "85","United Arab Emirates","867","83600","0.0103708" 87 | "86","Germany","3621","357114","0.0101396" 88 | "87","Cameroon","4591","475442","0.0096563" 89 | "88","C?te d'Ivoire","3110","322463","0.0096445" 90 | "89","Paraguay","3920","406752","0.0096373" 91 | "90","Gabon","2551","267668","0.0095305" 92 | "91","Malaysia","3147","330803","0.0095132" 93 | "92","Thailand","4863","513120","0.0094773" 94 | "93","Nicaragua","1231","130373","0.0094421" 95 | "94","Greece","1228","131957","0.0093061" 96 | "95","Poland","2788","312685","0.0089163" 97 | "96","Ghana","2094","238533","0.0087787" 98 | "97","Tunisia","1424","163610","0.0087036" 99 | "98","Burma","5876","676578","0.0086849" 100 | "99","Uruguay","1564","181034","0.0086393" 101 | "100","Pakistan","6774","796095","0.0085090" 102 | "101","Afghanistan","5529","652230","0.0084771" 103 | "102","Central African Republic","5203","622984","0.0083517" 104 | "103","Iraq","3650","438317","0.0083273" 105 | "104","Chile","6171","756102","0.0081616" 106 | "105","Finland","2690","338424","0.0079486" 107 | "106","Norway","2551","323802","0.0078783" 108 | "107","Zimbabwe","3066","390757","0.0078463" 109 | "108","Ecuador","2010","256369","0.0078403" 110 | "109","Ukraine","4663","603500","0.0077266" 111 | "110","Turkmenistan","3736","488100","0.0076542" 112 | "111","Zambia","5667","752612","0.0075298" 113 | "112","South Sudan","4797","644329","0.0074450" 114 | "113","Dominican Republic","360","48671","0.0073966" 115 | "114","Panama","555","75417","0.0073591" 116 | "115","Botswana","4015","582000","0.0068986" 117 | "116","Italy","1932","301336","0.0064114" 118 | "117","Bolivia","6743","1098581","0.0061379" 119 | "118","Kenya","3477","580367","0.0059910" 120 | "119","Mali","7243","1240192","0.0058402" 121 | "120","Mozambique","4571","801590","0.0057024" 122 | "121","Venezuela","4993","912050","0.0054745" 123 | "122","Colombia","6004","1141748","0.0052586" 124 | "123","Mongolia","8220","1564110","0.0052554" 125 | "124","France","2889","551500","0.0052384" 126 | "125","Qatar","60","11586","0.0051787" 127 | "126","Ireland","360","70273","0.0051229" 128 | "127","Sweden","2233","450295","0.0049590" 129 | "128","Mauritania","5074","1025520","0.0049477" 130 | "129","Ethiopia","5328","1104300","0.0048248" 131 | "130","Namibia","3936","824268","0.0047751" 132 | "131","Chad","5968","1284000","0.0046480" 133 | "132","Democratic Republic of the Congo","10730","2344858","0.0045760" 134 | "133","Morocco","2018","446550","0.0045191" 135 | "134","Niger","5697","1267000","0.0044964" 136 | "135","India","14103.1","3166414","0.0044540" 137 | "136","Oman","1374","309500","0.0044394" 138 | "137","Kazakhstan","12012","2724900","0.0044082" 139 | "138","Nigeria","4047","923768","0.0043810" 140 | "139","Peru","5536","1285216","0.0043074" 141 | "140","Angola","5198","1246700","0.0041694" 142 | "141","Tanzania","3861","945087","0.0040853" 143 | "142","South Africa","4862","1221037","0.0039819" 144 | "143","Spain","1918","505992","0.0037906" 145 | "144","Somalia","2340","637657","0.0036697" 146 | "145","Sudan","6764","1861484","0.0036337" 147 | "146","Argentina","9665","2780400","0.0034761" 148 | "147","Turkey","2648","783562","0.0033794" 149 | "148","Yemen","1746","527968","0.0033070" 150 | "149","Iran","5440","1648195","0.0033006" 151 | "150","Algeria","6343","2381741","0.0026632" 152 | "151","Egypt","2665","1002000","0.0026597" 153 | "152","Libya","4348","1759540","0.0024711" 154 | "153","South Korea","238","99828","0.0023841" 155 | "154","People's Republic of China","22147","9596961","0.0023077" 156 | "155","Mexico","4353","1964375","0.0022160" 157 | "156","Saudi Arabia","4431","2149690","0.0020612" 158 | "157","Papua New Guinea","820","462840","0.0017717" 159 | "158","Brazil","14691","8514877","0.0017253" 160 | "159","Denmark","68","43094","0.0015779" 161 | "160","Western Sahara","404","266000","0.0015188" 162 | "161","United Kingdom","360","242900","0.0014821" 163 | "162","Indonesia","2830","1910931","0.0014810" 164 | "163","United States","12034","9526468","0.0012632" 165 | "164","Russia","20017","17098242","0.0011707" 166 | "165","Canada","8893","9984670","0.0008907" 167 | "166","Sri Lanka","0.1","65610","0.0000015" 168 | "167","American Samoa (United States)","0","199","0" 169 | "168","Anguilla (United Kingdom)","0","91","0" 170 | "169","Antigua and Barbuda","0","442","0" 171 | "170","Aruba (Netherlands)","0","180","0" 172 | "171","Australia","0","7692024","0" 173 | "172","Bahamas","0","13943","0" 174 | "173","Bahrain","0","765","0" 175 | "174","Barbados","0","430","0" 176 | "175","Bermuda (United Kingdom)","0","54","0" 177 | "176","Bouvet Island","0","49","0" 178 | "177","British Indian Ocean Territory (United Kingdom)","0","60","0" 179 | "178","British Virgin Islands (United Kingdom)","0","151","0" 180 | "179","Cape Verde","0","4033","0" 181 | "180","Cayman Islands (United Kingdom)","0","264","0" 182 | "181","Christmas Island (Australia)","0","135","0" 183 | "182","Cocos (Keeling) Islands (Australia)","0","14","0" 184 | "183","Comoros","0","1862","0" 185 | "184","Cook Islands","0","236","0" 186 | "185","Cuba","0","109886","0" 187 | "186","Cura?ao (Netherlands)","0","444","0" 188 | "187","Dominica","0","751","0" 189 | "188","Falkland Islands (United Kingdom)","0","12173","0" 190 | "189","Faroe Islands (Denmark)","0","1393","0" 191 | "190","Federated States of Micronesia","0","702","0" 192 | "191","Fiji","0","18272","0" 193 | "192","French Polynesia (France)","0","4000","0" 194 | "193","French Southern and Antarctic Lands (France)","0","7747","0" 195 | "194","Greenland (Denmark)","0","2166086","0" 196 | "195","Grenada","0","344","0" 197 | "196","Guadeloupe (France)","0","1630","0" 198 | "197","Guam (United States)","0","549","0" 199 | "198","Guernsey (United Kingdom)","0","78","0" 200 | "199","Heard Island and McDonald Islands (Australia)","0","412","0" 201 | "200","Iceland","0","103000","0" 202 | "201","Isle of Man (United Kingdom)","0","572","0" 203 | "202","Jamaica","0","10991","0" 204 | "203","Japan","0","377930","0" 205 | "204","Jersey (United Kingdom)","0","116","0" 206 | "205","Kiribati","0","811","0" 207 | "206","Madagascar","0","587041","0" 208 | "207","Maldives","0","300","0" 209 | "208","Malta","0","316","0" 210 | "209","Marshall Islands","0","181","0" 211 | "210","Martinique (France)","0","1128","0" 212 | "211","Mauritius","0","2040","0" 213 | "212","Mayotte (France)","0","374","0" 214 | "213","Montserrat (United Kingdom)","0","102","0" 215 | "214","Nauru","0","21","0" 216 | "215","New Caledonia (France)","0","18575","0" 217 | "216","New Zealand","0","270467","0" 218 | "217","Niue (New Zealand)","0","260","0" 219 | "218","Norfolk Island (Australia)","0","36","0" 220 | "219","Northern Mariana Islands (United States)","0","464","0" 221 | "220","Palau","0","459","0" 222 | "221","Philippines","0","300000","0" 223 | "222","Pitcairn Islands (United Kingdom)","0","47","0" 224 | "223","Puerto Rico (United States)","0","8870","0" 225 | "224","R?union (France)","0","2513","0" 226 | "225","Saint Barth?lemy (France)","0","21","0" 227 | "226","Saint Helena, Ascension and Tristan da Cunha (United Kingdom)","0","308","0" 228 | "227","Saint Kitts and Nevis","0","261","0" 229 | "228","Saint Lucia","0","616","0" 230 | "229","Saint Pierre and Miquelon (France)","0","242","0" 231 | "230","Saint Vincent and the Grenadines","0","389","0" 232 | "231","Samoa","0","2842","0" 233 | "232","S?o Tom? and Pr?ncipe","0","964","0" 234 | "233","Seychelles","0","452","0" 235 | "234","Singapore","0","710","0" 236 | "235","Solomon Islands","0","28896","0" 237 | "236","South Georgia and the South Sandwich Islands (United Kingdom)","0","3903","0" 238 | "237","Svalbard (Norway)","0","62045","0" 239 | "238","Taiwan","0","36193","0" 240 | "239","Tokelau (New Zealand)","0","12","0" 241 | "240","Tonga","0","747","0" 242 | "241","Trinidad and Tobago","0","5130","0" 243 | "242","Turks and Caicos Islands (United Kingdom)","0","948","0" 244 | "243","Tuvalu","0","26","0" 245 | "244","United States Virgin Islands (United States)","0","347","0" 246 | "245","Vanuatu","0","12189","0" 247 | "246","Wallis and Futuna (France)","0","142","0" 248 | --------------------------------------------------------------------------------