├── .export └── common_profile.pqlib.pq ├── .github └── workflows │ └── PushGitLogger.yml ├── .gitignore ├── .output └── PowerQueryLib.pq ├── .vscode ├── launch.json ├── snippets.code-snippets └── tasks.json ├── Docs ├── How to document Power Query functions.md ├── List_Constants-All.csv ├── List_Functions-All.csv ├── List_Numbers.csv ├── List_Text.csv ├── List_Types.csv ├── Power query - Data type limits.md ├── Reference-Datatype-Conversions.md ├── Special-Common-Arguments.md ├── Table-Join-Algorithms.md ├── images │ ├── searching_language_csv.png │ └── using_Ninmonkey.PowerQueryLib.png └── regular expressions.md ├── ErrorRecord.Grammar.md ├── Examples-Blog ├── Custom Functions │ └── Part 1 - Using Optional Parameters.pq └── Using Web APIs in Power Query │ ├── Part 01 - Capturing-Response-Metadata.pbix │ ├── Part 01 - Capturing-Response-Metadata.pq │ └── img │ └── Part 01 - Capturing-Response-Metadata.png ├── Examples-🐒 Ninmonkey.PowerQueryLib.code-workspace ├── Examples ├── Cleanup-text-to-percentage-example.pq ├── Common_Imports.pq ├── Export-Culture │ ├── Export-DateTime-FormatStrings.ps1 │ ├── csv │ │ └── NamedDateFormatStrings.csv │ └── readme.md ├── For-Loops │ ├── Images │ │ └── screenshot_table_addcolumn1.png │ ├── Matrix and Vector Multiplication.pbix │ └── Matrix and Vector Multiplication.pq ├── Functions With Optional Default Values.pq.md ├── HowTo-Conditionally-Skip-Queries-Saving-Time.pq ├── Import.PowerQueryLib.pq ├── Logical.FromText-wip.pq ├── Merging Queries as One Library.ps1 ├── ODBC-limit-offset-clause.pq ├── Readme.Each-Synactic-Sugar.md ├── Readme.For-Loops-in-PowerQuery.md ├── Readme.md ├── Todo.md ├── Using_Write.Html_and_StringBuilder.md └── Write Power Query Source Code.ps1 ├── LICENSE ├── README-to include next.md ├── README.ValidatorsToWriteNext.md ├── README.md ├── Readme.OtherLinks.md ├── Template-Import-ninlib.pbix ├── Template-Import-ninlib.pq ├── Todo.md ├── WIP ├── 2020-08.pq ├── Combine Varargs as text.pq ├── Date.ToText ⇽ custom date formatting string ┐2021-01-29.pbix ├── DiagnosticsLog2.example.pq ├── Function-Varargs.pq ├── Functions-To-Experiment.pq ├── Inspect Text as Codepoints.pq ├── Inspect-Ast-Pair.pq ├── Inspect.Anything.pq ├── QueryWebAPI-Automatic-Schema.pq ├── Record.Schema.pq ├── Rendering codepoint info.dax ├── Table.MergeDimensionTable and To.DimensionTable.pq ├── Text.ToUnorderedList.pq ├── bad-sentence-capitalization.pq ├── buffer-find-empty-tables.pq ├── convertTo_RelativePath.doesnt_actually_invoke_the_final_yet.pq ├── custom-table-name-transformer.pq ├── detailed-uncompress-blob-zip.pq ├── diff-table-schemas.pq ├── int32-to-byte-and-back.pq ├── merge │ ├── List.SummaryRecurse-rewrite2022.pq │ └── sorta-lib-with-helpers.pq ├── old.summarize_query.pq ├── others │ ├── iso-week-number.2021.pq │ └── iso-week-number.fixed.pq ├── references-from-docs │ └── Diagnostics.pqm ├── sample-of-ui-supported-enter-data.pq ├── sanity-test │ ├── 2021-08-failed-sanity-list.pq │ └── 2021-08-ration-list.sanity-tests.pq ├── test--assert-types.pq ├── to-refactor-split-string-on-char-transition.pq ├── transform-many-example-months-and-number-formats.pq ├── transform_columns_whitespaceToNull.pq ├── uses-extra-metadata-for-author-etc_replace-values-with-non-text.pq └── validate_record_for_function_calls.pq ├── build ├── 2022-01.ninlib.pq ├── 2024-04.ninlib.pq └── auto-export │ └── 2024-06-02.ninlib.pq ├── source-before-2024 ├── Text │ ├── Text.MatchesAnyOf.CI.pq.off │ └── Text.TransformRegexReplace.pq ├── meta │ ├── shared - List Constants - All.pq │ ├── shared - List Function - Datasources.pq │ ├── shared - List Function - Not Datasource.pq │ ├── shared - List Functions - All.pq │ ├── shared - List Globals.pq │ ├── shared - List List.pq │ ├── shared - List Numbers.pq │ ├── shared - List Records.pq │ ├── shared - List Tables.pq │ ├── shared - List Text.pq │ └── shared - List Types.pq ├── random │ └── List.RandomItem.pq └── web │ ├── WebRequest.ToRecord.pq │ └── WebRequest.old.pq ├── source-dax ├── Conditional-Colors-Dax-CheatSheet.md ├── ConditionalColors-UseMinMax-For-Gradients.dax ├── Convert-ControlChars-to-Symbols.dax ├── Convert-Hex-Dec-Text.dax ├── Find-RowIntegrity-Violations.dax ├── Format-Percent-Without-Math-Or-P.dax ├── Safely Render UniChar -- replacing with symbols.dax └── new-syntax-datetime-literals-in-strings-are-true-datetime-values.dax ├── source-excel └── GetAbsoluteWorkbookPath.md ├── source-modules ├── Alias │ └── Alias.Text.pq ├── Color.Constants.pq ├── DebugUtils.module.pq ├── Encoding.Constants.pq ├── StringBuilder.module.pq ├── Unicode.Constants.pq └── Write.Html.module.pq ├── source-pwsh ├── Nin.PqLib.psd1 ├── Nin.PqLib.psm1 ├── old-example-src │ ├── Text to PowerQueryLiterals.ps1 │ ├── __init__.ps1 │ └── format-pqlibDate.ps1 ├── readme.build.ps1 └── tests │ └── Get-NativeCommand.tests.ps1 ├── source ├── Assert │ ├── Assert.ColumnsExist.pq │ ├── Assert.ListLength.pq │ └── Columns.ThatExist.pq ├── Binary.ToBase64.pq ├── CoerceTo.Table.pq ├── CoerceTo.Table.succinct.pq ├── Color │ └── Rgb.FromHexString.pq ├── ConvertTo.Markdown.pq ├── DateTable_FromDates.backup.pq ├── DateTable_FromDates.pq ├── DateTime.FromUnixTime.pq ├── DateTime.ToOData.pq ├── Diagnostic.FormatDetailedLog.pq ├── ErrorRecord.Format.pq ├── Errors │ ├── Convert.ScriptExtent.FromError.pq │ ├── Err.ExpandErrorRecord.Verbose.pq │ ├── Err.ExpandErrorRecord.pq │ └── Err.InvalidColumnNames.pq ├── Evaluate │ ├── EvalQuery.pq │ └── Example │ │ ├── EvalQuery.pq.example │ │ └── Import.PowerQueryLib.pq.example ├── Folder.FilesFormatted.pq ├── Grouping │ └── GroupBy.ShowCounts.pq ├── Html │ ├── Examples │ │ └── Write.Html.pq.example │ └── Write.Html.pq ├── IP.DottedDecimalFromList.pq ├── Inspect.MetaOfType.pq ├── Inspect_TableColumn.pq ├── List.AllDates.pq ├── List.Combine_BitFlags.pq ├── List.ContinuousDates.pq ├── List.Schema.pq ├── List.Summarize.pq ├── List.SummarizeListOfRecords.pq ├── List │ └── List.SelectBySuffix.pq ├── Matrix.MultiplyVectorDotProduct.pq ├── Number.From_TextWithBase.pq ├── Number.ToHexString.pq ├── Number │ ├── Duration.FromCustomText.pq │ ├── Number.FromHexString.pq │ └── Number.ToHexString.pq ├── ParameterQuery.Summary.pq ├── Query.Summarize.pq ├── Record.Summarize.pq ├── Serialize.ExtendedType.pq ├── Serialize.List.pq ├── Serialize.Text.pq ├── Sketch-SummarizeError.pq ├── Splitter.SplitOnCaseChange.pq ├── Splitter │ └── Splitter.SplitByDigitWithSuffix.pq ├── SummarizeRec.pq ├── Table.ColumnContainsNonBlank.pq ├── Table.ContinuousDates.pq ├── Table.FindBlankColumns.pq ├── Table.FindNotDistinctRows.pq ├── Table.FindSingleValueColumns.pq ├── Table.FromMashupLog.pq ├── Table.RemoveBlankColumns.pq ├── Table.SelectRemovedColumns.pq ├── Table.ToJson.pq ├── Text │ ├── Example │ │ └── Text.JoinString.pq.example │ ├── Format.JoinBR.pq │ ├── Format.JoinNewLine.pq │ ├── Format.ShowBlank.pq │ ├── Format.SimplifyJson.pq │ ├── Format.ToJsonText.pq │ ├── Text.AnyMatches.pq │ ├── Text.Contains.CI.pq │ ├── Text.EndsWith.CI.pq │ ├── Text.FormatCsv.pq │ ├── Text.FormatList.pq │ ├── Text.IsNullOrWhitespace.pq │ ├── Text.JoinString.pq │ ├── Text.JsonToPowerQuery.pq │ ├── Text.MatchesAny.pq │ ├── Text.PositionOf.CI.pq │ ├── Text.PositionOf.pq │ ├── Text.RemoveDiacritics.pq │ ├── Text.ReplaceFirstOnly.pq │ ├── Text.ReplaceMany.pq │ ├── Text.WordWrap.pq │ ├── TransformTo.TextList.pq │ └── XRay.pq ├── Type.ToText.pq ├── Type.ToText_simple.pq ├── Unicode.GenerateRange.pq ├── Validate_TableSchema.old.pq ├── Validate_TableSchema.pq ├── Value.ToPowerQuery.pq ├── Xml │ └── Table.AutoExpandTableColumn.pq ├── alias │ ├── Inspect.Metadata.pq │ ├── Inspect.Type.pq │ ├── alias_typeOf.pq │ ├── default_alias_list.pq │ └── mdt.pq ├── docs │ ├── Format.DocExpand.pq │ └── Generate-Docs-JoinAlgorithm.pq ├── inspect │ ├── Inspect.Function.pq │ └── function │ │ └── inspect_func_enumFieldNames.pq ├── old.Inspect.Metadata.pq ├── prototype │ ├── Error.Summarize.pq │ └── Import_PowerQueryLib.pq ├── random │ ├── Compare.RandomInt64.pq │ ├── List.RandomIndex.pq │ ├── List.RandomItem.pq │ ├── Random.Currency.pq │ └── Random.Int64.pq ├── readme.ByPrefix.md ├── readme.md ├── sketch-error-summary.md ├── test │ ├── summarize-schema.pq │ ├── test_All.pbix │ ├── test_All.pbix-all.pq │ ├── test_DateTime_FromUnixTime.pq │ ├── test_DateTime_ToOData.pbix │ ├── test_DateTime_ToOData.pq │ └── test_ListAsText.old.pq └── web │ ├── Html.GenerateSelectorList.pq │ ├── Html.GetScalar.pq │ ├── ImageUrl.ToBase64Text.pq │ ├── Web.SimpleRequest.pq │ ├── Web.SimpleRequest.tests.pq │ ├── WebRequest-ExtraDebugDetails.pq │ ├── WebRequest.pq │ ├── WebRequest_Simple.before_docs.pq │ ├── WebRequest_Simple.pq │ ├── ai-request-llama-llm.pq │ ├── img │ └── WebRequest-ExtraDebugDetails.pq.png │ └── waitForResult.pq ├── src-new-text-replacemany-documented.2022-07.pq ├── tabular └── Export-BestPractice-Results.cs ├── template ├── Hi world connector.pq ├── Power Query function with Documentation.pq ├── Reused Allowed Values - template.md ├── maybe dupe - Hi world connector.pq └── table_constructors.md ├── testing ├── AsanaWebRequest.pq ├── PowerQuery-documentation-grammar.md ├── Summarize_Record.recursion.tests.pq ├── Test-Web.Contents-SSL-Errors.pq ├── Unit-Test-Table-Constructor-Args.pbix ├── syntax-highlight-stress-other.pq └── syntax-highlight-stress-test.pq ├── todo.snippets.pq.md ├── util ├── Find-PowerBIPaths.ps1 ├── Invoke-Build.old ├── Invoke-Build.old.ps1 ├── Invoke-BuildPowerQueryLib.ps1 ├── common_profile.ps1 ├── common_profile_web.ps1 ├── logs │ └── parse-vscode_pq.ps1 ├── profile │ ├── Invoke-BuildAdventCode.ps1 │ ├── Invoke-BuildFull.ps1 │ ├── Invoke-BuildPQParser.ps1 │ └── import-pqlib-error-details.pq └── profile_blog.ps1 ├── wip - left off here.md ├── 🐒 -c- 2022- mini-workspace.PowerQueryLib.code-workspace ├── 🐒 Ninmonkey.PowerQueryLib-SingleFolder.code-workspace └── 🐒 Ninmonkey.PowerQueryLib.code-workspace /.github/workflows/PushGitLogger.yml: -------------------------------------------------------------------------------- 1 | name: demo List Files 2 | run-name: ${{ github.actor }} is testing out GitHub Actions 🚀 3 | on: [push] 4 | jobs: 5 | Explore-GitHub-Actions: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." 9 | - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" 10 | - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." 11 | - name: Check out repository code 12 | uses: actions/checkout@v3 13 | - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." 14 | - run: echo "🖥️ The workflow is now ready to test your code on the runner." 15 | - name: Run GitLogger 16 | uses: GitLogging/GitLoggerAction@main 17 | - name: List files in the repository 18 | run: | 19 | ls ${{ github.workspace }} 20 | - run: echo "🍏 This job's status is ${{ job.status }}." 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | todo.snippets.md 2 | **/*.lnk 3 | source/test/old-iterations/* 4 | .output/* 5 | -------------------------------------------------------------------------------- /.output/PowerQueryLib.pq: -------------------------------------------------------------------------------- 1 | /* PowerQueryLib : v 0.1 2 | Generated on: 10/16/2021 6:37 PM 3 | Source: 4 | https://github.com/ninmonkey/Ninmonkey.PowerQueryLib 5 | Jake Bolton ninmonkeys@gmail.com 6 | */let 7 | Metadata = [ 8 | LastExecution = DateTime.FixedLocalNow(), 9 | PQLib = "0.1", 10 | GeneratedOn = "2021-10-16T18:37:23.5699656-05:00", 11 | Commit = "d4ae4e4c7518fd39e8589900efe3310af934623b", 12 | BuildArgIncludes = "", 13 | BuildArgExcludes = "'.*', '$test_', 'wordwrap', 'regex', 'shared -', 'inspect', 'WebRequest', '\.old\.pq$'" 14 | ], 15 | FinalRecord = [ 16 | 17 | ] 18 | in 19 | FinalRecord 20 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | "version": "2.0.0", 4 | "tasks": [ 5 | { 6 | "label": "Build Library", 7 | "type": "shell", 8 | "command": "pwsh.exe -f './util/Invoke-BuildPowerQueryLib.ps1'", 9 | "problemMatcher": [], 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /Docs/List_Text.csv: -------------------------------------------------------------------------------- 1 | "Name","Value" 2 | "Culture.Current","en-US" 3 | "ODataOmitValues.Nulls","nulls" 4 | "WebMethod.Delete","DELETE" 5 | "WebMethod.Get","GET" 6 | "WebMethod.Head","HEAD" 7 | "WebMethod.Patch","PATCH" 8 | "WebMethod.Post","POST" 9 | "WebMethod.Put","PUT" -------------------------------------------------------------------------------- /Docs/List_Types.csv: -------------------------------------------------------------------------------- 1 | "Name","Empty For Github" 2 | "AccessControlEntry.ConditionContextType","" 3 | "AccessControlEntry.Type","" 4 | "AccessControlKind.Type","" 5 | "Any.Type","" 6 | "Binary.Type","" 7 | "BinaryEncoding.Type","" 8 | "BinaryOccurrence.Type","" 9 | "Byte.Type","" 10 | "ByteOrder.Type","" 11 | "Character.Type","" 12 | "Compression.Type","" 13 | "CsvStyle.Type","" 14 | "Currency.Type","" 15 | "Date.Type","" 16 | "DateTime.Type","" 17 | "DateTimeZone.Type","" 18 | "Day.Type","" 19 | "Decimal.Type","" 20 | "Double.Type","" 21 | "Duration.Type","" 22 | "EmigoDataSourceConnector.NavigationFunctionType","" 23 | "ExtraValues.Type","" 24 | "Function.Type","" 25 | "GroupKind.Type","" 26 | "Guid.Type","" 27 | "HiveProtocol.Type","" 28 | "Identity.Type","" 29 | "IdentityProvider.Type","" 30 | "Int16.Type","" 31 | "Int32.Type","" 32 | "Int64.Type","" 33 | "Int8.Type","" 34 | "JoinAlgorithm.Type","" 35 | "JoinKind.Type","" 36 | "JoinSide.Type","" 37 | "LimitClauseKind.Type","" 38 | "List.Type","" 39 | "Logical.Type","" 40 | "MissingField.Type","" 41 | "None.Type","" 42 | "Null.Type","" 43 | "Number.Type","" 44 | "Occurrence.Type","" 45 | "ODataOmitValues.Type","" 46 | "Order.Type","" 47 | "Password.Type","" 48 | "Percentage.Type","" 49 | "Precision.Type","" 50 | "QuoteStyle.Type","" 51 | "Record.Type","" 52 | "RelativePosition.Type","" 53 | "RoundingMode.Type","" 54 | "SapBusinessWarehouseExecutionMode.Type","" 55 | "SapHanaDistribution.Type","" 56 | "SapHanaRangeOperator.Type","" 57 | "Single.Type","" 58 | "SparkProtocol.Type","" 59 | "Table.Type","" 60 | "Text.Type","" 61 | "TextEncoding.Type","" 62 | "Time.Type","" 63 | "TraceLevel.Type","" 64 | "Type.Type","" 65 | "Uri.Type","" 66 | "WebMethod.Type","" -------------------------------------------------------------------------------- /Docs/Power query - Data type limits.md: -------------------------------------------------------------------------------- 1 | # Power Query: Data types 2 | 3 | Notes on data types in powerquery. It includes `primative types` and custom types. 4 | 5 | 6 | ## `number` 7 | 8 | Docs: 9 | > docs: A `number` is represented with **at least the precision** of a `Double` (but may retain more precision). The `Double` representation is congruent with the `IEEE 64-bit double` precision standard for binary floating point arithmetic 10 | > `Double` representation have an approximate dynamic range from `5.0 × 10−324` to `1.7 × 10308` with a **precision of 15-16 digits** 11 | 12 | 13 | 14 | ## `Decimal.Type` 15 | 16 | Docs: 17 | > representation have an **approximate dynamic range** from `5.0 × 10−324` to `1.7 × 10308` with a 18 | **precision of 15-16 digits** 19 | 20 | 21 | ## `duration` 22 | 23 | full value is signed `int64` ? 24 | 25 | Docs: 26 | > A `duration` value stores an opaque representation of the distance between two points on a timeline **measured 100-nanosecond ticks**. The magnitude of a duration can be either positive or negative, with positive values denoting progress forwards in time and negative values denoting progress backwards in time. 27 | > The **minimum value** that can be stored in a duration is `-9,223,372,036,854,775,808 ticks`, or `10,675,199 days 2 hours 48 minutes 05.4775808 seconds` backwards in time 28 | > The **maximum value** that can be stored in a duration is `9,223,372,036,854,775,807 ticks`, or `10,675,199 days 2 hours 48 minutes 05.4775807 seconds` forwards in time. 29 | 30 | 31 | ### .net test 32 | 33 | ```powershell 34 | 🐒> [timespan]::MaxValue.tostring() 35 | 10675199.02:48:05.4775807 36 | 37 | 🐒> [timespan]::MaxValue 38 | 39 | Days : 10675199 40 | Hours : 2 41 | Minutes : 48 42 | Seconds : 5 43 | Milliseconds : 477 44 | Ticks : 9223372036854775807 45 | TotalDays : 10675199.1167301 46 | TotalHours : 256204778.801522 47 | TotalMinutes : 15372286728.0913 48 | TotalSeconds : 922337203685.478 49 | TotalMilliseconds : 922337203685477 50 | 51 | # h1 "FromUnixTimeSeconds( $unixTime )" 52 | $unixTime = 1604073449 53 | [DateTimeOffset]::FromUnixTimeSeconds( $unixTime ) 54 | 55 | # h1 "FromUnixTimeMilliseconds( $unixTime )" 56 | [DateTimeOffset]::FromUnixTimeMilliseconds( $unixTime ) 57 | ``` -------------------------------------------------------------------------------- /Docs/Reference-Datatype-Conversions.md: -------------------------------------------------------------------------------- 1 | - [PowerBI / DAX](#powerbi--dax) 2 | - [Power Query](#power-query) 3 | - [SQL](#sql) 4 | 5 | ## PowerBI / DAX 6 | ## Power Query 7 | ## SQL -------------------------------------------------------------------------------- /Docs/Special-Common-Arguments.md: -------------------------------------------------------------------------------- 1 | # CompareCriteria, EquationCriteria 2 | 3 | ## `Comparison` criteria 4 | 5 | See [Value.Compare](https://docs.microsoft.com/en-us/powerquery-m/value-compare) 6 | 7 | ```ts 8 | Value.Compare( 9 | value1 as any, 10 | value2 as any, 11 | optional precision as nullable number 12 | ) as number 13 | ``` 14 | 15 | `Comparison` criterion` can be provided as either of the following values: 16 | 17 | - A number value to specify a sort order. See sort order in the parameter values section above. 18 | - To **compute a key** to be used for sorting, a **`function` of 1 argument** can be used. 19 | - To **both select a key and control order**, `comparison criterion` can be a list containing the key and order. 20 | - To completely control the `comparison`, a `function` of 2 arguments can be used that returns `-1, 0, or 1` given the **relationship between the left and right inputs**. Value.Compare is a method that can be used to delegate this logic. 21 | 22 | For examples, see description of Table.Sort. 23 | 24 | ## `Count` or `Condition` critieria 25 | 26 | This **criteria** is generally used in ordering or row operations. It determines the number of rows returned in the table and can take two forms, a number or a `condition`: 27 | 28 | 1. A number indicates how many values to return inline with the appropriate `function` 29 | 1. If a `condition` is specified, the rows containing values that initially meet the `condition` is returned. 30 | 1. Once a value fails the `condition`, no further values are considered. 31 | 32 | ## `Equation` criteria 33 | 34 | See [Value.Equals](https://docs.microsoft.com/en-us/powerquery-m/value-equals) 35 | 36 | ```ts 37 | Value.Equals( 38 | value1 as any, 39 | value2 as any, 40 | optional precision as nullable number 41 | ) as logical 42 | ``` 43 | 44 | Equation **criteria** for tables can be specified as either a 45 | 46 | 1. A `function` value that is either 47 | A key selector that determines the column in the table to apply the `equality criteria`, or 48 | A `comparer function` that is used to specify the kind of `comparison` to apply. Built in `comparer functions` can be specified, see section for `Comparer functions`. 49 | 50 | 2. A list of the columns in the table to apply the `equality criteria` 51 | 52 | -------------------------------------------------------------------------------- /Docs/Table-Join-Algorithms.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Docs/Table-Join-Algorithms.md -------------------------------------------------------------------------------- /Docs/images/searching_language_csv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Docs/images/searching_language_csv.png -------------------------------------------------------------------------------- /Docs/images/using_Ninmonkey.PowerQueryLib.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Docs/images/using_Ninmonkey.PowerQueryLib.png -------------------------------------------------------------------------------- /Docs/regular expressions.md: -------------------------------------------------------------------------------- 1 | # Regular Expressions 2 | 3 | 4 | ## Convert copy->pasting columns from PBIX 5 | 6 | `csv` files are generated using 7 | 8 | ```ps1 9 | # for two columns 10 | $in = '(.*)\s+(.*)' 11 | $replace = '"$1","$2"' 12 | ``` 13 | -------------------------------------------------------------------------------- /Examples-Blog/Custom Functions/Part 1 - Using Optional Parameters.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | About: Compare different optional parameter types 4 | From this blog post: https://ninmonkeys.com/blog/2024/06/05/power-query-functions-part1-using-optional-parameters 5 | source: https://github.com/ninmonkey/Ninmonkey.PowerQueryLib/blob/bf4e1d63b8bb9ffe3b3e4df0d687435bb88be30d/Examples-Blog/Custom%20Functions/Part%201%20-%20Using%20Optional%20Parameters.pq 6 | 7 | 8 | lets call Text.Combine() to test declaring optional parameters: 9 | 10 | For this version: 11 | - you can pass a null value for a separator 12 | - always requires you pass 2 parameters 13 | */ 14 | Join_Nullable = (texts as list, separator as nullable text) => 15 | Text.Combine( texts, separator ), 16 | 17 | /* 18 | For this version: 19 | - you can pass a null value for a separator 20 | - you can skip the second parameter 21 | - 'optional' parameters are automatically 'nullable', 22 | so you can drop the 'nullable' part 23 | 24 | This is how library functions have multiple call signatures 25 | Power Query defines one function 26 | 27 | Other languages let you define multiple functions with shared name 28 | Based on the argument types, it'll call a different overloaded function 29 | */ 30 | Join_Optional = (texts as list, optional separator as text) => 31 | Text.Combine( texts, separator ), 32 | 33 | Summary = [ 34 | chars = { "a".."h" }, // example array of strings 35 | 36 | // this version lets you pass an explicit null value 37 | Nullable_1 = Join_Nullable( chars, ", " ), 38 | Nullable_2 = Join_Nullable( chars, null ), 39 | 40 | // but it requires you to pass something. It doesn't let you omit a parameter 41 | Nullable_3 = Join_Nullable( chars ), 42 | 43 | // this version lets you pass an explicit null value 44 | // or drop the parameter completely 45 | Join_Optional_1 = Join_Optional( chars, ", " ), 46 | Join_Optional_2 = Join_Optional( chars, null ), 47 | Join_Optional_3 = Join_Optional( chars ) 48 | ] 49 | in Summary 50 | -------------------------------------------------------------------------------- /Examples-Blog/Using Web APIs in Power Query/Part 01 - Capturing-Response-Metadata.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Examples-Blog/Using Web APIs in Power Query/Part 01 - Capturing-Response-Metadata.pbix -------------------------------------------------------------------------------- /Examples-Blog/Using Web APIs in Power Query/Part 01 - Capturing-Response-Metadata.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | This is a stand alone example showing how to read response metadata 4 | See: https://ninmonkeys.com/blog/2024/05/31/power-query-capturing-response-metadata-from-web-contents 5 | */ 6 | RequestSummary = [ 7 | base_url = "https://httpbin.org", 8 | // if not set, these functions default to TextEncoding.Utf8 . 9 | // That's the most common encoding for the web 10 | RawBytes = Web.Contents( base_url, [ RelativePath = "/json" ]), // type is: binary|error 11 | ResponseMeta = Value.Metadata( RawBytes ), // type is: record 12 | Json = Json.Document( RawBytes ), // type is: record/list/error 13 | 14 | // We Can Detect if JSON parsed correctly 15 | IsJson = not (try Json)[HasError], // type is: logical 16 | 17 | // Decoding the response as raw text is useful for debugging. 18 | // say the api returns HTML when you expect JSON. That causes Json.Document to error 19 | // RawText still works, making it easy to check if it's actually returning CSV/JSON/HTML/etc. 20 | RawText = Text.FromBinary( RawBytes ), // type is: text/error 21 | 22 | // Lets strip newlines, making it easier to preview text in the UI 23 | TextWithoutNewLines = Text.Remove( RawText, { "#(lf)", "#(cr)" } ), // type is: text 24 | 25 | // Now lets grab important fields by drilling into the ResponseMeta record 26 | 27 | // Http Status Codes are explained here: 28 | StatusCode = ResponseMeta[Response.Status]?, // type is: text 29 | 30 | // Content.Uri is a function that returns the full filepath for files, or url for web requests 31 | FullRequestUrl = ResponseMeta[Content.Uri]() // type is: text 32 | ] 33 | in 34 | RequestSummary -------------------------------------------------------------------------------- /Examples-Blog/Using Web APIs in Power Query/img/Part 01 - Capturing-Response-Metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Examples-Blog/Using Web APIs in Power Query/img/Part 01 - Capturing-Response-Metadata.png -------------------------------------------------------------------------------- /Examples/Cleanup-text-to-percentage-example.pq: -------------------------------------------------------------------------------- 1 | let 2 | enableTest = false, 3 | Percent_FromText_Wrapped = (source as text) as number => 4 | // input: "(50.87%)" 5 | // output: Percentage.Type 6 | let 7 | trim_list = {"(", ")"}, 8 | strip = Text.Trim(source, trim_list), 9 | perc = Percentage.From(strip) 10 | in 11 | perc, 12 | 13 | Percent_FromText_Division = (source as text) as number => 14 | // input: "117/230" 15 | // output: Percentage.Type 16 | let 17 | crumbs = Text.Split(source, "/"), 18 | perc = Percentage.From( 19 | Number.From( crumbs{0} ) / Number.From( crumbs{1} ) 20 | ) 21 | in 22 | perc, 23 | Percentage_FromText = (source as text) as number => 24 | /* 25 | input: "117/230" 26 | input: "(40.83%)" 27 | */ 28 | let 29 | containsDivide = Text.Contains(source, "/"), 30 | result = 31 | if containsDivide then Percent_FromText_Division(source) 32 | else Percent_FromText_Wrapped(source) 33 | in 34 | result, 35 | 36 | tests = [ 37 | fromParens = Percentage_FromText("(20.45%)"), 38 | fromDivide = Percentage_FromText("120/345") 39 | ], 40 | Final = if enableTest then tests else Percentage_FromText 41 | in 42 | Final -------------------------------------------------------------------------------- /Examples/Common_Imports.pq: -------------------------------------------------------------------------------- 1 | let 2 | Version = lib[Lib], 3 | // local references to "import" functions into the current scope 4 | List.Summarize = lib[List.Summarize], 5 | Record.Summarize = lib[Record.Summarize], 6 | Table.Summarize = lib[Table.Summarize], 7 | // future: auto-expand docstrings for function table as a lib summary 8 | // libSummary_table = List.Summarize(Functions), 9 | summaryTable = Record.ToTable( lib ) 10 | 11 | in 12 | summaryTable -------------------------------------------------------------------------------- /Examples/Export-Culture/Export-DateTime-FormatStrings.ps1: -------------------------------------------------------------------------------- 1 | $ModulePath = Join-Path $PSScriptRoot '../../source-pwsh/Nin.PqLib.psd1' | Get-Item -ea 'stop' 2 | Import-Module $ModulePath -PassThru # -Force 3 | $ExportConfig = @{ 4 | ExportExcel = $false 5 | ExportCsv = $true 6 | ExportJson = $false 7 | } 8 | 9 | $query ??= Get-PqLibNamedDateFormatStrings -CultureName (Get-Culture -ListAvailable) 10 | 11 | # | Export-Csv -Path 12 | # |select -first 20 | Ft -AutoSize 13 | # $cachedAllCultsList| some 32 14 | 15 | mkdir -ea ignore (Join-path $PSScriptRoot 'csv') 16 | mkdir -ea ignore (Join-path $PSScriptRoot 'excel') 17 | mkdir -ea ignore (Join-path $PSScriptRoot 'json') 18 | 19 | $PathCsv = Join-Path $PSScriptRoot '/csv/NamedDateFormatStrings.csv' 20 | $PathExcel = Join-Path $PSScriptRoot '/excel/NamedDateFormatStrings.xlsx' 21 | $PathJson = Join-Path $PSScriptRoot '/json/NamedDateFormatStrings.json' 22 | 23 | if( $ExportConfig.ExportCsv ) { 24 | $query 25 | | Select-Object CultureName, Name, FormatString 26 | | Export-Csv -Path $PathCsv -Encoding utf8BOM # excel requires UTF8BOM if unicode 27 | 28 | $PathCsv | Get-Item | Join-String -f 'Wrote: ' | write-verbose -verbose 29 | } 30 | -------------------------------------------------------------------------------- /Examples/Export-Culture/readme.md: -------------------------------------------------------------------------------- 1 | [Go Back](../Readme.md) 2 | 3 | - [Exporting Cultures Examples](#exporting-cultures-examples) 4 | 5 | ## Exporting Cultures Examples 6 | 7 | [Named DateTime FormatStrings by Culture.csv](./csv/NamedDateFormatStrings.csv) was built with this command: 8 | 9 | ```powershell 10 | Import-Module Nin.PqLib 11 | 12 | Get-PqLibNamedDateFormatStrings -CultureName (Get-Culture -ListAvailable) 13 | | Select-Object CultureName, Name, FormatString 14 | | Export-Csv -Path 'export.csv' -Encoding utf8BOM # excel requires UTF8BOM if unicode 15 | ``` -------------------------------------------------------------------------------- /Examples/For-Loops/Images/screenshot_table_addcolumn1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Examples/For-Loops/Images/screenshot_table_addcolumn1.png -------------------------------------------------------------------------------- /Examples/For-Loops/Matrix and Vector Multiplication.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Examples/For-Loops/Matrix and Vector Multiplication.pbix -------------------------------------------------------------------------------- /Examples/Functions With Optional Default Values.pq.md: -------------------------------------------------------------------------------- 1 | ```sdfs 2 | ``` 3 | 4 | 5 | - 1] the syntax to declare a default, optional value named 'encoding' 6 | 7 | - 2] if 'encoding' is missing or null, then set the value to Utf8. 8 | 9 | - 3] you can "declare" local variables by using an inner let-expression 10 | 11 | - 4] Local variables 12 | 13 | 14 | ```sql 15 | let 16 | /* this demonstrates a few things: 17 | 18 | the variable 'declaring 'encoding' inside the inner let expression is like a local variable 19 | the variable is "shadowing" the variable of the outer scope. 20 | 21 | */ 22 | TextToBytes = (source as text, optional encoding as nullable number) => 23 | let 24 | encoding = encoding ?? TextEncoding.Utf8, 25 | bytes = Text.ToBinary(source, encoding ) 26 | in 27 | bytes, 28 | 29 | // Custom3 = Value.Metadata( Text.ToBinary ), 30 | Example = [ 31 | defaults = TextToBytes("hi #(0001F412) world!"), // 14 bytes 32 | ascii = TextToBytes("hi #(0001F412) world!", TextEncoding.Ascii ), // 12 bytes 33 | 34 | // encoding a monkey U+1f412 as ascii, will destroy it. 35 | #"round trip: ascii" = Text.FromBinary(ascii, TextEncoding.Ascii), 36 | 37 | // encoding as utf8 works 38 | #"round trip: utf8" = Text.FromBinary(defaults, TextEncoding.Utf8) 39 | ] 40 | 41 | in 42 | Example 43 | ``` 44 | -------------------------------------------------------------------------------- /Examples/HowTo-Conditionally-Skip-Queries-Saving-Time.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | See more: 4 | 5 | - [Dynamic, Lazy Records](https://bengribaudo.com/blog/2021/08/09/5941/dynamic-lazy-records) 6 | - [For fun, a deeper dive](https://bengribaudo.com/blog/2023/03/03/7292/lazy-streamed-immutable-try-building-a-table] 7 | 8 | About: 9 | this example shows that the api call can be skipped if it's not referenced. 10 | Set SkipLoad to true or false, and refresh. 11 | */ 12 | 13 | delay = #duration(0,0,0, 3), 14 | SkipLoad = true, 15 | 16 | // if they are called, it will delay for 3 seconds to simulate a query or web request 17 | // skip load ends up skipping the query, and the 3 second delay 18 | 19 | 20 | // sleep for delay Seconds and return a string 21 | Api1 = () => Function.InvokeAfter( 22 | () => "Api1 finished", delay ), 23 | 24 | // for fun, this version uses a string template to create the error rmessage 25 | Api2 =() => Function.InvokeAfter( 26 | () => 27 | Text.Format( 28 | "Api2 finished, at #[dt]", 29 | [ dt = DateTime.LocalNow() ] 30 | ), 31 | delay ), 32 | 33 | 34 | // the important part of the query: 35 | ApiList = { Api1(), Api2() }, 36 | Query2 = { "some", "Fallback" }, 37 | Final = if SkipLoad = true then Query2 else ApiList 38 | in 39 | Final 40 | -------------------------------------------------------------------------------- /Examples/Logical.FromText-wip.pq: -------------------------------------------------------------------------------- 1 | let 2 | runTest = false, 3 | Logical_FromText_WR = (item as text) as logical => 4 | // input: R => true 5 | // input: W => false 6 | if item = "R" then true else false, 7 | 8 | Logical_FromText_YN = (item as text) as logical => 9 | // input: Y => true 10 | // input: any => false 11 | if item = "Y" then true else false, 12 | 13 | Logical_FromTextAny = (item as text) as logical => 14 | let 15 | hasWR = 16 | List.AnyTrue( 17 | { 18 | Text.Contains(item, "R", Comparer.OrdinalIgnoreCase), 19 | Text.Contains(item, "W", Comparer.OrdinalIgnoreCase) 20 | } 21 | ), 22 | 23 | result = 24 | if hasWR then Logical_FromText_WR(item) 25 | else Logical_FromText_YN(item) 26 | in 27 | result, 28 | tests = [ 29 | #"R should be True" = Logical_FromText_WR("R"), 30 | #"W should be False" = Logical_FromText_WR("W"), 31 | #"Y should be True" = Logical_FromText_YN("Y"), 32 | #"other should be False" = Logical_FromText_YN(""), 33 | #"Auto R" = Logical_FromTextAny("R"), 34 | #"Auto W" = Logical_FromTextAny("W"), 35 | #"Auto Y" = Logical_FromTextAny("Y"), 36 | #"Auto other" = Logical_FromTextAny("") 37 | ], 38 | final = 39 | if runTest then tests 40 | else Logical_FromTextAny 41 | in 42 | final -------------------------------------------------------------------------------- /Examples/Merging Queries as One Library.ps1: -------------------------------------------------------------------------------- 1 | 'Running: {0}' -f $PSCommandPath | Write-verbose -verbose 2 | $impo_Splat = @{ ErrorAction = 'stop' ; PassThru = $true } 3 | $ModulePath = Get-Item 'H:\data\2024\tabular\Ninmonkey.PowerQueryLib\source-pwsh\Nin.PqLib.psd1' 4 | $AppRoot = Get-Item -ea 'stop' $PSScriptRoot 5 | @( 6 | Import-module @impo_splat Pansies 7 | Import-module @impo_splat -Force $ModulePath 8 | ) | Join-String -p { $_.Name, $_.Version } -sep ', ' 9 | 10 | $Config = @{ 11 | AppRoot = $AppRoot 12 | ImportRoot = Join-Path $AppRoot '../source' | Get-Item -ea 'stop' 13 | AutoExportRoot = Join-path $AppRoot '../build/auto-export' | Get-Item -ea 'stop' 14 | } 15 | $Color = @{ 16 | Fg = '#47c589' 17 | Green = '#47c589' 18 | H1 = '#fea9aa' 19 | Red = '#fea9aa' 20 | } 21 | 22 | $Config | Ft -auto 23 | $RegexMustMatch = @( 24 | # 'Html' 25 | 'Random' 26 | ) 27 | $select_pq = Find-PqLibSources -RootDir $Config.ImportRoot -Regex $RegexMustMatch 28 | 29 | 'Final Files' | write-host -fg $Color.H1 30 | $select_pq -join "`n" 31 | 32 | $dynamicName = Join-String -f '{0}.ninlib.pq' -In (get-date).tostring('yyyy-MM-dd') 33 | $newExport = Join-path $Config.AutoExportRoot $DynamicName 34 | hr 35 | gc ($select_pq | select -First 1) | sc -Path $newExport 36 | $newExport = $newExport | Get-Item 37 | 'wrote: {0}' -f $NewExport | write-host -fg $Color.Red 38 | 39 | Get-PqLibManifestInfo|ft -AutoSize 40 | 41 | return 42 | 43 | $all_pq_files = FindPowerQuerySources -RootDir $Config.ImportRoot 44 | $Patterns = @( 45 | 'Html.Write' 46 | ) 47 | 48 | $select_pq = $all_pq_files | ?{ 49 | $Patterns.Where({ $_.Name -match $_ }, 'first', 1) 50 | $_.name -match $Patterns 51 | } 52 | $select_pq -join "`n" 53 | -------------------------------------------------------------------------------- /Examples/ODBC-limit-offset-clause.pq: -------------------------------------------------------------------------------- 1 | 2 | source: 3 | 4 | // when LIMIT but not OFFSET is supported: 5 | LimitClause1 = (skip, take) => 6 | if (skip > 0) then error "Skip/Offset not supported" 7 | else 8 | [ 9 | Text = Text.Format("LIMIT #{0}", {take}), 10 | Location = "AfterQuerySpecification" 11 | ]; 12 | 13 | // LIMIT and OFFSET 14 | LimitClause2 = (skip, take) => 15 | let 16 | offset = if (skip > 0) then Text.Format("OFFSET #{0} ROWS", {skip}) else "", 17 | limit = if (take <> null) then Text.Format("LIMIT #{0}", {take}) else "" 18 | in 19 | [ 20 | Text = Text.Format("#{0} #{1}", {offset, limit}), 21 | Location = "AfterQuerySpecification" 22 | ]; -------------------------------------------------------------------------------- /Examples/Readme.Each-Synactic-Sugar.md: -------------------------------------------------------------------------------- 1 | 2 | ## Walkthrough, Example converting `each` functions 3 | Short version: 4 | ```ts 5 | List.Generate(() => 0, each _ > 5, each _ + 1) 6 | ``` 7 | ```ts 8 | //formatted 9 | numbers = List.Generate( 10 | () => 0, 11 | each _ < 5, 12 | each _ + 1, 13 | each _ 14 | ) 15 | ``` 16 | Same results, without the each-sugar 17 | ```ts 18 | numbers = List.Generate( 19 | () => 0, // initialize 20 | (x) => x < 5, // test condition 21 | (x) => x + 1, // increment/update 22 | (x) => x // optional transform 23 | ) 24 | ``` 25 | ```ts 26 | squares = List.Generate( 27 | () => 0, // initialize 28 | (x) => x < 5, // test condition 29 | (x) => x + 1 // increment/update 30 | (x) => x * x // transform 31 | ) 32 | ``` 33 | ### Explicit `Each` functions 34 | 35 | `each` is synactic sugar. To be explicit: 36 | 37 | ```ts 38 | each 39 | _ < 5 40 | ``` 41 | is equivalent to 42 | ```ts 43 | (x as any) as any => 44 | x < 5 45 | ``` -------------------------------------------------------------------------------- /Examples/Readme.md: -------------------------------------------------------------------------------- 1 | # Examples from other sources -------------------------------------------------------------------------------- /Examples/Todo.md: -------------------------------------------------------------------------------- 1 | ## Examples checklist 2 | 3 | - [ ] List.TransformMany 4 | - [ ] List.Transform 5 | - [ ] List.Accumulate (word wrap) 6 | - [ ] List.Repeat 7 | - [x] List.Generate (for loop) 8 | - [ ] List.Generate : For looping API until paging is done 9 | - [ ] or List.Accumulate 10 | - [ ] constructor `{0..100}` or `{x..y}` or `{"a".."b"}` 11 | - [ ] Value.WaitFor 12 | - [ ] other [List.*](https://docs.microsoft.com/en-us/powerquery-m/list-functions) ? 13 | - [ ] List 14 | - [ ] List.Dates, List.DateTimes, List.DateTimeZones 15 | - [ ] List.Times 16 | - [ ] List.Durations 17 | - [ ] List.Generate 18 | - [ ] List.Numbers 19 | - [ ] List.Random 20 | - [ ] Record 21 | - [ ] TransformFields 22 | - [ ] [Table](https://docs.microsoft.com/en-us/powerquery-m/table-functions) 23 | - [ ] Join, group, expandcols, expand rows, 24 | - [ ] AlternateRows, FromPartitions, Partition, PartitionValues 25 | - [ ] Range, Repeat, Table.AlternateRows, 26 | - [ ] DuplicateColumn, AddColumn, Pivot, 27 | - [ ] Unpivot, Pivot, UnpivotOther 28 | - [ ] `AddColumn`, `AddJoinColumn`, 29 | - [ ] columns to record, addjoincolumn, addkey 30 | - [ ] Table.FilterWithDataTable (internal) 31 | - [ ] `NestedJoin`, SplitColumn 32 | - [ ] `TransformColumns` 33 | - [ ] `TransformColumnTypes` 34 | - [ ] `TransformRows` 35 | - [ ] `Transpose` 36 | -------------------------------------------------------------------------------- /Examples/Write Power Query Source Code.ps1: -------------------------------------------------------------------------------- 1 | 'Running: {0}' -f $PSCommandPath | Write-verbose -verbose 2 | $impo_Splat = @{ ErrorAction = 'stop' ; PassThru = $true } 3 | $ModulePath = Get-Item 'H:\data\2024\tabular\Ninmonkey.PowerQueryLib\source-pwsh\Nin.PqLib.psd1' 4 | $AppRoot = Get-Item -ea 'stop' $PSScriptRoot 5 | @( 6 | Import-module @impo_splat Pansies 7 | Import-module @impo_splat -Force $ModulePath 8 | ) | Join-String -p { $_.Name, $_.Version } -sep ', ' 9 | 10 | $Config = @{ 11 | AppRoot = $AppRoot 12 | ImportRoot = Join-Path $AppRoot '../source' | Get-Item -ea 'stop' 13 | AutoExportRoot = Join-path $AppRoot '../build/auto-export' | Get-Item -ea 'stop' 14 | } 15 | $Color = @{ 16 | Fg = '#47c589' 17 | Green = '#47c589' 18 | H1 = '#fea9aa' 19 | Red = '#fea9aa' 20 | } 21 | 22 | Get-PqLibManifestInfo|ft -AutoSize 23 | $Config | Ft -auto 24 | 25 | h1 'write' 26 | $origPq = 27 | @' 28 | let 29 | stuff = "1232", 30 | other = [ name = "bob" ] 31 | in 32 | other 33 | '@ 34 | 35 | Write-PqWrapLetExpression -KeyName 'what' -OutputType Let -RawContent $origPq 36 | 37 | return 38 | 39 | 40 | $RegexMustMatch = @( 41 | # 'Html' 42 | 'Random' 43 | ) 44 | $select_pq = Find-PqLibSources -RootDir $Config.ImportRoot -Regex $RegexMustMatch 45 | 46 | 'Final Files' | write-host -fg $Color.H1 47 | $select_pq -join "`n" 48 | 49 | $dynamicName = Join-String -f '{0}.ninlib.pq' -In (get-date).tostring('yyyy-MM-dd') 50 | $newExport = Join-path $Config.AutoExportRoot $DynamicName 51 | hr 52 | gc ($select_pq | select -First 1) | sc -Path $newExport 53 | $newExport = $newExport | Get-Item 54 | 'wrote: {0}' -f $NewExport | write-host -fg $Color.Red 55 | 56 | 57 | return 58 | 59 | $all_pq_files = FindPowerQuerySources -RootDir $Config.ImportRoot 60 | $Patterns = @( 61 | 'Html.Write' 62 | ) 63 | 64 | $select_pq = $all_pq_files | ?{ 65 | $Patterns.Where({ $_.Name -match $_ }, 'first', 1) 66 | $_.name -match $Patterns 67 | } 68 | $select_pq -join "`n" 69 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020-2024 Jake Bolton (ninmonkey) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README-to include next.md: -------------------------------------------------------------------------------- 1 | First: 2 | - [ ] fix 3 | - [ ] `Test_TableFrom` / to 4 | - [ ] `TableFrom` / to 5 | - [ ] SummarizeList( sorted list as text ) 6 | 7 | 8 | 9 | ## Examples in: Ninmonkey.PowerQueryLib-ExampleReports 10 | 11 | **TextAsList.pbix** 12 | 13 | - SerializeList, TypeAsText, ListAsText 14 | 15 | **IP Addresses.pbix** 16 | 17 | - 18 | - convert from DecimalIP, DottetDecimalIP, and BinaryIP addresses 19 | - IP Decimal bitwise math in **DAX** 20 | 21 | **Random Sales** 22 | 23 | - Random.Currency(), Random.Int() 24 | - Integer rounding modes 25 | 26 | **Enter Data** 27 | 28 | - Uses compressed JSON tables 29 | 30 | **Importing Raw Text Files.pbix** 31 | 32 | - see `ConvertTableFromText = (filepath as text, splitCharacter as text, linesPerRecord as number, optional encoding as nullable number) as table =>` 33 | 34 | **All HTTP Request types.pbix** 35 | 36 | - uses `HTTP` `GET`, `POST`, `PUT`, `PATCH`, etc... 37 | -------------------------------------------------------------------------------- /README.ValidatorsToWriteNext.md: -------------------------------------------------------------------------------- 1 | # About: Wishlist / or Ideas of PowerQuery Validation Scripts 2 | 3 | Potential future rules/parsers. 4 | 5 | ## Implementation 6 | 7 | Either: 8 | - [ ] PQ on the CLI, and/or PQ Parser 9 | - [ ] Tabular editor C# rules using Tabular2/3 ( or even by itself ) 10 | 11 | ## Fatal/Mandatory 12 | 13 | - [ ] ScriptIsSynacticallyValid / no parse errors 14 | 15 | ## Quality / Preference 16 | 17 | - [ ] either `some_table` or `someTable` or `SomeTable` or `Some.Table` or `#"Some Table" 18 | - [ ] pick one, and be consistant 19 | 20 | ## Special warnings, not always fatal 21 | 22 | - [ ] nested `each function`, warn may be missing 23 | - [ ] like the linter message: You used `stuff` did you mean `Stuff` 24 | - [ ] Identifier references are missing ( ie: 2nd query in the advanced editor ) 25 | - [ ] Invisible Datetime Intelligence Tables found 26 | 27 | # Validate Parameters in functions 28 | 29 | - [ ] Inspect existing connector's code for patterns 30 | 31 | ## Text 32 | 33 | - [ ] WarnOnInvisibleCharacters: invisible control chars should give a warning 34 | 35 | # Wanted 36 | 37 | - [ ] Assert `t` is one of union `Union([int, text, Currency.Type])` 38 | 39 | ## Numerical 40 | 41 | - [ ] DisallowDefaultValueAsAggregate : Undo any non-numerical 42 | - [ ] ValidateRange(min,max) numerical ranges 43 | - [ ] Assert List has N count items 44 | - [ ] Assert List is all of the same type 45 | 46 | ### coercion 47 | 48 | 49 | Resolve.Text = 50 | if text return 51 | else Text.From(..) 52 | 53 | ### ex 54 | 55 | fn_Example( source as list, ...) 56 | Assert.CompatibleWithUnion( source, Union(int | text | currency) ) 57 | 58 | 59 | ### ex 60 | 61 | fn_AverageSales( source as list ): 62 | entire list is of `{ Currency.Type }` 63 | not just a `{ Int64 }` 64 | -------------------------------------------------------------------------------- /Readme.OtherLinks.md: -------------------------------------------------------------------------------- 1 | Misc. links that are interesting 2 | 3 | ## querypower.com 4 | 5 | - [Power Query matrix multiplication](https://querypower.com/2017/02/24/powerquery-matrix-multiplication/) 6 | - [R.Execute\(\) the Swiss Army knife of Power Query](https://querypower.com/2017/03/11/r-execute-the-swiss-army-knife-of-power-query/] 7 | - [Exporting Power Query tables to SQL Server](https://querypower.com/2017/04/04/exporting-power-query-tables-to-sql-server/) -------------------------------------------------------------------------------- /Template-Import-ninlib.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/Template-Import-ninlib.pbix -------------------------------------------------------------------------------- /Todo.md: -------------------------------------------------------------------------------- 1 | # Todo 2 | 3 | ## first 4 | 5 | - [ ] automate generation of [template\Reused Allowed Values - template.md](template\Reused Allowed Values - template.md) 6 | 7 | - [ ] Documentation generator 8 | - [ ] autodetect variable names, and values, serialize as text 9 | 10 | ## Table schema related functions 11 | 12 | - [ ] 13 | 14 | ## collect 15 | 16 | - [ ] DecimalToDottedDecimal 17 | -[ ] DecimalToBinary / int32 18 | 19 | ## ListAsText 20 | 21 | 1. replace nested lists and records with '{...}' const 22 | 2. recurse ListAsText 23 | 24 | -------------------------------------------------------------------------------- /WIP/2020-08.pq: -------------------------------------------------------------------------------- 1 | // IPDottedToDecimal 2 | // IPDottedToDecimal 3 | let 4 | /* 5 | 6 | input: DottedDecimal 7 | "192.168.1.0" 8 | output: Decimal [ 32-bit ] 9 | 3232235776 10 | */ 11 | fn_IPDottedToDecimal = (IPString as text) as number => let 12 | /* 13 | todo: error check 14 | - list length exactly 4 15 | - all are number convertable 16 | - all within 1 byte 17 | */ 18 | TextList = Text.Split(IPString, "."), 19 | DecimalList = List.Transform( 20 | TextList, 21 | each Number.FromText(_, null) 22 | ), 23 | ByteString = Binary.FromList( DecimalList ), 24 | Decimal = BinaryFormat.UnsignedInteger32( ByteString ) 25 | in 26 | Decimal 27 | 28 | in fn_IPDottedToDecimal 29 | 30 | // ConvertNumberToBytes 31 | let 32 | ConvertNumberToBytes = (num as number, numBytes as number) as list => let 33 | /* 34 | input: 3232240641 35 | output: { 192, 168, 20, 1 } 36 | */ 37 | 38 | 39 | byteString = List.Generate( 40 | () => 8 * (numBytes - 1), 41 | each _ >= 0, 42 | each _ - 8, 43 | (bits as number) => 44 | Number.BitwiseAnd( 45 | 255, 46 | Number.BitwiseShiftRight( num, bits ) 47 | ) 48 | ) 49 | in 50 | byteString 51 | in ConvertNumberToBytes 52 | 53 | // IPDecimalToDotted 54 | let 55 | IPDecimalToDotted = (DecimalIP as number) as text => let 56 | /* 57 | input: 58 | 3232240641 59 | output: 60 | 192.168.20.1 61 | */ 62 | 63 | // ListOfDecimal = ConvertNumberToBytes(DecimalIP, 4), 64 | 65 | /* hard coded method */ 66 | ListDecimal = { 67 | Number.BitwiseAnd( 68 | 255, 69 | Number.BitwiseShiftRight( DecimalIP, 24 )), 70 | 71 | Number.BitwiseAnd( 72 | 255, 73 | Number.BitwiseShiftRight( DecimalIP, 16 )), 74 | Number.BitwiseAnd( 75 | 255, 76 | Number.BitwiseShiftRight( DecimalIP, 8 )), 77 | 78 | Number.BitwiseAnd( 79 | 255, 80 | DecimalIP) 81 | } 82 | in 83 | ListAsIp( ListDecimal ) 84 | 85 | in IPDecimalToDotted 86 | -------------------------------------------------------------------------------- /WIP/Combine Varargs as text.pq: -------------------------------------------------------------------------------- 1 | let 2 | TextFromRecurse = (object as any) as text => 3 | let 4 | scalar = if object is list 5 | then "{ " & Text.Combine( 6 | List.Transform( 7 | object, 8 | // each "Item," 9 | each @TextFromRecurse(_) 10 | ), 11 | ", " 12 | ) & "} " 13 | else object, 14 | 15 | output = Text.From(scalar) 16 | in 17 | output, 18 | 19 | CombineVararg = 20 | Function.From( 21 | type function() as list, 22 | each TextFromRecurse(_) 23 | ), 24 | 25 | results = [ 26 | basic = CombineVararg("a", "b", 3, TextEncoding.Utf16), 27 | nested = CombineVararg("a", "b", 3, {3..6}), 28 | nested2 = CombineVararg( 29 | "a", "b", 30 | DateTime.LocalNow(), 31 | 3, {3..6} 32 | ) 33 | 34 | ] 35 | in 36 | results -------------------------------------------------------------------------------- /WIP/Date.ToText ⇽ custom date formatting string ┐2021-01-29.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/WIP/Date.ToText ⇽ custom date formatting string ┐2021-01-29.pbix -------------------------------------------------------------------------------- /WIP/Function-Varargs.pq: -------------------------------------------------------------------------------- 1 | let 2 | TextFromRecurse = (object as any) as text => 3 | let 4 | scalar = if object is list 5 | then "{ " & Text.Combine( 6 | List.Transform( 7 | object, 8 | // each "Item," 9 | each @TextFromRecurse(_) 10 | ), 11 | ", " 12 | ) & "} " 13 | else object, 14 | 15 | output = Text.From(scalar) 16 | in 17 | output, 18 | 19 | CombineVararg = 20 | Function.From( 21 | type function() as list, 22 | each TextFromRecurse(_) 23 | ), 24 | 25 | results = [ 26 | basic = CombineVararg("a", "b", 3, TextEncoding.Utf16), 27 | nested = CombineVararg("a", "b", 3, {3..6}), 28 | nested2 = CombineVararg( 29 | "a", "b", 30 | DateTime.LocalNow(), 31 | 3, {3..6} 32 | ) 33 | 34 | ] 35 | in 36 | results -------------------------------------------------------------------------------- /WIP/Inspect Text as Codepoints.pq: -------------------------------------------------------------------------------- 1 | let 2 | transformChar = (char as text) => [ 3 | Codepoint = Character.ToNumber(char), 4 | Text = char 5 | ], 6 | 7 | transformString = (source as text) as list => 8 | // list codepoints 9 | let 10 | charList = Text.ToList(source), 11 | codepoints = List.Transform( 12 | charList, 13 | (char as text) as record => transformChar(char) 14 | ) 15 | in 16 | codepoints, 17 | 18 | inspectText = (source as text) as table => 19 | let 20 | charRecords = transformString(source), 21 | t = Table.FromRecords( 22 | charRecords, 23 | type table[Codepoint = number, Text = text], 24 | MissingField.Error 25 | ) 26 | in 27 | t, 28 | 29 | 30 | example = inspectText(" Foo, #(tab)bar#(cr,lf).") 31 | 32 | in 33 | example -------------------------------------------------------------------------------- /WIP/Inspect-Ast-Pair.pq: -------------------------------------------------------------------------------- 1 | let 2 | Docs = ItemExpression.From, 3 | Ast1 = ItemExpression.From(each _ <> null), 4 | Ast2 = ItemExpression.From( (x as number) as number => Int64.From(x, RoundingMode.ToEven ) 5 | Left = Ast1[Left], 6 | 7 | inspectAstPair = (source as any) as any => 8 | let 9 | x = Record.FromList(source) 10 | in 11 | x, 12 | 13 | // Record.FromList({1, "Bob", "123-4567"}, {"CustomerID", "Name", "Phone"}) 14 | // Record.FromList({1, "Bob", "123-4567"}, type [CustomerID = number, Name = text, Phone = number]) 15 | // Custom1 = Record.ToTable(Left), 16 | doc = Record.FromList, 17 | z = Type.Facets() 18 | doc2 = Record.FromTable, 19 | Custom1 = Record.FromList( Left, {}), 20 | a = DirectQueryCapabilities.From, 21 | // Value.ResourceExpression() 22 | x = Facts.Summarize(), 23 | y = Type.ReplaceFacets, 24 | // a3 = Roundingmod, 25 | a1 = Type.FunctionParameters, 26 | a32 = Type.FunctionRequiredParameters, 27 | // a3 = Type. * generate this , qer = Typee 28 | e = Type.Facets, 29 | 30 | 31 | z = FactCheck, 32 | insp = inspectAstPair( Left ), 33 | Custom2 = TableToJson( Left ) 34 | 35 | // insp = Inspect 36 | in 37 | Custom2 38 | 39 | 40 | -------------------------------------------------------------------------------- /WIP/Inspect.Anything.pq: -------------------------------------------------------------------------------- 1 | let 2 | EnableTests = false, 3 | // table 4 | ins = Inspect.Object, 5 | Inspect.Object = (source as any, optional options as nullable record) as any => 6 | let 7 | simpleOutput = options[simpleOutput]? ?? false, 8 | // should I catch exception? 9 | results_basic = [ 10 | Metadata = Value.Metadata( target ), 11 | Facets = Type.Facets( target ) 12 | ], 13 | results_detailed = [ 14 | ResourceExpression = Value.ResourceExpression( target ), 15 | DirectQueryCapabilities = DirectQueryCapabilities.From( target ) 16 | ] 17 | // Record.Combine() 18 | in 19 | if simpleOutput then results_basic 20 | else Record.Combine( 21 | {results_basic, results_detailed}), 22 | 23 | 24 | // q = Value.Compare() 25 | targets = [ 26 | JoinAlgorithm.Type = JoinAlgorithm.Type, 27 | DirectQueryCapabilities.From = DirectQueryCapabilities.From, 28 | Value.Compare = Value.Compare, // type: comparisonCriteria 29 | equate = "?" // type: equationCriteria 30 | ], 31 | 32 | test = Inspect.Object( {"a".."4"}), 33 | test2 = Inspect.Object( Rations ), 34 | test3 = Inspect.Object( GenerateDocs_OnAlgoType ), 35 | 36 | // target = targets{0}, 37 | target = Rations, 38 | 39 | test_results = [ 40 | Metadata = Value.Metadata( target ), 41 | ResourceExpression = Value.ResourceExpression( target ), 42 | DirectQueryCapabilities = DirectQueryCapabilities.From( target ), 43 | Facets = Type.Facets( target ) 44 | ], 45 | 46 | samplesList = { 47 | RoundingMode.ToEven, 48 | RoundingMode.Type 49 | }, 50 | samplesDict = [ 51 | maybe1 = ins( JoinAlgorithm.Type, [simpleOutput = true]), 52 | maybe2 = ins( JoinAlgorithm.Type ), 53 | maybe3 = ins( test_results ) 54 | ], 55 | 56 | 57 | 58 | tab2 = DirectQueryCapabilities.From( target ), 59 | 60 | final = if EnableTests then test_results else Inspect.Object 61 | in 62 | final -------------------------------------------------------------------------------- /WIP/Record.Schema.pq: -------------------------------------------------------------------------------- 1 | // summarizes schema and data types, in comparison to Summarize_Record() -------------------------------------------------------------------------------- /WIP/Rendering codepoint info.dax: -------------------------------------------------------------------------------- 1 | 2 | define 3 | // U+2400 aka 9216 aka '␀' is the safe symbol to display null U+0 values 4 | var StrNullSymbol = 9216 // 0x2400 '␀' is the safe symbol for null 0x0 5 | 6 | // this is the old one, new one has more ideomatic fallback 7 | 8 | measure Named_Codepoints[Step1] = 9 | var cur = SELECTEDVALUE( Named_Codepoints[Codepoint], StrNullSymbol) 10 | var render = "'" & UNICHAR(106) & "' should be 'j'" 11 | return render 12 | 13 | /* 14 | should render using the column codepoint 15 | error: 16 | argument of UNICHAR is wrong type, or out of bound 17 | */ 18 | 19 | measure Named_Codepoints[RenderDax] = 20 | var cur = SELECTEDVALUE( Named_Codepoints[Codepoint], StrNullSymbol) 21 | var render = UNICHAR( cur ) 22 | return render 23 | 24 | measure Named_Codepoints[RenderDax_works] = 25 | var cur = SELECTEDVALUE( Named_Codepoints[Codepoint], StrNullSymbol) 26 | var render = UNICHAR( 9332 ) 27 | return render 28 | 29 | EVALUATE 30 | SUMMARIZECOLUMNS( 31 | Named_Codepoints[Codepoint], 32 | Named_Codepoints[Render], 33 | "max", Max( Named_Codepoints[Codepoint] ), 34 | "step1", [Step1] 35 | //, "actual", [RenderDax] // commenting this, removes all errors 36 | , "Working", [RenderDax_works] 37 | 38 | ) 39 | ORDER BY 40 | Named_Codepoints[Codepoint] ASC, 41 | Named_Codepoints[Render] ASC 42 | -------------------------------------------------------------------------------- /WIP/Table.MergeDimensionTable and To.DimensionTable.pq: -------------------------------------------------------------------------------- 1 | let 2 | 3 | x1 = 4 | let 5 | Table.MergeDimensionTable = (source as table, dim as table, optional options as nullable record 6 | ) => 7 | let 8 | t_join = Table.NestedJoin( source, {"Phone"}, dim, {"Phone"}, "Result", JoinKind.LeftOuter ), 9 | t_expand = Table.ExpandTableColumn( t_join, "Result", {"Id"}, {"Phone Id"}) 10 | in t_expand 11 | 12 | in 13 | Table.MergeDimensionTable, 14 | 15 | 16 | x2 = let 17 | Source = Table.FromRecords({ 18 | [ Phone = "1234", User = "Bob" ], 19 | [ Phone = "1234", User = "Jen" ], 20 | [ Phone = "2234", User = "Fred" ], 21 | [ Phone = "3234", User = "Penny"], 22 | [ Phone = "4234", User = "Gary" ] 23 | }, type table[Phone = text, User = text], MissingField.Error), 24 | 25 | Table.ToDimensionTable = (source as table, column as text) => 26 | let t0 = Table.Distinct( Table.SelectColumns(source, column, MissingField.Error) ), 27 | col_id = Table.AddIndexColumn(t0, "Id", 0) 28 | in col_id, 29 | 30 | dimPhone = Table.ToDimensionTable( Source, "Phone" ), 31 | dimUser = Table.ToDimensionTable( Source, "User" ), 32 | // t0 = Table.Distinct( Table.SelectColumns( Source, {"Phone"}) ), 33 | // dimTable_Phone = Table.AddIndexColumn(t0, "Index", 0, 1, Int64.Type) 34 | Summary = [ 35 | Source = Source, 36 | dimPhone = dimPhone, 37 | dimUser = dimUser 38 | ] 39 | in 40 | Summary 41 | in 42 | x2 -------------------------------------------------------------------------------- /WIP/Text.ToUnorderedList.pq: -------------------------------------------------------------------------------- 1 | let 2 | Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("i45WKkmtKFGK1YlWysnMS1UwAjNz84tSFYpLStPSwFyv1OJipdhYAA==", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Line = _t]), 3 | 4 | Str = [ 5 | Bullet = "•", 6 | Newline = "#(cr,lf)" 7 | ], 8 | Text.ToUnorderedList = (source as list) as text => 9 | let 10 | prefix = Str[Bullet] & " ", 11 | items = List.Transform( 12 | source, 13 | each Prefix & Text.From(_) 14 | ) 15 | x = z 16 | in 17 | "final", 18 | #"transform text" = Table.TransformColumns(Source,{{ 19 | "Line", 20 | each Str[Bullet] & " " & _, 21 | type text}} 22 | ), 23 | 24 | MergedText = Text.Combine( #"transform text"[Line], Str[Newline] ), 25 | final = #"transform text" 26 | // #"transform text" = Table.TransformColumns(Source,{{"Line", Text.Trim, type text}}) 27 | in 28 | final -------------------------------------------------------------------------------- /WIP/convertTo_RelativePath.doesnt_actually_invoke_the_final_yet.pq: -------------------------------------------------------------------------------- 1 | convertTo_RelativePath = (source as text, optional options as nullable record) => 2 | /* 3 | todo: 4 | - [ ] attempt to resolve using the longest replacements first 5 | - [ ] optional username 6 | */ 7 | let 8 | userName = options[Name]? ?? "c:\users\cppmo_000", 9 | replacements = { 10 | [ 11 | From = "C:\Users\cppmo_000\SkyDrive\Documents", 12 | To = "$Env:UserProfile\SkyDrive\Documents" 13 | ], 14 | [ 15 | From = "C:\Users\cppmo_000", 16 | To = "$Env:UserProfile" 17 | ], 18 | [ 19 | From = "C:\Users\cppmo_000\AppData\Roaming", 20 | To = "$Env:AppData" 21 | ] 22 | }, 23 | // silly inline example 24 | remapUser = List.Transform( 25 | replacements, 26 | (item) => 27 | let 28 | after = Text.Replace( item[From]?, "c:\users\cppmo_000", userName ) 29 | in 30 | Record.Combine({ 31 | item, 32 | [From = after] 33 | }) 34 | ), 35 | 36 | 37 | final = ... 38 | in 39 | if false then final else [ 40 | replacements = replacements, 41 | remapUser = remapUser, 42 | final = final, 43 | source = source, 44 | other = "fds" 45 | ], 46 | tests = [ 47 | appdata_roam = convertTo_RelativePath( "C:\Users\cppmo_000\AppData\Roaming" ), 48 | docs = convertTo_RelativePath( "C:\Users\cppmo_000\SkyDrive\Documents" ) 49 | ], -------------------------------------------------------------------------------- /WIP/custom-table-name-transformer.pq: -------------------------------------------------------------------------------- 1 | let 2 | z = 3 | (name as text) as text => 4 | let //original: https://bengribaudo.com/blog/2018/05/18/4447/automating-column-name-renames 5 | #"Split into Parts" = Text.Split(name, "_"), 6 | #"Change Case" = 7 | (input as text) as text => 8 | if Comparer.Equals(Comparer.OrdinalIgnoreCase, "id", input) 9 | then Text.Upper(input) 10 | else Text.Proper(input), 11 | #"Transformed Parts" = List.Transform(#"Split into Parts", #"Change Case"), 12 | Result = Text.Combine(#"Transformed Parts", " ") 13 | in 14 | Result 15 | in 16 | z 17 | 18 | 19 | // usage 20 | 21 | let 22 | Source = ..., 23 | #"Transform Column Names" = Table.TransformColumnNames(Source, #"Column Name Transformer") 24 | in 25 | #"Transform Column Names" 26 | -------------------------------------------------------------------------------- /WIP/detailed-uncompress-blob-zip.pq: -------------------------------------------------------------------------------- 1 | 2 | 3 | https://community.powerbi.com/t5/Community-Blog/Working-With-Zip-Files-in-Power-Query/ba-p/1190186 4 | // https://en.wikipedia.org/wiki/Zip_(file_format)#Structure 5 | 6 | and 7 | https://blog.crossjoin.co.uk/2015/12/08/working-with-compression-in-power-query-and-power-bi-desktop/ 8 | 9 | 10 | (ZIPFile) => 11 | let 12 | //read the entire ZIP file into memory - we'll use it often so this is worth it 13 | Source = Binary.Buffer(File.Contents(ZIPFile)), 14 | // get the full size of the ZIP file 15 | Size = Binary.Length(Source), 16 | //Find the start of the central directory at the sixth to last byte 17 | Directory = BinaryFormat.Record([ 18 | MiscHeader=BinaryFormat.Binary(Size-6), 19 | Start=BinaryFormat.ByteOrder(BinaryFormat.UnsignedInteger32, ByteOrder.LittleEndian) 20 | ]) , 21 | Start = Directory(Source)[Start], 22 | //find the first entry in the directory and get the compressed file size 23 | FirstDirectoryEntry = BinaryFormat.Record([ 24 | MiscHeader=BinaryFormat.Binary(Start+20), 25 | 26 | ... 27 | 28 | see url -------------------------------------------------------------------------------- /WIP/int32-to-byte-and-back.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* go from int -> binarystr, and back */ 3 | 4 | listIpDec = {192, 168, 5, 85}, 5 | nums = [ 6 | a = Byte.From( listIpDec{0} ), 7 | b = Byte.From( listIpDec{1} ), 8 | c = Byte.From( listIpDec{2} ), 9 | d = Byte.From( listIpDec{3} ) 10 | ], 11 | 12 | // both work 13 | byteStringIp = #binary(listIpDec), 14 | byteStringIpFromList = Binary.FromList(listIpDec), 15 | 16 | // bytes to 32bit int 17 | UInt32 = BinaryFormat.UnsignedInteger32( byteStringIp ), 18 | 19 | // FailByteRead = Int32.From(byteStringIp, null, RoundingMode.Down), 20 | // fromfromint = Byte.From(binUint32) //Binary.From(binUint32) 21 | 22 | // t = byteStringIpFromList( fromfromint ), 23 | 24 | // binary to list of integers 25 | ListFormatByteString = BinaryFormat.List(BinaryFormat.Byte), 26 | listIntFromByteString = ListFormatByteString( byteStringIp ), 27 | 28 | Summary = [ 29 | listIpDec = ListAsText( listIpDec ), 30 | // z = #binary(binUint32), 31 | byteStringIp = byteStringIp, 32 | ListFormatByteString = ListFormatByteString, 33 | listIntFromByteString = ListAsText( listIntFromByteString ), 34 | UInt32 = UInt32, 35 | nums = nums 36 | ] 37 | in 38 | Summary 39 | -------------------------------------------------------------------------------- /WIP/others/iso-week-number.2021.pq: -------------------------------------------------------------------------------- 1 | //datacornering.com 2 | 3 | let 4 | Source = List.Dates(DateTime.Date(DateTime.LocalNow()), 365, #duration(1, 0, 0, 0)), 5 | #"Converted to Table" = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error), 6 | #"Renamed Columns" = Table.RenameColumns(#"Converted to Table",{{"Column1", "Date"}}), 7 | #"Changed Type" = Table.TransformColumnTypes(#"Renamed Columns",{{"Date", type date}}), 8 | #"Weekday Number" = Table.AddColumn(#"Changed Type", "Weekday Number", each Date.DayOfWeek([Date], Day.Monday)+1), 9 | 10 | #"ISO Week Number" = Table.AddColumn(#"Weekday Number", "ISO Week Number", each if 11 | Number.RoundDown((Date.DayOfYear([Date])-(Date.DayOfWeek([Date], Day.Monday)+1)+10)/7)=0 12 | 13 | then 14 | Number.RoundDown((Date.DayOfYear(#date(Date.Year([Date])-1,12,31))-(Date.DayOfWeek(#date(Date.Year([Date])-1,12,31), Day.Monday)+1)+10)/7) 15 | 16 | else if 17 | (Number.RoundDown((Date.DayOfYear([Date])-(Date.DayOfWeek([Date], Day.Monday)+1)+10)/7)=53 18 | and (Date.DayOfWeek(#date(Date.Year([Date]),12,31), Day.Monday)+1<4)) 19 | 20 | then 21 | 1 22 | 23 | else 24 | Number.RoundDown((Date.DayOfYear([Date])-(Date.DayOfWeek([Date], Day.Monday)+1)+10)/7)) 25 | 26 | in 27 | #"ISO Week Number" -------------------------------------------------------------------------------- /WIP/others/iso-week-number.fixed.pq: -------------------------------------------------------------------------------- 1 | // originally: 2 | let 3 | todo = "Cleanup" , 4 | Source = List.Dates(DateTime.Date(DateTime.LocalNow()), 365, #duration(1, 0, 0, 0)), 5 | #"Converted to Table" = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error), 6 | #"Renamed Columns" = Table.RenameColumns(#"Converted to Table", {{"Column1", "Date"}}), 7 | #"Changed Type" = Table.TransformColumnTypes(#"Renamed Columns", {{"Date", type date}}), 8 | #"Weekday Number" = Table.AddColumn(#"Changed Type", "Weekday Number", each Date.DayOfWeek([Date], Day.Monday) + 1), 9 | #"ISO Week Number" = Table.AddColumn( 10 | #"Weekday Number", 11 | "ISO Week Number", 12 | each 13 | if Number.RoundDown((Date.DayOfYear([Date]) - (Date.DayOfWeek([Date], Day.Monday) + 1) + 10) / 7) = 0 then 14 | Number.RoundDown( 15 | ( 16 | Date.DayOfYear(#date(Date.Year([Date]) - 1, 12, 31)) - ( 17 | Date.DayOfWeek(#date(Date.Year([Date]) - 1, 12, 31), Day.Monday) + 1 18 | ) + 10 19 | ) / 7 20 | ) 21 | else if ( 22 | Number.RoundDown((Date.DayOfYear([Date]) - (Date.DayOfWeek([Date], Day.Monday) + 1) + 10) / 7) = 53 23 | and (Date.DayOfWeek(#date(Date.Year([Date]), 12, 31), Day.Monday) + 1 < 4) 24 | ) then 25 | 1 26 | else 27 | Number.RoundDown((Date.DayOfYear([Date]) - (Date.DayOfWeek([Date], Day.Monday) + 1) + 10) / 7) 28 | ) 29 | in 30 | todo 31 | -------------------------------------------------------------------------------- /WIP/sample-of-ui-supported-enter-data.pq: -------------------------------------------------------------------------------- 1 | let 2 | Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("i45WCsnPyc5MzVPSUTI0sjQBUkbGBgYGSrE60UoupXmpQAFTUyOQuImxpZFSbCwA", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [#" Book Name" = _t, Year = _t, Sales = _t]) 3 | in 4 | Source -------------------------------------------------------------------------------- /WIP/sanity-test/2021-08-ration-list.sanity-tests.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Sanity = let 3 | var = [ 4 | dimIdList = #"dim Unit"[Unit Id], 5 | factIdList = Rations[Unit Id], 6 | source = Rations 7 | ], 8 | sanityTest = List.Transform( 9 | var[factIdList], 10 | (item) => 11 | let 12 | exists = List.Contains( var[dimIdList], item) 13 | in 14 | if exists then item 15 | else error Error.Record( 16 | "SanityFailed", "Id exists in fact table but not dim table!", item ) 17 | ), 18 | base_table = Table.FromList(sanityTest, Splitter.SplitByNothing(), type table[Id = Int64.Type], null, ExtraValues.Error), 19 | #"Kept Errors" = Table.SelectRowsWithErrors(base_table), 20 | 21 | error_integrity = 22 | if Table.RowCount( #"Kept Errors" ) > 0 then 23 | error Table.First( #"Kept Errors" ) 24 | else base_table, 25 | 26 | error_isDistinct = 27 | if Table.IsDistinct( var[source], {"Date", "Unit Id"} ) then true 28 | else error Error.Record("TableNotDistinct", "Columns {Date, Unit Id}", var[source] ), 29 | 30 | error_list = List.Select( 31 | { try error_integrity, try error_isDistinct}, 32 | (item) => item[HasError] 33 | ), 34 | 35 | error_table = Table.FromRecords(error_list), 36 | #"Added Custom" = Table.AddColumn(error_table, "MaybeError", each if not [HasError] then "ok" else error Error.Record("had error", "error", error [Error] )) 37 | in 38 | #"Added Custom" 39 | 40 | // sanityTable = Table.FromList( 41 | // sanityTest 42 | 43 | // ), 44 | 45 | // final = Table.AddColumn( 46 | // sanityTable, "Valid", 47 | // each 48 | // if [Exists] then true else 49 | // error Error.Record("SanityFailed", "Id exists in fact table but not dim table!", [Id] )), 50 | // #"Kept Errors" = Table.SelectRowsWithErrors(final) 51 | 52 | 53 | // in 54 | // #"Kept Errors" -------------------------------------------------------------------------------- /WIP/test--assert-types.pq: -------------------------------------------------------------------------------- 1 | FinalResult = 1, 2 | 3 | Assert.TypeIs = (source as any, expected as type) as any => 4 | <# 5 | 6 | primitive type is compatible? 7 | Should.BeOfType( #date(2020, 10, 10), type date ) //# true 8 | Should.BeOfType( #date(2020, 10, 10), type datetime ) // # 9 | 10 | primitive type is exactly of type 11 | #> 12 | 13 | // todo: 14 | ..., 15 | 16 | -------------------------------------------------------------------------------- /WIP/transform-many-example-months-and-number-formats.pq: -------------------------------------------------------------------------------- 1 | // let 2 | // b = List.TransformMany( 3 | // {1..12}, 4 | // (x) => { 5 | // #date(2016, x, 1) 6 | // }, 7 | // (x, y) => 8 | // [ x = x, y = y, z = Date.MonthName( y ) ] 9 | // ), 10 | // b2 = List.TransformMany( 11 | // {1..12}, 12 | // (x) => { 13 | // #date(2016, x, 1) 14 | // }, 15 | // (x, y) => 16 | // Table.FromRecords( 17 | // { 18 | // [ x = x, y = y, z = Date.MonthName( y ) ] 19 | // } 20 | // ) 21 | // ), 22 | // b_t = Table.FromRecords( b ), 23 | // Summary = [ 24 | // b = b2, 25 | // a = b_t 26 | // ] 27 | // in 28 | // Summary 29 | // also many unit test? 30 | 31 | let 32 | a = List.TransformMany( 33 | {1..12}, 34 | (x) => { 35 | #date(2016, x, 1) 36 | }, 37 | (x, y) => 38 | Text.Combine( 39 | { 40 | Number.ToText(x), " - ", 41 | Text.Lower(Date.MonthName( y )) 42 | }, "" 43 | ) 44 | ), 45 | b = List.TransformMany( 46 | {1..12}, 47 | (x) => { 48 | #date(2016, x, 1) 49 | }, 50 | (x, y) => 51 | [ x = x, y = y, z = Date.MonthName( y ) ] 52 | ), 53 | b_t = Table.FromRecords( b ) 54 | in 55 | b_t -------------------------------------------------------------------------------- /WIP/validate_record_for_function_calls.pq: -------------------------------------------------------------------------------- 1 | let 2 | todo = " 3 | about: validate optional record args in Function 4 | 5 | 1] All(options.keys in AllowedKeys) 6 | 2] assert ( options.keys notin AllowedKeys ).count = 0 7 | 3] error 8 | InvalidKeyException, 9 | keyname 10 | supported keys are ..., 11 | 12 | 4] invalid value type 13 | InvalidTypeError, 14 | Key[Foo] type != required type 15 | expects value = typename 16 | " 17 | in 18 | todo 19 | -------------------------------------------------------------------------------- /build/auto-export/2024-06-02.ninlib.pq: -------------------------------------------------------------------------------- 1 | /* 2 | Name: 3 | List_Random 4 | 5 | Description: 6 | Randomly select items in a list [2021-04-24] 7 | 8 | If a list has 3 elements, this generates a random int 9 | in the range [0-2] (inclusive), then returns list{i} 10 | 11 | Alias: 12 | python uses the name: random.choice( ) 13 | 14 | Example: 15 | 16 | List_Random( {"Dog", "Cat", "Fish"} ) 17 | // returns: "Cat" 18 | 19 | future: 20 | - [ ] validate empty list, or null, or null in list 21 | - [ ] nicer user-facing errors 22 | */ 23 | 24 | (source as list) as any => 25 | let 26 | maxIndex = List.Count(source) - 1, 27 | offset = List.Random(1){0} * maxIndex, 28 | r = Number.Round( offset, null, null ), 29 | name = source{r} 30 | in 31 | name 32 | -------------------------------------------------------------------------------- /source-before-2024/Text/Text.MatchesAnyOf.CI.pq.off: -------------------------------------------------------------------------------- 1 | // let 2 | // // todo: double check that this isn't fully replaced by Text.MatchesAny.pq 3 | // // this is a smpler case if someone wants it as a reference 4 | // // see also: Text.Contains.CI, Text.PositionOf.CI, Text.MatchesAnyOf.CI 5 | // // was it to show a list of list of tests, or a one final logical ? 6 | // Text.MatchesAnyOf.CI = ( 7 | // sourceList as text, 8 | // patternList as list 9 | 10 | // ) as logical => [ 11 | // find = List.Select( 12 | // sourceList, 13 | // (source) => List.AnyTrue( 14 | // List.Transform( 15 | // patternList, 16 | // (pattern) => 17 | // Text.Contains( source, pattern, Comparer.OrdinalIgnoreCase ) 18 | // ) 19 | // ) 20 | // ), 21 | // return = List.Count( find ) > 0 22 | 23 | // ][return] 24 | // in 25 | // Text.MatchesAnyOf.CI -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Constants - All.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List_Constants-All.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , 4 | each List.Contains( 5 | { "text", "number" }, 6 | [TypeNameText] ) 7 | ), 8 | t1 = Table.RemoveColumns( 9 | Source, 10 | {"TypeNameText"}), 11 | 12 | t2 = Table.Sort(t1, 13 | {{"Name", Order.Ascending}, 14 | {"Value", Order.Ascending}} 15 | ) 16 | in 17 | t2 18 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Function - Datasources.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List Datasurce Functions.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "Function" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | { {"Name", Order.Ascending}}), 11 | #"Added Custom" = Table.SelectRows( t2, each Function.IsDataSource( [Value] ) ) 12 | in 13 | #"Added Custom" 14 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Function - Not Datasource.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List Functions NotDatasource.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "Function" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | { {"Name", Order.Ascending}}), 11 | #"Added Custom" = Table.SelectRows( t2, each not Function.IsDataSource( [Value] ) ) 12 | in 13 | #"Added Custom" 14 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Functions - All.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List Functions - All.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "Function" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | { {"Name", Order.Ascending}}), 11 | #"Added Custom" = Table.AddColumn(t2, "IsDataSource", each Function.IsDataSource( [Value] )) 12 | in 13 | #"Added Custom" 14 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Globals.pq: -------------------------------------------------------------------------------- 1 | let 2 | Source = #shared, 3 | t1 = Record.ToTable(Source), 4 | #"Remove Self" = Table.SelectRows( t1, each [Name] <> "List Globals" ), 5 | t2 = Table.Sort( 6 | #"Remove Self", 7 | {{"Name", Order.Ascending}}), 8 | 9 | colTypeNameText = Table.AddColumn( 10 | t2, 11 | "TypeNameText", 12 | each Type.ToText( Value.Type([Value]) ), 13 | type text) 14 | in 15 | colTypeNameText 16 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List List.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List Lists.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "list" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | { {"Name", Order.Ascending}, {"Value", Order.Ascending}}), 11 | t3 = Table.RemoveColumns( t2, {"Value"}) 12 | in 13 | t3 14 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Numbers.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'ListNumbers.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "number" ), 4 | t1 = Table.TransformColumnTypes( 5 | Source, 6 | {{"Value", type number}}), 7 | t2 = Table.RemoveColumns(t1,{"TypeNameText"}), 8 | Final = Table.Sort( t2, 9 | {{"Name", Order.Ascending}, {"Value", Order.Ascending}}) 10 | in 11 | Final 12 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Records.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List Record.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "Record" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | { {"Name", Order.Ascending}, {"Value", Order.Ascending}}) 11 | in 12 | t2 13 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Tables.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List Tables.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "Table" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | { {"Name", Order.Ascending}}), 11 | 12 | colSchema = Table.AddColumn( 13 | t2, "Schema", each 14 | Table.Schema( [Value] )), 15 | 16 | t3 = Table.RemoveColumns( colSchema, {"Value"}) 17 | in 18 | t3 19 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Text.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'List Text.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "text" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | { {"Name", Order.Ascending}, {"Value", Order.Ascending}}) 11 | in 12 | t2 13 | -------------------------------------------------------------------------------- /source-before-2024/meta/shared - List Types.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* this generates 'ListTypes.csv' */ 3 | Source = Table.SelectRows( #"List Globals" , each [TypeNameText] = "type" ), 4 | t1 = Table.RemoveColumns( 5 | Source, 6 | {"Value", "TypeNameText"}), 7 | 8 | t2 = Table.Sort( 9 | t1, 10 | {{"Name", Order.Ascending}}) 11 | in 12 | t2 13 | -------------------------------------------------------------------------------- /source-before-2024/random/List.RandomItem.pq: -------------------------------------------------------------------------------- 1 | /* 2 | Name: 3 | List_Random 4 | 5 | Description: 6 | Randomly select items in a list [2021-04-24] 7 | 8 | If a list has 3 elements, this generates a random int 9 | in the range [0-2] (inclusive), then returns list{i} 10 | 11 | Alias: 12 | python uses the name: random.choice( ) 13 | 14 | Example: 15 | 16 | List_Random( {"Dog", "Cat", "Fish"} ) 17 | // returns: "Cat" 18 | 19 | future: 20 | - [ ] validate empty list, or null, or null in list 21 | - [ ] nicer user-facing errors 22 | */ 23 | 24 | (source as list) as any => 25 | let 26 | maxIndex = List.Count(source) - 1, 27 | offset = List.Random(1){0} * maxIndex, 28 | r = Number.Round( offset, null, null ), 29 | name = source{r} 30 | in 31 | name -------------------------------------------------------------------------------- /source-before-2024/web/WebRequest.ToRecord.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Get metadata for a Web.Contents call 3 | WebRequest_ToRecord = (response as binary) as record => 4 | let 5 | metaData = Value.Metadata( response ), 6 | maybeErr = try metaData[Content.Type], 7 | r2 = Record.AddField(metaData, "Binary", response, null ), 8 | r3 = Record.AddField(r2, "Response.Error", 9 | if maybeErr[HasError] then maybeErr[Error] else null, 10 | null 11 | ), 12 | r4 = Record.AddField(r3, "Url", metaData[Content.Uri]() ) 13 | in 14 | r4 15 | in 16 | WebRequest_ToRecord 17 | -------------------------------------------------------------------------------- /source-dax/ConditionalColors-UseMinMax-For-Gradients.dax: -------------------------------------------------------------------------------- 1 | [ HSL Hue from Sales ] = 2 | // simpler version, which scales hue from 0 degrees to maxHue 3 | // outputs a gradient like: 4 | // hsl( 0, 80%, 78%) 5 | // hsl( 40, 80%, 78%) 6 | // hsl( 160, 80%, 78%) 7 | var maxHue = 160 8 | 9 | // Can this ALL() be rewritten to run faster? 10 | VAR CurrentValue = SELECTEDVALUE( Sale[Value], 0 ) 11 | VAR MinValue = CALCULATE( MIN(Sale[Value]), All( Sale[Value] ) ) 12 | VAR MaxValue = CALCULATE( MAX(Sale[Value]), All( Sale[Value] ) ) 13 | 14 | var relativeValue = 15 | DIVIDE( 16 | CurrentValue - MinValue, 17 | MaxValue - MinValue, 18 | 0 19 | ) 20 | 21 | // outputs the color string: "hsl( 60, 80%, 78% )" 22 | var renderInt = format( (relativeValue * maxHue), "#,0", "en-us" ) 23 | var finalColor = "hsl( " & renderInt & ", 80%, 80%)" 24 | return finalColor 25 | -------------------------------------------------------------------------------- /source-dax/Convert-ControlChars-to-Symbols.dax: -------------------------------------------------------------------------------- 1 | Calculated Rune = 2 | /* 3 | Docs: 4 | Extra links if you're curious 5 | 6 | [NonChars](https://www.unicode.org/faq/private_use.html#nonchar1) 7 | [C0 and C1 Control Chars](https://en.wikipedia.org/wiki/C0_and_C1_control_codes) 8 | [TOC of blocks / index of all groups](https://www.compart.com/en/unicode/plane/U+0000) 9 | [surrogate pairs @ globalization/encoding](https://docs.microsoft.com/en-us/globalization/encoding/surrogate-pairs) 10 | 11 | unicode-max: 1114111 0x10ffff 12 | [c0]: [0x0, 0x1f] 13 | space: 32 / 0x20 14 | del 127 / 0x7f 15 | 0x2421 = 9249 ␡ 16 | 17 | Block: 18 | "Ascii" / "latin": 0x0000..0x007f 19 | "latin-1 suppliment" 0x0080..0x00ff 20 | "Control Char Symbols": 0x2400..0x243f 21 | 22 | Planes: 23 | Private Use Areas plane 15 (U+F0000-U+FFFFD) 24 | Private Use 2 plane 16 (U+100000-U+10FFFD) 25 | 26 | Supplementary 27 | Supplementary Multilingual Plane (SMP) 0x10000..0x1FFFF 28 | Supplementary Ideographic Plane (SIP)- 0x20000..0x2FFFF 29 | Supplementary Special-purpose Plane (SSP) 0xe0000..0xEFFFF 30 | 31 | 32 | Verified 33 | 0..32, 34 | 127, 35 | */ 36 | var curCodepoint = SELECTEDVALUE( Number[Codepoint] ) 37 | var x = IFERROR( 38 | SWITCH( 39 | TRUE(), 40 | // curCodepoint < 32, "Control Chars", // if you want to just cut it off 41 | curCodepoint < 33, 42 | UNICHAR( curCodepoint + 9216 ), // = 0x2400 43 | curCodepoint = 127, 44 | UNICHAR( 9249 ), 45 | curCodepoint >= 1114111, 46 | "Too Big", // 0x10ffff 47 | IFERROR( 48 | UNICHAR( curCodepoint ), "Failed!" 49 | ) 50 | ) 51 | 52 | , "OuterError") 53 | return x -------------------------------------------------------------------------------- /source-dax/Find-RowIntegrity-Violations.dax: -------------------------------------------------------------------------------- 1 | select 2 | [Database_name] , [Dimension_Name] , [RIVIOLATION_COUNT] from $SYSTEM.DISCOVER_STORAGE_TABLES 3 | WHERE 4 | [RIVIOLATION_COUNT] > 0 -------------------------------------------------------------------------------- /source-dax/Format-Percent-Without-Math-Or-P.dax: -------------------------------------------------------------------------------- 1 | [ Percent Message ] = 2 | /* 3 | Render percentage, because the normal "p" format string 4 | is missing. Instead you can use "%", 5 | which sort of is the culture-invariant version of "p" 6 | 7 | (no multiplication needed) to render percentagel, because the normal "p" format string is missing 8 | explained here: https://docs.microsoft.com/en-us/dax/format-function-dax 9 | 10 | Example 11 | Input Output 12 | 0.0 0.0% 13 | 0.5 50.0% 14 | 1.0 100.0% 15 | 16 | related docs about culture invariants: 17 | https://docs.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo?view=net-6.0#invariant-neutral-and-specific-cultures 18 | https://docs.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.invariantculture?view=net-6.0 19 | */ 20 | var something = SELECTEDVALUE( Range[Value] ) 21 | var render = FORMAT( something, "##0.0%" ) 22 | return 23 | "Hit Rate: " & render 24 | -------------------------------------------------------------------------------- /source-dax/Safely Render UniChar -- replacing with symbols.dax: -------------------------------------------------------------------------------- 1 | RenderFromInt = 2 | /* 3 | a safe version of UniChar(), 4 | [1] Normally UniChar() can crash the visual if certain codepoints are used (control chars) 5 | [2] and then visualizes invisble control chars by using their symbol equivalents 6 | instead of blanks 7 | 8 | 9 | 0 => ␀ ( null ) 10 | 10 => ␊ ( newline) 11 | 12 | Normally unicode is based in hex, AFAIK you can't currently use hex literals in DAX 13 | To convert codepoints to their Control Symbols, add 0x2400 14 | ex: 15 | null is 0 + 0x2400, newline is 10 + 0x2400 16 | 17 | high Surrogate pairs (U+D800–U+DBFF) 18 | Low Surrogate (U+DC00–U+DFFF) "🐍" 19 | ref: 20 | del => https://www.compart.com/en/unicode/U+2421 21 | https://en.wikipedia.org/wiki/C0_and_C1_control_codes 22 | https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane 23 | */ 24 | var symbolOffset = 9216 // 0x2400 25 | var unicodeMaxValue = 1114111 // 0x10ffff 26 | var codepoint = 27 | SELECTEDVALUE( main[Codepoint] ) 28 | 29 | var isSurrogatePair = // pbi is rendering blocks, but doesn't actually error 30 | codepoint >= 55296 && codepoint <= 57343 // 0xd800 .. 0xdfff 31 | var isCtrl_C0 = 32 | codepoint >= 0 && codepoint <= 31 // 0x7f 33 | 34 | var isCtrl_C1 = 35 | codepoint >= 128 && codepoint <= 159 // 0x80 - 0x9f 36 | 37 | var isOutOfBounds = // not used currently, because the snake will catch it 38 | codepoint < 0 || codepoint > unicodeMaxValue 39 | 40 | var isCtrl = 41 | isCtrl_C0 || isCtrl_C1 42 | 43 | var codepointOffset = if( 44 | isCtrl, 45 | codepoint + symbolOffset, 46 | codepoint ) 47 | 48 | var maybeRender = 49 | IFERROR( 50 | UNICHAR( codepointOffset ) , "🐍" ) // 😭" 51 | 52 | return maybeRender -------------------------------------------------------------------------------- /source-dax/new-syntax-datetime-literals-in-strings-are-true-datetime-values.dax: -------------------------------------------------------------------------------- 1 | [ People ] = DATATABLE( 2 | "When", DATETIME, "Name", STRING, 3 | { { dt"2020-03-03", "Bob" }, 4 | { dt"2020-03-03 11:45:22", "Jed" }, 5 | { dt"2022-12-18 00:00:00", "Bob" }, 6 | // { dt"2020-03-03 31:45:22", "Jed" }, // uncomment to see thrown error 7 | { dt"2022-12-18", "Jen" } 8 | }) -------------------------------------------------------------------------------- /source-excel/GetAbsoluteWorkbookPath.md: -------------------------------------------------------------------------------- 1 | This parses `cell()` output to get the **absolute filepath of the current workbook** 2 | This is one way you can have powerquery autodetect absolute filepaths 3 | ``` 4 | = Cell( "filename" ) 5 | ``` 6 | The raw output 7 | ``` 8 | c:\data\excel\[Example Sales Table.xlsx]Sheet2 9 | ``` 10 | Note: You could drop the worksheet name, then replace `[]` with `''`. 11 | I split this into steps to show how you can use intermediate calculations as variables 12 | without polluting the scope using named cells 13 | 14 | ```sql 15 | = LET( 16 | rawName, CELL("filename"), 17 | 18 | folder, 19 | TEXTBEFORE( rawName, "[", 1 ), 20 | 21 | file, 22 | TEXTBEFORE( 23 | TEXTAFTER( rawName, "[", 1 ), 24 | "]", -1 25 | ), 26 | 27 | fullName, 28 | CONCAT(folder, file), 29 | 30 | fullName ) 31 | ``` 32 | Our final output: 33 | ``` 34 | c:\data\excel\Example Sales Table.xlsx 35 | ``` 36 | -------------------------------------------------------------------------------- /source-modules/Alias/Alias.Text.pq: -------------------------------------------------------------------------------- 1 | [ 2 | FmtList = Text.FormatList meta [ AliasTo = "Text.FormatList", NinLibType = "Alias" ], 3 | Csv = Text.FormatCsv, 4 | Format.Csv = Csv, 5 | Fmt.Csv = Csv, 6 | 7 | TransformTo.TextList = List.TransformTo.ListOfText, 8 | ListOf.Text = List.TransformTo.ListOfText, 9 | As.TextList = List.TransformTo.ListOfText, 10 | 11 | List.TransformTo.ListOfText = TransformTo.TextList 12 | 13 | 14 | /* 15 | old aliases I experimentede with: 16 | 17 | AsList.Text = AsTextList, 18 | AsList.OfText = TextList = AsTextList, 19 | To.TextList = Transform.TextList, 20 | AsTextList = As.TextList, 21 | To.TextList = As.TextList, 22 | 23 | 24 | in: { "a", "b" } 25 | out: "a#(cr,lf)b" 26 | 27 | Format.JoinDocNewline = Format.JoinBR, 28 | 29 | Join.NewLine = Format.JoinNewLine, 30 | Join.NL = Join.NewLine, 31 | FmtNL = Join.NewLine, 32 | 33 | 34 | // # attributes could auto-strip these out based on a config value, that reads metadata 35 | ListOf.Text = TransformTo.TextList, 36 | As.TextList = ListOf.Text, 37 | // AsTextList = ListOf.Text, 38 | // To.TextList = ListOf.Text, 39 | // ToList.OfText = ListOf.Text, 40 | AsList.OfText = ListOf.Text, 41 | TextList = ListOf.Text, 42 | Transform.TextList = ListOf.Text, 43 | // AsList.Text = ListOf.Text, 44 | Format.ControlChar = Format.ShowBlank, 45 | Fcc = Format.ShowBlank, 46 | Format.Newline = Format.ShowBlank, 47 | Format.Blank = Format.ShowBlank, 48 | 49 | */ 50 | ] -------------------------------------------------------------------------------- /source-modules/Color.Constants.pq: -------------------------------------------------------------------------------- 1 | let 2 | Constant.Colors = [ 3 | Red = "#FF0000", 4 | Green = "#00FF00", 5 | Blue = "#0000FF", 6 | Yellow = "#FFFF00", 7 | Cyan = "#00FFFF", 8 | Magenta = "#FF00FF", 9 | White = "#FFFFFF", 10 | Black = "#000000" 11 | ] 12 | in 13 | Constant.Colors -------------------------------------------------------------------------------- /source-modules/Encoding.Constants.pq: -------------------------------------------------------------------------------- 1 | [ 2 | TextEncoding.Cyrillic = 28595 3 | ] -------------------------------------------------------------------------------- /source-modules/StringBuilder.module.pq: -------------------------------------------------------------------------------- 1 | [ 2 | /* 3 | About: Stand alone StringBuilder inspired functions 4 | 5 | See Also: 6 | Write.Html.module.pq 7 | 8 | */ 9 | // optional aggressive Aliases 10 | Sb.JoinDelim = StringBuilder.JoinDelimiter, 11 | Sb.Format = StringBuilder.Format, 12 | Sb.JoinFormat = StringBuilder.JoinFormat, 13 | Sb.JoinLineEnding = StringBuilder.JoinLineEnding, 14 | Sb.JoinNL = StringBuilder.JoinLineEnding, 15 | 16 | Sb.Str = [ 17 | LineEnding = "#(cr,lf)", 18 | NullStr = "#(2400)", 19 | SpaceStr = "#(2420)", 20 | Tab = "#(tab)" 21 | ], 22 | 23 | StringBuilder.JoinLineEnding = ( 24 | items as list 25 | ) as text => [ 26 | return = Text.Combine( items, Sb.Str[LineEnding] ) 27 | ][return], 28 | 29 | StringBuilder.JoinDelimiter = ( 30 | items as list, 31 | separator as text 32 | ) as text => [ 33 | return = Text.Combine( items, separator ) 34 | 35 | ][return], 36 | /* 37 | like StringBuilder.JoinFormat, 38 | but this accepts a single value verses a list 39 | */ 40 | StringBuilder.Format = ( 41 | content as text, 42 | optional options as nullable record 43 | ) as text => [ 44 | return = StringBuilder.JoinFormat( {content}, options ) 45 | ][return], 46 | 47 | /* 48 | Valid options are any combination of these: 49 | - Prefix, 50 | - Suffix 51 | - Separator 52 | */ 53 | StringBuilder.JoinFormat = ( 54 | items as list, 55 | optional options as nullable record 56 | 57 | ) as text => [ 58 | formatStr = "#[prefix]#[content]#[suffix]", 59 | 60 | renderLines = List.Transform( 61 | items, 62 | (line) => Text.Format( 63 | formatStr, [ 64 | prefix = options[Prefix]? ?? "", 65 | content = line, 66 | suffix = options[Suffix]? ?? "" 67 | ] 68 | ) 69 | ), 70 | 71 | return = Text.Combine( 72 | renderLines, 73 | options[Separator]? ?? Sb.Str[LineEnding] 74 | ) 75 | 76 | ][return], 77 | 78 | ] -------------------------------------------------------------------------------- /source-pwsh/old-example-src/Text to PowerQueryLiterals.ps1: -------------------------------------------------------------------------------- 1 | class RuneToPqLiteral { 2 | [string] $Source 3 | [int] $Codepoint 4 | [string] $Hex 5 | 6 | # hidden $First 7 | 8 | hidden [string] $DisplayCodepoint 9 | 10 | RuneToPqLiteral ( [string] $Rune ) { 11 | $first, $rest = $Rune.EnumerateRunes() 12 | if ($rest.count -gt 0) { throw 'nyi IEnumerable list of runes' } 13 | 14 | $This.Source = $Rune 15 | # $This.First = $first 16 | $This.Codepoint = $first.Value 17 | $This.Hex = '{0:x}' -f @($This.Codepoint) 18 | $This.DisplayCodepoint = '0x' + $this.Hex 19 | # just do 1 for now 20 | # foreach($Rune in $InputText.EnumerateRunes()) { 21 | 22 | # } 23 | $null -eq 2 24 | } 25 | 26 | [string] ToString() { 27 | return '' 28 | } 29 | 30 | [int] Length () { 31 | return $this.Source.EnumerateRunes().Value.Count 32 | } 33 | 34 | 35 | } 36 | 37 | function StringToPqLiteral { 38 | param( 39 | [Parameter(ValueFromPipeline)] 40 | $InputText 41 | ) 42 | 43 | process { 44 | # $ErrorActionPreference = 'break' 45 | $InputText.EnumerateRunes() | ForEach-Object { 46 | [RuneToPqLiteral]::new($_) 47 | } 48 | # $ErrorActionPreference = 'continue' 49 | } 50 | } 51 | 52 | 53 | $sample = ' 🙊🐛🐈' 54 | 55 | StringToPqLiteral $sample 56 | -------------------------------------------------------------------------------- /source-pwsh/old-example-src/__init__.ps1: -------------------------------------------------------------------------------- 1 | using namespace System.Text.StringBuilder 2 | #Requires -Version 7.0 3 | #Requires -Module Ninmonkey.Console 4 | # Requires -Module Dev.Nin 5 | 6 | Import-Module Ninmonkey.Console, Dev.nin -wa ignore 7 | 8 | $ModuleConfig = @{ 9 | 10 | } 11 | $eaStop = @{ ErrorAction = 'stop' } 12 | 13 | $script:__Paths = @{ 14 | RepoRoot = Get-Item @eaStop "$Env:UserProfile/SkyDrive/Documents/2021/Power BI/My_Github/Ninmonkey.PowerQueryLib" 15 | } 16 | $__paths = @{ 17 | OutputRoot = Get-Item @eaStop Join-path $__paths.RepoRoot '.output' 18 | ExportRoot = Get-Item @eaStop Join-path $__paths.RepoRoot '.export' 19 | PowerQuerySourceRoot = Get-Item @eaStop Join-path $__paths.RepoRoot 'source' 20 | } 21 | function foo { 22 | <# 23 | .synpopsis 24 | what 25 | #> 26 | [CmdletBinding()] 27 | param() 28 | 29 | 30 | } 31 | function Set-PqLibOption { 32 | <# 33 | .synopsis 34 | set options 35 | #> 36 | param( 37 | [Parameter(Mandatory)] 38 | [hashtable]$Options 39 | ) 40 | 41 | } 42 | function Get-PqLibOption { 43 | <# 44 | .synopsis 45 | get options 46 | #> 47 | [CmdletBinding()] 48 | param() 49 | $opt = @{ 50 | PsTypeName = 'Nin.PqLib.Options' 51 | Paths = $script:__Paths 52 | } 53 | return [pscustomobject]$Opt 54 | } 55 | 56 | 57 | $Config = @{ 58 | AutoOpenEditor = $false 59 | AutoRoot = Get-Item -ea stop (Join-Path $PSScriptRoot '..') 60 | } 61 | $Config += @{ 62 | AutoExportRoot = Join-Path $Config.AutoRoot '.output' 63 | } 64 | 65 | -------------------------------------------------------------------------------- /source-pwsh/old-example-src/format-pqlibDate.ps1: -------------------------------------------------------------------------------- 1 | 'Original command' 2 | $d = [datetime]'2024-03-13' 3 | $cults = Get-culture -ListAvailable 4 | $cults | %{ 5 | $Cult = $_ 6 | $d.ToString( $_.DateTimeFormat.ShortDatePattern, $cult ) 7 | # .DateTimeFormat.ShortDatePattern 8 | # $d.ToShortDateString() } ) 9 | } 10 | 11 | function Format-PqLibDate { 12 | 13 | param( 14 | [ValidateSet( 15 | 'ShortDate', 'ShortTime' 16 | )] 17 | [Alias('Template', 'FStr')] 18 | [Parameter(Mandatory)] 19 | [string]$Format 20 | ) 21 | 22 | $FormatStr = switch($Format) { 23 | 'ShortDate' { 'yyyy-MM-dd' } 24 | 'ShortTime' { 'HH:mm:ss' } 25 | default { throw "ShouldNeverReachException: Unhandled Format '$Format'" } 26 | } 27 | 28 | $d = [datetime]'2024-03-13' 29 | $cults = Get-culture -ListAvailable 30 | $cults | %{ 31 | $d.ToString( $_.DateTimeFormat.ShortDatePattern,(get-culture 'de' )) 32 | # .DateTimeFormat.ShortDatePattern 33 | # $d.ToShortDateString() } ) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /source-pwsh/tests/Get-NativeCommand.tests.ps1: -------------------------------------------------------------------------------- 1 | Import-Module Pester -ea 'stop' -MinimumVersion '5.0' -PassThru 2 | Import-Module (Join-Path $PSScriptRoot '../Nin.PqLib.psd1' ) -Force -ea 'continue' 3 | 4 | Describe 'Nin.PqLib NativeCommands' { 5 | BeforeAll { 6 | if((gcm -ea 'ignore' 'git' -CommandType Application).count -eq 0) { 7 | write-warning '"git" may not be in the path. Expect tests to fail' 8 | } 9 | } 10 | 11 | Context 'GetNativeCommand' { 12 | it 'IsA ApplicationInfo' { 13 | Get-PqLibNativeCommand -CommandName 'git' 14 | | Should -BeOfType ([System.Management.Automation.ApplicationInfo]) 15 | } 16 | it 'Throws If Missing' { 17 | { Get-PqLibNativeCommand -CommandName 'DoesNotExist' } 18 | | Should -Throw -Because 'Cmd Path is not real' 19 | } 20 | } 21 | Context 'GetNativeCommandFast' { 22 | it 'IsA String' { 23 | Get-PqLibNativeCommandFast -CommandName 'git' 24 | | Should -BeOfType ([string]) 25 | } 26 | it 'Finds git' { 27 | (Get-PqLibNativeCommandFast -CommandName 'git').count -gt 0 28 | | Should -Be $true -Because 'Expect at least one result' 29 | } 30 | Context 'TestsIfExists' { 31 | it 'Return IsA bool' { 32 | Get-PqLibNativeCommandFast -TestIfExists -CommandName 'TestIfExists' 33 | | Should -BeOfType ([bool]) -Because 'Should always be bool' 34 | Get-PqLibNativeCommandFast -TestIfExists -CommandName 'TestIfExists' 35 | | Should -BeOfType ([bool]) -Because 'Should always be bool' 36 | } 37 | it 'Never Throws' { 38 | { Get-PqLibNativeCommandFast -TestIfExists -CommandName 'git' } 39 | | Should -not -throw -because 'Should Never Throw when using Fast Test if Exists' 40 | { Get-PqLibNativeCommandFast -TestIfExists -CommandName 'TestIfExists' } 41 | | Should -not -throw -because 'Should Never Throw when using Fast Test if Exists' 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /source/Assert/Assert.ListLength.pq: -------------------------------------------------------------------------------- 1 | let 2 | // If length was expected, return it, otherwise throw for unexpected values 3 | // todo future: Add a version that returns a logic instead. TryAssert.ListLength ? 4 | Assert.ListLength = ( 5 | source as list, 6 | expectedLength as number 7 | ) as list => [ 8 | actualLength = List.Count( source ), 9 | shouldThrow = actualLength <> expectedLength, 10 | 11 | finalAssert = 12 | if not shouldThrow then source 13 | else error [ 14 | Message.Format = "InvalidLengthException: Expected list length of {0}, but found {1} for list: {2}", 15 | Message.Parameters = { expectedLength, actualLength, source } 16 | ] 17 | 18 | return = source 19 | 20 | ][return] 21 | 22 | in 23 | Assert.ListLength -------------------------------------------------------------------------------- /source/Assert/Columns.ThatExist.pq: -------------------------------------------------------------------------------- 1 | 2 | let 3 | /* 4 | Asserts if columns are valid, return their names as a list 5 | otherwise throw 6 | 7 | in: { "Name", "Id" } 8 | out: good 9 | { "Name", "Id"} 10 | out: bad 11 | error Exception for missing columns 12 | 13 | */ 14 | Columns.ThatExist = ( 15 | source as table, 16 | columnNames as list 17 | ) as any => [ 18 | 19 | actual_columnNames = Table.ColumnNames(source), 20 | all_exist = List.ContainsAll( actual_columnNames, columnNames, Comparer.Ordinal ), 21 | assert = all_exist, 22 | valid_data = columnNames, 23 | 24 | error_missingMandatory = 25 | // The ErrorRecord from Err.InvalidColumnNames contains 26 | // Message.Parameters, if you want to drill down into columnNames, and table 27 | Err.InvalidColumnNames( source, columnNames ) 28 | meta [ 29 | NinAssertName = "Columns.ThatExist", 30 | Activity = Diagnostics.ActivityId() ], 31 | return = 32 | if assert then valid_data 33 | else error error_missingMandatory 34 | 35 | ][return] 36 | in 37 | Columns.ThatExist -------------------------------------------------------------------------------- /source/Binary.ToBase64.pq: -------------------------------------------------------------------------------- 1 | let 2 | Binary_ToBase64 = (bytes as binary) as text => 3 | /* 4 | About: 2021-04-22 5 | encode any file to base64 6 | 7 | future: include metadata of file as meta? or return a record? 8 | */ 9 | let 10 | encodedText = Binary.ToText( bytes, BinaryEncoding.Base64 ) 11 | in 12 | encodedText 13 | 14 | in 15 | Binary_ToBase64 -------------------------------------------------------------------------------- /source/CoerceTo.Table.pq: -------------------------------------------------------------------------------- 1 | // CoerceToTable 2 | let 3 | CoerceToTable = 4 | (source as any) as any => 5 | // as table => 6 | let 7 | // currently these are redundant as case statements, but I may use multiple later 8 | isRecord = Type.Is(sourceType, Record.Type), 9 | isTable = Type.Is(sourceType, Table.Type), 10 | isText = Type.Is(sourceType, Text.Type), 11 | sourceType = Value.Type(source), 12 | other = 13 | error 14 | Error.Record( 15 | "UnhandledTypeConversion", 16 | "UnhandledType", 17 | sourceType 18 | ) 19 | in 20 | if false then 21 | null 22 | else if isRecord then 23 | Record.ToTable(source) as table 24 | else if isText then 25 | #table(1, {source}) as table 26 | else if isTable then 27 | source as table 28 | else 29 | try other catch (e) => e[Message] 30 | in 31 | CoerceToTable -------------------------------------------------------------------------------- /source/CoerceTo.Table.succinct.pq: -------------------------------------------------------------------------------- 1 | // CoerceToTable 2 | let 3 | // sugar to convert types to table, else text 4 | CoerceToTable = 5 | (source as any) as any => // as table => 6 | let 7 | sourceType = Value.Type(source) 8 | in 9 | if Type.Is(sourceType, Record.Type) then 10 | Record.ToTable(source) as table 11 | else if Type.Is(sourceType, Table.Type) then 12 | source as table 13 | else if Type.Is(sourceType, Text.Type) then 14 | #table(1, {source}) as table 15 | else 16 | error Error.Record( "UnhandledTypeConversion", "UnhandledType", sourceType ) 17 | in 18 | CoerceToTable -------------------------------------------------------------------------------- /source/Color/Rgb.FromHexString.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | converts string to a bunch of formats 4 | - components as [int] 5 | - components as RGBFloat [ 0..1.0 ] 6 | - components as Hex [ 0x00..0xFF ] 7 | - full hex string 8 | - int64, int32 9 | */ 10 | Rgb.FromHexString = (source as text, optional options as nullable record) as record => [ 11 | /* 12 | future: add support for 13 | - [ ] 2, 4, 6, 8 digits 14 | - [ ] strip prefixes like 0x or # 15 | - [ ] alpha 16 | - [ ] int32 17 | - [ ] int64 18 | - [ ] as range 19 | */ 20 | Len = Text.Length( source ), 21 | pairs = List.Split( Text.ToList( source ), 2 ), 22 | pairStrings = List.Transform( pairs, each Text.Combine(_, "") ), 23 | 24 | // Assert.ValidLength = List.Contains({2, 4, 6}, Len), 25 | // List.Transform( List.Split( source, 2 ), each Number.FromHexString(_) ) 26 | components = [ 27 | RgbHexStr = Text.Combine( {"#", RHex, GHex, BHex}, ""), 28 | RHex = pairStrings{0}?, 29 | GHex = pairStrings{1}?, 30 | BHex = pairStrings{2}?, 31 | // AHex = null, 32 | // A = null, 33 | R = Number.FromHexString( RHex ), 34 | G = Number.FromHexString( GHex ), 35 | B = Number.FromHexString( BHex ), 36 | 37 | RFloat = R / 255, 38 | GFloat = G / 255, 39 | BFloat = B / 255 40 | 41 | ], 42 | Assert.ValidLength = List.Contains({6}, Len), 43 | ret = 44 | if Assert.ValidLength then components 45 | else error [ 46 | Message.Format = "Message #{0}", 47 | Message.Parameters = { source } 48 | ] 49 | // List.Transform( List.Split( source, 2 ), each Number.FromHexString(_) ) 50 | // List.Transform( List.Split( Text.ToList("feaa99"), 2 ), each Text.Combine(_, "")) 51 | 52 | ][ret] 53 | in 54 | Rgb.FromHexString -------------------------------------------------------------------------------- /source/ConvertTo.Markdown.pq: -------------------------------------------------------------------------------- 1 | let 2 | SampleInput = GenerateDocs_OnType, // as table, 3 | source = SampleInput, 4 | 5 | Table_ToMarkdown = (source as table) as any => // text ? 6 | let 7 | headerRow = Format_RowsToMarkdown( Table.ColumnNames(source) ), 8 | tableBody = generate_lines(source), 9 | finalText = Text.Combine( 10 | {headerRow, tableBody}, "#(cr,lf)") 11 | in 12 | finalText, 13 | 14 | 15 | /* 16 | 17 | Input: "foo", "bar" 18 | Output: | foo | bar | 19 | future: 20 | tie into the better text coercion funcs 21 | */ 22 | Format_RowsToMarkdown = (row as record) as any => // text 23 | let 24 | values = Record.FieldValues(row), 25 | joinPipe = Text.Combine(values, " | "), 26 | finalString = Text.Combine({"|", joinPipe, "|"}, " ") 27 | in 28 | finalString, 29 | /* 30 | 31 | Input: "foo", "bar" 32 | Output: | foo | bar | 33 | future: 34 | tie into the better text coercion funcs 35 | */ 36 | GenerateRows = (source as table) as list => 37 | let 38 | generate_lines = Table.TransformRows( 39 | source, Format_RowsToMarkdown 40 | ) 41 | in 42 | generate_lines, 43 | 44 | 45 | 46 | test = Table_ToMarkdown( SampleInput ) 47 | in 48 | test -------------------------------------------------------------------------------- /source/DateTable_FromDates.backup.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | Input: 4 | Table with at least a [Date] column 5 | output: 6 | Adds columns for: [Year], [Month], [Day], [Week of Year], [Date Id] key 7 | 8 | future: 9 | optionally add time when datetime is the source 10 | 11 | */ 12 | DateTable.FromDates_Impl = () => 13 | error Error.Record("NYI", "First write ListAll.Dates.pq"), 14 | old_version_Date.TableFromDates = (source as table) as table => 15 | let 16 | // isDatetime = ..., // if datetime, or datetimezone, 17 | source = 18 | if Table.HasColumns( source, {"Date"} ) 19 | then source 20 | else error Error.Record( 21 | "InvalidArgument", "Expects a table with a column named 'Date'", source 22 | ), 23 | 24 | base =List.ContinuousDates( source[Date] ), 25 | col_Year = Table.AddColumn(base, "Year", (_) as number => Date.Year([Date]), Int64.Type), 26 | col_Month = Table.AddColumn(col_Year, "Month", (_) as number => Date.Month([Date]), Int64.Type), 27 | col_Day = Table.AddColumn(col_Month, "Day", (_) as number => Date.Day([Date]), Int64.Type), 28 | col_WeekOfYear = Table.AddColumn(col_Day, "Week of Year", (_) as number => Date.WeekOfYear([Date]), Int64.Type), 29 | col_Index = Table.AddIndexColumn(col_WeekOfYear, "Date Id", 0, 1, Int64.Type), 30 | verifyDistict = Table.IsDistinct(col_Index, {"Date"}), 31 | Final = 32 | if verifyDistict then col_Index 33 | else error Error.Record( 34 | "TableNotDistinct", "Using column 'date'", col_Index ) 35 | in 36 | Final 37 | 38 | in 39 | Value.ReplaceType( DateTable.FromDates_Impl, DateTable.FromDates_Type1 ) 40 | 41 | a -------------------------------------------------------------------------------- /source/DateTable_FromDates.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | Input: 4 | Table with at least a [Date] column 5 | output: 6 | Adds columns for: [Year], [Month], [Day], [Week of Year], [Date Id] key 7 | 8 | future: 9 | optionally add time when datetime is the source 10 | 11 | */ 12 | DateTable.FromDates_Impl = () => 13 | error Error.Record("NYI", "First write ListAll.Dates.pq"), 14 | old_version_Date.TableFromDates = (source as table) as table => 15 | let 16 | // isDatetime = ..., // if datetime, or datetimezone, 17 | source = 18 | if Table.HasColumns( source, {"Date"} ) 19 | then source 20 | else error Error.Record( 21 | "InvalidArgument", "Expects a table with a column named 'Date'", source 22 | ), 23 | 24 | base =List.ContinuousDates( source[Date] ), 25 | col_Year = Table.AddColumn(base, "Year", (_) as number => Date.Year([Date]), Int64.Type), 26 | col_Month = Table.AddColumn(col_Year, "Month", (_) as number => Date.Month([Date]), Int64.Type), 27 | col_Day = Table.AddColumn(col_Month, "Day", (_) as number => Date.Day([Date]), Int64.Type), 28 | col_WeekOfYear = Table.AddColumn(col_Day, "Week of Year", (_) as number => Date.WeekOfYear([Date]), Int64.Type), 29 | col_Index = Table.AddIndexColumn(col_WeekOfYear, "Date Id", 0, 1, Int64.Type), 30 | verifyDistict = Table.IsDistinct(col_Index, {"Date"}), 31 | Final = 32 | if verifyDistict then col_Index 33 | else error Error.Record( 34 | "TableNotDistinct", "Using column 'date'", col_Index ) 35 | in 36 | Final 37 | 38 | in 39 | Value.ReplaceType( DateTable.FromDates_Impl, DateTable.FromDates_Type1 ) 40 | 41 | -------------------------------------------------------------------------------- /source/DateTime.FromUnixTime.pq: -------------------------------------------------------------------------------- 1 | let 2 | DateTime_FromUnixTime = (unixTime as number, optional mode as text) as datetime => 3 | /* 4 | Convert Unix date times to a `datetime` 5 | modes 6 | "s" is seconds 7 | "ms" is milliseconds 8 | "ns" is nanoseconds 9 | 10 | To use the epoch "12:00 midnight, January 1, 1601", use the functions 11 | DateTime.FromFileTime 12 | DateTimeZone.FromFileTime 13 | 14 | see also: 15 | https://en.wikipedia.org/wiki/Unix_time#UTC_basis 16 | .net class for duration: https://docs.microsoft.com/en-us/dotnet/api/system.timespan?view=netcore-3.1 17 | 18 | when out of range, the user facing error is: 19 | > An error occurred in the ‘DateTime_FromUnixTime’ query. Expression.Error: The Duration operation failed because the resulting duration falls outside the range of allowed values. 20 | */ 21 | 22 | let 23 | UnixEpoch = #datetime(1970, 1, 1, 0, 0, 0), 24 | modifier 25 | = if mode = "s" or mode = "sec" then 1 26 | else if mode = "ms" then 1e3 27 | else if mode = "ns" then 1e9 28 | else 1, 29 | 30 | offsetDuration = #duration(0, 0, 0, (unixTime / modifier) ), 31 | tryDatetime = try UnixEpoch + offsetDuration, 32 | result = tryDatetime 33 | in 34 | if result[HasError] then error result[Error] else result[Value] 35 | 36 | in 37 | DateTime_FromUnixTime -------------------------------------------------------------------------------- /source/DateTime.ToOData.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | 4 | See also: 5 | .\source\test\test_DateTime_ToOData.pq 6 | 7 | Description: 8 | 9 | Converts Power Query `date` and `datetimes` to OData `datetime` and `datetimeoffset` 10 | 11 | Examples: 12 | 9/8/2020 2:01:50 PM DateTime Local 13 | 9/8/2020 2:01:50 PM -05:00 DateTimeZone Local 14 | 9/8/2020 7:23:18 PM +00:00 DateTimeZone Utc 15 | 16 | Output: 17 | datetime'2020-09-08T14:01:49' 18 | datetimeoffset'2020-09-08T14:01:49' 19 | datetimeoffset'2020-09-08T19:23:17' 20 | */ 21 | 22 | DateTime_ToOData = (dates as any) as text => 23 | // verify that culture doesn't change when using format "s" 24 | let 25 | formattedType = if dates is datetime then 26 | "datetime'" & DateTime.ToText( dates, "s", null ) 27 | else if dates is datetimezone then 28 | "datetimeoffset'" & DateTimeZone.ToText( dates, "s", null ) 29 | else 30 | error Error.Record( 31 | "InvalidType", "Expected, datetime, or datetimezone", 32 | Value.Type( dates ) 33 | ), 34 | formatted = formattedType & "'" 35 | in 36 | formatted 37 | in 38 | DateTime_ToOData 39 | -------------------------------------------------------------------------------- /source/Diagnostic.FormatDetailedLog.pq: -------------------------------------------------------------------------------- 1 | // from PBI traces, the one named 'detailed' 2 | Diagnostic.FormatDetailedLog = (source as table, optional CurrentActivityId as nullable any, 3 | optional options as nullable record 4 | ) as table => 5 | let 6 | options = Record.Combine({ defaults, (options ?? []) }), 7 | defaults = [ 8 | FilterByUserQuery = true, 9 | CurrentActivityId = CurrentActivityId ?? null 10 | ], 11 | 12 | select_activityId = 13 | if options[CurrentActivityId] = null then source 14 | else Table.SelectRows( source, each ([Activity Id] = options[CurrentActivityId] )), 15 | 16 | select_isUserQuery = 17 | if not options[FilterByUserQuery] then select_activityId 18 | else Table.SelectRows( select_activityId, each [Is User Query] = true ) 19 | in 20 | select_isUserQuery -------------------------------------------------------------------------------- /source/ErrorRecord.Format.pq: -------------------------------------------------------------------------------- 1 | let 2 | ErrorRecord.Format = (source as record, optional options as nullable record) as text => 3 | let 4 | maybeDetail = source[Detail]? ?? [], 5 | prefix = Text.Combine({ 6 | (source[Reason]? ?? ""), 7 | (source[Message]? ?? "") 8 | }, "#(cr,lf) "), 9 | formatted = Text.Format( 10 | (source[Message.Format]? ?? ""), 11 | (source[Message.Parameters]? ?? null) 12 | ), 13 | combineAll = Text.Combine({ 14 | maybeDetail, prefix, formatted 15 | }, "#(cr,lf)"), 16 | summary = [ 17 | nyi = "func not finished", 18 | combineAll = combineAll ] 19 | in 20 | summary 21 | // combineAll meta [Exception = source] 22 | in 23 | ErrorRecord.Format -------------------------------------------------------------------------------- /source/Errors/Convert.ScriptExtent.FromError.pq: -------------------------------------------------------------------------------- 1 | [ 2 | Convert.ScriptExtent.FromError = (err as any) => [ 3 | Split.ScriptExtent = Splitter.SplitTextByEachDelimiter({"[", ",", "-", ",", "]"}, QuoteStyle.None), 4 | lineData = Split.ScriptExtent( err[Message] ), 5 | return = [ 6 | StartLineNumber = Number.FromText( lineData{1}? ), 7 | StartColumnNumber = Number.FromText( lineData{2}? ), 8 | EndLineNumber = Number.FromText( lineData{3}? ), 9 | EndColumnNumber = Number.FromText( lineData{4}? ), 10 | RemainingMessage = lineData{5}?, // shouldn't be more than 1 index? 11 | Reason = err[Reason], 12 | Message = err[Message], 13 | ErrorRecord = err, 14 | ErrorDetailsJson = [ 15 | target = ErrorRecord[Detail]?, 16 | json = Json.FromValue( target, TextEncoding.Utf8 ), 17 | string = try Text.FromBinary( json, TextEncoding.Utf8 ) catch (e) => null 18 | ][string], 19 | RawText = err[Message] 20 | ] 21 | ][return] 22 | ] -------------------------------------------------------------------------------- /source/Errors/Err.ExpandErrorRecord.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | About: 4 | accepts an error record merging its data and metadata as one record 5 | see also: 6 | Err.ExpandErrorRecord.pq - start here, this is a lot simpler to use 7 | Err.ExpandErrorRecord.Verbose.pq - when you want an excessive amount of details 8 | */ 9 | 10 | Err.ExpandErrorRecord = ( 11 | exception as any, 12 | optional options as nullable record 13 | ) => [ 14 | Json = (source as any) as text => Text.FromBinary( Json.FromValue( source ) ), 15 | // future: auto coerce exceptions and error records, so that either can be passed 16 | // isUncaught = (try exception)[HasError], 17 | // exception = 18 | // if not isUncaught 19 | // then exception 20 | // else try exception, 21 | 22 | // exception = try exception catch (e) => e, 23 | return = [ 24 | // Er_Type = Value.Type( exception ), // is alway a record? 25 | Err = exception, 26 | Err.Meta = Value.Metadata( Err ), 27 | ErTypeMeta = Value.Metadata( Value.Type( Err ) ), 28 | ErMeta_KeyNames = Text.Combine( Record.FieldNames( Err.Meta ), ", " ), 29 | 30 | Reason = Err[Reason]? ?? null, 31 | Message = Err[Message]? ?? null, 32 | Detail = Err[Detail]? ?? null, 33 | Message.Format = Err[Message.Format]? ?? null, 34 | Message.Parameters = Err[Message.Parameters]? ?? null, 35 | Message.Parameters_Json = 36 | Json( Message.Parameters ), 37 | 38 | 39 | Expression.Stack = Err.Meta[Expression.Stack]? ?? null, 40 | Expression.Stack.Json = Json( Expression.Stack ), 41 | Expression.Stack.JsonLines = 42 | Text.Combine( 43 | List.Transform( 44 | Expression.Stack, 45 | (item) => Json( item ) 46 | ), 47 | "#(cr,lf)" ), 48 | 49 | ActivityId = Diagnostics.ActivityId(), 50 | ModuleVersion = Module.Versions() 51 | ] 52 | 53 | ][return] 54 | in 55 | Err.ExpandErrorRecord -------------------------------------------------------------------------------- /source/Errors/Err.InvalidColumnNames.pq: -------------------------------------------------------------------------------- 1 | let 2 | // creates a new ErrorRecord fo you to throw 3 | 4 | Err.InvalidColumnNames = ( 5 | source as table, 6 | columnNames as list, 7 | optional options as nullable record 8 | ) as record => [ 9 | 10 | // or table source could be name, 11 | // assert = if sourceOrName is table then 12 | // Columns.ThatExist( sourceOrName, columnNames ) 13 | // else 14 | // true, 15 | // assertAllowedKeys = {"Reason", "Detail", "Message.Parameters", "Message.Format" }, 16 | // tableName = if source is text then source else Expression.Identifier(source), 17 | FmtList = TransformTo.TextList, 18 | FmtNL = Format.JoinNewLine, 19 | MissingColumns = List.RemoveItems( columnNames, Table.ColumnNames( source) ), 20 | err = [ 21 | Reason = options[Reason]? ?? options[ExceptionName]? ?? 22 | "InvalidColumnsException", 23 | 24 | Detail = options[Detail]? ?? 25 | "Exact column names did not match. Verify capitalization and whitespace", 26 | 27 | Message.Parameters = { 28 | source, // AsText? 29 | FmtList( columnNames ), 30 | FmtList( Table.ColumnNames(source) ), 31 | FmtList( MissingColumns ) 32 | }, 33 | Message.Format = options[Message.Format]? ?? 34 | FmtNL({ "", 35 | "RequiredParameterMissingValues ColumnNames: ", 36 | "Table: #{0}", 37 | "Wanted: ", 38 | " #{1}", 39 | "Found: ", 40 | " #{2}", 41 | "Missing: ", 42 | " #{3}" 43 | }) 44 | ] 45 | ][err] 46 | 47 | in Err.InvalidColumnNames -------------------------------------------------------------------------------- /source/Evaluate/EvalQuery.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | ## About 4 | 5 | executes an external power query text file 6 | 7 | [1] You can edit 'c:\myquery.pq' in an external editor like VS Code 8 | [2] Save, then hit "Refresh" or "Apply" in Power BI 9 | 10 | This is nice for local development. , lets you dynically load powerquery files for local developement 11 | 12 | 13 | ## Valid Keys for 'options' Record : [ 14 | Environment: defaults to #shared 15 | Encoding : defaults to Utf8 16 | ] 17 | 18 | */ 19 | EvalQuery.Type = type function ( 20 | filePath as text, 21 | optional options as nullable record 22 | ) as any meta [ 23 | Documentation.Name = "EvalQuery", 24 | Documentation.LongDescription = Text.Combine({ 25 | "Execute a local .pq file", "", "useful for importing modules as one query","" 26 | }, "
") 27 | ], 28 | 29 | EvalQuery.Impl = ( 30 | filePath as text, 31 | optional options as nullable record 32 | 33 | ) as any => [ 34 | 35 | encoding = options[Encoding]? ?? TextEncoding.Utf8, 36 | environment = options[Environment]? ?? #shared, 37 | bytes = File.Contents( filePath ), 38 | lines = Text.FromBinary( bytes, encoding ), 39 | return = Expression.Evaluate( lines, environment ) 40 | ][return] 41 | in 42 | Value.ReplaceType( EvalQuery.Impl, EvalQuery.Type ) 43 | -------------------------------------------------------------------------------- /source/Evaluate/Example/EvalQuery.pq.example: -------------------------------------------------------------------------------- 1 | EvalQuery( "c:\data\Ninmonkey.PowerQueryLib\build\2024-04.ninlib.pq" ) 2 | -------------------------------------------------------------------------------- /source/Grouping/GroupBy.ShowCounts.pq: -------------------------------------------------------------------------------- 1 | let 2 | GroupBy.ShowCounts = ( 3 | source as table, 4 | columnNames as list 5 | 6 | ) as table => [ 7 | 8 | columnNames = Columns.ThatExist(source, columnNames ), 9 | grouped = Table.Group( source, columnNames, { 10 | { "Distinct", 11 | each Table.RowCount( Table.Distinct( _, columnNames ) ), 12 | Int64.Type }, 13 | 14 | { "Count", 15 | each Table.RowCount(_), 16 | Int64.Type } 17 | }), 18 | return = grouped 19 | 20 | ][return] 21 | 22 | /* 23 | // distinct_per_pair = Table.Group(Source, {"Species", "Region"}, {{"Distinct", each Table.RowCount(Table.Distinct(_, {"Species", "Region"})), Int64.Type}, {"Count", each Table.RowCount(_), Int64.Type}}) 24 | 25 | 26 | */ 27 | in GroupBy.ShowCounts -------------------------------------------------------------------------------- /source/IP.DottedDecimalFromList.pq: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Input: 4 | ListAsIp( {192, 168, 5, 85} ) 5 | Output: 6 | 192.168.5.85 7 | */ 8 | let 9 | IP.DottedDecimalFromList = (num as list) as text => let 10 | isValid = try if List.Count(num) = 4 then 11 | true 12 | else 13 | Error.Record( 14 | "Bad value", "Expected list length of 4", ListAsText(num) 15 | ), 16 | string = ListAsText(num, ".", "", "") 17 | in 18 | if isValid[HasError] then 19 | isValid[Error] 20 | else 21 | string 22 | 23 | in 24 | IP.DottedDecimalFromList 25 | -------------------------------------------------------------------------------- /source/Inspect.MetaOfType.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Inspect type 3 | FuncDef = (x as any) as record => 4 | let 5 | t = Value.Type(x), 6 | md = Value.Metadata(x), 7 | mdt = Value.Metadata( Value.Type(x) ), 8 | mdt_invert = Value.Type( Value.Metadata ), 9 | mergedMeta = Record.Combine({ 10 | (md ?? []), (mdt ?? []) 11 | }) 12 | in 13 | [ 14 | TypeName = Type.ToText( t ), 15 | Type = t, 16 | Metadata = md, 17 | TypeMetadata = mdt, 18 | TypeMetadata_Invert = mdt_invert, 19 | MergedMetadata = mergedMeta 20 | 21 | ], 22 | 23 | // Inspect type 24 | FuncType = type function (input as any) as record 25 | meta [ 26 | Documentation.Name = "Metadata of Type", 27 | Documentation.LongDescription = "Sugar for the Metadata of an object's type", 28 | Documentation.Examples = { 29 | [ 30 | Description = "Type of a function", 31 | Code = "Record.FieldNames( Inspect.MetaOfType( BinaryEncoding.Base64 ) )", 32 | Result = "{ ""Documentation.Name"", ""Documentation.Description"", ""Documentation.LongDescription"", ""Documentation.Category"", ""Documentation.Examples"" }" 33 | ] 34 | } 35 | ], 36 | // enableTesting = false, 37 | // testResults = Record.FieldNames(Inspect.MetaOfType(BinaryEncoding.Base64)), 38 | Inspect.MetaOfType = Value.ReplaceType(FuncDef, FuncType) 39 | in 40 | Inspect.MetaOfType -------------------------------------------------------------------------------- /source/List.AllDates.pq: -------------------------------------------------------------------------------- 1 | let 2 | List.AllDates_Impl = (source as list) as table => 3 | /* 4 | Generate all dates between 2 or more dates. (order does not matter) 5 | rewrite/rename: 'List.ContinuousDates.pq' 6 | Source: 7 | input: 8 | a list or table table column of dates 9 | output: 10 | table of continuous [Date]s., for a Date table 11 | future: 12 | auto detect if arg is a table or a list 13 | */ 14 | let 15 | // source = List.Buffer(source), // maybe? 16 | 17 | // future: assert compat with dates, 18 | validArgs = source is list and first is date and last is date, 19 | first = List.Min(source) as date, 20 | last = List.Max(source) as date, 21 | days = { Number.From(first)..Number.From(last) }, 22 | extra = [ ValidArgs = validArgs, first = first, last = last, days = days ], 23 | 24 | Datefrom = DateTime.Date() 25 | in 26 | extra, 27 | // a 28 | 29 | 30 | // let 31 | // first = List.Min(source), 32 | // last = List.Max(source), 33 | // days = { Number.From(first)..Number.From(last) }, 34 | 35 | // baseDates = List.Transform( 36 | // days, each Date.From(_) ), 37 | 38 | // FinalTable = Table.FromList( 39 | // baseDates, Splitter.SplitByNothing(), 40 | // type table[Date = date], null, ExtraValues.Error 41 | // ) 42 | // in 43 | // FinalTable, 44 | 45 | // show_example = false, 46 | // example = 47 | // let 48 | // sample = { #date(2010,1,9), #date(2010,1,3), #date(2010,1,5) }, 49 | // result = List.AllDates_Impl(sample) 50 | // in 51 | // result, 52 | 53 | // FinalResult = 54 | // if show_example then example else List.AllDates_Impl 55 | 56 | in 57 | FinalResult -------------------------------------------------------------------------------- /source/List.Combine_BitFlags.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Combine all numbers by bitwise-OR-ing them togetherj 3 | List.Combine_BitFlags = (source as list) as number => 4 | let 5 | joined_flags = List.Accumulate( 6 | source, 0, 7 | (state, current) => Number.BitwiseOr( state, current ) 8 | ) 9 | in 10 | joined_flags 11 | in 12 | List.Combine_BitFlags -------------------------------------------------------------------------------- /source/List.ContinuousDates.pq: -------------------------------------------------------------------------------- 1 | let 2 | 3 | /* 4 | Generate all dates for a date table 5 | Source: 6 | 7 | category: data generation? dates 8 | input: 9 | a list of dates 10 | */ 11 | List.ContinuousDates.Type = type function (source as list) as list meta [ 12 | Documentation.Name = "List.ContinuousDates", 13 | Documentation.LongDescription = "Generate a continuous list of dates from a list of dates, useful for date tables", 14 | Documentation.Examples = {[ 15 | Description = "Generate a continuous list of dates", 16 | Code = "List.ContinuousDates( { #date(2021, 1, 1), #date(2021, 1, 3)} )", 17 | Result = "Table.FromRecords({ [ Date = #date(2021, 1, 1)], [ Date = #date(2021, 1, 2)], [ Date = #date(2021, 1, 3)])" 18 | ]} 19 | ], 20 | List.ContinuousDates.Func = (source as list) as list => let 21 | first = List.Min(source), 22 | last = List.Max(source), 23 | days = { Number.From(first)..Number.From(last) }, 24 | 25 | baseDates = List.Transform( 26 | days, each Date.From(_) ) 27 | in 28 | baseDates 29 | 30 | in Value.ReplaceType( 31 | List.ContinuousDates.Func, List.ContinuousDates.Type ) 32 | -------------------------------------------------------------------------------- /source/List.Schema.pq: -------------------------------------------------------------------------------- 1 | let 2 | List.Schema = ( items as list ) as table => 3 | let 4 | recordList = List.Transform( 5 | items, 6 | (item as any) as record => [ 7 | Type = Type.ToText( Value.Type(item) ), 8 | Value = item 9 | ] 10 | ), 11 | schemaTable = Table.FromRecords( 12 | recordList, 13 | type table[Type = text, Value = any] 14 | ) 15 | 16 | in schemaTable 17 | in 18 | List.Schema 19 | -------------------------------------------------------------------------------- /source/List.Summarize.pq: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | options: 4 | Suffix type: text default: "{" 5 | Prefix type: text default: "{" 6 | Separator type: text default: { 7 | 8 | Note: 9 | 10 | */ 11 | let 12 | List.Summarize = ( 13 | values as list, 14 | optional options as nullable record 15 | ) as text => let 16 | Separator = options[Separator]? ?? ", ", 17 | Prefix = options[Prefix]? ?? "{", 18 | Suffix = options[Suffix]? ?? "}", 19 | joined = List.Accumulate( 20 | values, "", 21 | (seed, item) => 22 | if seed = "" then Text.From(item) 23 | else seed & Separator & Text.From(item)), 24 | 25 | output = Prefix & joined & Suffix 26 | in 27 | output 28 | in 29 | List.Summarize -------------------------------------------------------------------------------- /source/List.SummarizeListOfRecords.pq: -------------------------------------------------------------------------------- 1 | let 2 | Summarize.ListOfRecord = (source as any) as any => // text 3 | let 4 | // x = source{0}, 5 | // result = Record.Summarize(x), 6 | listOfRecords = source, 7 | listOfText = List.Transform(source, each Record.Summarize(_)), 8 | finalSummary = List.Summarize(listOfText) 9 | in 10 | finalSummary, 11 | 12 | 13 | fin = Summarize.ListOfRecord 14 | in 15 | fin -------------------------------------------------------------------------------- /source/List/List.SelectBySuffix.pq: -------------------------------------------------------------------------------- 1 | let 2 | // expects a list of pairs of 2 elements, first is the digit, 2nd is the unit 3 | 4 | List.SelectBySuffix.Func = ( items as list, unit as text ) as any => 5 | List.First( 6 | List.Select( items, 7 | (i) => i{1} = unit ), null ){0}?, 8 | 9 | List.SelectBySuffix.Type = type function ( 10 | items as (type { text }), 11 | unit as text 12 | ) as text meta [ 13 | Documentation.Name = "List.SelectBySuffix", 14 | Documentation.LongDescription = Text.Combine({ 15 | "After calling Splitter.SplitDigitWithSuffix(), this function will choose one item from the pair", "", "#(tab)Example: For the text '2m 3s' find '2' if unit is 'm'" 16 | }, "
"), 17 | Documentation.Examples = { 18 | [ 19 | Description = "Selects the first item in a list that ends with the specified suffix", 20 | Code = Format.DocExpand( "List.SelectBySuffix( { {␞2␞, ␞d␞}, {␞3␞, ␞h␞} }, ␞h␞)" ), 21 | Result = Format.DocExpand("3") 22 | ], 23 | [ 24 | Description = "Selects the first item in a list that ends with the specified suffix", 25 | Code = Format.DocExpand( "List.SelectBySuffix( { {␞2␞, ␞d␞}, {␞3␞, ␞h␞} }, ␞d␞)" ), 26 | Result = Format.DocExpand("2") 27 | ] 28 | } 29 | ] 30 | in 31 | Value.ReplaceType( List.SelectBySuffix.Func, List.SelectBySuffix.Type ) 32 | 33 | -------------------------------------------------------------------------------- /source/Matrix.MultiplyVectorDotProduct.pq: -------------------------------------------------------------------------------- 1 | let 2 | // multiply a table matrix with by vector 3 | // vector can be a table or record 4 | // example: 5 | MatrixVectorDot = (matrix as table, vector as any) => [ 6 | vector = // ensure it's a record 7 | if vector is record then vector 8 | else Table.First( vector ), 9 | 10 | vectorList = List.Buffer( Record.ToList( vector ) ), 11 | vectorLen = List.Count( vectorList ), 12 | 13 | applyRows = Table.TransformRows( 14 | matrix, 15 | (row) => calcRow( row ) 16 | ), 17 | 18 | calcRow = (row as record) as number => [ 19 | rowList = Record.ToList( row ), 20 | rowLen = List.Count( rowList ), 21 | products = List.Transform( 22 | { 0 .. ( vectorLen - 1 ) }, 23 | (pos) => 24 | rowList{pos} * vectorList{pos} 25 | ), 26 | 27 | return = 28 | if rowLen <> vectorLen then error [ 29 | // allow them to drill down and inspect exact values that caused the error 30 | Message.Format = "Matrix row and Vector Len are not equal! row #{0} and Vector #{1}", 31 | Message.Parameters = { rowList, vectorList } 32 | ] else 33 | List.Sum( products ) 34 | ][return], 35 | return = applyRows 36 | ][return] 37 | in 38 | MatrixVectorDot -------------------------------------------------------------------------------- /source/Number.From_TextWithBase.pq: -------------------------------------------------------------------------------- 1 | let 2 | enableTest = false, 3 | /* 4 | better naming idea? 5 | 6 | Number.FromTextWithBaseWithBase 7 | TextTo_NumberWithBase 8 | Number.FromTextWithBase 9 | Number.FromTextWithBaseBase 10 | Text.To_NumberWithBase 11 | 12 | */ 13 | 14 | Number.FromTextWithBase = (Input as text, optional Base as number) as number => 15 | // original from: 16 | let 17 | Symbols = {"0".."9"} & {"A".."F"}, 18 | AsReverseList = List.Reverse(Text.ToList(Text.Upper(Input))), 19 | AsNumbers = List.Transform( 20 | AsReverseList, 21 | each List.PositionOf(Symbols, _) 22 | ), 23 | DigitConversion = List.Generate( 24 | () => 0, 25 | each _ < List.Count(AsNumbers), 26 | each _ + 1, 27 | each AsNumbers{_} * Number.Power(Base ?? 16, _) 28 | ), 29 | // Result = try List.Sum(DigitConversion) otherwise null 30 | Result = try List.Sum(DigitConversion) otherwise error Error.Record( 31 | "TypeConversionFailed", "Unable to convert", 32 | [ 33 | Input = Input, 34 | Base = Base, 35 | DigitConversion = DigitConversion 36 | ] 37 | ) 38 | in 39 | Result, 40 | 41 | tests = { 42 | Number.FromTextWithBase("0101", 2) = 5, 43 | Number.FromTextWithBase("ff", 16) = 255 44 | }, 45 | Final = 46 | if enableTest then tests else Number.FromTextWithBase 47 | in 48 | Final -------------------------------------------------------------------------------- /source/Number.ToHexString.pq: -------------------------------------------------------------------------------- 1 | 2 | let 3 | /* 255 => 0xff */ 4 | Number_ToHexString = ( num as number, optional options as nullable record) as text => 5 | let 6 | defaults = [ 7 | PadStart = true, 8 | Prefix = true, 9 | Columns = 8 10 | ], 11 | options = Record.Combine({defaults, options ?? []}), 12 | prefix = if options[Prefix]? then "0x" else "", 13 | numString = Number.ToText(num, "x"), 14 | 15 | paddedString = 16 | if options[PadStart]? 17 | then Text.PadStart( numString, "0", options[Columns]? ?? 8) 18 | else numString, 19 | 20 | finalText = prefix & numString 21 | in 22 | finalText 23 | in 24 | Number_ToHexString 25 | -------------------------------------------------------------------------------- /source/Number/Duration.FromCustomText.pq: -------------------------------------------------------------------------------- 1 | let 2 | Duration.FromCustomText.Type = type function ( 3 | source as duration 4 | ) as duration meta [ 5 | Documentation.Name = "Duration.FromCustomText", 6 | Documentation.LongDescription = Text.Combine({ 7 | "Call Splitter.SplitDigitWithSuffix().","", 8 | "Pick out pairs of 'd', 'h', 'm', 's' and 'd' using List.SelectBySuffix()", 9 | "returns type duration" 10 | }, "
") 11 | ], 12 | 13 | Duration.FromCustomText.Func = (source as text) as duration => [ 14 | clean = Text.Trim( Text.Lower( source, Culture.Current ) ), 15 | s_segs = Text.Split(clean, " "), 16 | s_pairs = List.Transform( s_segs, (pair) => Splitter.SplitDigitWithSuffix( pair )), 17 | 18 | Days = 19 | Number.From( List.SelectBySuffix( s_pairs, "d" ) ) ?? 0, 20 | Hours = 21 | Number.From( List.SelectBySuffix( s_pairs, "h" ) ) ?? 0, 22 | Minutes = 23 | Number.From( List.SelectBySuffix( s_pairs, "m" ) ) ?? 0, 24 | Seconds = 25 | Number.From( List.SelectBySuffix( s_pairs, "s" ) ) ?? 0, 26 | 27 | inst = 28 | #duration( Days, Hours, Minutes, Seconds ) 29 | ][inst] 30 | in 31 | Value.ReplaceType( Duration.FromCustomText.Func, Duration.FromCustomText.Type) 32 | // Duration.FromCustomText = Value.ReplaceType( Duration.FromCustomText.Func, Duration.FromCustomText.Type), -------------------------------------------------------------------------------- /source/Number/Number.FromHexString.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | Converts from hex encoded, plain-text values into numbers 4 | 5 | in: "2432" out: 9266 (which is 0x2432) 6 | 7 | note: It uses Eval. I think 8 | 9 | see also: Number.ToHexString, Number.FromHexString 10 | future improvement, I think I can make it stricter by wrapping it as a constant first 11 | psuedo code: 12 | 13 | Expression.Evaluate( Expression.Constant('0x' + "2400") ), 14 | 15 | */ 16 | Number.FromHexString = (source as text) as number => [ 17 | hasPrefix = Text.StartsWith( source, "0x", Comparer.OrdinalIgnoreCase ), 18 | withPrefix = 19 | if hasPrefix 20 | then source else "0x" & source, 21 | 22 | // asConstant = Expression.Constant( withPrefix ), 23 | ret = Expression.Evaluate( withPrefix, [] ) 24 | ][ret] 25 | 26 | in Number.FromHexString -------------------------------------------------------------------------------- /source/Number/Number.ToHexString.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | in: 0x2400 out: 2400 4 | in: 203 out: cb 5 | in: 203, [Prefix = true] 6 | out: 0xcb 7 | 8 | see also: Number.ToHexString, Number.FromHexString 9 | */ 10 | Number.ToHexString = ( 11 | source as number, 12 | optional options as nullable record 13 | 14 | ) as text => [ 15 | 16 | config = Record.Combine({ [ 17 | Prefix = false 18 | ], 19 | (options ?? []) 20 | }), 21 | hex = Number.ToText( source, "x" ), 22 | return = 23 | if config[Prefix] then // options[Prefix]? then 24 | "0x" & hex 25 | else 26 | hex 27 | ][return] 28 | 29 | in 30 | Number.ToHexString -------------------------------------------------------------------------------- /source/ParameterQuery.Summary.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Type.ToText = lib[Type.ToText], 3 | 4 | // todo: next: Autodetect cyclical refs 5 | ParameterQuery.Summary = (environment as nullable record, ignoreQueries as list) as any => 6 | let 7 | ignoreQueries = ignoreQueries ?? {"lib", "Readme", "SummarizeReportParams" }, 8 | source = environment ?? #sections[Section1], 9 | removeCyclicalRef = Record.RemoveFields( source , ignoreQueries ), 10 | 11 | baseTable = Record.ToTable( removeCyclicalRef ), 12 | #"Added Custom" = Table.AddColumn(baseTable, 13 | "Typename", 14 | (_) as text => 15 | Type.ToText( Value.Type( [Value] ) ), 16 | type text 17 | ), 18 | 19 | getMetadata = Table.AddColumn( 20 | #"Added Custom", "Meta", 21 | each Value.Metadata( [Value] ) 22 | ), 23 | 24 | // met = Value.Metadata( [Value] ), 25 | #"Filtered Rows" = getMetadata 26 | // ]), 27 | // #"Expanded Meta" = Table.ExpandRecordColumn(#"Added Custom1", "Meta", {"mtype", "met"}, {"mtype", "met"}), 28 | // #"Expanded met" = Table.ExpandRecordColumn(#"Expanded Meta", "met", {"IsParameterQuery", "List", "DefaultValue", "Type", "IsParameterQueryRequired", "ExpressionIdentifier"}, {"met.IsParameterQuery", "met.List", "met.DefaultValue", "met.Type", "met.IsParameterQueryRequired", "met.ExpressionIdentifier"}), 29 | // #"Filtered Rows" = Table.SelectRows(#"Expanded met", each ([met.IsParameterQuery] = true)) 30 | 31 | // in [ t1 = Custom5, fin = #"Filtered Rows" ] 32 | in 33 | #"Filtered Rows" 34 | in 35 | ParameterQuery.Summary 36 | -------------------------------------------------------------------------------- /source/Serialize.List.pq: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Description: 4 | 5 | 6 | see: 7 | ListAsText() 8 | 9 | original code based on: 10 | 11 | */ 12 | let 13 | 14 | Documentation = type function ( 15 | anyList as (type number meta [ 16 | Documentation.FieldCaption = "First Number", 17 | Documentation.FieldDescription = "[Field A Desc] text (tooltip?)", 18 | // use a multi-line input box 19 | Formatting.IsMultiLine = false, 20 | // monospace font 21 | Formatting.IsCode = false, 22 | // shown as faded input in text box, else: 23 | Documentation.SampleValues = {1, 9}, 24 | // as a dropdown, but not enforced 25 | Documentation.AllowedValues = {34, 99} 26 | ]) 27 | ) 28 | as table meta [ 29 | Documentation.Name = "[Name] SumNumbers", 30 | Documentation.LongDescription = "[LongDesc] Adds two numbers", 31 | Documentation.Examples = {[ 32 | Description = "Sum two numbers", 33 | Code = "let sum = Func(1, 3) in sum", 34 | Result = "4" 35 | ]} 36 | ], 37 | 38 | 39 | /* Optional parameter: Is this being used as part of a function signature? */ 40 | Serialize.List = (anyList as list) as text => "{" & 41 | List.Accumulate(anyList, "", 42 | (seed,item) => 43 | if seed="" then Text.From(item) 44 | else seed & ", " & Text.From(item)) & 45 | "} " 46 | in 47 | Serialize.List 48 | -------------------------------------------------------------------------------- /source/Serialize.Text.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | from: https://github.com/microsoft/DataConnectors/blob/master/samples/UnitTesting/UnitTesting.query.pq 4 | */ 5 | Serialize.Text = (x) => let 6 | escapeText = (n as number) as text => "#(#)(" & Text.PadStart(Number.ToText(n, "X", "en-US"), 4, "0") & ")" 7 | in 8 | List.Accumulate( 9 | List.Transform( 10 | Text.ToList(x), 11 | (c) => let n = Character.ToNumber(c) in 12 | if n = 9 then "#(#)(tab)" else 13 | if n = 10 then "#(#)(lf)" else 14 | if n = 13 then "#(#)(cr)" else 15 | if n = 34 then """""" else 16 | if n = 35 then "#(#)(#)" else 17 | if n < 32 then escapeText(n) else 18 | if n < 127 then Character.FromNumber(n) else 19 | escapeText(n) 20 | ), 21 | "", 22 | (s, i) => s & i 23 | ) 24 | in 25 | Serialize.Text 26 | -------------------------------------------------------------------------------- /source/Splitter.SplitOnCaseChange.pq: -------------------------------------------------------------------------------- 1 | let 2 | // 2023-08-27 3 | // maybe this should be one function, that returns the correct variant 4 | // essentially an enum -> switch 5 | Summary = [ 6 | SplitOnCaseChange_UpperToLower = 7 | Splitter.SplitTextByCharacterTransition( 8 | {{"A".."Z"}, {"a".."z"}}), 9 | 10 | SplitOnCaseChange_LowerToUpper = 11 | Splitter.SplitTextByCharacterTransition( 12 | {{"a".."z"}, {"A".."Z"}}), 13 | 14 | SplitOnCaseChange_Any = 15 | Splitter.SplitTextByCharacterTransition( 16 | {"A".."Z", "a".."z"}, {"a".."z", "A".."Z"}) 17 | ] 18 | in 19 | Summary -------------------------------------------------------------------------------- /source/Splitter/Splitter.SplitByDigitWithSuffix.pq: -------------------------------------------------------------------------------- 1 | let 2 | // values like "10s" or "3m" 3 | Splitter.SplitDigitWithSuffix = // Splitter.SplitTextByDigitWithSuffixTransition = 4 | Splitter.SplitTextByCharacterTransition( 5 | {"0".."9"}, (c) => not List.Contains({"0".."9"}, c) ) 6 | 7 | 8 | in Splitter.SplitDigitWithSuffix -------------------------------------------------------------------------------- /source/Table.ColumnContainsNonBlank.pq: -------------------------------------------------------------------------------- 1 | 2 | Table.ColumnContainsNonBlank = (source as table, columnName as text) as logical => 3 | // Test.ColumnIsBlank 4 | // future: source could optionally be a list, then drop the colukmn name args 5 | let 6 | col = Table.Column(source, columnName), 7 | containsNonBlank = List.MatchesAny( col, each _ <> null) 8 | in 9 | containsNonBlank, -------------------------------------------------------------------------------- /source/Table.ContinuousDates.pq: -------------------------------------------------------------------------------- 1 | let 2 | List.ContinuousDates = (source as list) as table => 3 | /* 4 | Generate all dates for a date table 5 | Source: 6 | input: 7 | a list or table table column of dates 8 | output: 9 | table of continuous [Date]s., for a Date table 10 | future: 11 | auto detect if arg is a table or a list 12 | */ 13 | let 14 | first = List.Min(source), 15 | last = List.Max(source), 16 | days = { Number.From(first)..Number.From(last) }, 17 | 18 | baseDates = List.Transform( 19 | days, each Date.From(_) ), 20 | 21 | FinalTable = Table.FromList( 22 | baseDates, Splitter.SplitByNothing(), 23 | type table[Date = date], null, ExtraValues.Error 24 | ) 25 | in 26 | FinalTable, 27 | 28 | show_example = false, 29 | example = 30 | let 31 | sample = { #date(2010,1,9), #date(2010,1,3), #date(2010,1,5) }, 32 | result = List.ContinuousDates(sample) 33 | in 34 | result, 35 | 36 | FinalResult = 37 | if show_example then example else List.ContinuousDates 38 | 39 | in 40 | FinalResult 41 | -------------------------------------------------------------------------------- /source/Table.FindBlankColumns.pq: -------------------------------------------------------------------------------- 1 | 2 | Table.FindBlankColumns = (source as table) as table => 3 | let 4 | columnNames = Table.ColumnNames(source), 5 | t = Table.FromList( 6 | columnNames, null, 7 | type table[ColumnName = text]), 8 | 9 | col_containsTest = Table.AddColumn( 10 | t, "ContainsNonBlank", 11 | (row) => 12 | Table.ContainsNonBlank( source, row[ColumnName] ), 13 | type text 14 | ) 15 | 16 | in 17 | col_containsTest, 18 | -------------------------------------------------------------------------------- /source/Table.FindNotDistinctRows.pq: -------------------------------------------------------------------------------- 1 | let 2 | Table_FindNotDistinctRows = (source as table, columnNames as list) as table => 3 | let 4 | grouped = Table.Group(source, columnNames, { 5 | { 6 | "Count", 7 | each Table.RowCount(_), Int64.Type}, 8 | { 9 | "Rows", each _, Value.Type(source) 10 | } 11 | } 12 | ), 13 | find_repeats = Table.SelectRows(grouped, each [Count] > 1) 14 | in 15 | find_repeats 16 | in 17 | Table_FindNotDistinctRows -------------------------------------------------------------------------------- /source/Table.FindSingleValueColumns.pq: -------------------------------------------------------------------------------- 1 | // find columns with non=-disstinct 2 | 3 | Table.FindSingleValueColumns = (source as table) as any => 4 | let 5 | columnNames = Table.ColumnNames(source), 6 | t = Table.FromList( 7 | columnNames, null, 8 | type table[ColumnName = text]), 9 | col_SingleOrNone = wip 10 | col_containsTest = Table.AddColumn( 11 | t, "SingleOr", 12 | (row) => 13 | // List.Count( Table.Column(source, ) ) 14 | List.Count( 15 | Table.Column( source, row[ColumnName] ) <= 1 16 | source, row[ColumnName] 17 | ), 18 | type text 19 | ) 20 | 21 | in 22 | col_containsTest, -------------------------------------------------------------------------------- /source/Table.RemoveBlankColumns.pq: -------------------------------------------------------------------------------- 1 | Table.RemoveBlankColumns = (source as table, optional options as nullable record) as any =>//as table => 2 | let 3 | blankColumns = Table.FindBlankColumns( source ), 4 | cols_to_keep = Table.SelectRows( blankColumns, each [ContainsNonBlank] = true )[ColumnName], 5 | drop_emptyCols = Table.SelectColumns( source, cols_to_keep ) 6 | in 7 | drop_emptyCols, -------------------------------------------------------------------------------- /source/Table.SelectRemovedColumns.pq: -------------------------------------------------------------------------------- 1 | let 2 | // For when you're using Table.SelectColumns(), easily view only the removed columns 3 | Table.SelectRemovedColumns = (source as table, previousTable as table) as any => 4 | let 5 | sourceCols = Table.ColumnNames(source), 6 | prevCols = Table.ColumnNames(previousTable), 7 | deltaCols = List.RemoveMatchingItems(sourceCols, prevCols), 8 | selectMissing = Table.SelectColumns( source, deltaCols ), 9 | return = [ 10 | deltaCols = deltaCols, 11 | selectMissing = selectMissing 12 | ] 13 | in 14 | selectMissing 15 | in 16 | Table.SelectRemovedColumns -------------------------------------------------------------------------------- /source/Table.ToJson.pq: -------------------------------------------------------------------------------- 1 | let 2 | Documentation = type function ( 3 | source as (type table meta [ 4 | Documentation.FieldCaption = "Input Table", 5 | Documentation.FieldDescription = "Input Table" 6 | ]), 7 | optional encoding as (type nullable number meta [ 8 | Documentation.FieldCaption = "Text Encoding", 9 | Documentation.FieldDescription = "Text Encoding", 10 | Documentation.AllowedValues = { TextEncoding.Ascii, TextEncoding.BigEndianUnicode, TextEncoding.Unicode, TextEncoding.Utf16, TextEncoding.Utf8, TextEncoding.Windows } 11 | ]) 12 | ) as table meta [ 13 | Documentation.Name = "TableToJson", 14 | Documentation.LongDescription = "Converts a table to JSON. (The reverse of using 'Enter Data'). Currently this does not save schema other than JSON types", 15 | Documentation.Examples = {[ 16 | Description = "Generate one value", 17 | Code = "TableToJson( #table({""Animal"", ""Id""}, {{""Cat"", 1}, {""Turtle"", 2}}) )", 18 | Result = "[{""Animal"":""Cat"",""Id"":1},{""Animal"":""Turtle"",""Id"":2}]" 19 | ]} 20 | ], 21 | 22 | TableToJson = (source as table, optional encoding as nullable number) as text => 23 | let 24 | encoding = encoding ?? TextEncoding.Utf8, 25 | bin = Json.FromValue(source, encoding), 26 | jsonAsText = Text.FromBinary(bin, encoding) 27 | in 28 | jsonAsText 29 | in 30 | Value.ReplaceType( TableToJson, Documentation) 31 | -------------------------------------------------------------------------------- /source/Text/Example/Text.JoinString.pq.example: -------------------------------------------------------------------------------- 1 | [ 2 | names = {"Jen", "Hubert", "Nobody", "Somebody" }, 3 | Default = Text.JoinString( names ) , 4 | AsCsv = Text.JoinString( names, [ Delimiter = ", "] ), 5 | AsTablePipes = Text.JoinString( names, [ 6 | Prefix = "| ", Delimiter = " | ", Suffix = " |" ] ), 7 | 8 | AsBullets = Text.JoinString( names, [ 9 | Prefix = " #(2022) ", 10 | Delimiter = " #(cr,lf) #(2022) " 11 | ]) 12 | ] -------------------------------------------------------------------------------- /source/Text/Format.JoinBR.pq: -------------------------------------------------------------------------------- 1 | let 2 | 3 | // See Write.Html.pq module for a better, nested approach to this function 4 | Format.JoinBR = (source as list) as text => [ 5 | ret = Text.Combine( As.TextList( source ), "#(cr,lf)
" ) 6 | ][ret] 7 | in 8 | Format.JoinBR -------------------------------------------------------------------------------- /source/Text/Format.JoinNewLine.pq: -------------------------------------------------------------------------------- 1 | let 2 | // previous name: List.JoinNewLine, Format.JoinNewLine, Join.NewLine 3 | Format.JoinNewLine = (source as list) as text => [ 4 | As.TextList = List.TransformTo.ListOfText, 5 | ret = Text.Combine( As.TextList( source ), "#(cr,lf)" ) //# Uni[CrLf] ) 6 | ][ret] 7 | in 8 | Format.JoinNewLine -------------------------------------------------------------------------------- /source/Text/Format.ShowBlank.pq: -------------------------------------------------------------------------------- 1 | let Format.ShowBlank = (source as list) as list => [ 2 | ret = List.ReplaceMatchingItems( source, 3 | { { "", "" }, 4 | { null, Uni[Null] } } 5 | ) 6 | ][ret] 7 | in Format.ShowBlank -------------------------------------------------------------------------------- /source/Text/Format.SimplifyJson.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | About: 4 | I wanted a quick way to visualize JSON 5 | Expanding JSON with newlines still has a lot of noice 6 | I am experimenting with something shorter like yaml, but not using yaml constraints 7 | this is overkill. 8 | */ 9 | Format.SimplfyJson = ( 10 | source as text, 11 | optional options as nullable record 12 | 13 | ) as text => [ 14 | 15 | // make defaults param 16 | options = options ?? [ 17 | PadKeyValue = true, 18 | KeyValueDelim = " ▸ ", 19 | PadBraceEnding = true, 20 | TrimDoubleQuotes = true, 21 | ReplaceDoubleQuoteString = " ", 22 | TrimPrefixSuffixBrace = false 23 | ], 24 | // Str = [ DoubleQuote = "#(0022)" ], 25 | keyValueDelim_1 = " ▸ ", 26 | keyValueDelim = ": ", 27 | acc0 = source, 28 | // acc1 = Text.Replace( acc0, "#(0022):{#(0022)", " => " ), 29 | acc1 = 30 | if not options[PadKeyValue] 31 | then acc0 32 | else 33 | Text.Replace( acc0, "#(0022):", options[KeyValueDelim] ), 34 | acc2 = 35 | if not options[TrimDoubleQuotes] 36 | then acc1 37 | else 38 | Text.Replace( acc1, "#(0022)", options[ReplaceDoubleQuoteString] ), 39 | acc3 = 40 | if not options[PadBraceEnding] then acc2 41 | else 42 | Text.Replace( acc2, "}", " }" ), 43 | acc4 = // this only strips the final instance because of acc3 running first 44 | // should instead be a replace first and last occurence instead 45 | if not options[TrimPrefixSuffixBrace] then acc3 46 | else 47 | [ 48 | trimSpace = Text.Trim( acc3 ), 49 | trimStart = Text.TrimStart( trimSpace, "{" ), 50 | trimEnd = Text.TrimEnd( trimStart, "}" ), 51 | return = trimEnd 52 | 53 | ][return], 54 | 55 | return = acc3 56 | 57 | ][return] 58 | in 59 | Format.SimplfyJson -------------------------------------------------------------------------------- /source/Text/Format.ToJsonText.pq: -------------------------------------------------------------------------------- 1 | let 2 | // takes any type, converts it to json, then expands json into multiple lines 3 | ToJson = ( 4 | source as any, // a value thats passed to Json.FromValue 5 | optional options as nullable record 6 | ) as text => [ 7 | 8 | // see also: Xray, ToJson, Csv 9 | defaults = [ 10 | Encoding = TextEncoding.Utf8, 11 | Expand = true 12 | ], 13 | config = Record.Combine({ 14 | options ?? [], 15 | defaults 16 | }), 17 | json = Text.FromBinary( Json.FromValue( source, config[Encoding] ) , config[Encoding] ), 18 | 19 | json_expanded = Text.Replace( json, "},{", "},#(cr,lf){"), 20 | 21 | return = 22 | if config[Expand] then json_expanded 23 | else json 24 | 25 | ][return] 26 | 27 | in ToJson -------------------------------------------------------------------------------- /source/Text/Text.AnyMatches.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | For a list of values, test any combination of: Text.StartsWith, Text.Contains, Text.EndsWith 4 | one or more matches returns true 5 | 6 | future: 7 | List.Generate/Accumulate to stop calls when one is already true 8 | */ 9 | #"Text.AnyMatch" = ( 10 | source as text, 11 | optional filtersStartsWith as nullable list, 12 | optional filterContainsWith as nullable list, 13 | optional filterEndsWith as nullable list, 14 | optional comparer as nullable function 15 | ) as logical => 16 | let 17 | comparer = comparer ?? Comparer.OrdinalIgnoreCase, 18 | filtersStartsWith = filtersStartsWith ?? {}, 19 | filterContainsWith = filterContainsWith ?? {}, 20 | filterEndsWith = filterEndsWith ?? {}, 21 | // x = FilterPrefix Text.Contains(source, Config), 22 | anyStart = List.Transform( 23 | filtersStartsWith, (i) => Text.StartsWith( source, i, comparer) 24 | ), 25 | anyContain = List.Transform( 26 | filterContainsWith, (i) => Text.Contains( source, i, comparer) 27 | ), 28 | anyEnd = List.Transform( 29 | filterEndsWith, (i) => Text.EndsWith( source, i, comparer) 30 | ), 31 | boolShouldFilter = List.AnyTrue( anyStart ) 32 | or List.AnyTrue( anyContain ) 33 | or List.AnyTrue( anyEnd ) 34 | in 35 | boolShouldFilter 36 | in 37 | #"Text.AnyMatch" 38 | -------------------------------------------------------------------------------- /source/Text/Text.Contains.CI.pq: -------------------------------------------------------------------------------- 1 | let 2 | // mainly for interactive use when debugging 3 | // long name: Text.Contains.CaseInsensitiveMatch 4 | Text.Contains.CI = (source as text, substring as text) as logical => 5 | // see also: Text.Contains.CI, Text.PositionOf.CI, Text.MatchesAnyOf.CI 6 | Text.Contains( source, substring, Comparer.OrdinalIgnoreCase ) 7 | in 8 | Text.Contains.CI -------------------------------------------------------------------------------- /source/Text/Text.EndsWith.CI.pq: -------------------------------------------------------------------------------- 1 | let 2 | // long name: Text.Contains.CaseInsensitiveMatch 3 | // mainly for interactive use when debugging 4 | Text.EndsWith.CI = (source as text, substring as text) as logical => 5 | // see also: Text.Contains.CI, Text.PositionOf.CI, Text.MatchesAnyOf.CI 6 | // Text.Contains( source, substring, Comparer.OrdinalIgnoreCase ) 7 | // if Text.EndsWith( imageUrl, ".png", Comparer.OrdinalIgnoreCase ) then 8 | Text.EndsWith( source, substring, Comparer.OrdinalIgnoreCase ) 9 | in 10 | Text.EndsWith.CI -------------------------------------------------------------------------------- /source/Text/Text.FormatCsv.pq: -------------------------------------------------------------------------------- 1 | [ 2 | Text.FormatCsv = (source as list, optional options as nullable record ) as text => [ 3 | culture = options[Culture]?, 4 | segments = List.Transform( source, each Text.From( _, culture ) ), 5 | return = Text.Combine( segments, ", " ) 6 | 7 | ][return] 8 | ] -------------------------------------------------------------------------------- /source/Text/Text.FormatList.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | in: { "a", 2, #date(2023,1,4) }, 4 | out: "{ a, 2, 2023-01-04 }" 5 | */ 6 | Text.FormatList = (source as list) as text => [ 7 | Config = [ 8 | BracePrefix = "{ ", 9 | BraceSuffix = " }", 10 | Separator = ", ", 11 | OuterSeparator = "" 12 | ], 13 | str_list = List.Transform( source, each Text.From(_) ), 14 | ret = Text.Combine( { 15 | Config[BracePrefix], 16 | Text.Combine( str_list, Config[Separator] ), 17 | Config[BraceSuffix] 18 | }, Config[OuterSeparator] ) 19 | ][ret] 20 | 21 | in 22 | Text.FormatList -------------------------------------------------------------------------------- /source/Text/Text.IsNullOrWhitespace.pq: -------------------------------------------------------------------------------- 1 | // Text_IsNullOrWhitespace 2 | let 3 | run_example = false, 4 | Text_IsNullOrWhitespace = (source as nullable text) as logical => 5 | /* True when source only contains: null or whitespace or control chars 6 | 7 | [string]::IsNullOrWhitespace() 8 | */ 9 | let 10 | trimClean = Text.Trim( Text.Clean( source ) ), 11 | len = Text.Length(trimClean) 12 | in 13 | source is null or len = 0, 14 | tests = { 15 | Text_IsNullOrWhitespace("#(tab)#(cr,lf)"), 16 | Text_IsNullOrWhitespace(" "), 17 | Text_IsNullOrWhitespace(""), 18 | Text_IsNullOrWhitespace(null) 19 | } 20 | in 21 | if run_example then tests else Text_IsNullOrWhitespace -------------------------------------------------------------------------------- /source/Text/Text.JoinString.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | About: Join a string with a delimiter, prefix, and suffix 4 | It was written as an example for using optional record parameters 5 | 6 | 7 | For more advanced joins, checkout these collections of functions: 8 | - Write.Html.module.pq 9 | - String.Builder.module.pq 10 | 11 | */ 12 | Text.JoinString = (strings as list, optional options as record) as text => let 13 | Prefix = options[Prefix]? ?? "", 14 | Suffix = options[Suffix]? ?? "", 15 | Delimiter = options[Delimiter]? ?? "," 16 | in 17 | Prefix & Text.Combine( strings, Delimiter ) & Suffix 18 | in 19 | Text.JoinString 20 | 21 | // Examples = [ 22 | 23 | // names = {"Jen", "Hubert", "Nobody", "Somebody" }, 24 | 25 | // Default = Text.JoinString( names ) , 26 | 27 | // AsCsv = Text.JoinString( names, [ Delimiter = ", "] ), 28 | 29 | // AsTablePipes = Text.JoinString( names, [ 30 | // Prefix = "| ", Delimiter = " | ", Suffix = " |" ] ), 31 | 32 | // AsBullets = Text.JoinString( names, [ 33 | // Prefix = " #(2022) ", 34 | // Delimiter = " #(cr,lf) #(2022) " 35 | // ]) 36 | // ] 37 | // in 38 | // Examples 39 | -------------------------------------------------------------------------------- /source/Text/Text.JsonToPowerQuery.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* converts JSON string to a pastable PowerQuery Literal 3 | Otherwise you have to escape values 4 | 5 | Usage: 6 | Text_JsonToPowerQuery( 7 | Table_ToJson( TableName ) 8 | ) 9 | 10 | */ 11 | Text.JsonToPowerQuery = (json as text) as text => 12 | let 13 | doubleQuote = """", 14 | escapeQuotes = Text.Replace( 15 | json, doubleQuote, 16 | ( doubleQuote & doubleQuote) 17 | ) 18 | in 19 | escapeQuotes 20 | in 21 | Text.JsonToPowerQuery -------------------------------------------------------------------------------- /source/Text/Text.PositionOf.CI.pq: -------------------------------------------------------------------------------- 1 | let 2 | // see also: Text.Contains.CI, Text.PositionOf.CI, Text.MatchesAnyOf.CI 3 | Text.PositionOf.CI = ( 4 | source as text, 5 | substring as text, 6 | optional options as nullable record 7 | ) as any => [ 8 | 9 | occurrence = options[Occurrence]? ?? Occurrence.First, 10 | comparer = options[Comparer]? ?? Comparer.OrdinalIgnoreCase, 11 | return = Text.PositionOf( source, substring, occurrence, comparer ) 12 | 13 | ][ return ] 14 | 15 | in 16 | Text.PositionOf.CI -------------------------------------------------------------------------------- /source/Text/Text.PositionOf.pq: -------------------------------------------------------------------------------- 1 | let Text.PositionOf.CI = (source as text, substring as text, optional options as nullable record) as any => [ 2 | // see also: Text.Contains.CI, Text.PositionOf.CI, Text.MatchesAnyOf.CI 3 | occurrence = options[Occurrence]? ?? Occurrence.First, 4 | comparer = options[Comparer]? ?? Comparer.OrdinalIgnoreCase, 5 | ret = Text.PositionOf( source, substring, occurrence, comparer ) 6 | ][ret] 7 | in Text.PositionOf.CI -------------------------------------------------------------------------------- /source/Text/Text.RemoveDiacritics.pq: -------------------------------------------------------------------------------- 1 | let 2 | // a simple way to remove accented characters is to encode as Cryillic 3 | // and then decode as ascii/utf8 4 | Text.RemoveDiacritics = (Text as text) as text => 5 | let 6 | TextEncoding.Cyrillic = 28595, 7 | bytes = Text.ToBinary( Text, TextEncoding.Cyrillic ), 8 | cleanText = Text.FromBinary( bytes ) 9 | 10 | in cleanText 11 | 12 | in 13 | Text.RemoveDiacritics 14 | -------------------------------------------------------------------------------- /source/Text/Text.ReplaceFirstOnly.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Like Text.Replace except that this replaces at most one single occurrence of oldValue 3 | Text.ReplaceFirstOnly = ( 4 | source as nullable text, 5 | oldValue as text, 6 | newValue as text 7 | ) as nullable text => [ 8 | 9 | found = Text.PositionOf( source, oldValue, 10 | Occurrence.First, Comparer.OrdinalIgnoreCase ), 11 | 12 | replaced = Text.ReplaceRange( 13 | source, found, 14 | Text.Length( oldValue ), 15 | newValue 16 | ), 17 | 18 | return = 19 | if source = null then null 20 | else if found = -1 then source 21 | else replaced 22 | 23 | ][return] 24 | in 25 | Text.ReplaceFirstOnly 26 | -------------------------------------------------------------------------------- /source/Text/Text.ReplaceMany.pq: -------------------------------------------------------------------------------- 1 | let 2 | Text.ReplaceMany 3 | = ( source as text, mapping as list ) as text => 4 | let 5 | result = List.Accumulate( 6 | mapping, source, 7 | ( state, cur ) => Text.Replace(state, cur{0}, cur{1} )) 8 | in 9 | result, 10 | Text.ReplaceMany.Type = type function( 11 | source as ( 12 | type text meta [ 13 | Documentation.FieldCaption = "source", 14 | Documentation.FieldDescription = "List to search and replace multiple values" 15 | ]), 16 | mapping as (type list meta[ 17 | Documentation.FieldCaption = "List of {oldText, newText} replacement Pairs", 18 | Documentation.FieldDescription = "A list of {oldText, newText} pairs, to search and then replace values" 19 | ])) 20 | as (type text meta [ 21 | Documentation.Name = "Text.ReplaceMany", 22 | Documentation.LongDescription = "A list of {oldText, newText} pairs,


#(cr,lf)#(cr,lf) to search case-sensitive, and then replace values", 23 | /* 24 | 25 | example: 26 | map = { 27 | {"cat hat", "cat in the hat"}, 28 | {"cat", "#(0001f408)"}, 29 | {"bat", "hat"}, 30 | {"THE", "the"} 31 | }, 32 | Text.ReplaceMany( "The cat hat") 33 | 34 | output: 35 | "The 🐈 in the hat" 36 | */ 37 | Documentation.Examples = { 38 | [ Description = "cat to emoji", 39 | Code = "{0..4}", 40 | Result = "none" ], 41 | [ Description = "cat to emoji", 42 | Code = "{0..4}", 43 | Result = "none" ] 44 | } 45 | ]) 46 | in 47 | Value.ReplaceType( Text.ReplaceMany, Text.ReplaceMany.Type ) -------------------------------------------------------------------------------- /source/Text/Text.WordWrap.pq: -------------------------------------------------------------------------------- 1 | 2 | let 3 | /* 4 | Example: 5 | 6 | wrapped = Text.WordWrap(LoremIpsum, 80) 7 | LoremIpsum = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed eu laoreet turpis. Curabitur lacinia, risus ut rhoncus mattis, turpis lorem iaculis justo, nec ultrices arcu erat vitae felis. Pellentesque vulputate efficitur scelerisque. Etiam bibendum dignissim mauris", 8 | */ 9 | 10 | // calculate length of string *after* the rightmost newline 11 | strNewline = "#(lf)", 12 | Text_LengthAfterNewline = (string as text) as number => 13 | let 14 | posLastNewline = Text.PositionOf(string, strNewline, Occurrence.Last), 15 | posOffset = if posLastNewline <> -1 then posLastNewline else 0, 16 | deltaLen = Text.Length(string) - posOffset 17 | in 18 | deltaLen, 19 | 20 | // word wraps text 21 | Text.WordWrap = (string as text, max_width as number) as text => 22 | let 23 | words = Text.Split(string, " "), 24 | accum_result = List.Accumulate( 25 | words, "", 26 | (state as text, current as text) as text => 27 | let 28 | len = Text_LengthAfterNewline(state) + Text.Length(current) + 1, 29 | maybeNewline = 30 | if len > max_width then strNewline else "", 31 | 32 | accum_string = Text.Combine({state & maybeNewline, current}, " ") 33 | in 34 | accum_string 35 | ) 36 | in 37 | accum_result 38 | in 39 | Text.WordWrap -------------------------------------------------------------------------------- /source/Text/TransformTo.TextList.pq: -------------------------------------------------------------------------------- 1 | let 2 | // in: List type { any } 3 | // out: List type { text } 4 | TransformTo.TextList = ( 5 | source as list, 6 | optional culture as nullable text 7 | 8 | ) as list => [ 9 | return = 10 | List.Transform( 11 | source, 12 | each Text.From(_, culture ?? null ) ) 13 | 14 | ][return] 15 | 16 | in 17 | TransformTo.TextList 18 | -------------------------------------------------------------------------------- /source/Text/XRay.pq: -------------------------------------------------------------------------------- 1 | let 2 | XRay = (source as any) as text => [ 3 | // see also: Xray, ToJson, Csv 4 | bytes = Json.FromValue( source, TextEncoding.Utf8 ), 5 | json = Text.FromBinary( bytes, TextEncoding.Utf8 ) 6 | ][json] 7 | 8 | /* or Jsonify 9 | 10 | // Convert to Json, then decode as a string. sometime useful because the UI auto decodes 11 | Jsonify = (source as any, optional options as nullable record) as text => let 12 | encoding = options[encoding]? ?? TextEncoding.Utf8, 13 | messageArgs = source, 14 | bytes = Json.FromValue( messageArgs, encoding ), 15 | render = Text.FromBinary( bytes, encoding) 16 | in render, 17 | */ 18 | 19 | in XRay -------------------------------------------------------------------------------- /source/Type.ToText_simple.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | dependencies: None 4 | */ 5 | Type.ToText.Type = type function( 6 | typeInfo as (type any meta [ 7 | Documentation.FieldCaption = "Object to test", 8 | Documentation.FieldDescription = "Value to compare type names against", 9 | Documentation.SampleValues = { 10 | 23.45, "string", DateTime.LocalNow() 11 | //, #table(type table[Text = text],{{"hi world"}}) 12 | } 13 | ]), 14 | optional options as (type record meta [ 15 | Documentation.FieldCaption = "for future formatting", 16 | Documentation.FieldDescription = "for future formatting", 17 | Documentation.SampleValues = { 18 | } 19 | ])) as table meta [ 20 | Documentation.Name = "Get Type Name", 21 | Documentation.LongDescription = "Returns type names as text, for the current value", 22 | Documentation.Examples = {[ 23 | Description = "Getting Type Names:", 24 | Code = "{ Type.ToText(3), Type.ToText( {0..4} ), Type.ToText(null) }", 25 | Result = "{ ""Number"", ""List"", ""None""}" 26 | ]} 27 | ], 28 | Type.ToText_impl = (typeInfo as any, optional options as nullable record) as text => 29 | let 30 | options = Record.Combine([], options), 31 | name = 32 | if typeInfo is null then "Null" 33 | else if typeInfo is type then "Type" 34 | else if typeInfo is binary then "Binary" 35 | else if typeInfo is number then "Number" 36 | else if typeInfo is function then "Function" 37 | else if typeInfo is list then "List" 38 | else if typeInfo is table then "Table" 39 | else if typeInfo is record then "Record" 40 | else "Other" meta [ ValueType = typeInfo ] 41 | in 42 | name, 43 | Type.ToText = Value.ReplaceType( Type.ToText_impl, Type.ToText.Type ) 44 | in 45 | Type.ToText 46 | -------------------------------------------------------------------------------- /source/Validate_TableSchema.pq: -------------------------------------------------------------------------------- 1 | // fn_ValidateSchemaTest 2 | let 3 | runTest = true, 4 | Sources = TestMismatchingDatattype, //#"📌auto-detected-type", 5 | Source_Wrong = Sources[SampleWrongColumnType], 6 | Source_Right = Sources[SampleWrongColumnType], 7 | validateSchema = (source as table) as table => 8 | let 9 | selectColumns = {"Name", "Position", "TypeName", "Kind", "IsNullable"}, 10 | firstValues = Table.First( source ), 11 | t0 = Table.Schema(source), 12 | t1 = Table.SelectColumns(t0, selectColumns), 13 | t2 = Table.AddColumn( t1, "Actual Type", 14 | (row) => Value.Type( Table.Column( source, row[Name]){0} ), 15 | Type.Type 16 | ), 17 | t3 = Table.AddColumn( t2, "Sample Value", 18 | (row) => Table.Column( source, row[Name] ){0} 19 | ), 20 | final = t3 21 | in 22 | final, 23 | 24 | 25 | test_case = validateSchema( Source_Wrong ), 26 | final = if runTest then test_case else validateSchema 27 | in 28 | final -------------------------------------------------------------------------------- /source/Xml/Table.AutoExpandTableColumn.pq: -------------------------------------------------------------------------------- 1 | 2 | // Someone asked for a way to auto expand xml columns 3 | // hard coded example filter 4 | Table.AutoExpandTableColumn = (source as table, columnName as text ) => 5 | let 6 | allNames = Table.ColumnNames( source ), 7 | filteredNames = List.Select( allNames, (item) => 8 | not Text.StartsWith( item, "http://", Comparer.OrdinalIgnoreCase ) ), 9 | 10 | prefixedNames = List.Transform( filteredNames, each expandColumnName & "." & _ ), 11 | return = Table.ExpandTableColumn( source, columnName, filteredNames, prefixedNames ) 12 | in 13 | return, 14 | -------------------------------------------------------------------------------- /source/alias/Inspect.Metadata.pq: -------------------------------------------------------------------------------- 1 | // md 2 | let 3 | // Inspect metadata 4 | FuncDef = (x as any) as any => Value.Metadata(x), // nullable record not guaranteed 5 | // Inspect metadata 6 | FuncType = type function (input as any) as any 7 | meta [ 8 | Documentation.Name = "Alias for Value.Metadata(x)", 9 | Documentation.LongDescription = "Alias for getting an object's metadata" 10 | // Documentation.Examples = { 11 | // [ 12 | // Description = "Metadata of object", 13 | // Code = "Inspect.Type( Value.Type )", 14 | // Result = "function" 15 | // ] 16 | // } 17 | ], 18 | enableTesting = false, 19 | testResults = Inspect.Metadata( Value.Type ), 20 | Inspect.Metadata = Value.ReplaceType(FuncDef, FuncType) 21 | in 22 | if not enableTesting then Inspect.Metadata else testResults -------------------------------------------------------------------------------- /source/alias/Inspect.Type.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Inspect type 3 | FuncDef = (x as any) => as type 4 | Value.Type(x), 5 | 6 | // Inspect type 7 | FuncType = type function (input as any) as any 8 | meta [ 9 | Documentation.Name = "Alias for Value.Type()", 10 | Documentation.LongDescription = "Alias for getting an object's type", 11 | Documentation.Examples = { 12 | [ 13 | Description = "Type of a function", 14 | Code = "Inspect.Type( Value.Type )", 15 | Result = "function" 16 | ] 17 | } 18 | ], 19 | enableTesting = false, 20 | testResults = Inspect.Type( Value.Type ), 21 | Inspect.Type = Value.ReplaceType(FuncDef, FuncType) 22 | in 23 | if not enableTesting then Inspect.Type else testResults -------------------------------------------------------------------------------- /source/alias/alias_typeOf.pq: -------------------------------------------------------------------------------- 1 | let 2 | t = Inspect.Type, 3 | todo = "Use functools.partial to make MetaOfType return less fields" 4 | in 5 | t -------------------------------------------------------------------------------- /source/alias/default_alias_list.pq: -------------------------------------------------------------------------------- 1 | let 2 | mapping = [ 3 | Join-String = Text.Combine, 4 | t = Value.Type, 5 | mdt = Inspect.Meta, 6 | // value.me = Value, 7 | 8 | 9 | // random = { 10 | // ..,... /// list of all funcs related to random 11 | // }, 12 | // findFunc = ..., // filter functions in current session with pattern 13 | 14 | // fmt_Hex = Number_ToHexString 15 | 16 | 17 | ] meta [ 18 | pqLibAlias = "Inspect.MetaOfType", 19 | IsAlias = true 20 | ] 21 | in 22 | mapping 23 | -------------------------------------------------------------------------------- /source/alias/mdt.pq: -------------------------------------------------------------------------------- 1 | let 2 | // alias source 3 | alias = Inspect.MetaOfType 4 | in 5 | alias meta [ 6 | "pqLibAlias" = "Inspect.MetaOfType", 7 | IsAlias = true 8 | ] -------------------------------------------------------------------------------- /source/docs/Format.DocExpand.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | sugar to make it easier to embed powerquery as a string literal 4 | 5 | It replaces '␞' with double quotes '"' 6 | 7 | input : List.SelectBySuffix( { {␞2␞, ␞d␞}, {␞3␞, ␞h␞} }, ␞h␞) 8 | output: List.SelectBySuffix( { {"2", "d"}, {"3", "h"} }, "h") 9 | */ 10 | Format.DocExpand.Type = type function ( source as text ) 11 | as text meta [ 12 | Documentation.Name = "Format.DocExpand", 13 | Documentation.LongDescription = Text.Combine({ 14 | "A helper function to generate help strings without crazy escaping.", 15 | "", 16 | "Input:", 17 | "", 18 | " = DocExpand( ""List.SelectBySuffix( { {␞2␞, ␞d␞}, {␞3␞, ␞h␞} }, ␞h␞)"" )", 19 | "", 20 | "Output:", 21 | "", 22 | " = 'List.SelectBySuffix( { {""2"", ""d""}, {""3"", ""h""} }, ""h"")""'", 23 | "", 24 | "returns type text" 25 | }, "#(cr,lf)
") 26 | ], 27 | 28 | Format.DocExpand.Func = (source as text) as text => 29 | Text.Replace( source, "␞", "#(0022)" ) 30 | in 31 | Value.ReplaceType( Format.DocExpand.Func, Format.DocExpand.Type ) -------------------------------------------------------------------------------- /source/docs/Generate-Docs-JoinAlgorithm.pq: -------------------------------------------------------------------------------- 1 | let 2 | // generates docs for join type. sorta generic. 3 | 4 | GenerateDocs_JoinAlgorithm = (sourceType as type) => 5 | let 6 | root = Value.Metadata( sourceType ), 7 | desc = root[Description]? ?? null, 8 | docsLong_Html = root[Documentation.LongDescription]? ?? null, 9 | Custom1 = Table, 10 | 11 | Html_simple = Html.Table( 12 | docsLong_Html, { { "Item", "li"} } 13 | ), 14 | Html_details = Html.Table( 15 | docsLong_Html, 16 | { { "li", "li", (x) as record => x, type record } } ), 17 | // The last list value appears to be the type of it? 18 | 19 | SelectedHtml = Html_simple, 20 | split_header = Table.SplitColumn( 21 | SelectedHtml, "Item", 22 | Splitter.SplitTextByEachDelimiter({": "}, QuoteStyle.Csv, false), 23 | {"JoinAlgorithm.Type", "Description"} 24 | ) 25 | in 26 | split_header, 27 | 28 | docs = GenerateDocs_JoinAlgorithm( JoinAlgorithm.Type ) 29 | in 30 | docs -------------------------------------------------------------------------------- /source/inspect/Inspect.Function.pq: -------------------------------------------------------------------------------- 1 | let 2 | //currently equal, will change. 3 | Inspect.Function = ... 4 | in 5 | Inspect.Function -------------------------------------------------------------------------------- /source/inspect/function/inspect_func_enumFieldNames.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | 4 | usually returns 5 | "Documentation.Name", "Documentation.Description", "Documentation.LongDescription", "Documentation.Category", "Documentation.Examples" 6 | */ 7 | enumerate_fn_fieldName = 8 | (source as function) as list => 9 | let 10 | target = Inspect.TypeMetadata( source ), 11 | target_md = target[TypeMetadata] //?? null 12 | in 13 | Record.FieldNames( target_md ) 14 | , 15 | test = enumerate_fn_fieldName( Web.Contents ) 16 | in 17 | test 18 | 19 | -------------------------------------------------------------------------------- /source/random/Compare.RandomInt64.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* Build a bunch of random numbers with different rounding modes to compare distribution statistics. */ 3 | Rand.Int64 = (min as number, max as number, optional roundingMode as number ) => 4 | Int64.From( 5 | Number.RandomBetween( min, max ), 6 | "en-us", 7 | roundingMode ?? RoundingMode.TowardZero 8 | ), 9 | 10 | /* a macro to convert to table */ 11 | TestOne = (items as list, optional label as text) as table => [ 12 | asTable = Table.FromList(items, Splitter.SplitByNothing(), null, null, ExtraValues.Error), 13 | with_type = Table.TransformColumnTypes( asTable , { { "Column1", Int64.Type } }), 14 | ret = Table.RenameColumns( with_type, { {"Column1", label ?? "Column1"} }, MissingField.Error ) 15 | ][ret], 16 | 17 | Oracle = (obj as any) as text => Text.FromBinary( Json.FromValue(obj) ), 18 | 19 | names = List.Buffer( {0..1000} ), 20 | maxIndex = List.Count( names ) - 1, 21 | 22 | // build tables 23 | many_up = List.Transform( names, each Rand.Int64( 0, maxIndex, RoundingMode.Up ) ), 24 | many_down = List.Transform( names, each Rand.Int64( 0, maxIndex, RoundingMode.Down ) ), 25 | 26 | many_to0 = List.Transform( names, each Rand.Int64( 0, maxIndex, RoundingMode.TowardZero ) ), 27 | many_away0 = List.Transform( names, each Rand.Int64( 0, maxIndex, RoundingMode.AwayFromZero ) ), 28 | many_toeven = List.Transform( names, each Rand.Int64( 0, maxIndex, RoundingMode.ToEven ) ), 29 | 30 | // drill down into one, then check column metrics 31 | summary = [ 32 | Up = TestOne( many_up, "Up" ), 33 | Down = TestOne( many_down, "Down" ), 34 | TowardZero = TestOne( many_to0, "TowardZero" ), 35 | AwayFromZero = TestOne( many_away0, "AwayFromZero" ), 36 | ToEven = TestOne( many_toeven, "ToEven" ) 37 | ] 38 | in 39 | summary 40 | -------------------------------------------------------------------------------- /source/random/List.RandomIndex.pq: -------------------------------------------------------------------------------- 1 | let // gets a random index within the bounds of a list 2 | List.RandomIndex = (source as list) as number => [ 3 | maxIndex = List.Count(source) - 1, 4 | return = Random.Int64(0, maxIndex) 5 | ][return] 6 | in 7 | List.RandomIndex -------------------------------------------------------------------------------- /source/random/List.RandomItem.pq: -------------------------------------------------------------------------------- 1 | let 2 | // returns a random item from a list 3 | List.Random.Item = (source as list) as any => [ 4 | which = List.Random.Index( source ), 5 | return = source{ which } 6 | ][return] 7 | in 8 | List.Random.Item -------------------------------------------------------------------------------- /source/random/Random.Currency.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | About: randomly generate currency within a range [2021-04-23] 4 | optional culture 5 | Return Type: [Currency] 6 | */ 7 | RandomCurrency = ( 8 | min as number, 9 | max as number, 10 | optional culture as nullable text, 11 | optional roundingMode as nullable number 12 | ) as number => 13 | Currency.From( 14 | Number.RandomBetween( min, max ), 15 | culture, 16 | RoundingMode.Down 17 | ) 18 | in 19 | RandomCurrency -------------------------------------------------------------------------------- /source/test/summarize-schema.pq: -------------------------------------------------------------------------------- 1 | let 2 | Source = AllRecords, 3 | Schema = Table.Schema( Source ), 4 | 5 | #"Drop Most Properties" = Table.ReorderColumns( 6 | Schema,{"Name", "TypeName", "Kind", "IsNullable", "Position", "NumericPrecisionBase", 7 | "NumericPrecision", "NumericScale", "DateTimePrecision", "MaxLength", "IsVariableLength", 8 | "NativeTypeName", "NativeDefaultExpression", "NativeExpression", "Description", "IsWritable", "FieldCaption"}), 9 | 10 | // summarizes column types and name as a string 11 | SummarizeSchemaColumn = 12 | (column as record, 13 | optional options as nullable record) as any => 14 | let 15 | options = Record.Combine({default, options ?? []}), 16 | default = [ 17 | Template = "[#[AscribedTypeName]]: #[ColName]#[Nullability] = #[TypeName]#[Nullability]" 18 | ], 19 | render = Text.Format( 20 | options[Template], 21 | [ 22 | AscribedTypeName = column[TypeName], 23 | TypeName = column[Kind], 24 | ColName = column[Name], 25 | Nullability = if column[IsNullable] then "?" else "" 26 | ] 27 | ) 28 | in 29 | render, 30 | 31 | #"Removed Other Columns" = Table.SelectColumns( 32 | #"Drop Most Properties",{"Name", "TypeName", "Kind", "IsNullable", "Position"}), 33 | 34 | #"Summarized Cols" = Table.AddColumn( 35 | #"Removed Other Columns", 36 | "Summary", 37 | (row as record) as any => 38 | try SummarizeSchemaColumn( row ) catch (e) => e, 39 | type text 40 | ), 41 | Summary = #"Summarized Cols", 42 | #"Reordered Columns" = Table.ReorderColumns(Summary,{"Summary", "Name", "TypeName", "Kind", "IsNullable", "Position"}) 43 | in 44 | #"Reordered Columns" -------------------------------------------------------------------------------- /source/test/test_All.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/source/test/test_All.pbix -------------------------------------------------------------------------------- /source/test/test_DateTime_ToOData.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/source/test/test_DateTime_ToOData.pbix -------------------------------------------------------------------------------- /source/test/test_DateTime_ToOData.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | Description 4 | Convert PowerQuery `datetime` and `datetimezone` to a OData format 5 | 6 | OData primatives: https://www.odata.org/documentation/odata-version-2-0/json-format/ 7 | 8 | Example: 9 | 9/8/2020 2:01:50 PM DateTime Local 10 | 9/8/2020 2:01:50 PM -05:00 DateTimeZone Local 11 | 9/8/2020 7:23:18 PM +00:00 DateTimeZone Utc 12 | 13 | Output: 14 | datetime'2020-09-08T14:01:49' 15 | datetimeoffset'2020-09-08T14:01:49' 16 | datetimeoffset'2020-09-08T19:23:17' 17 | */ 18 | Source = #table( 19 | type table[Label = text, Format = text, Example = text, Input = any], 20 | { 21 | // "there is no 'date' primative type?", 22 | { 23 | "DT Fixed local", 24 | "datetime'yyyy-mm-ddThh:mm[:ss[.fffffff]]", 25 | "datetime'2000-12-12T12:00'", 26 | DateTime.FixedLocalNow() 27 | }, 28 | { 29 | "DTZ Fixed Local", 30 | "..", 31 | "datetimeoffset'2002-10-10T17:00:00Z'", 32 | DateTimeZone.FixedLocalNow() 33 | }, 34 | { 35 | "DTZ Fixed UTC", 36 | "..", 37 | "datetimeoffset'2002-10-10T17:00:00Z'", 38 | DateTimeZone.FixedUtcNow() 39 | } 40 | }), 41 | 42 | colUsingFunction = Table.AddColumn( 43 | Source, 44 | "Results", 45 | each DateTime_ToOData( [Input] ), 46 | type text ), 47 | 48 | Final = colUsingFunction 49 | 50 | in 51 | Final 52 | -------------------------------------------------------------------------------- /source/test/test_ListAsText.old.pq: -------------------------------------------------------------------------------- 1 | let 2 | items = {7, 10, 40, 3}, 3 | Summary = [ 4 | asText = ListAsText(items), 5 | a2 = ListAsText(items, "."), 6 | a3 = ListAsText(items, ".", "<-"), 7 | a4 = ListAsText(items, ".", "<-", "->"), 8 | a5List = ListAsText(items, "#(cr)• ", "• ", ""), 9 | a6 = ListAsText(items, ".", "IP addr = ", "") 10 | ] 11 | in 12 | Summary 13 | -------------------------------------------------------------------------------- /source/web/Html.GenerateSelectorList.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | latest version: 4 | */ 5 | GenerateSelectorList_Function = ( 6 | selector as text, 7 | count as number 8 | ) as list => let 9 | numbers = List.Numbers( 1, count, 1), 10 | ListResult = List.Transform( 11 | numbers, 12 | each { 13 | "Column" & Text.From(_), 14 | selector & " :nth-child(" & Text.From(_) & ")" 15 | } 16 | ) 17 | in 18 | ListResult, 19 | 20 | GenerateSelectorList_Type = 21 | type function ( 22 | selector as (type text meta [ 23 | Documentation.FieldCaption = "CSS Selector to prefix generated children", 24 | Documentation.FieldDescription = "CSS Selector to prefix generated children", 25 | Documentation.SampleValues = { "TABLE.table > * > TR >" }, 26 | Formatting.IsMultiLine = true 27 | ]), 28 | count as (type number meta[ 29 | Documentation.FieldCaption = "Number of Columns", 30 | Documentation.FieldDescription = "Number of Columns", 31 | Documentation.SampleValues = {3, 5, 10} 32 | 33 | ]) 34 | ) as list meta [ 35 | Documentation.Name = "Html.GenerateSelectorList", 36 | Documentation.LongDescription = "Dynamically generates a list of CSS selectors. 37 | Used to scrape tables with Web.Page()", 38 | Documentation.Examples = {[ 39 | Description = "Generate one value", 40 | Code = " 41 | Html.GenerateSelectorList( 42 | ""TABLE.table > * > TR >"", 43 | 3 44 | )", 45 | Result = " 46 | { 47 | { 48 | ""Column1"", 49 | ""TABLE.table > * > TR > :nth-child(1)"" 50 | }, 51 | { 52 | ""Column2"", 53 | ""TABLE.table > * > TR > :nth-child(2)"" 54 | }, 55 | { 56 | ""Column3"", 57 | ""TABLE.table > * > TR > :nth-child(3)"" 58 | } 59 | } 60 | " 61 | ]} 62 | 63 | ], 64 | Html.GenerateSelectorList = Value.ReplaceType( GenerateSelectorList_Function, GenerateSelectorList_Type ) 65 | in 66 | Html.GenerateSelectorList -------------------------------------------------------------------------------- /source/web/Html.GetScalar.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | find a better name that describes it will not return a collection? 4 | Html.FirstValue to be consistant ? 5 | */ 6 | Html.GetScalar = (html as any, selector as text, 7 | optional default as nullable any) as text => 8 | let 9 | html = Html.Table( 10 | html, 11 | { 12 | { "title", selector } 13 | }, 14 | null 15 | ) 16 | in 17 | Table.FirstValue(html, default) 18 | 19 | in 20 | Html.GetScalar -------------------------------------------------------------------------------- /source/web/ImageUrl.ToBase64Text.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | given a web url to an image 4 | convert it to base64 text in a format expected by browsers 5 | and Power BI Columns 6 | 7 | todo future: add gif, svg, etc 8 | */ 9 | Image.ToBase64Text = (imageUrl as text) as any => [ 10 | bytes = Web.Contents( imageUrl ), 11 | bytesAsText = Binary.ToText( bytes, BinaryEncoding.Base64 ), 12 | 13 | mime_prefix = 14 | if Text.EndsWith( imageUrl, ".png", Comparer.OrdinalIgnoreCase ) 15 | then 16 | "data:image/png;base64, " 17 | else if ( 18 | Text.EndsWith( imageUrl, ".jpeg", Comparer.OrdinalIgnoreCase ) or 19 | Text.EndsWith( imageUrl, ".jpg", Comparer.OrdinalIgnoreCase ) 20 | ) 21 | then 22 | "data:image/jpg;base64, " 23 | else 24 | error [ 25 | Message.Format = "Unhandled Image type: {0}", 26 | Message.Parameters = { imageUrl } ], 27 | 28 | return = mime_prefix & bytesAsText 29 | 30 | ][return] 31 | 32 | in Image.ToBase64Text -------------------------------------------------------------------------------- /source/web/WebRequest.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | Yet another web request implementation 4 | */ 5 | WebRequest = ( 6 | urlBase, // text or uri 7 | config as record, 8 | optional options as nullable record 9 | ) as record => 10 | let return = [ 11 | defaults = [ Buffer = false ], 12 | options = Record.Combine({ defaults, options ?? [] }), 13 | 14 | bytes = 15 | // this could be simplifeid with a shadowed ref 16 | if options[Buffer]? ?? false 17 | then Binary.Buffer( Web.Contents( urlBase, config ) ) 18 | else Web.Contents( urlBase, config ), 19 | 20 | Data = if not IsJson then null else Json{0}?[Data]?, 21 | AsText = Text.FromBinary( bytes, TextEncoding.Utf8 ), 22 | Json = Json.Document( bytes, TextEncoding.Utf8 ), 23 | IsJson = not (try Json)[HasError]?, 24 | Meta = Value.Metadata( bytes ), 25 | 26 | Response.Status = Meta[Response.Status], 27 | Infer.ContentType = Binary.InferContentType( bytes ), 28 | TotalBytes = Binary.ApproximateLength( bytes ), 29 | Url = Meta[Content.Uri]() 30 | 31 | ], returnFields = { 32 | "Response.Status", 33 | "Url", "Infer.ContentType", "TotalBytes", 34 | "Meta", "Data", "AsText", "Json", "IsJson", 35 | "bytes" } 36 | in 37 | Record.SelectFields(return, returnFields, MissingField.Error), 38 | 39 | in WebRequest -------------------------------------------------------------------------------- /source/web/img/WebRequest-ExtraDebugDetails.pq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/source/web/img/WebRequest-ExtraDebugDetails.pq.png -------------------------------------------------------------------------------- /source/web/waitForResult.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* from: https://docs.microsoft.com/en-us/power-query/waitretry#putting-it-all-together */ 3 | waitForResult = Value.WaitFor( 4 | (iteration) => 5 | let 6 | result = Web.Contents(url, [ManualStatusHandling = {500}]), 7 | buffered = Binary.Buffer(result), 8 | status = Value.Metadata(result)[Response.Status], 9 | actualResult = if status = 500 then null else buffered 10 | in 11 | actualResult, 12 | (iteration) => #duration(0, 0, 0, Number.Power(2, iteration)), 13 | 5) 14 | in 15 | waitForResult -------------------------------------------------------------------------------- /src-new-text-replacemany-documented.2022-07.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | doc tips: 4 | 5 | documentation on the types uses for documentation 6 | https://docs.microsoft.com/en-us/power-query/handlingdocumentation 7 | 8 | from: 9 | https://bengribaudo.com/blog/2021/03/17/5523/power-query-m-primer-part20-metadata 10 | > A note about Documentation.Name: This value should probably match the name of the identifier you assign your function when you define it. For example, if you call your function SomeFunction, then when you set Documentation.Name, you should probably set its value to “SomeFunction”, as well. 11 | 12 | */ 13 | 14 | 15 | Text.ReplaceMany 16 | = ( source as text, mapping as list ) as text => 17 | let 18 | result = List.Accumulate( 19 | mapping, source, 20 | ( state, cur ) => Text.Replace(state, cur{0}, cur{1} )) 21 | in 22 | result, 23 | Text.ReplaceMany.Type = type function( 24 | source as ( 25 | type text meta [ 26 | Documentation.FieldCaption = "source", 27 | Documentation.FieldDescription = "List to search and replace multiple values" 28 | ]), 29 | mapping as (type list meta[ 30 | Documentation.FieldCaption = "List of {oldText, newText} replacement Pairs", 31 | Documentation.FieldDescription = "A list of {oldText, newText} pairs, to search and then replace values" 32 | ])) 33 | as (type text meta [ 34 | Documentation.Name = "Text.ReplaceMany", 35 | Documentation.LongDescription = "A list of {oldText, newText} pairs,


#(cr,lf)#(cr,lf) to search and then replace values", 36 | Documentation.Examples = {[ 37 | Description = "cat to emoji", 38 | Code = "{0..4}", 39 | Result = "none" 40 | ]} 41 | ]) 42 | in 43 | Value.ReplaceType( Text.ReplaceMany, Text.ReplaceMany.Type ) -------------------------------------------------------------------------------- /tabular/Export-BestPractice-Results.cs: -------------------------------------------------------------------------------- 1 | using TabularEditor.BestPracticeAnalyzer; 2 | 3 | // # source: https://www.elegantbi.com/post/exportbparesults 4 | 5 | var bpa = new Analyzer(); 6 | bpa.SetModel(Model); 7 | 8 | var sb = new System.Text.StringBuilder(); 9 | string newline = Environment.NewLine; 10 | 11 | sb.Append("RuleCategory" + '\t' + "RuleName" + '\t' + "ObjectName" + '\t' + "ObjectType" + '\t' + "RuleSeverity" + '\t' + "HasFixExpression" + newline); 12 | 13 | foreach (var a in bpa.AnalyzeAll().ToList()) 14 | { 15 | sb.Append(a.Rule.Category + '\t' + a.RuleName + '\t' + a.ObjectName + '\t' + a.ObjectType + '\t' + a.Rule.Severity + '\t' + a.CanFix + newline); 16 | } 17 | 18 | sb.Output(); 19 | -------------------------------------------------------------------------------- /template/Hi world connector.pq: -------------------------------------------------------------------------------- 1 | section HelloWorld; 2 | 3 | [DataSource.Kind="HelloWorld", Publish="HelloWorld.Publish"] 4 | shared HelloWorld.Contents = (optional message as text) => 5 | let 6 | message = if (message <> null) then message else "Hello world" 7 | in 8 | message; 9 | 10 | HelloWorld = [ 11 | Authentication = [ 12 | Implicit = [] 13 | ], 14 | Label = Extension.LoadString("DataSourceLabel") 15 | ]; 16 | 17 | HelloWorld.Publish = [ 18 | Beta = true, 19 | ButtonText = { Extension.LoadString("FormulaTitle"), Extension.LoadString("FormulaHelp") }, 20 | SourceImage = HelloWorld.Icons, 21 | SourceTypeImage = HelloWorld.Icons 22 | ]; 23 | 24 | HelloWorld.Icons = [ 25 | Icon16 = { Extension.Contents("HelloWorld16.png"), Extension.Contents("HelloWorld20.png"), Extension.Contents("HelloWorld24.png"), Extension.Contents("HelloWorld32.png") }, 26 | Icon32 = { Extension.Contents("HelloWorld32.png"), Extension.Contents("HelloWorld40.png"), Extension.Contents("HelloWorld48.png"), Extension.Contents("HelloWorld64.png") } 27 | ]; 28 | 29 | 30 | // Use this file to write queries to test your data connector 31 | // let 32 | // result = PQ_Hi_World.Contents() 33 | // in 34 | // result 35 | 36 | -------------------------------------------------------------------------------- /template/Power Query function with Documentation.pq: -------------------------------------------------------------------------------- 1 | let 2 | /* 3 | ref: 4 | - https://docs.microsoft.com/en-us/power-query/handlingdocumentation 5 | - https://bengribaudo.com/blog/2021/03/17/5523/power-query-m-primer-part20-metadata#function-parameter-documentation 6 | 7 | AllowedValues may be dynamic: 8 | https://ssbi-blog.de/blog/technical-topics-english/writing-documentation-for-custom-m-functions-part3/ 9 | */ 10 | 11 | Documentation = type function ( 12 | a as (type number meta [ 13 | Documentation.FieldCaption = "First Number", 14 | Documentation.FieldDescription = "[Field A Desc] text (tooltip?)", 15 | // use a multi-line input box 16 | Formatting.IsMultiLine = false, 17 | // monospace font 18 | Formatting.IsCode = false, 19 | // shown as faded input in text box, else: 20 | Documentation.SampleValues = {1, 9}, 21 | // as a dropdown, but not enforced 22 | Documentation.AllowedValues = {34, 99} 23 | 24 | ]), 25 | b as (type number meta [ 26 | Documentation.FieldCaption = "Second Number", 27 | Documentation.FieldDescription = "[Field B Desc] text (tooltip?)", 28 | Formatting.IsMultiLine = true, 29 | Formatting.IsCode = false, 30 | Documentation.SampleValues = {4, 7} 31 | ]) 32 | ) as table meta [ 33 | Documentation.Name = "[Name] SumNumbers", 34 | Documentation.LongDescription = "[LongDesc] Adds two numbers", 35 | Documentation.Examples = {[ 36 | Description = "Sum two numbers", 37 | Code = "let sum = Func(1, 3) in sum", 38 | Result = "4" 39 | ]} 40 | ], 41 | 42 | SumNum = (a as number, b as number) as number => 43 | a + b 44 | in 45 | Value.ReplaceType( SumNum, Documentation) 46 | -------------------------------------------------------------------------------- /template/Reused Allowed Values - template.md: -------------------------------------------------------------------------------- 1 | # Common 'allowed values' 2 | 3 | These are common types to reuse in `Documentation.AllowedValues` blocks 4 | 5 | ## RoundingMode 6 | 7 | ```pq 8 | { RoundingMode.AwayFromZero, RoundingMode.Down, RoundingMode.ToEven, RoundingMode.TowardZero, RoundingMode.Up } 9 | ``` 10 | 11 | ## Text Encoding 12 | 13 | ```pq 14 | { TextEncoding.Ascii, TextEncoding.BigEndianUnicode, TextEncoding.Unicode, TextEncoding.Utf16, TextEncoding.Utf8, TextEncoding.Windows } 15 | ``` 16 | 17 | ## Numbers and Precision 18 | 19 | ```pq 20 | { Precision.Double, Precision.Decimal } 21 | ``` 22 | 23 | ## Byte Order (BOM) 24 | 25 | ```pq 26 | { ByteOrder.BigEndian, ByteOrder.LittleEndian } 27 | ``` 28 | 29 | ## Occurrence 30 | 31 | ```pq 32 | { Occurrence.Optional, Occurrence.Repeating, Occurrence.Required } 33 | ``` 34 | 35 | ## BinaryOccurence 36 | 37 | ```pq 38 | { BinaryOccurrence.Optional, BinaryOccurrence.Repeating, BinaryOccurrence.Required } 39 | ``` 40 | 41 | ## BinaryEncoding 42 | 43 | ```p1 44 | { BinaryEncoding.Base64, BinaryEncoding.Hex } 45 | ``` 46 | 47 | ## Compression 48 | 49 | ```pq 50 | { Compression.Brotli, Compression.Deflate, Compression.GZip, Compression.LZ4, Compression.None, Compression.Snappy, Compression.Zstandard } 51 | ``` 52 | 53 | ## Web Method 54 | 55 | ```pq 56 | { WebMethod.Delete, WebMethod.Get, WebMethod.Head, WebMethod.Patch, WebMethod.Post, WebMethod.Put } 57 | ``` 58 | 59 | ## Csv Style 60 | 61 | ```pq 62 | CsvStyle.QuoteAfterDelimiter, CsvStyle.QuoteAlways 63 | ``` 64 | 65 | ## Days 66 | 67 | ```pq 68 | { Day.Friday, Day.Monday, Day.Saturday, Day.Sunday, Day.Thursday, Day.Tuesday, Day.Wednesday } 69 | ``` 70 | 71 | ## Extra Values 72 | 73 | ```pq 74 | { ExtraValues.Error, ExtraValues.Ignore, ExtraValues.List } 75 | ``` 76 | -------------------------------------------------------------------------------- /template/maybe dupe - Hi world connector.pq: -------------------------------------------------------------------------------- 1 | section HelloWorldWithDocs; 2 | 3 | [DataSource.Kind="HelloWorldWithDocs", Publish="HelloWorldWithDocs.Publish"] 4 | shared HelloWorldWithDocs.Contents = Value.ReplaceType(HelloWorldImpl, HelloWorldType); 5 | 6 | HelloWorldType = type function ( 7 | message as (type text meta [ 8 | Documentation.FieldCaption = "Message", 9 | Documentation.FieldDescription = "Text to display", 10 | Documentation.SampleValues = {"Hello world", "Hola mundo"} 11 | ]), 12 | optional count as (type number meta [ 13 | Documentation.FieldCaption = "Count", 14 | Documentation.FieldDescription = "Number of times to repeat the message", 15 | Documentation.AllowedValues = { 1, 2, 3 } 16 | ])) 17 | as table meta [ 18 | Documentation.Name = "Hello - Name", 19 | Documentation.LongDescription = "Hello - Long Description", 20 | Documentation.Examples = {[ 21 | Description = "Returns a table with 'Hello world' repeated 2 times", 22 | Code = "HelloWorldWithDocs.Contents(""Hello world"", 2)", 23 | Result = "#table({""Column1""}, {{""Hello world""}, {""Hello world""}})" 24 | ],[ 25 | Description = "Another example, new message, new count!", 26 | Code = "HelloWorldWithDocs.Contents(""Goodbye"", 1)", 27 | Result = "#table({""Column1""}, {{""Goodbye""}})" 28 | ]} 29 | ]; 30 | 31 | HelloWorldImpl = (message as text, optional count as number) as table => 32 | let 33 | _count = if (count <> null) then count else 5, 34 | listOfMessages = List.Repeat({message}, _count), 35 | table = Table.FromList(listOfMessages, Splitter.SplitByNothing()) 36 | in 37 | table; 38 | 39 | // Data Source Kind description 40 | HelloWorldWithDocs = [ 41 | Authentication = [ 42 | Anonymous = [] 43 | ]//, 44 | //Label = "Hello World With Docs" 45 | ]; 46 | 47 | // Data Source UI publishing description 48 | HelloWorldWithDocs.Publish = [ 49 | Beta = true, 50 | Category = "Other", 51 | ButtonText = { "Hello World With Docs", "Provides an example of how to provide function documentation" } 52 | ]; 53 | -------------------------------------------------------------------------------- /template/table_constructors.md: -------------------------------------------------------------------------------- 1 | # table examples for snippets 2 | 3 | ```pq 4 | t0 = #table(type table[Name=text], {}), 5 | t1 = #table(null, {}), 6 | t2 = #table(null, null) 7 | 8 | t3 = TableFromRecords(...) # my fav 9 | 10 | ``` -------------------------------------------------------------------------------- /testing/AsanaWebRequest.pq: -------------------------------------------------------------------------------- 1 | 2 | // Get metadata for a Web.Contents call 3 | WebRequest_ToRecord = (response as binary) as record => 4 | let 5 | metaData = Value.Metadata( response ), 6 | maybeErr = try metaData[Content.Type], 7 | r2 = Record.AddField(metaData, "Binary", response, null ), 8 | r3 = Record.AddField(r2, "Response.Error", 9 | if maybeErr[HasError] then maybeErr[Error] else null, 10 | null 11 | ), 12 | result = Record.AddField(r3, "Url", metaData[Content.Uri]() ) 13 | in 14 | result, 15 | 16 | Asana_WebRequest = (path as text, optional options as nullable record) 17 | as any => 18 | let 19 | /* options 20 | Query, ManualStatusHandling, Headers 21 | */ 22 | headers = [ 23 | Authorization = "Bearer " & "Token" 24 | ], 25 | options = [ 26 | RelativePath = "api/1.0/" & path, 27 | Query = options[Query]? ?? null, 28 | ManualStatusHandling = options[ManualStatusHandling]? ?? {404, 400, 500}, 29 | Headers = options[Headers]? ?? headers 30 | // ApiKeyName = "Bearer" 31 | ], 32 | response = Web.Contents( "https://app.asana.com", options), 33 | maybeJson = try Json.Document(response, 65001), 34 | json = 35 | if maybeJson[HasError] then maybeJson[Error][Message] 36 | else maybeJson[Value], 37 | result = [ 38 | Response = response, 39 | Json = json 40 | ] 41 | in 42 | result, 43 | 44 | // try_request = Asana_WebRequest(shared_options), 45 | try_request = Asana_WebRequest("users/me"), 46 | // try_request = Asana_WebRequest(shared_options), 47 | Json = try_request[Json], 48 | // try_request = Asana_WebRequest(shared_options), 49 | Value = Json[Value], 50 | // try_request = Asana_WebRequest(shared_options), 51 | data = Value[data] 52 | in 53 | Asana_WebRequest 54 | -------------------------------------------------------------------------------- /testing/PowerQuery-documentation-grammar.md: -------------------------------------------------------------------------------- 1 | ## Important 2 | 3 | - https://docs.microsoft.com/en-us/power-query/handlingdocumentation 4 | - Intro to docs metadata: https://bengribaudo.com/blog/2021/03/17/5523/power-query-m-primer-part20-metadata 5 | 6 | ## Related: 7 | - https://bengribaudo.com/blog/2020/06/02/5259/power-query-m-primer-part18-type-system-iii-custom-types 8 | - https://bengribaudo.com/blog/2020/09/03/5408/power-query-m-primer-part19-type-system-iv-ascription-conformance-and-equalitys-strange-behaviors 9 | 10 | 11 | ## Partial lexical grammar of PQ documentation 12 | 13 | - Todo: Script that mines the names of all metadata on built-in connectors 14 | - this is to detect any missing `Documentation.` keys for function metadata 15 | Abbreviation 16 | `Documentation.Examples` as `D.Examples` 17 | 18 | ### Basic pseudo lexical grammar 19 | 20 | ```yml 21 | type Function: 22 | | D.Examples 23 | | D.LongDescription 24 | | D.Name 25 | 26 | type Parameter: 27 | | D.AllowedValues 28 | | D.FieldCaption 29 | | D.FieldDescription 30 | | D.SampleValues 31 | | Formatting.IsCode 32 | | Formatting.IsMultiLine 33 | ``` 34 | 35 | ### expanded with types 36 | 37 | ```yml 38 | type Function: 39 | | D.Examples 40 | ListOf: 41 | type [ Description, Code, Result ] 42 | | D.LongDescription 43 | text 44 | | D.Name 45 | text 46 | 47 | type Parameter: 48 | | D.AllowedValues 49 | ListOf: 50 | type any 51 | 52 | | D.FieldCaption 53 | text 54 | | D.FieldDescription 55 | text 56 | | D.SampleValues 57 | ListOf: 58 | type any 59 | | Formatting.IsCode 60 | logical 61 | | Formatting.IsMultiLine 62 | logical 63 | ``` -------------------------------------------------------------------------------- /testing/Summarize_Record.recursion.tests.pq: -------------------------------------------------------------------------------- 1 | 2 | let 3 | /* 4 | 5 | output: 6 | 7 | As Column [Value] 8 | [ a = 10, c = 4 ] 9 | [ a = 93.45, b = 1/1/2031 ] 10 | [ a = [ a = 10, c = 4 ], b = [ b = 3 ] ] 11 | 12 | As Row itself 13 | [ As Column [Value] = [ a = 10, c = 4 ], Name = flat, Value = [ a = 10, c = 4 ] ] 14 | [ As Column [Value] = [ a = 93.45, b = 1/1/2031 ], Name = withdate, Value = [ a = 93.45, b = 1/1/2031 ] ] 15 | [ As Column [Value] = [ a = [ a = 10, c = 4 ], b = [ b = 3 ] ], Name = nested, Value = [ a = [ a = 10, c = 4 ], b = [ b = 3 ] ] ]` 16 | 17 | */ 18 | r1 = [ a = 10, c = 4 ], 19 | r2 = [ b = 3 ], 20 | r3 = [ z = 9, a = 30 ], 21 | r4 = [ a = 93.45 , b = #date(2031, 1,1) ], 22 | r5 = [ a = r1, b = r2 ], 23 | tests_list = { 24 | [ arg1 = r1, arg2 = r2 ], 25 | [ arg1 = r1, arg2 = r3 ], 26 | [ arg1 = r2, arg2 = r3 ], 27 | [ arg1 = r1, arg2 = r3 ] 28 | }, 29 | 30 | summarizeArgs = [ 31 | flat = r1, 32 | withdate = r4, 33 | nested = r5 34 | ], 35 | t_summary = Record.ToTable(summarizeArgs), 36 | as_row = Table.AddColumn( 37 | t_summary, "As Column [Value]", 38 | (row) as text => Summarize_Record(row[Value]), type text ), 39 | 40 | as_self = Table.AddColumn( 41 | as_row, "As Row itself", 42 | (row) as text => Summarize_Record(row), type text ), 43 | final = as_self 44 | in 45 | final -------------------------------------------------------------------------------- /testing/Unit-Test-Table-Constructor-Args.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninmonkey/Ninmonkey.PowerQueryLib/8bb1f4d7f43e9989722cf56f359c6ddc1ad3bc10/testing/Unit-Test-Table-Constructor-Args.pbix -------------------------------------------------------------------------------- /testing/syntax-highlight-stress-other.pq: -------------------------------------------------------------------------------- 1 | let 2 | q =3, // e=4, 3 | T.Wfd = List.Count = q = e = 3 4 | in 5 | [ q=q, lc = List.Count , j =false][[q],[lc]] -------------------------------------------------------------------------------- /testing/syntax-highlight-stress-test.pq: -------------------------------------------------------------------------------- 1 | /* 2 | this code is terrible on purpose. 3 | it's meant to stress test syntax highlighting 4 | It is syntactically valid query 5 | Foo.bar( , ) 6 | Table.List( Tbar.Casdf(...) ) 7 | ) 8 | see more: 9 | */ 10 | let 11 | T.Wfd = List.Count = q = e = 3, 12 | l = TA.A.Wfd(), 13 | l3 = T.Wfd(), 14 | l7 = T.Aa(), 15 | l9 = T.Aaf(), 16 | l2 = TA.AWfd(), 17 | T.L.P = List.Count, 18 | j = T.L(...), q = T.L.P( ), 19 | q1 = #"List."( each 0, each _ < 1, each _ ), 20 | z = table.list(df), 21 | #"TA.A.Wfd" = "#(cr,lf)#""Table.List(""", 22 | #" Table.List(bar Table.List({3,45,5}))" = 345, 23 | a = Table.List, 24 | b = Table(), 25 | now = #duration(0,0,0,1), t = #table( type table[ Id = Int64.Type, Nested = Table.Type], {} ), 26 | r = Table.Combine({null, #"Some.Stepame"( ... ) }), 27 | r2 = Table.Combine( 28 | {null, #"Some.Stepame"( ... ) } 29 | ), 30 | r3 = {null, #"Some.Stepame"( ... ) }, // #"Table.List"#".List", 31 | #"[ T.Bar(" = [ fd = List.Bar( List.Transform( {9..4}, each _ + 3 )), When = DateTime.FixedLocalNow() ], 32 | // c = List.Generate(...) ], 33 | zed = List.Transform( source, (item) => item * 3 ) + Bar.FromText( " Table.List(" & ".." ), 34 | /* 35 | Text.Type, TA.A.Wfd, 36 | Text.Type 37 | */ 38 | q3 = foo.bar(sdf), 39 | #"q35" = Bcat( { 234 } ), 40 | z2 = Bcat(), 41 | g = Table.Bar(dsf) 42 | 43 | 44 | in [g=Table.RowCount( #table(null, {{null}}))] -------------------------------------------------------------------------------- /todo.snippets.pq.md: -------------------------------------------------------------------------------- 1 | snippets to write first 2 | snippets 3 | 4 | - [ ] new buffer 5 | - [ ] export env to json, with version numbers 6 | - [ ] table join types 7 | - [ ] documentation helper 8 | 9 | ## textmate grammar 10 | 11 | 12 | todo: powerquery 13 | 14 | snippets 15 | new buffer 16 | table join types 17 | documentation helper 18 | 19 | update the grammar 20 | https://github.com/microsoft/vscode-powerquery/issues/66#issuecomment-739428136 21 | 22 | https://github.com/microsoft/vscode-powerquery/tree/master/syntaxes 23 | 24 | original repo: https://github.com/microsoft/powerquery-language -------------------------------------------------------------------------------- /util/profile/import-pqlib-error-details.pq: -------------------------------------------------------------------------------- 1 | let 2 | // Source = Folder.Contents(#"PQLib Path"), #"pqlib pq" = Source{[Name="pqlib.pq"]}, 3 | bytes = File.Contents( #"PQLib Path" ), 4 | raw_text = Text.FromBinary( bytes, TextEncoding.Utf8 ), 5 | final_evaluation = try Expression.Evaluate( raw_text , #shared ), 6 | 7 | FormatError = (source as any) as any => 8 | error "NYI: refactor the formatting logic", 9 | 10 | error_ast = final_evaluation[Error], 11 | error_json = Text.FromBinary( Json.FromValue( final_evaluation[Error] ), TextEncoding.Utf8 ), 12 | summary_message = Text.Replace( error_ast[Message], "] ", "]#(cr,lf)"), 13 | 14 | // converts strings "[171-18-171-28]" to structured location data 15 | error_location 16 | = let 17 | rawText = Text.AfterDelimiter( Text.BeforeDelimiter( error_ast[Message], "]"), "["), 18 | pairs = Text.Split( rawText, "-"), 19 | return = [ 20 | StartLine = Number.From( Text.BeforeDelimiter( pairs{0}, "," )), 21 | StartColumn = Number.From( Text.AfterDelimiter( pairs{0}, "," )), 22 | EndLine = Number.From( Text.BeforeDelimiter( pairs{1}, "," )), 23 | EndColumn = Number.From( Text.AfterDelimiter( pairs{1}, "," )) 24 | ] 25 | in 26 | return, 27 | 28 | actual_lines = Lines.FromText(raw_text), 29 | code_at_location = Lines.ToText( List.Range( actual_lines, 30 | error_location[StartLine] - 1, error_location[EndLine] + 1) ), 31 | 32 | formatted_errors = Table.FromRecords( 33 | {[ 34 | Message = summary_message, 35 | Location = error_location, 36 | Code = code_at_location, 37 | json = error_json 38 | ]}, 39 | type table [ 40 | Message = text, 41 | Code = text, 42 | json = text, 43 | Location = Record.Type 44 | ], 45 | MissingField.Error 46 | ), 47 | 48 | return 49 | = if not final_evaluation[HasError] 50 | then final_evaluation[Value] 51 | else formatted_errors 52 | in 53 | return -------------------------------------------------------------------------------- /wip - left off here.md: -------------------------------------------------------------------------------- 1 | 2 | clean up queries, like running web.contents ? 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | Example snippets 18 | 19 | ## `Let` expression 20 | 21 | ```ts 22 | (_) as any => 23 | x = Table.SelectRows(source, each [x] ) 24 | // to 25 | (_) as any => 26 | let 27 | selection = 10 28 | in 29 | selection 30 | 31 | ``` 32 | 33 | ## convert to `function` expression 34 | 35 | ```ts 36 | 37 | cat = 30, 38 | zed = ... 39 | // to 40 | cat = (_) => {selection}, 41 | 42 | or 43 | cat = ($1 as ax) => 44 | let 45 | $2 = 30 46 | in 47 | $2, 48 | 49 | 50 | or 51 | ``` 52 | --------------------------------------------------------------------------------