├── .github └── workflows │ ├── scala-release.yml │ └── scala.yml ├── .gitignore ├── .scalafix.conf ├── .scalafmt.conf ├── LICENSE ├── Makefile ├── README.md ├── build.sbt ├── images ├── cloudtags.png ├── created-examples-trend.png └── testable-examples-status.png ├── package.sbt ├── project ├── build.properties └── plugins.sbt ├── publish.sbt ├── src ├── main │ ├── resources │ │ └── reference.conf │ ├── scala │ │ └── fr │ │ │ └── janalyse │ │ │ └── cem │ │ │ ├── Configuration.scala │ │ │ ├── Execute.scala │ │ │ ├── FileSystemService.scala │ │ │ ├── Main.scala │ │ │ ├── RemoteGithubOperations.scala │ │ │ ├── RemoteGitlabOperations.scala │ │ │ ├── RemoteOperations.scala │ │ │ ├── Synchronize.scala │ │ │ ├── model │ │ │ ├── Change.scala │ │ │ ├── CodeExample.scala │ │ │ ├── CodeExampleMetaData.scala │ │ │ ├── Overview.scala │ │ │ ├── RemoteExample.scala │ │ │ ├── RemoteExampleState.scala │ │ │ └── RunStatus.scala │ │ │ └── tools │ │ │ ├── DescriptionTools.scala │ │ │ ├── GitMetaData.scala │ │ │ ├── GitOps.scala │ │ │ ├── Hashes.scala │ │ │ └── HttpTools.scala │ └── twirl │ │ └── fr.janalyse.cem.templates │ │ └── ExamplesOverviewTemplate.scala.txt └── test │ └── scala │ └── fr │ └── janalyse │ └── cem │ ├── FileSystemServiceStub.scala │ ├── RemoteGithubOperationsSpec.scala │ ├── SynchronizeSpec.scala │ ├── model │ └── CodeExampleSpec.scala │ └── tools │ ├── DescriptionToolsSpec.scala │ └── HashesSpec.scala ├── tracing.sbt └── version.sbt /.github/workflows/scala-release.yml: -------------------------------------------------------------------------------- 1 | name: Scala CI Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Setup JDK 17 | uses: actions/setup-java@v4 18 | with: 19 | distribution: temurin 20 | java-version: 17 21 | 22 | - name: Setup sbt launcher 23 | uses: sbt/setup-sbt@v1 24 | 25 | - name: Get the version 26 | id: get_version 27 | run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v} 28 | 29 | - name: Run tests 30 | run: sbt test 31 | 32 | - name: Package 33 | run: sbt universal:packageZipTarball 34 | 35 | - name: Copy artifact 36 | run: cp target/universal/*.tgz code-examples-manager-${{steps.get_version.outputs.VERSION}}.tgz 37 | 38 | - name: Upload a Build Artifact 39 | uses: actions/upload-artifact@v4 40 | with: 41 | name: code-examples-manager-${{steps.get_version.outputs.VERSION}}.tgz 42 | path: code-examples-manager-${{steps.get_version.outputs.VERSION}}.tgz 43 | 44 | - name: GitHub Releases 45 | uses: fnkr/github-action-ghr@v1.3 46 | env: 47 | GHR_PATH: code-examples-manager-${{steps.get_version.outputs.VERSION}}.tgz 48 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 49 | -------------------------------------------------------------------------------- /.github/workflows/scala.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: 4 | branches: [ master ] 5 | push: 6 | branches: [ master ] 7 | paths-ignore: 8 | - 'README.md' 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v4 15 | - name: Setup JDK 16 | uses: actions/setup-java@v4 17 | with: 18 | distribution: temurin 19 | java-version: 17 20 | - name: Setup sbt launcher 21 | uses: sbt/setup-sbt@v1 22 | - name: Build and Test 23 | run: sbt test 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Security purposes 2 | .envrc 3 | 4 | # temporary files 5 | *~ 6 | *.log 7 | nohup.out 8 | *.swp 9 | .attach_pid* 10 | 11 | # build related 12 | target/ 13 | project/target 14 | metals.sbt 15 | 16 | # IDE related 17 | .metals/ 18 | .bloop/ 19 | .bsp/ 20 | .vscode/ 21 | .idea/ 22 | 23 | tmp-*.gif 24 | tmp-*.png 25 | 26 | # application related 27 | private-application*.conf* 28 | .lmdb/ 29 | -------------------------------------------------------------------------------- /.scalafix.conf: -------------------------------------------------------------------------------- 1 | rules = [ 2 | DisableSyntax, 3 | RedundantSyntax, 4 | //ExplicitResultTypes // not yet compatible with Scala3 5 | ] 6 | DisableSyntax.noVars = true 7 | DisableSyntax.noThrows = true 8 | DisableSyntax.noNulls = true 9 | DisableSyntax.noReturns = true 10 | DisableSyntax.noWhileLoops = true 11 | DisableSyntax.noAsInstanceOf = true 12 | DisableSyntax.noIsInstanceOf = true 13 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 3.9.4 2 | runner.dialect = scala3 3 | align.preset = most 4 | maxColumn = 200 5 | assumeStandardLibraryStripMargin = true 6 | align.stripMargin = true 7 | indent.defnSite = 2 8 | 9 | //align.tokens.add = [ 10 | // {code = "=", owner = "Term.Arg.Named"} 11 | //] 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache NON-AI License, Version 2.0 2 | 3 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 4 | 5 | 1. Definitions. 6 | 7 | “License” shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 8 | 9 | “Licensor” shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 10 | 11 | “Legal Entity” shall mean the union of the acting entity and all other entities that control, are controlled by, 12 | or are under common control with that entity. For the purposes of this definition, “control” means (i) the power, direct or indirect, 13 | to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) 14 | or more of the outstanding shares, or (iii) beneficial ownership of such entity. 15 | 16 | “You” (or “Your”) shall mean an individual or Legal Entity exercising permissions granted by this License. 17 | 18 | “Source” form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, 19 | and configuration files. 20 | 21 | “Object” form shall mean any form resulting from mechanical transformation or translation of a Source form, including 22 | but not limited to compiled object code, generated documentation, and conversions to other media types. 23 | 24 | “Work” shall mean the work of authorship, whether in Source or Object form, made available under the License, 25 | as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 26 | 27 | “Derivative Works” shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and 28 | for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. 29 | For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) 30 | to the interfaces of, the Work and Derivative Works thereof. 31 | 32 | “Contribution” shall mean any work of authorship, including the original version of the Work and any modifications or additions to 33 | that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or 34 | by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, “submitted” 35 | means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to 36 | communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, 37 | the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated 38 | in writing by the copyright owner as “Not a Contribution.” 39 | 40 | “Contributor” shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and 41 | subsequently incorporated within the Work. 42 | 43 | 2. Grant of Copyright License. 44 | 45 | Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, 46 | no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, 47 | sublicense, and distribute the Work and such Derivative Works in Source or Object form, 48 | under the following conditions: 49 | 50 | 2.1. You shall not use the Covered Software in the creation of an Artificial Intelligence training dataset, 51 | including but not limited to any use that contributes to the training or development of an AI model or algorithm, 52 | unless You obtain explicit written permission from the Contributor to do so. 53 | 54 | 2.2. You acknowledge that the Covered Software is not intended for use in the creation of an Artificial Intelligence training dataset, 55 | and that the Contributor has no obligation to provide support or assistance for any use that violates this license. 56 | 57 | 3. Grant of Patent License. 58 | 59 | Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, 60 | royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, 61 | and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are 62 | necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which 63 | such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) 64 | alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, 65 | then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 66 | 67 | 4. Redistribution. 68 | 69 | You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, 70 | and in Source or Object form, provided that You meet the following conditions: 71 | 72 | 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and 73 | 74 | 2. You must cause any modified files to carry prominent notices stating that You changed the files; and 75 | 76 | 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, 77 | and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 78 | 79 | 4. If the Work includes a “NOTICE” text file as part of its distribution, then any Derivative Works that You distribute 80 | must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that 81 | do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed 82 | as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; 83 | or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. 84 | The contents of the NOTICE file are for informational purposes only and do not modify the License. 85 | 86 | You may add Your own attribution notices within Derivative Works that You distribute, 87 | alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as 88 | modifying the License. 89 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, 90 | reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, 91 | and distribution of the Work otherwise complies with the conditions stated in this License. 92 | 93 | 5. Submission of Contributions. 94 | 95 | Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor 96 | shall be under the terms and conditions of this License, without any additional terms or conditions. 97 | Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed 98 | with Licensor regarding such Contributions. 99 | 100 | 6. Trademarks. 101 | 102 | This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, 103 | except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 104 | 105 | 7. Disclaimer of Warranty. 106 | 107 | Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) 108 | on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, 109 | any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. 110 | You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated 111 | with Your exercise of permissions under this License. 112 | 113 | 8. Limitation of Liability. 114 | 115 | In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, 116 | unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, 117 | shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages 118 | of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages 119 | for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), 120 | even if such Contributor has been advised of the possibility of such damages. 121 | 122 | 9. Accepting Warranty or Additional Liability. 123 | 124 | While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, 125 | warranty, indemnity, or other liability obligations and/or rights consistent with this License. 126 | However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, 127 | not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, 128 | or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 129 | 130 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | @sbt universal:packageZipTarball 3 | @echo "packages are generated in target/universal directory :" 4 | @ls target/universal/*.tgz 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CEM - Code Examples Manager [![][CodeExamplesManagerImg]][CodeExamplesManagerLnk] ![Scala CI][scalaci-master] 2 | 3 | Code example manager (CEM) is a software managing your notes, scripts and code examples. 4 | It provides publish mechanisms to [github.com][githubcom] (as [gists][gists]) or 5 | [gitlab.com][gitlabcom] (as [snippets][snippets]). It also automates execution for 6 | testable examples, this is a quite useful to manage a high number of examples. 7 | 8 | All my notes, scripts and code examples (my programming knowledge base) are now managed using this tool, 9 | you can take a look to **[my public gists overview on github][mygists]** to illustrate the 10 | publishing work achieved by CEM. 11 | 12 | Current [Code example manager (CEM)][cem] implementation is just a command line tool 13 | which compare locally available examples with already published ones in order to find 14 | what it should do (add, update, do nothing). 15 | 16 | ![](images/cloudtags.png) 17 | 18 | ## Why ? 19 | 20 | Code examples are very important, each example is most of the time designed to focus 21 | on a particular feature/characteristic of a programming language, a library or a framework. 22 | They help us to quickly test, experiment and remember how bigger project or at least some 23 | parts of them are working. 24 | 25 | **[See the rules for good code examples][rules] for more information.** 26 | 27 | Managing hundreds of published code example files as gists (github) and/or snippets (gitlab) 28 | is really not easy and time-consuming, in particular if you want to keep them up to date. This 29 | is the main issue addressed by this software. 30 | 31 | - My open-source examples evolution trend : 32 | ![](images/created-examples-trend.png) 33 | - and the execution status trend for executable/testable ones : 34 | ![](images/testable-examples-status.png) 35 | 36 | As you can see through the previous charts, once you have industrialized your notes and code 37 | examples, analytics on your examples become quite easy, and a lot of advanced features become 38 | possible... So stay connected to this project ;) 39 | 40 | ## Quick start 41 | 42 | No particular prerequisites, just a Java >=8 JVM available, and 43 | it will run on your Linux, Windows or MacOSX 44 | 45 | Instructions example with github.com publishing configuration : 46 | - Install the [coursier][csget] from @alxarchambault to automate 47 | the download/update/start of code-examples-manager directly from 48 | maven repositories 49 | - Customize your configuration (see below for token configuration) 50 | ``` 51 | export CEM_SEARCH_ROOTS="/home/myuser/myexamples" 52 | export CEM_GITHUB_TOKEN="xxxxx" 53 | ``` 54 | - Create an example file in `/home/myuser/myexamples` such as `hello.md` 55 | ``` 56 | 63 | # Hello world ! 64 | this is just an example 65 | ``` 66 | - Run the following command from your terminal (`cs` is the [coursier][cs] CLI command): 67 | ``` 68 | cs launch fr.janalyse:code-examples-manager_3:2.4.0 69 | ``` 70 | - you can even use `cs launch fr.janalyse:code-examples-manager_3:latest.release` to always use the latest release 71 | - current release is : [![][CodeExamplesManagerImg]][CodeExamplesManagerLnk] 72 | - Check the command output to get the overview URL 73 | 74 | 75 | ## Code examples 76 | 77 | In order to be published your code examples must come with a description header 78 | inserted using single line comments. You must provide a unique identifier (UUID) 79 | to each of your example, as well as a summary and publish keywords which define 80 | remote destinations. 81 | 82 | Example for languages using `//` for line comments : 83 | ```scala 84 | // summary : Simplest scalatest test framework usage. 85 | // keywords : scala, scalatest, pi, @testable 86 | // publish : gist 87 | // authors : David Crosson 88 | // license : Apache 89 | // id : d24d8cb3-45c0-4d88-b033-7fae2325607b 90 | // created-on : 2020-05-31T19:54:52Z 91 | // run-with : scala-cli $file 92 | 93 | // --------------------- 94 | //> using scala "3.1.1" 95 | //> using lib "org.scalatest::scalatest:3.2.10" 96 | // --------------------- 97 | 98 | import org.scalatest._, matchers.should.Matchers._ 99 | 100 | math.Pi shouldBe 3.14d +- 0.01d 101 | ``` 102 | 103 | Supported keys in description header are the following : 104 | - **`summary`** : example summary in one line. 105 | - **`keywords`** : keywords describing your code features (comma separated). Some reserved keywords : 106 | - `@testable` : allow automatic execution 107 | - `@fail` : the example is expected to fail when executed 108 | - `@exclusive` : all testable examples with this flag will be run sequentially (for scripts which open server sockets for example) 109 | - **`publish`** : publish destination keywords (comma separated) 110 | - the default configuration file provide those activation keywords : 111 | - `gist` : for github.com 112 | - `snippet` : for gitlab.com 113 | - **`authors`** : code example authors list (comma separated). 114 | - **`license`** : the example license. 115 | - **`id`** : UUID for this code example. Generated using such commands : 116 | - with linux command : `uuidgen` 117 | - with [scala-cli][scl] : `scala-cli https://gist.github.com/dacr/87c9636a6d25787d7c274b036d2a8aad` 118 | - with [ammonite][amm] : `amm -c 'println(java.util.UUID.randomUUID.toString)'` 119 | - **`attachments`** : List of secondary files (comma separated) which must be published with the current one 120 | - must be placed in the same directory in your local storage 121 | - secondary files do not require any headers 122 | - **`created-on`** : The ISO8601 date when this example has been created. Generated using such commands : 123 | - with linux command : `date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"` 124 | - with [scala-cli][scl] : `scala-cli https://gist.github.com/dacr/4298fce08e12ba76ab91e9766be52acb` 125 | - with [ammonite][amm] : `amm -c 'println(java.time.Instant.now.toString)'` 126 | - **`run-with`** : command used to execute this example 127 | - Only examples with `@testable` keywords are eligible for automated execution 128 | - on execution the exit code is used to compute execution success or failure 129 | - Use `$file` (or `$scriptFile`) for example filename substitution 130 | - **`test-with`** : Command to test the example 131 | - When `@testable` is set as keyword 132 | - When your example is a "blocking service", you can specify an external command to test it 133 | - for example : `test-with : curl http://127.0.0.1:8080/docs` 134 | 135 | ## CEM operations 136 | 137 | Code examples manager operations : 138 | - It reads its configuration 139 | - It searches for code examples from the given directories roots 140 | - select files matching the search pattern and not involved in the ignore mask 141 | - Selects code examples if and only if they contain a unique identifier (UUID) 142 | - It publishes or updates remote code examples to remote destinations 143 | - the code example publish scope (`publish` keyword) select target destinations 144 | - comma separated publish activation keyword (`activation-keyword` parameter in configuration) 145 | - It adds or updates a global overview of all published examples for a given destination 146 | - this summary has its own UUID defined in the configuration file 147 | 148 | ## Configuration 149 | 150 | The configuration relies on configuration files, a default one named `reference.conf` is provided. 151 | This [default configuration file][referenceconf] defines default values and default behaviors and 152 | allow a simple configuration way based on environment variables which override default values. 153 | 154 | ### Simplified configuration 155 | 156 | | env or property name | description | default value 157 | |----------------------------|---------------------------------------------------------------|--------------------------- 158 | | CEM_CONFIG_FILE | Your custom advanced configuration file (optional) | *undefined* 159 | | CEM_SUMMARY_TITLE | The generated summary title for all published examples | Examples knowledge base 160 | | CEM_SEARCH_ROOTS | Examples search roots (comma separated) | "" 161 | | CEM_SEARCH_PATTERN | Examples files regular expression pattern | ".*" 162 | | CEM_SEARCH_IGNORE_MASK | Ignore file regular expression | "(/[.]bsp)|(/[.]scala.*)|([.]png$)" 163 | | CEM_CHAR_ENCODING | Chararacter encoding for your examples or notes | "UTF-8" 164 | | CEM_GITHUB_ENABLED | To enable or disable standard GITHUB support | true 165 | | CEM_GITHUB_ACTIVATION_KEY | Example publish keyword for github | "gist" 166 | | CEM_GITHUB_TOKEN | Github authentication token for gists API access | *more information below* 167 | | CEM_GITHUB_API | Github API http end point | "https://api.github.com" 168 | | CEM_GITLAB_ENABLED | To enable or disable standard GITLAB support | true 169 | | CEM_GITLAB_ACTIVATION_KEY | Example publish keyword for the gitlab | "snippet" 170 | | CEM_GITLAB_TOKEN | gitlab authentication token for snippets API access | *more information below* 171 | | CEM_GITLAB_API | Gitlab API http end point | "https://gitlab.com/api/v4" 172 | | CEM_GITLAB_VISIBILITY | Gitlab published examples chosen visibility | "public" 173 | 174 | Configuration examples : 175 | ```shell 176 | export CEM_SEARCH_ROOTS="/tmp/someplace,/tmp/someotherplace" 177 | export CEM_SEARCH_PATTERN="[.](sc)|(sh)|(md)|(jsh)$" 178 | export CEM_GITHUB_TOKEN="fada-fada-fada-fada" 179 | ``` 180 | 181 | ### Advanced configuration 182 | 183 | Take a look to the [configuration file][referenceconf] for more information about advanced configuration. 184 | 185 | Once CEM installed you can modify the provided `conf/application.conf` file (whose content is by default 186 | the same as the default [reference.conf][referenceconf] file), remember that any unset parameter in `application.conf` 187 | will default to the value defined in `reference.conf`. 188 | 189 | Note : It is also possible to provide a custom configuration file through the `config.file` java property or the 190 | `CEM_CONFIG_FILE` environment variable. 191 | 192 | ## Authentication tokens 193 | 194 | ### Gitlab authentication token configuration 195 | 196 | Get an access token from gitlab : 197 | - Go to your user **settings** 198 | - Select **Access tokens** 199 | - Add a **Personal access token** 200 | - Enable scopes : `api` and `read_user` 201 | - setup your `CEM_GITLAB_TOKEN` environment variable or `token` parameter in your configuration file 202 | with the generated token 203 | - **Keep it carefully as it is not possible to retrieve it later.** 204 | - **And of course KEEP IT SECURE** 205 | 206 | ### Github authentication token configuration 207 | 208 | Get an access token from github.com : 209 | - Got to your user **settings** 210 | - Select **Developer settings** 211 | - Select **Personal access tokens** 212 | - Then **generate new token** 213 | - Enable scopes : `gist` and `read:user` 214 | - setup your `CEM_GITHUB_TOKEN` environment variable or `token` parameter in your configuration file 215 | with the generated token, the value shown within curl json response 216 | - **Keep it carefully as it is not possible to retrieve it later.** 217 | - **And of course KEEP IT SECURE** 218 | 219 | ## Project history 220 | 221 | - 2019-06 - PoC#1 example proof of concept 222 | - 2019-07 - PoC#2 example proof of concept (in prod) 223 | - 2019-08 - Switch to a real world project 224 | - 2019-09 - In prod for my own usage 225 | - 2020-07 - First public release 226 | - 2021-05 - Full refactoring to use [ZIO][zio] - pure functional 227 | - 2021-06 - Migration to Scala3 228 | - 2021-12 - PoC#3 Search & Execution engines 229 | - 2022-01 - Migrate to ZIO2 and add support for attachments 230 | - 2023-04 - Use ZIO standard configuration 231 | - 2023-05 - Add ZIO LMDB for local storage cache AND data sharing with external applications 232 | 233 | [CodeExamplesManager]: https://github.com/dacr/code-examples-manager 234 | [CodeExamplesManagerImg]: https://img.shields.io/maven-central/v/fr.janalyse/code-examples-manager_3.svg 235 | [CodeExamplesManagerLnk]: https://mvnrepository.com/artifact/fr.janalyse/code-examples-manager 236 | [scalaci-master]: https://github.com/dacr/code-examples-manager/workflows/Scala%20CI/badge.svg 237 | [mygists]: https://gist.github.com/c071a7b7d3de633281cbe84a34be47f1 238 | [cem]: https://github.com/dacr/code-examples-manager 239 | [amm]: https://ammonite.io/ 240 | [scl]: https://scala-cli.virtuslab.org/ 241 | [githubcom]: https://github.com/ 242 | [gitlabcom]: https://gitlab.com/ 243 | [snippets]: https://docs.gitlab.com/ce/user/snippets.html 244 | [gists]: https://docs.github.com/en/github/writing-on-github/creating-gists 245 | [uuid-sc]: https://gist.github.com/dacr/87c9636a6d25787d7c274b036d2a8aad 246 | [scala]: https://www.scala-lang.org/ 247 | [lihaoyi]: https://github.com/lihaoyi 248 | [ac2019]: https://www.alpescraft.fr/edition_2019/ 249 | [ac2019talk]: https://www.youtube.com/watch?v=61AGIBdG7YE 250 | [referenceconf]: https://github.com/dacr/code-examples-manager/blob/master/src/main/resources/reference.conf 251 | [latest]: https://github.com/dacr/code-examples-manager/releases/latest 252 | [rules]: https://github.com/dacr/the-rules-for-good-code-examples 253 | [cs]: https://get-coursier.io/ 254 | [csget]: https://get-coursier.io/docs/cli-installation 255 | [zio]: https://zio.dev/ -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "code-examples-manager" 2 | organization := "fr.janalyse" 3 | description := "Tool to manage set of code examples : synchronize and publish, automated execution, ..." 4 | 5 | licenses += "NON-AI-APACHE2" -> url(s"https://github.com/non-ai-licenses/non-ai-licenses/blob/main/NON-AI-APACHE2") 6 | 7 | scalaVersion := "3.6.4" 8 | 9 | scalacOptions += "-Xkind-projector:underscores" 10 | 11 | lazy val versions = new { 12 | val sttp = "3.11.0" 13 | val zio = "2.1.17" 14 | val zionio = "2.0.2" 15 | val zioproc = "0.7.2" 16 | val zioconfig = "4.0.4" 17 | val ziologging = "2.5.0" 18 | val ziolmdb = "2.0.1" 19 | val naturalsort = "1.0.7" 20 | val jgit = "7.2.0.202503040940-r" 21 | } 22 | 23 | libraryDependencies ++= Seq( 24 | "dev.zio" %% "zio" % versions.zio, 25 | "dev.zio" %% "zio-test" % versions.zio % Test, 26 | "dev.zio" %% "zio-test-junit" % versions.zio % Test, 27 | "dev.zio" %% "zio-test-sbt" % versions.zio % Test, 28 | "dev.zio" %% "zio-test-scalacheck" % versions.zio % Test, 29 | "dev.zio" %% "zio-streams" % versions.zio, 30 | "dev.zio" %% "zio-nio" % versions.zionio, 31 | "dev.zio" %% "zio-process" % versions.zioproc, 32 | "dev.zio" %% "zio-logging" % versions.ziologging, 33 | "dev.zio" %% "zio-logging-slf4j-bridge" % versions.ziologging, 34 | "dev.zio" %% "zio-config" % versions.zioconfig, 35 | "dev.zio" %% "zio-config-typesafe" % versions.zioconfig, 36 | "dev.zio" %% "zio-config-magnolia" % versions.zioconfig, 37 | "fr.janalyse" %% "zio-lmdb" % versions.ziolmdb, 38 | "com.softwaremill.sttp.client3" %% "async-http-client-backend-zio" % versions.sttp, 39 | "com.softwaremill.sttp.client3" %% "zio-json" % versions.sttp, 40 | "fr.janalyse" %% "naturalsort" % versions.naturalsort, 41 | "org.eclipse.jgit" % "org.eclipse.jgit" % versions.jgit 42 | ) 43 | 44 | testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework") 45 | 46 | TwirlKeys.templateImports += "fr.janalyse.cem.model._" 47 | 48 | mainClass := Some("fr.janalyse.cem.Main") 49 | 50 | // ZIO-LMDB requires special authorization at JVM level 51 | ThisBuild / fork := true 52 | ThisBuild / javaOptions ++= Seq("--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED") 53 | 54 | enablePlugins(SbtTwirl) 55 | 56 | homepage := Some(url("https://github.com/dacr/code-examples-manager")) 57 | scmInfo := Some(ScmInfo(url(s"https://github.com/dacr/code-examples-manager.git"), s"git@github.com:dacr/code-examples-manager.git")) 58 | developers := List( 59 | Developer( 60 | id = "dacr", 61 | name = "David Crosson", 62 | email = "crosson.david@gmail.com", 63 | url = url("https://github.com/dacr") 64 | ) 65 | ) 66 | -------------------------------------------------------------------------------- /images/cloudtags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dacr/code-examples-manager/ce5fa6106e3bcf10ccb96e6d3568ccccb98998ab/images/cloudtags.png -------------------------------------------------------------------------------- /images/created-examples-trend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dacr/code-examples-manager/ce5fa6106e3bcf10ccb96e6d3568ccccb98998ab/images/created-examples-trend.png -------------------------------------------------------------------------------- /images/testable-examples-status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dacr/code-examples-manager/ce5fa6106e3bcf10ccb96e6d3568ccccb98998ab/images/testable-examples-status.png -------------------------------------------------------------------------------- /package.sbt: -------------------------------------------------------------------------------- 1 | enablePlugins(JavaAppPackaging) 2 | 3 | Universal / mappings += { 4 | // we are using the reference.conf as default application.conf 5 | // the user can override settings here 6 | val conf = (Compile / resourceDirectory).value / "reference.conf" 7 | conf -> "conf/application.conf" 8 | } 9 | 10 | // add jvm parameter for typesafe config 11 | bashScriptExtraDefines += """addJava "-Dconfig.file=${app_home}/../conf/application.conf"""" 12 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | # suppress inspection "UnusedProperty" for whole file 2 | sbt.version=1.10.11 3 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") 2 | addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") 3 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") 4 | addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") 5 | addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") 6 | addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.2") 7 | addSbtPlugin("org.playframework.twirl" % "sbt-twirl" % "2.0.8") 8 | -------------------------------------------------------------------------------- /publish.sbt: -------------------------------------------------------------------------------- 1 | pomIncludeRepository := { _ => false } 2 | publishMavenStyle := true 3 | Test / publishArtifact := false 4 | releaseCrossBuild := true 5 | versionScheme := Some("semver-spec") 6 | 7 | publishTo := { 8 | // For accounts created after Feb 2021: 9 | // val nexus = "https://s01.oss.sonatype.org/" 10 | val nexus = "https://oss.sonatype.org/" 11 | if (isSnapshot.value) Some("snapshots" at nexus + "content/repositories/snapshots") 12 | else Some("releases" at nexus + "service/local/staging/deploy/maven2") 13 | } 14 | 15 | releasePublishArtifactsAction := PgpKeys.publishSigned.value 16 | 17 | releaseTagComment := s"Releasing ${(ThisBuild / version).value}" 18 | releaseCommitMessage := s"Setting version to ${(ThisBuild / version).value}" 19 | releaseNextCommitMessage := s"[ci skip] Setting version to ${(ThisBuild / version).value}" 20 | 21 | import ReleaseTransformations.* 22 | releaseProcess := Seq[ReleaseStep]( 23 | checkSnapshotDependencies, 24 | inquireVersions, 25 | runClean, 26 | runTest, 27 | setReleaseVersion, 28 | commitReleaseVersion, 29 | tagRelease, 30 | publishArtifacts, 31 | releaseStepCommand("sonatypeReleaseAll"), 32 | setNextVersion, 33 | commitNextVersion, 34 | pushChanges 35 | ) 36 | -------------------------------------------------------------------------------- /src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | // Copyright 2023 David Crosson 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | code-examples-manager-config { 16 | // ================================================================================= 17 | summary { 18 | title = "Examples knowledge base" 19 | title = ${?CEM_SUMMARY_TITLE} 20 | } 21 | // ================================================================================= 22 | examples { 23 | // comma separated list of paths where to search recursively for examples 24 | search-root-directories = "" 25 | search-root-directories = ${?CEM_SEARCH_ROOTS} 26 | // search only for files which match this regexp pattern 27 | search-only-pattern = ".*" 28 | search-only-pattern = ${?CEM_SEARCH_PATTERN} 29 | // ignore sub-directories which match this regexp pattern 30 | search-ignore-mask = "(/[.]bsp)|(/[.]scala.*)|([.]png$)" 31 | search-ignore-mask = ${?CEM_SEARCH_IGNORE_MASK} 32 | // examples character encoding 33 | char-encoding = "UTF-8" 34 | char-encoding = ${?CEM_CHAR_ENCODING} 35 | } 36 | // ================================================================================= 37 | // each adapter is taken into account if and only if enabled is true && token is defined 38 | publish-adapters { 39 | // ----------------------------------------------------------------------------- 40 | // Configuration defaults for github.com 41 | github-com-gists { 42 | enabled = true 43 | enabled = ${?CEM_GITHUB_ENABLED} 44 | kind = "github" 45 | activation-keyword = "gist" 46 | activation-keyword = ${?CEM_GITHUB_ACTIVATION_KEY} 47 | api-end-point = "https://api.github.com" 48 | api-end-point = ${?CEM_GITHUB_API} 49 | overview-uuid = "fecafeca-feca-feca-feca-fecafecafeca" 50 | token = ${?CEM_GITHUB_TOKEN} 51 | filename-rename-rules {} 52 | } 53 | // ----------------------------------------------------------------------------- 54 | // Configuration defaults for gitlab.com 55 | gitlab-com-snippets { 56 | enabled = true 57 | enabled = ${?CEM_GITLAB_ENABLED} 58 | kind = "gitlab" 59 | activation-keyword = "snippet" 60 | activation-keyword = ${?CEM_GITLAB_ACTIVATION_KEY} 61 | api-end-point = "https://gitlab.com/api/v4" 62 | api-end-point = ${?CEM_GITLAB_API} 63 | overview-uuid = "cafecafe-cafe-cafe-cafe-cafecafecafe" 64 | default-visibility = "public" 65 | default-visibility = ${?CEM_GITLAB_VISIBILITY} 66 | token = ${?CEM_GITLAB_TOKEN} 67 | filename-rename-rules { 68 | //// DISABLED AS scala-cli behaves differently when executing .scala versus .sc 69 | //scala-files { // rename .sc scala file to an extension known by gitlab (for colorization) 70 | // from = "^(.*)[.]sc$" 71 | // to = "$1.scala" 72 | //} 73 | //java-files { // rename .jsh java file to an extension known by gitlab (for colorization) 74 | // from = "^(.*)[.]jsh$" 75 | // to = "$1.java" 76 | //} 77 | } 78 | } 79 | } 80 | meta-info { 81 | // populated externally during the build, see tracing.sbt file 82 | } 83 | } 84 | 85 | lmdb { 86 | name = code-examples-manager-data 87 | sync = false 88 | } 89 | 90 | logger { 91 | #format = "%highlight{%timestamp{yyyy-MM-dd'T'HH:mm:ssZ} %fixed{7}{%level} [%fiberId] %name:%line %message %kvs %cause}" 92 | format = "%highlight{%level [%fiberId] \"%message\" %spans %kvs %cause}}" 93 | } 94 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/Configuration.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem 17 | 18 | import zio.* 19 | 20 | import scala.util.Properties.* 21 | import zio.config.* 22 | import zio.config.magnolia.* 23 | import zio.config.typesafe.* 24 | 25 | import java.io.File 26 | import scala.util.matching.Regex 27 | 28 | case class ExamplesConfig( 29 | searchRootDirectories: String, 30 | searchOnlyPattern: Option[String], 31 | searchIgnoreMask: Option[String], 32 | charEncoding: String 33 | ) { 34 | def searchOnlyPatternRegex(): Option[Regex] = searchOnlyPattern.filterNot(_.trim.isEmpty).map(_.r) 35 | def searchIgnoreMaskRegex(): Option[Regex] = searchIgnoreMask.filterNot(_.trim.isEmpty).map(_.r) 36 | } 37 | 38 | case class RenameRuleConfig( 39 | from: String, 40 | to: String 41 | ) { 42 | def rename(input: String): String = { 43 | if (input.matches(from)) { 44 | input.replaceAll(from, to) 45 | } else input 46 | } 47 | } 48 | 49 | case class PublishAdapterConfig( 50 | enabled: Boolean, 51 | kind: String, 52 | activationKeyword: String, 53 | apiEndPoint: String, 54 | overviewUUID: String, 55 | token: Option[String], 56 | defaultVisibility: Option[String], 57 | filenameRenameRules: Map[String, RenameRuleConfig] 58 | ) { 59 | def targetName = s"$kind/$activationKeyword" 60 | } 61 | 62 | // Automatically populated by the build process from a generated config file 63 | case class MetaConfig( 64 | projectName: Option[String], 65 | projectGroup: Option[String], 66 | projectPage: Option[String], 67 | projectCode: Option[String], 68 | buildVersion: Option[String], 69 | buildDateTime: Option[String], 70 | buildUUID: Option[String], 71 | contactEmail: Option[String] 72 | ) { 73 | def name: String = projectName.getOrElse("code-examples-manager") 74 | def code: String = projectName.getOrElse("cem") 75 | def version: String = buildVersion.getOrElse("x.y.z") 76 | def dateTime: String = buildDateTime.getOrElse("?") 77 | def uuid: String = buildUUID.getOrElse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") 78 | def projectURL: String = projectPage.getOrElse("https://github.com/dacr") 79 | def contact: String = contactEmail.getOrElse("crosson.david@gmail.com") 80 | } 81 | 82 | case class SummaryConfig( 83 | title: String 84 | ) 85 | 86 | case class CodeExampleManagerConfig( 87 | examples: ExamplesConfig, 88 | publishAdapters: Map[String, PublishAdapterConfig], 89 | metaInfo: MetaConfig, 90 | summary: SummaryConfig 91 | ) 92 | 93 | case class ApplicationConfig( 94 | codeExamplesManagerConfig: CodeExampleManagerConfig 95 | ) 96 | 97 | object ApplicationConfig { 98 | val config : Config[ApplicationConfig] = deriveConfig[ApplicationConfig].mapKey(toKebabCase) 99 | } 100 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/Execute.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem 2 | 3 | import zio.* 4 | import zio.lmdb.LMDB 5 | import zio.stream.* 6 | import zio.stream.ZPipeline.{splitLines, utf8Decode} 7 | import zio.process.* 8 | import zio.json.* 9 | import zio.ZIOAspect.* 10 | 11 | import java.util.concurrent.TimeUnit 12 | import java.time.OffsetDateTime 13 | import java.util.UUID 14 | import fr.janalyse.cem.model.* 15 | import zio.nio.file.Path 16 | import zio.Schedule.Decision 17 | 18 | case class RunFailure( 19 | message: String 20 | ) 21 | case class RunResults( 22 | command: List[String], 23 | exitCode: Int, 24 | output: String 25 | ) 26 | 27 | object Execute { 28 | val timeoutDuration = Duration(60, TimeUnit.SECONDS) 29 | val testStartDelay = Duration(500, TimeUnit.MILLISECONDS) 30 | val defaultParallelismLevel = 8 31 | 32 | def makeCommandProcess(command: List[String], workingDir: Path) = { 33 | val results = for { 34 | executable <- ZIO.from(command.headOption).orElseFail(RunFailure("Example command is invalid")) 35 | arguments = command.drop(1) 36 | process <- ZIO.acquireRelease( 37 | Command(executable, arguments*) 38 | .redirectErrorStream(true) 39 | .workingDirectory(workingDir.toFile) 40 | .run 41 | .mapError(err => RunFailure(s"Command error ${err.toString}")) 42 | )(process => process.killTreeForcibly.tapError(err => ZIO.logError(err.toString)).ignore) 43 | stream = process.stdout.stream.via(utf8Decode >>> splitLines) 44 | mayBeOutputLines <- stream.runCollect.disconnect 45 | .timeout(timeoutDuration) 46 | .mapError(err => RunFailure(s"Couldn't collect outputs\n${err.toString}")) 47 | outputText = mayBeOutputLines.map(chunks => chunks.mkString("\n")).getOrElse("") 48 | exitCode <- process.exitCode.mapError(err => RunFailure(outputText + "\n" + err.toString)) 49 | } yield RunResults(command, exitCode.code, outputText) 50 | 51 | ZIO.scoped(results) 52 | } 53 | 54 | def makeRunCommandProcess(example: CodeExample) = { 55 | for { 56 | exampleFilePath <- ZIO.fromOption(example.filepath).orElseFail(RunFailure("Example has no path for its content")) 57 | workingDir <- ZIO.fromOption(exampleFilePath.parent).orElseFail(RunFailure("Example file path content has no parent directory")) 58 | absoluteFileName <- exampleFilePath.toAbsolutePath.orElseFail(RunFailure("Example absolute file path error")) 59 | command <- ZIO 60 | .from( 61 | example.runWith 62 | .map(_.replaceAll("[$]scriptFile", absoluteFileName.toString)) 63 | .map(_.replaceAll("[$]file", absoluteFileName.toString)) 64 | .map(_.split("\\s+").toList) 65 | ) 66 | .orElseFail(RunFailure(s"Example ${example.uuid} as no run-with directive")) 67 | results <- makeCommandProcess(command, workingDir) @@ annotated("/run-command" -> command.mkString(" ")) 68 | } yield results 69 | } 70 | 71 | def makeTestCommandProcess(example: CodeExample) = { 72 | for { 73 | exampleFilePath <- ZIO.fromOption(example.filepath).orElseFail(RunFailure("Example has no path for its content")) 74 | workingDir <- ZIO.fromOption(exampleFilePath.parent).orElseFail(RunFailure("Example file path content has no parent directory")) 75 | command <- ZIO.succeed(example.testWith.getOrElse(s"sleep ${timeoutDuration.getSeconds()}").trim.split("\\s+").toList) 76 | results <- makeCommandProcess(command, workingDir) @@ annotated("/test-command" -> command.mkString(" ")) 77 | } yield results 78 | } 79 | 80 | def runExample(example: CodeExample, runSessionDate: OffsetDateTime, runSessionUUID: UUID) = { 81 | val result = 82 | for { 83 | startTimestamp <- Clock.currentDateTime 84 | runEffect = makeRunCommandProcess(example).disconnect 85 | .timeout(timeoutDuration) 86 | testEffect = makeTestCommandProcess(example) 87 | .filterOrElseWith(result => result.exitCode == 0)(result => ZIO.fail(RunFailure(s"test code is failing + ${result.output}"))) 88 | .retry( 89 | (Schedule.exponential(1.second) && Schedule.recurs(6)) 90 | .onDecision((state, out, decision) => 91 | decision match { 92 | case Decision.Done => ZIO.logError("No more retry attempt !") 93 | case Decision.Continue(interval) => ZIO.logWarning(s"Failed, will retry at ${interval.start}") 94 | } 95 | ) 96 | ) 97 | .disconnect 98 | .delay(testStartDelay) 99 | .timeout(timeoutDuration) 100 | results <- runEffect.raceFirst(testEffect).either 101 | duration <- Clock.instant.map(i => i.toEpochMilli - startTimestamp.toInstant.toEpochMilli) 102 | timeout = results.exists(_.isEmpty) 103 | output = results.toOption.flatten.map(_.output).getOrElse("") 104 | exitCodeOption = results.toOption.flatten.map(_.exitCode) 105 | success = exitCodeOption.exists(_ == 0) || (example.shouldFail && exitCodeOption.exists(_ != 0)) 106 | runState = if timeout then "timeout" else if success then "success" else "failure" 107 | _ <- if (results.isLeft) ZIO.logError(s"""Couldn't execute either run or test part\n${results.swap.toOption.getOrElse("")}""") else ZIO.succeed(()) 108 | _ <- if (!success) ZIO.logWarning(s"example run $runState\nFailed cause:\n$output") else ZIO.log("example run success") 109 | runStatus = RunStatus( 110 | example = example, 111 | exitCodeOption = exitCodeOption, 112 | // stdout = output, 113 | stdout = output.take(1024), // truncate the output as some scripts may generate a lot of data !!! 114 | startedTimestamp = startTimestamp, 115 | duration = duration, 116 | runSessionDate = runSessionDate, 117 | runSessionUUID = runSessionUUID, 118 | success = success, 119 | timeout = timeout, 120 | runState = runState 121 | ) 122 | _ <- upsertRunStatus(runStatus) 123 | } yield runStatus 124 | 125 | // ZIO.logAnnotate("/file", example.filename)(result) 126 | result 127 | } 128 | 129 | def runTestableExamples(runnableExamples: List[CodeExample], parallelism: Int) = { 130 | val execStrategy = ExecutionStrategy.ParallelN(parallelism) 131 | for { 132 | runSessionDate <- Clock.currentDateTime 133 | startEpoch <- Clock.instant.map(_.toEpochMilli) 134 | runSessionUUID = UUID.randomUUID() 135 | // runStatuses <- ZIO.foreachExec(runnableExamples)(execStrategy)(example => runExample(example, runSessionDate, runSessionUUID)) 136 | runStatuses <- ZIO.foreachExec(runnableExamples)(execStrategy) { example => 137 | ZIO.logSpan("/run") { 138 | runExample(example, runSessionDate, runSessionUUID) 139 | @@ annotated("/uuid" -> example.uuid.toString, "/file" -> example.filename) 140 | } 141 | } 142 | successes = runStatuses.filter(_.success) 143 | failures = runStatuses.filterNot(_.success) 144 | endEpoch <- Clock.instant.map(_.toEpochMilli) 145 | durationSeconds = (endEpoch - startEpoch) / 1000 146 | _ <- reportInLog(runStatuses, durationSeconds) 147 | } yield runStatuses 148 | } 149 | 150 | def reportInLog(results: List[RunStatus], durationSeconds: Long) = { 151 | val successes = results.filter(_.success) 152 | val failures = results.filterNot(_.success) 153 | for { 154 | _ <- if (failures.size > 0) 155 | ZIO.logError( 156 | failures // runStatuses 157 | .sortBy(s => (s.success, s.example.filepath.map(_.toString))) 158 | .map(state => s"""${if (state.success) "OK" else "KO"} : ${state.example.filepath.get} : ${state.example.summary.getOrElse("")}""") 159 | .mkString("\n", "\n", "") 160 | ) 161 | else ZIO.log("ALL examples executed with success :)") 162 | _ <- ZIO.log(s"${successes.size} successes") 163 | _ <- ZIO.log(s"${failures.size} failures") 164 | _ <- ZIO.log(s"${results.size} runnable examples (with scala-cli) in ${durationSeconds}s") 165 | 166 | } yield () 167 | } 168 | 169 | def upsertRunStatus(result: RunStatus) = { 170 | val collectionName = "run-statuses" 171 | val key = result.example.uuid.toString 172 | val examplePath = result.example.filepath.getOrElse(Path(result.example.filename)) 173 | for { 174 | collection <- LMDB 175 | .collectionGet[RunStatus](collectionName) 176 | .orElse(LMDB.collectionCreate[RunStatus](collectionName)) 177 | .mapError(th => ExampleStorageIssue(examplePath, s"Storage issue with collection $collectionName")) 178 | _ <- collection 179 | .upsertOverwrite(key, result) 180 | .mapError(th => ExampleStorageIssue(examplePath, s"Couldn't upsert anything in collection $collectionName")) 181 | } yield () 182 | } 183 | 184 | def executeEffect(keywords: Set[String] = Set.empty): ZIO[FileSystemService & LMDB, Throwable | ExampleIssue, List[RunStatus]] = ZIO.logSpan("/runs") { 185 | for { 186 | _ <- ZIO.log("Searching examples...") 187 | examples <- Synchronize.examplesCollect 188 | filteredExamples = examples.filter(example => keywords.isEmpty || example.keywords.intersect(keywords) == keywords) 189 | testableExamples = filteredExamples.filter(_.runWith.isDefined).filter(_.isTestable) 190 | (exclusiveRunnableExamples, runnableExamples) = testableExamples.partition(_.isExclusive) 191 | _ <- ZIO.log(s"Found ${examples.size} examples with ${testableExamples.size} marked as testable") 192 | startEpoch <- Clock.instant.map(_.toEpochMilli) 193 | exclusiveRunnableResultsFiber <- runTestableExamples(exclusiveRunnableExamples, 1).fork 194 | runnableResultsFiber <- runTestableExamples(runnableExamples, defaultParallelismLevel).fork 195 | exclusiveRunnableResults <- exclusiveRunnableResultsFiber.join 196 | runnableResults <- runnableResultsFiber.join 197 | endEpoch <- Clock.instant.map(_.toEpochMilli) 198 | durationSeconds = (endEpoch - startEpoch) / 1000 199 | results = exclusiveRunnableResults ++ runnableResults 200 | // _ <- ZIO.foreach(results)(result => upsertRunStatus(result)) 201 | _ <- reportInLog(results, durationSeconds) 202 | } yield results 203 | } 204 | 205 | } 206 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/FileSystemService.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem 2 | 3 | import fr.janalyse.cem.model.CodeExample 4 | import zio.* 5 | import zio.nio.charset.Charset 6 | import zio.nio.file.* 7 | import zio.stream.* 8 | 9 | import java.io.{File, IOException} 10 | import java.nio.file.attribute.BasicFileAttributes 11 | import scala.util.matching.Regex 12 | 13 | trait FileSystemService: 14 | def readFileContent(inputPath: Path): Task[String] 15 | def readFileLines(inputPath: Path, maxLines: Option[Int] = None): Task[List[String]] 16 | def searchFiles(searchRoot: Path, searchOnlyRegex: Option[Regex], ignoreMaskRegex: Option[Regex]): Task[List[Path]] 17 | 18 | object FileSystemService: 19 | def readFileContent(inputPath: Path): ZIO[FileSystemService, Throwable, String] = 20 | ZIO.serviceWithZIO(_.readFileContent(inputPath)) 21 | 22 | def readFileLines(inputPath: Path, maxLines: Option[Int] = None): ZIO[FileSystemService, Throwable, List[String]] = 23 | ZIO.serviceWithZIO(_.readFileLines(inputPath, maxLines)) 24 | 25 | def searchFiles(searchRoot: Path, searchOnlyRegex: Option[Regex], ignoreMaskRegex: Option[Regex]): ZIO[FileSystemService, Throwable, List[Path]] = 26 | ZIO.serviceWithZIO(_.searchFiles(searchRoot, searchOnlyRegex, ignoreMaskRegex)) 27 | 28 | def live = ZLayer.fromZIO( 29 | for { 30 | applicationConfig <- ZIO.config(ApplicationConfig.config) 31 | } yield FileSystemServiceImpl(applicationConfig) 32 | ) 33 | 34 | class FileSystemServiceImpl(applicationConfig: ApplicationConfig) extends FileSystemService: 35 | 36 | override def readFileContent(inputPath: Path): Task[String] = 37 | for 38 | charset <- ZIO.attempt(Charset.forName(applicationConfig.codeExamplesManagerConfig.examples.charEncoding)) 39 | content <- Files.readAllBytes(inputPath) 40 | yield String(content.toArray, charset.name) 41 | 42 | override def readFileLines(inputPath: Path, maxLines: Option[Int]): Task[List[String]] = 43 | for 44 | charset <- ZIO.attempt(Charset.forName(applicationConfig.codeExamplesManagerConfig.examples.charEncoding)) 45 | stream = Files.lines(inputPath, charset) 46 | selectedStream = maxLines.map(n => stream.take(n)).getOrElse(stream) 47 | lines <- selectedStream.runCollect 48 | yield lines.toList 49 | 50 | def searchPredicate(searchOnlyRegex: Option[Regex], ignoreMaskRegex: Option[Regex])(path: Path, attrs: BasicFileAttributes): Boolean = 51 | attrs.isRegularFile && 52 | (ignoreMaskRegex.isEmpty || ignoreMaskRegex.get.findFirstIn(path.toString).isEmpty) && 53 | (searchOnlyRegex.isEmpty || searchOnlyRegex.get.findFirstIn(path.toString).isDefined) 54 | 55 | override def searchFiles(searchRoot: Path, searchOnlyRegex: Option[Regex], ignoreRegex: Option[Regex]): Task[List[Path]] = 56 | for { 57 | searchPath <- ZIO.attempt(searchRoot) 58 | stream = Files.find(searchPath, 10)(searchPredicate(searchOnlyRegex, ignoreRegex)) 59 | foundFiles <- stream.runCollect 60 | } yield foundFiles.toList 61 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/Main.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem 2 | 3 | import zio.* 4 | import zio.config.* 5 | import zio.config.typesafe.* 6 | import zio.config.magnolia.* 7 | import zio.logging.{ConsoleLoggerConfig, LogFormat, consoleLogger} 8 | import zio.logging.slf4j.bridge.Slf4jBridge 9 | import sttp.client3.asynchttpclient.zio.AsyncHttpClientZioBackend 10 | import zio.Runtime.removeDefaultLoggers 11 | import zio.lmdb.{LMDB, LMDBConfig} 12 | 13 | object Main extends ZIOAppDefault { 14 | 15 | private def loadTypesafeBasedConfigData(configFileOption: Option[String]) = { 16 | import com.typesafe.config.{Config, ConfigFactory} 17 | val metaDataConfig = ConfigFactory.load("cem-meta.conf") // TODO - TO REMOVE - very old stuff 18 | val applicationConfig = configFileOption 19 | .map(f => ConfigFactory.parseFile(new java.io.File(f))) 20 | .getOrElse(ConfigFactory.load()) 21 | ConfigFactory 22 | .empty() 23 | .withFallback(applicationConfig) 24 | .withFallback(metaDataConfig) 25 | .resolve() 26 | } 27 | 28 | val configProviderLogic = for { 29 | configFileEnvOption <- System.env("CEM_CONFIG_FILE") 30 | configFilePropOption <- System.property("CEM_CONFIG_FILE") 31 | configFileOption = configFileEnvOption.orElse(configFilePropOption) 32 | tconfig <- ZIO.attempt(loadTypesafeBasedConfigData(configFileOption)) 33 | configProvider = ConfigProvider.fromTypesafeConfig(tconfig) 34 | } yield { 35 | configProvider 36 | } 37 | 38 | val configLayer = ZLayer.fromZIO(configProviderLogic.map(provider => Runtime.setConfigProvider(provider))).flatten 39 | 40 | override val bootstrap = configLayer ++ ( removeDefaultLoggers >>> configLayer >>> consoleLogger() >>> Slf4jBridge.initialize) 41 | 42 | val httpClientLayer = AsyncHttpClientZioBackend.layer() 43 | 44 | override def run = 45 | getArgs 46 | .flatMap(args => 47 | val chosenBehavior = args.toList match { 48 | case "run" :: keywords => 49 | Execute 50 | .executeEffect(keywords.toSet) 51 | .flatMap(status => ZIO.cond(status.forall(_.success), "All examples are successful", "Some examples has failed")) 52 | .flatMap(message => ZIO.log(message)) 53 | 54 | case "version" :: _ => 55 | Synchronize.versionEffect 56 | .flatMap(versionInfo => Console.printLine(versionInfo)) 57 | 58 | case "stats" :: _ => 59 | Synchronize.examplesCollect 60 | .flatMap(examples => Synchronize.statsEffect(examples)) 61 | .flatMap(statsInfo => Console.printLine(statsInfo)) 62 | 63 | case "publish" :: _ | _ => 64 | Synchronize.synchronizeEffect.unit 65 | } 66 | chosenBehavior.unit.provide(httpClientLayer, FileSystemService.live, LMDB.live, Scope.default) 67 | ) 68 | } 69 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/RemoteGithubOperations.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem 17 | 18 | import zio.* 19 | import zio.json.* 20 | import zio.json.ast.{Json, JsonCursor} 21 | import zio.json.ast.Json.* 22 | import zio.logging.* 23 | import zio.json.{DeriveJsonDecoder, DeriveJsonEncoder, JsonDecoder, JsonEncoder} 24 | import sttp.model.Uri 25 | import sttp.client3.* 26 | import sttp.client3.SttpBackend 27 | import sttp.client3.ziojson.* 28 | import java.util.UUID 29 | 30 | import fr.janalyse.cem.model.* 31 | import fr.janalyse.cem.model.WhatToDo.* 32 | import fr.janalyse.cem.tools.DescriptionTools 33 | import fr.janalyse.cem.tools.DescriptionTools.remoteExampleFileRename 34 | import fr.janalyse.cem.tools.HttpTools.{uriParse, webLinkingExtractNext} 35 | 36 | object RemoteGithubOperations { 37 | type SttpClient = SttpBackend[Task, Any] 38 | 39 | case class GithubUser( 40 | login: String, // user name in APIs 41 | name: String, 42 | id: Int, 43 | public_gists: Int, 44 | private_gists: Int, 45 | followers: Int, 46 | following: Int 47 | ) 48 | object GithubUser { 49 | implicit val decoder: JsonDecoder[GithubUser] = DeriveJsonDecoder.gen 50 | implicit val encoder: JsonEncoder[GithubUser] = DeriveJsonEncoder.gen 51 | } 52 | 53 | case class GistFileInfo( 54 | filename: String, 55 | `type`: String, 56 | language: Option[String], 57 | raw_url: String, 58 | size: Int 59 | ) 60 | object GistFileInfo { 61 | implicit val decoder: JsonDecoder[GistFileInfo] = DeriveJsonDecoder.gen 62 | implicit val encoder: JsonEncoder[GistFileInfo] = DeriveJsonEncoder.gen 63 | } 64 | 65 | case class GistInfo( 66 | id: String, 67 | description: String, 68 | html_url: String, 69 | public: Boolean, 70 | files: Map[String, GistFileInfo] 71 | ) 72 | object GistInfo { 73 | implicit val decoder: JsonDecoder[GistInfo] = DeriveJsonDecoder.gen 74 | implicit val encoder: JsonEncoder[GistInfo] = DeriveJsonEncoder.gen 75 | } 76 | 77 | case class GistCreateResponse( 78 | id: String, 79 | html_url: String 80 | ) 81 | object GistCreateResponse { 82 | implicit val decoder: JsonDecoder[GistCreateResponse] = DeriveJsonDecoder.gen 83 | implicit val encoder: JsonEncoder[GistCreateResponse] = DeriveJsonEncoder.gen 84 | } 85 | 86 | case class GistUpdateResponse( 87 | id: String, 88 | html_url: String 89 | ) 90 | object GistUpdateResponse { 91 | implicit val decoder: JsonDecoder[GistUpdateResponse] = DeriveJsonDecoder.gen 92 | implicit val encoder: JsonEncoder[GistUpdateResponse] = DeriveJsonEncoder.gen 93 | } 94 | 95 | def githubInjectAuthToken[A, B](request: Request[A, B], tokenOption: Option[String]) = { 96 | val base = request.header("Accept", "application/vnd.github.v3+json") 97 | tokenOption.fold(base)(token => base.header("Authorization", s"token $token")) 98 | } 99 | 100 | def githubUser(adapterConfig: PublishAdapterConfig): RIO[SttpClient, GithubUser] = { 101 | import adapterConfig.apiEndPoint 102 | for { 103 | backend <- ZIO.service[SttpBackend[Task, Any]] 104 | apiURI <- uriParse(s"$apiEndPoint/user") 105 | query = basicRequest.get(apiURI).response(asJson[GithubUser]) 106 | responseBody <- backend.send(githubInjectAuthToken(query, adapterConfig.token)).map(_.body).absolve 107 | } yield responseBody 108 | } 109 | 110 | def githubRemoteExamplesStatesFetch(adapterConfig: PublishAdapterConfig): RIO[SttpClient, Iterable[RemoteExampleState]] = { 111 | 112 | def worker(uri: Uri): RIO[SttpClient, Iterable[GistInfo]] = { 113 | for { 114 | backend <- ZIO.service[SttpBackend[Task, Any]] 115 | query = basicRequest.get(uri).response(asJson[Vector[GistInfo]]) 116 | _ <- ZIO.log(s"${adapterConfig.targetName} : Fetching from $uri") 117 | response <- backend.send(githubInjectAuthToken(query, adapterConfig.token)) 118 | gists <- ZIO.fromEither(response.body) 119 | nextLinkOption = response.header("Link").flatMap(webLinkingExtractNext) 120 | nextUriOption <- ZIO.foreach(nextLinkOption)(link => uriParse(link)) 121 | nextGists <- ZIO.foreach(nextUriOption)(uri => worker(uri)).map(_.getOrElse(Iterable.empty)) 122 | } yield gists ++ nextGists 123 | } 124 | 125 | val perPage = 100 126 | for { 127 | userLogin <- githubUser(adapterConfig).map(_.login) 128 | link = s"${adapterConfig.apiEndPoint}/users/$userLogin/gists?page=1&per_page=$perPage" 129 | uri <- uriParse(link) 130 | gists <- worker(uri).retry(Schedule.exponential(100.millis, 2).jittered && Schedule.recurs(6)) 131 | } yield githubRemoteGistsToRemoteExampleState(gists) 132 | } 133 | 134 | def githubRemoteGistsToRemoteExampleState(gists: Iterable[GistInfo]): Iterable[RemoteExampleState] = { 135 | for { 136 | gist <- gists 137 | desc = gist.description 138 | (uuid, hash) <- DescriptionTools.extractMetaDataFromDescription(desc) 139 | url = gist.html_url 140 | files = gist.files 141 | } yield { 142 | RemoteExampleState( 143 | remoteId = gist.id, 144 | description = desc, 145 | url = url, 146 | files = files.keys.toList, 147 | uuid = UUID.fromString(uuid), // TODO FIX IT AS IT CAN FAIL !!!!!! 148 | hash = hash 149 | ) 150 | } 151 | } 152 | 153 | def buildAddRequestBody(adapterConfig: PublishAdapterConfig, todo: AddExample, description: String): Json = { 154 | val remoteMainFilename = remoteExampleFileRename(todo.example.filename, adapterConfig) 155 | val publicBool = adapterConfig.defaultVisibility.map(_.trim.toLowerCase == "public").getOrElse(true) 156 | 157 | val files: List[(String, Json)] = List( 158 | remoteMainFilename -> Obj( 159 | "filename" -> Str(remoteMainFilename), 160 | "content" -> Str(todo.example.content) 161 | ) 162 | ) ++ todo.example.attachments.map { case (filename, content) => 163 | val remoteFilename = remoteExampleFileRename(filename, adapterConfig) 164 | remoteFilename -> Obj( 165 | "filename" -> Str(remoteFilename), 166 | "content" -> Str(content) 167 | ) 168 | } 169 | 170 | Obj( 171 | "description" -> Str(description), 172 | "public" -> Bool(publicBool), 173 | "files" -> Obj(files*) 174 | ) 175 | } 176 | 177 | def githubRemoteExampleAdd(adapterConfig: PublishAdapterConfig, todo: AddExample): RIO[SttpClient, RemoteExample] = { 178 | for { 179 | backend <- ZIO.service[SttpBackend[Task, Any]] 180 | apiURI <- uriParse(s"${adapterConfig.apiEndPoint}/gists") 181 | example = todo.example 182 | description <- ZIO.getOrFail(DescriptionTools.makeDescription(example)) 183 | requestBody = buildAddRequestBody(adapterConfig, todo, description) 184 | query = basicRequest.post(apiURI).body(requestBody).response(asJson[GistCreateResponse]) 185 | response <- backend.send(githubInjectAuthToken(query, adapterConfig.token)).map(_.body).absolve 186 | id = response.id 187 | url = response.html_url 188 | _ <- ZIO.log(s"""${adapterConfig.targetName} : ADDED ${todo.uuid} - ${example.summary.getOrElse("")} - $url""") 189 | } yield RemoteExample( 190 | todo.example, 191 | RemoteExampleState( 192 | remoteId = id, 193 | description = description, 194 | url = url, 195 | files = List(todo.example.filename) ++ todo.example.attachments.keys, 196 | uuid = todo.uuid, 197 | hash = example.hash 198 | ) 199 | ) 200 | } 201 | 202 | def buildUpdateRequestBody(adapterConfig: PublishAdapterConfig, todo: UpdateRemoteExample, description: String): Json = { 203 | val remoteFiles = todo.state.files.toSet 204 | val remoteMainFilename = remoteExampleFileRename(todo.example.filename, adapterConfig) 205 | val remoteOrphanFiles = (remoteFiles - remoteMainFilename) -- ( 206 | todo.example.attachments.keys.map(f => remoteExampleFileRename(f, adapterConfig)) 207 | ) 208 | 209 | val files: List[(String, Json)] = List( 210 | remoteMainFilename -> Obj( 211 | "filename" -> Str(remoteMainFilename), 212 | "content" -> Str(todo.example.content) 213 | ) 214 | ) ++ todo.example.attachments.map { case (filename, content) => 215 | val remoteFilename = remoteExampleFileRename(filename, adapterConfig) 216 | remoteFilename -> Obj( 217 | "filename" -> Str(remoteFilename), 218 | "content" -> Str(content) 219 | ) 220 | } ++ remoteOrphanFiles.map(name => name -> Obj()) 221 | 222 | Obj( 223 | "description" -> Str(description), 224 | "files" -> Obj(files*) 225 | ) 226 | } 227 | 228 | def githubRemoteExampleUpdate(adapterConfig: PublishAdapterConfig, todo: UpdateRemoteExample): RIO[SttpClient, RemoteExample] = { 229 | for { 230 | backend <- ZIO.service[SttpBackend[Task, Any]] 231 | gistId = todo.state.remoteId 232 | apiURI <- uriParse(s"${adapterConfig.apiEndPoint}/gists/$gistId") 233 | example = todo.example 234 | description <- ZIO.getOrFail(DescriptionTools.makeDescription(example)) 235 | requestBody = buildUpdateRequestBody(adapterConfig, todo, description) 236 | query = basicRequest.post(apiURI).body(requestBody).response(asJson[GistUpdateResponse]) 237 | authedQuery = githubInjectAuthToken(query, adapterConfig.token) 238 | response <- backend.send(authedQuery).map(_.body).absolve 239 | id = response.id 240 | url = response.html_url 241 | _ <- ZIO.log(s"""${adapterConfig.targetName} : UPDATED ${todo.uuid} - ${example.summary.getOrElse("")} - $url""") 242 | } yield RemoteExample( 243 | todo.example, 244 | RemoteExampleState( 245 | remoteId = id, 246 | description = description, 247 | url = url, 248 | files = List(todo.example.filename) ++ todo.example.attachments.keys, 249 | uuid = todo.uuid, 250 | hash = example.hash 251 | ) 252 | ) 253 | } 254 | 255 | def githubRemoteExampleChangesApply(adapterConfig: PublishAdapterConfig)(todo: WhatToDo): RIO[SttpClient, Option[RemoteExample]] = { 256 | todo match { 257 | case _: UnsupportedOperation => ZIO.succeed(None) 258 | case _: OrphanRemoteExample => ZIO.succeed(None) 259 | case _: DeleteRemoteExample => ZIO.succeed(None) // TODO - Add support for delete operation 260 | case KeepRemoteExample(uuid, example, state) => ZIO.succeed(Some(RemoteExample(example, state))) 261 | case exampleTODO: UpdateRemoteExample => githubRemoteExampleUpdate(adapterConfig, exampleTODO).asSome 262 | case exampleTODO: AddExample => githubRemoteExampleAdd(adapterConfig, exampleTODO).asSome 263 | } 264 | } 265 | 266 | def githubRemoteExamplesChangesApply(adapterConfig: PublishAdapterConfig, todos: Iterable[WhatToDo]): RIO[SttpClient, Iterable[RemoteExample]] = { 267 | ZIO.foreach(todos)(githubRemoteExampleChangesApply(adapterConfig)).map(_.flatten) 268 | } 269 | 270 | } 271 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/RemoteGitlabOperations.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem 17 | 18 | import zio.* 19 | import zio.json.{DeriveJsonDecoder, DeriveJsonEncoder, JsonDecoder, JsonEncoder} 20 | import sttp.model.Uri 21 | import sttp.client3.* 22 | import sttp.client3.SttpBackend 23 | import sttp.client3.ziojson.* 24 | import java.util.UUID 25 | 26 | import fr.janalyse.cem.model.* 27 | import fr.janalyse.cem.model.WhatToDo.* 28 | import fr.janalyse.cem.tools.DescriptionTools 29 | import fr.janalyse.cem.tools.DescriptionTools.remoteExampleFileRename 30 | import fr.janalyse.cem.tools.HttpTools.{uriParse, webLinkingExtractNext} 31 | 32 | import java.time.OffsetDateTime 33 | 34 | object RemoteGitlabOperations { 35 | type SttpClient = SttpBackend[Task, Any] 36 | 37 | case class SnippetAuthor( 38 | id: Long, 39 | name: String, 40 | username: String, 41 | state: String, 42 | avatar_url: Option[String], 43 | web_url: Option[String] 44 | ) 45 | 46 | object SnippetAuthor { 47 | implicit val decoder: JsonDecoder[SnippetAuthor] = DeriveJsonDecoder.gen 48 | implicit val encoder: JsonEncoder[SnippetAuthor] = DeriveJsonEncoder.gen 49 | } 50 | 51 | case class SnippetFileInfo( 52 | path: String, 53 | raw_url: String 54 | ) 55 | 56 | object SnippetFileInfo { 57 | implicit val decoder: JsonDecoder[SnippetFileInfo] = DeriveJsonDecoder.gen 58 | implicit val encoder: JsonEncoder[SnippetFileInfo] = DeriveJsonEncoder.gen 59 | } 60 | 61 | case class SnippetInfo( 62 | id: Long, 63 | title: String, 64 | file_name: String, 65 | files: List[SnippetFileInfo], 66 | description: String, 67 | visibility: String, 68 | author: SnippetAuthor, 69 | updated_at: OffsetDateTime, 70 | created_at: OffsetDateTime, 71 | web_url: String, 72 | raw_url: String 73 | ) 74 | object SnippetInfo { 75 | implicit val decoder: JsonDecoder[SnippetInfo] = DeriveJsonDecoder.gen 76 | implicit val encoder: JsonEncoder[SnippetInfo] = DeriveJsonEncoder.gen 77 | } 78 | 79 | case class SnippetFileAdd( 80 | file_path: String, 81 | content: String 82 | ) 83 | 84 | object SnippetFileAdd { 85 | implicit val decoder: JsonDecoder[SnippetFileAdd] = DeriveJsonDecoder.gen 86 | implicit val encoder: JsonEncoder[SnippetFileAdd] = DeriveJsonEncoder.gen 87 | } 88 | 89 | case class SnippetAddRequest( 90 | title: Option[String], 91 | description: String, 92 | visibility: String, 93 | files: List[SnippetFileAdd] 94 | ) 95 | 96 | object SnippetAddRequest { 97 | implicit val decoder: JsonDecoder[SnippetAddRequest] = DeriveJsonDecoder.gen 98 | implicit val encoder: JsonEncoder[SnippetAddRequest] = DeriveJsonEncoder.gen 99 | } 100 | 101 | case class SnippetFileChange( 102 | action: String, // "create" | "update" | "delete" | "move" 103 | file_path: Option[String], 104 | previous_path: Option[String], 105 | content: Option[String] 106 | ) 107 | 108 | object SnippetFileChange { 109 | implicit val decoder: JsonDecoder[SnippetFileChange] = DeriveJsonDecoder.gen 110 | implicit val encoder: JsonEncoder[SnippetFileChange] = DeriveJsonEncoder.gen 111 | } 112 | 113 | case class SnippetUpdateRequest( 114 | id: String, 115 | title: Option[String], 116 | description: String, 117 | visibility: String, 118 | files: List[SnippetFileChange] 119 | ) 120 | 121 | object SnippetUpdateRequest { 122 | implicit val decoder: JsonDecoder[SnippetUpdateRequest] = DeriveJsonDecoder.gen 123 | implicit val encoder: JsonEncoder[SnippetUpdateRequest] = DeriveJsonEncoder.gen 124 | } 125 | 126 | def gitlabInjectAuthToken[A, B](request: Request[A, B], tokenOption: Option[String]) = { 127 | val base = request.header("Content-Type", "application/json") 128 | tokenOption.fold(base)(token => base.header("Authorization", s"Bearer $token")) 129 | } 130 | 131 | def gitlabRemoteExamplesStatesFetch(adapterConfig: PublishAdapterConfig): RIO[SttpClient, Iterable[RemoteExampleState]] = { 132 | import adapterConfig.apiEndPoint 133 | 134 | def worker(uri: Uri): RIO[SttpClient, Iterable[SnippetInfo]] = { 135 | for { 136 | backend <- ZIO.service[SttpBackend[Task, Any]] 137 | query = basicRequest.get(uri).response(asJson[Vector[SnippetInfo]]) 138 | _ <- ZIO.log(s"${adapterConfig.targetName} : Fetching from $uri") 139 | response <- backend.send(gitlabInjectAuthToken(query, adapterConfig.token)) 140 | snippets <- ZIO.fromEither(response.body) 141 | nextLinkOption = response.header("Link").flatMap(webLinkingExtractNext) 142 | nextUriOption <- ZIO.foreach(nextLinkOption)(link => uriParse(link)) 143 | nextSnippets <- ZIO.foreach(nextUriOption)(uri => worker(uri)).map(_.getOrElse(Iterable.empty)) 144 | } yield snippets ++ nextSnippets 145 | } 146 | 147 | val perPage = 100 148 | for { 149 | uri <- uriParse(s"$apiEndPoint/snippets?page=1&per_page=$perPage") 150 | snippets <- worker(uri).retry(Schedule.exponential(100.millis, 2).jittered && Schedule.recurs(6)) 151 | } yield gitlabRemoteGistsToRemoteExampleState(snippets) 152 | } 153 | 154 | def gitlabRemoteGistsToRemoteExampleState(snippets: Iterable[SnippetInfo]): Iterable[RemoteExampleState] = { 155 | for { 156 | snippet <- snippets 157 | desc = snippet.description 158 | (uuid, hash) <- DescriptionTools.extractMetaDataFromDescription(desc) 159 | url = snippet.web_url 160 | filename = snippet.file_name 161 | files = snippet.files 162 | } yield { 163 | RemoteExampleState( 164 | remoteId = snippet.id.toString, 165 | description = desc, 166 | url = url, 167 | files = files.map(_.path), 168 | uuid = UUID.fromString(uuid), 169 | hash = hash 170 | ) 171 | } 172 | } 173 | 174 | def buildAddRequestBody(adapterConfig: PublishAdapterConfig, todo: AddExample, description: String): SnippetAddRequest = SnippetAddRequest( 175 | title = todo.example.summary, 176 | description = description, 177 | visibility = adapterConfig.defaultVisibility.getOrElse("public"), 178 | files = List( 179 | SnippetFileAdd( 180 | file_path = remoteExampleFileRename(todo.example.filename, adapterConfig), 181 | content = todo.example.content 182 | ) 183 | ) ++ todo.example.attachments.map { case (filename, content) => 184 | SnippetFileAdd( 185 | file_path = remoteExampleFileRename(filename, adapterConfig), 186 | content = content 187 | ) 188 | } 189 | ) 190 | 191 | def gitlabRemoteExampleAdd(adapterConfig: PublishAdapterConfig, todo: AddExample): RIO[SttpClient, RemoteExample] = { 192 | for { 193 | backend <- ZIO.service[SttpBackend[Task, Any]] 194 | apiURI <- uriParse(s"${adapterConfig.apiEndPoint}/snippets") 195 | example = todo.example 196 | description <- ZIO.getOrFail(DescriptionTools.makeDescription(example)) 197 | requestBody = buildAddRequestBody(adapterConfig, todo, description) 198 | query = basicRequest.post(apiURI).body(requestBody).response(asJson[SnippetInfo]) 199 | response <- backend.send(gitlabInjectAuthToken(query, adapterConfig.token)).map(_.body).absolve 200 | id = response.id.toString 201 | url = response.web_url 202 | _ <- ZIO.log(s"""${adapterConfig.targetName} : ADDED ${todo.uuid} - ${example.summary.getOrElse("")} - $url""") 203 | } yield RemoteExample( 204 | todo.example, 205 | RemoteExampleState( 206 | remoteId = id, 207 | description = description, 208 | url = url, 209 | files = List(todo.example.filename) ++ todo.example.attachments.keys, 210 | uuid = todo.uuid, 211 | hash = example.hash 212 | ) 213 | ) 214 | } 215 | 216 | def buildUpdateRequestBody(adapterConfig: PublishAdapterConfig, todo: UpdateRemoteExample, description: String): SnippetUpdateRequest = { 217 | val remoteFiles = todo.state.files.toSet 218 | val remoteMainFilename = remoteExampleFileRename(todo.example.filename, adapterConfig) 219 | val remoteOrphanFiles = (remoteFiles - remoteMainFilename) -- ( 220 | todo.example.attachments.keys.map(f => remoteExampleFileRename(f, adapterConfig)) 221 | ) 222 | 223 | val updatedFiles = List( 224 | SnippetFileChange( 225 | action = if (remoteFiles.contains(remoteMainFilename)) "update" else "create", 226 | file_path = Some(remoteMainFilename), 227 | previous_path = None, 228 | content = Some(todo.example.content) 229 | ) 230 | ) ++ todo.example.attachments.map { case (filename, content) => 231 | val remoteFilename = remoteExampleFileRename(filename, adapterConfig) 232 | SnippetFileChange( 233 | action = if (remoteFiles.contains(remoteFilename)) "update" else "create", 234 | file_path = Some(remoteFilename), 235 | previous_path = None, 236 | content = Some(content) 237 | ) 238 | } ++ remoteOrphanFiles.map(name => 239 | SnippetFileChange( 240 | action = "delete", 241 | file_path = Some(name), 242 | previous_path = None, 243 | content = None 244 | ) 245 | ) 246 | 247 | SnippetUpdateRequest( 248 | id = todo.state.remoteId, 249 | title = todo.example.summary, 250 | description = description, 251 | visibility = adapterConfig.defaultVisibility.getOrElse("public"), 252 | files = updatedFiles 253 | ) 254 | } 255 | 256 | def gitlabRemoteExampleUpdate(adapterConfig: PublishAdapterConfig, todo: UpdateRemoteExample): RIO[SttpClient, RemoteExample] = { 257 | for { 258 | backend <- ZIO.service[SttpBackend[Task, Any]] 259 | snippetId = todo.state.remoteId 260 | apiURI <- uriParse(s"${adapterConfig.apiEndPoint}/snippets/$snippetId") 261 | example = todo.example 262 | description <- ZIO.getOrFail(DescriptionTools.makeDescription(example)) 263 | requestBody = buildUpdateRequestBody(adapterConfig, todo, description) 264 | query = basicRequest.put(apiURI).body(requestBody).response(asJson[SnippetInfo]) 265 | authedQuery = gitlabInjectAuthToken(query, adapterConfig.token) 266 | response <- backend.send(authedQuery).map(_.body).absolve 267 | id = response.id.toString 268 | url = response.web_url 269 | _ <- ZIO.log(s"""${adapterConfig.targetName} : UPDATED ${todo.uuid} - ${example.summary.getOrElse("")} - $url""") 270 | } yield RemoteExample( 271 | todo.example, 272 | RemoteExampleState( 273 | remoteId = id, 274 | description = description, 275 | url = url, 276 | files = List(todo.example.filename) ++ todo.example.attachments.keys, 277 | uuid = todo.uuid, 278 | hash = example.hash 279 | ) 280 | ) 281 | } 282 | 283 | def gitlabRemoteExampleChangesApply(adapterConfig: PublishAdapterConfig)(todo: WhatToDo): RIO[SttpClient, Option[RemoteExample]] = { 284 | todo match { 285 | case _: UnsupportedOperation => ZIO.succeed(None) 286 | case _: OrphanRemoteExample => ZIO.succeed(None) 287 | case _: DeleteRemoteExample => ZIO.succeed(None) // TODO - Add support for delete operation 288 | case KeepRemoteExample(uuid, example, state) => ZIO.succeed(Some(RemoteExample(example, state))) 289 | case exampleTODO: UpdateRemoteExample => gitlabRemoteExampleUpdate(adapterConfig, exampleTODO).asSome 290 | case exampleTODO: AddExample => gitlabRemoteExampleAdd(adapterConfig, exampleTODO).asSome 291 | } 292 | } 293 | 294 | def gitlabRemoteExamplesChangesApply(adapterConfig: PublishAdapterConfig, todos: Iterable[WhatToDo]): RIO[SttpClient, Iterable[RemoteExample]] = { 295 | ZIO.foreach(todos)(gitlabRemoteExampleChangesApply(adapterConfig)).map(_.flatten) 296 | } 297 | 298 | } 299 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/RemoteOperations.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem 17 | 18 | import zio.* 19 | import sttp.client3.SttpBackend 20 | import fr.janalyse.cem.model.* 21 | import fr.janalyse.cem.model.WhatToDo.* 22 | 23 | object RemoteOperations { 24 | type SttpClient = SttpBackend[Task, Any] 25 | 26 | def remoteExampleStatesFetch(adapterConfig: PublishAdapterConfig): RIO[SttpClient, Iterable[RemoteExampleState]] = { 27 | for { 28 | _ <- ZIO.log(s"${adapterConfig.targetName} : Fetching already published examples") 29 | states <- 30 | if (adapterConfig.kind == "github") RemoteGithubOperations.githubRemoteExamplesStatesFetch(adapterConfig) 31 | else if (adapterConfig.kind == "gitlab") RemoteGitlabOperations.gitlabRemoteExamplesStatesFetch(adapterConfig) 32 | else ZIO.fail(new Exception(s"${adapterConfig.targetName} : Unsupported adapter kind ${adapterConfig.kind}")) 33 | } yield states 34 | } 35 | 36 | def remoteExamplesChangesApply(adapterConfig: PublishAdapterConfig, todos: Iterable[WhatToDo]): RIO[SttpClient, Iterable[RemoteExample]] = { 37 | for { 38 | //_ <- ZIO.log(s"${adapterConfig.targetName} : Applying changes") 39 | //_ <- ZIO.log(s"${adapterConfig.targetName} : To add count ${todos.count(_.isInstanceOf[AddExample])}") 40 | //_ <- ZIO.log(s"${adapterConfig.targetName} : To update count ${todos.count(_.isInstanceOf[UpdateRemoteExample])}") 41 | //_ <- ZIO.log(s"${adapterConfig.targetName} : To keep count ${todos.count(_.isInstanceOf[KeepRemoteExample])}") 42 | remoteExamples <- 43 | if (adapterConfig.kind == "github") RemoteGithubOperations.githubRemoteExamplesChangesApply(adapterConfig, todos) 44 | else if (adapterConfig.kind == "gitlab") RemoteGitlabOperations.gitlabRemoteExamplesChangesApply(adapterConfig, todos) 45 | else ZIO.fail(new Exception(s"${adapterConfig.targetName} : Unsupported adapter kind ${adapterConfig.kind}")) 46 | } yield remoteExamples 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/Synchronize.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem 17 | 18 | import zio.* 19 | import zio.stream.* 20 | import zio.config.* 21 | import zio.nio.file.* 22 | import zio.nio.charset.Charset 23 | import zio.lmdb.LMDB 24 | 25 | import java.nio.file.attribute.BasicFileAttributes 26 | import java.nio.file.{FileVisitOption, OpenOption, StandardOpenOption} 27 | import java.util.UUID 28 | import zio.nio.file.Files 29 | import sttp.client3.asynchttpclient.zio.AsyncHttpClientZioBackend 30 | import sttp.client3.SttpBackend 31 | import fr.janalyse.cem.model.* 32 | import fr.janalyse.cem.model.WhatToDo.* 33 | 34 | import scala.util.Success 35 | import scala.util.matching.Regex 36 | 37 | object Synchronize { 38 | type SttpClient = SttpBackend[Task, Any] 39 | type SearchRoot = Path 40 | 41 | def findExamplesFromSearchRoot( 42 | searchRoot: SearchRoot, 43 | searchOnlyRegex: Option[Regex], 44 | ignoreMaskRegex: Option[Regex] 45 | ): ZIO[FileSystemService & LMDB, Throwable, List[Either[ExampleIssue, CodeExample]]] = { 46 | for { 47 | foundFiles <- FileSystemService.searchFiles(searchRoot, searchOnlyRegex, ignoreMaskRegex) 48 | examples <- ZIO.foreach(foundFiles)(path => CodeExample.buildFromFile(path, searchRoot).either) 49 | } yield examples 50 | } 51 | 52 | def examplesValidSearchRoots(searchRootDirectories: String): RIO[Any, List[SearchRoot]] = { 53 | for { 54 | roots <- ZIO.attempt(searchRootDirectories.split("""\s*,\s*""").toList.map(_.trim).map(r => Path(r))) 55 | validRoots <- ZIO.filter(roots)(root => Files.isDirectory(root)) 56 | } yield validRoots 57 | } 58 | 59 | def examplesCollectFor(searchRoots: List[SearchRoot]): ZIO[FileSystemService & LMDB, ExampleIssue | Throwable, List[CodeExample]] = { 60 | for { 61 | examplesConfig <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.examples) 62 | searchOnlyPattern <- ZIO.attempt(examplesConfig.searchOnlyPatternRegex()) 63 | searchIgnorePattern <- ZIO.attempt(examplesConfig.searchIgnoreMaskRegex()) 64 | foundExamplesList <- ZIO.foreach(searchRoots)(fromRoot => findExamplesFromSearchRoot(fromRoot, searchOnlyPattern, searchIgnorePattern)) 65 | foundExamples = foundExamplesList.flatten 66 | validExamples = foundExamples.collect { case Right(example) => example } 67 | invalidExamples = foundExamples.collect { case Left(example) => example } 68 | _ <- ZIO.foreach(invalidExamples)(issue => ZIO.logWarning(issue.toString)) 69 | _ <- examplesCheckCoherency(foundExamples) 70 | } yield validExamples 71 | } 72 | 73 | def examplesCheckCoherency(candidates: List[Either[ExampleIssue, CodeExample]]): Task[Unit] = { 74 | val examples = candidates.collect { case Right(example) => example } 75 | val invalidExamples = candidates.collect { case Left(issue) => issue } 76 | val invalidContent = invalidExamples.collect { case issue: ExampleContentIssue => issue } 77 | val groupedInvalid = invalidExamples.groupBy(_.getClass.getName) 78 | val publishable = examples.filter(_.isPublishable) 79 | val duplicatedUUIDs = examples.groupBy(_.uuid).filter((_, found) => found.size > 1) 80 | val duplicatedSummaries = examples.groupBy(_.summary).collect { case (summary, examples) if summary.isDefined && examples.size > 1 => summary.get -> examples } 81 | val testable = examples.filter(example => example.isTestable && example.runWith.isDefined) 82 | val publishTargets = examples.flatMap(_.publish).distinct 83 | val missingRunWith = examples.filter(ex => ex.isTestable && ex.runWith.isEmpty) 84 | val examplesByPublishTargets = 85 | publishTargets 86 | .map(target => target -> publishable.filter(_.publish.contains(target))) 87 | .groupMapReduce { case (target, v) => target } { case (target, v) => v.toSet } { case (l, r) => r ++ l } 88 | for { 89 | _ <- ZIO.logInfo(s"found ${candidates.size} examples candidates") 90 | _ <- ZIO.logInfo(s"found ${examples.size} valid examples") 91 | _ <- ZIO.logInfo(s"found ${publishTargets.size} publishing targets : ${publishTargets.mkString(", ")}") 92 | _ <- ZIO.logInfo(s"found ${examplesByPublishTargets.map((k, v) => s"$k:${v.size}").mkString(", ")} examples by targets") 93 | _ <- ZIO.logInfo(s"found ${duplicatedUUIDs.size} duplicated UUIDs in valid examples") 94 | _ <- ZIO.logInfo(s"found issues count : ${groupedInvalid.map((kind, errs) => kind + ":" + errs.size).mkString(", ")}") 95 | _ <- ZIO.logInfo(s"found invalid content for ${invalidContent.size} examples : ${invalidContent.map(_.filepath).mkString(",")}") 96 | _ <- ZIO.logInfo(s"found ${publishable.size} publishable distinct examples ") 97 | _ <- ZIO.logInfo(s"found ${examples.map(_.publish.distinct.size).sum} publishable examples (1 example can be published several times)") 98 | _ <- ZIO.logInfo(s"found ${examples.filter(_.createdOn.isEmpty).size} without explicit created-on field") 99 | _ <- ZIO.logInfo(s"found ${testable.size} testable and executable examples using run-with directive") 100 | _ <- ZIO.logInfo(s"found ${missingRunWith.size} testable examples without run-with instruction") 101 | _ <- ZIO.logInfo(s"add runWith to those examples${missingRunWith.mkString("\n -", "\n -", "")}") 102 | _ <- ZIO.cond(duplicatedUUIDs.size == 0, (), RuntimeException(s"Duplicated UUIDs ${duplicatedUUIDs.keys.mkString(",")}")) // TODO enhance error management 103 | _ <- ZIO.cond(duplicatedSummaries.size == 0, (), RuntimeException(s"Duplicated summaries ${duplicatedSummaries.keys.mkString(",")}")) // TODO enhance error management 104 | } yield () 105 | } 106 | 107 | val examplesCollect: ZIO[FileSystemService & LMDB, ExampleIssue | Throwable, List[CodeExample]] = ZIO.logSpan("/collect") { 108 | for { 109 | searchRootDirectories <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.examples.searchRootDirectories) 110 | searchRoots <- examplesValidSearchRoots(searchRootDirectories) 111 | _ <- ZIO.log(s"Searching examples in ${searchRoots.mkString(",")}") 112 | localExamples <- examplesCollectFor(searchRoots) 113 | } yield localExamples 114 | } 115 | 116 | def computeWorkToDo(examples: Iterable[CodeExample], states: Iterable[RemoteExampleState]): List[WhatToDo] = { 117 | val statesByUUID = states.map(state => state.uuid -> state).toMap 118 | val examplesByUUID = examples.map(example => example.uuid -> example).toMap 119 | val examplesUUIDs = examplesByUUID.keys.toSet 120 | val examplesTriple = { // (exampleUUID, foundLocalExample, foundRemoteExampleState) 121 | examples 122 | .map(example => (example.uuid, Some(example), statesByUUID.get(example.uuid))) ++ 123 | states 124 | .filterNot(state => examplesUUIDs.contains(state.uuid)) 125 | .map(state => (state.uuid, examplesByUUID.get(state.uuid), Some(state))) 126 | } 127 | examplesTriple.toSet.toList.collect { 128 | case (uuid, None, Some(state)) => OrphanRemoteExample(uuid, state) 129 | case (uuid, Some(example), None) => AddExample(uuid, example) 130 | case (uuid, Some(example), Some(state)) if example.hash == state.hash => KeepRemoteExample(uuid, example, state) 131 | case (uuid, Some(example), Some(state)) if example.hash != state.hash => UpdateRemoteExample(uuid, example, state) 132 | case (uuid, y, z) => UnsupportedOperation(uuid, y, z) 133 | } 134 | } 135 | 136 | def checkRemote(adapterConfig: PublishAdapterConfig)(todo: WhatToDo): RIO[Any, Unit] = { 137 | val overviewUUID = UUID.fromString(adapterConfig.overviewUUID) 138 | todo match { 139 | case UnsupportedOperation(uuidOption, exampleOption, stateOption) => ZIO.log(s"${adapterConfig.targetName} : Invalid input $uuidOption - $exampleOption - $stateOption") 140 | case OrphanRemoteExample(uuid, state) if uuid != overviewUUID => ZIO.log(s"${adapterConfig.targetName} : Found orphan example $uuid - ${state.description} - ${state.url}") 141 | case _: OrphanRemoteExample => ZIO.unit 142 | case _: KeepRemoteExample => ZIO.unit 143 | case _: UpdateRemoteExample => ZIO.unit 144 | case _: AddExample => ZIO.unit 145 | case _: DeleteRemoteExample => ZIO.unit 146 | } 147 | } 148 | 149 | def checkCoherency(adapterConfig: PublishAdapterConfig, todos: Iterable[WhatToDo]): RIO[Any, Unit] = 150 | ZIO.foreach(todos)(checkRemote(adapterConfig)).unit 151 | 152 | def examplesPublishToGivenAdapter( 153 | examples: Iterable[CodeExample], 154 | adapterConfig: PublishAdapterConfig 155 | ): RIO[SttpClient, Unit] = { 156 | val examplesToSynchronize = examples.filter(_.publish.contains(adapterConfig.activationKeyword)) 157 | if (!adapterConfig.enabled || examplesToSynchronize.isEmpty || adapterConfig.token.isEmpty) ZIO.unit 158 | else { 159 | for { 160 | remoteStates <- RemoteOperations.remoteExampleStatesFetch(adapterConfig) 161 | _ <- ZIO.log(s"${adapterConfig.targetName} : Found ${remoteStates.size} already published artifacts") 162 | _ <- ZIO.log(s"${adapterConfig.targetName} : Found ${examplesToSynchronize.size} synchronisable examples") 163 | todos = computeWorkToDo(examplesToSynchronize, remoteStates) 164 | _ <- checkCoherency(adapterConfig, todos) 165 | remoteExamples <- RemoteOperations.remoteExamplesChangesApply(adapterConfig, todos) 166 | // _ <- ZIO.log(s"${adapterConfig.targetName} : Build examples summary") 167 | overviewOption <- Overview.makeOverview(remoteExamples, adapterConfig) 168 | // _ <- ZIO.log(s"${adapterConfig.targetName} : Publish examples summary") 169 | overviewTodo = computeWorkToDo(overviewOption, remoteStates) 170 | remoteOverview <- RemoteOperations.remoteExamplesChangesApply(adapterConfig, overviewTodo) 171 | _ <- ZIO.foreach(remoteOverview.headOption)(publishedOverview => ZIO.log(s"${adapterConfig.targetName} : Summary available at ${publishedOverview.state.url}")) 172 | } yield () 173 | } 174 | } 175 | 176 | def examplesPublish(examples: Iterable[CodeExample]): RIO[SttpClient, Unit] = ZIO.logSpan("/publish") { 177 | for { 178 | adapters <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.publishAdapters) 179 | _ <- ZIO.foreachPar(adapters.toList) { case (adapterName, adapterConfig) => 180 | examplesPublishToGivenAdapter(examples, adapterConfig) 181 | } 182 | } yield () 183 | } 184 | 185 | def countExamplesByPublishKeyword(examples: Iterable[CodeExample]): Map[String, Int] = { 186 | examples 187 | .flatMap(example => example.publish.map(key => key -> example)) 188 | .groupMap { case (key, _) => key } { case (_, ex) => ex } 189 | .map { case (key, examples) => key -> examples.size } 190 | } 191 | 192 | def statsEffect(examples: List[CodeExample]) = ZIO.logSpan("stats") { 193 | for { 194 | metaInfo <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.metaInfo) 195 | version = metaInfo.version 196 | appCode = metaInfo.code 197 | appName = metaInfo.name 198 | oldestCreated = examples.filter(_.createdOn.isDefined).minBy(_.createdOn) 199 | latestCreated = examples.filter(_.createdOn.isDefined).maxBy(_.createdOn) 200 | latestUpdated = examples.filter(_.lastUpdated.isDefined).maxBy(_.lastUpdated) 201 | message = 202 | s"""$appCode $appName version $version 203 | |Found ${examples.size} available locally for synchronization purpose 204 | |Found ${examples.count(_.publish.size > 0)} distinct publishable examples 205 | |Oldest example : ${oldestCreated.createdOn.get} ${oldestCreated.filename} 206 | |Latest example : ${latestCreated.createdOn.get} ${latestCreated.filename} 207 | |Latest updated : ${latestUpdated.lastUpdated.get} ${latestUpdated.filename} 208 | |Available by publishing targets : ${countExamplesByPublishKeyword(examples).toList.sorted.map { case (k, n) => s"$k:$n" }.mkString(", ")} 209 | |Defined keywords count : ${examples.flatMap(_.keywords).toSet.size} 210 | |Defined keywords : ${examples.flatMap(_.keywords).distinct.sorted.mkString(",")} 211 | |""".stripMargin 212 | } yield message 213 | } 214 | 215 | val versionEffect = 216 | for { 217 | metaInfo <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.metaInfo) 218 | version = metaInfo.version 219 | appName = metaInfo.name 220 | appCode = metaInfo.code 221 | projectURL = metaInfo.projectURL 222 | message = 223 | s"""$appCode $appName version $version 224 | |$appCode project page $projectURL 225 | |$appCode contact email = ${metaInfo.contactEmail} 226 | |$appCode build Version = ${metaInfo.buildVersion} 227 | |$appCode build DateTime = ${metaInfo.buildDateTime} 228 | |$appCode build UUID = ${metaInfo.buildUUID} 229 | |""".stripMargin 230 | } yield message 231 | 232 | def synchronizeEffect: ZIO[SttpClient & FileSystemService & LMDB, ExampleIssue | Throwable, Unit] = { 233 | ZIO.logSpan("/synchronize") { 234 | for { 235 | metaInfo <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.metaInfo) 236 | appName = metaInfo.name 237 | version <- versionEffect 238 | _ <- ZIO.log(s"\n$version") 239 | examples <- examplesCollect 240 | stats <- statsEffect(examples) 241 | _ <- ZIO.log(s"\n$stats") 242 | _ <- examplesPublish(examples) 243 | } yield () 244 | } 245 | } 246 | 247 | } 248 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/model/Change.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.model 17 | 18 | import java.util.UUID 19 | 20 | enum WhatToDo { 21 | case OrphanRemoteExample(uuid: UUID, state: RemoteExampleState) 22 | case DeleteRemoteExample(uuid: UUID, state: RemoteExampleState) 23 | case AddExample(uuid: UUID, example: CodeExample) 24 | case KeepRemoteExample(uuid: UUID, example: CodeExample, state: RemoteExampleState) 25 | case UpdateRemoteExample(uuid: UUID, example: CodeExample, state: RemoteExampleState) 26 | case UnsupportedOperation(uuid: UUID, exampleOption: Option[CodeExample], stateOption: Option[RemoteExampleState]) 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/model/CodeExample.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.model 17 | 18 | import fr.janalyse.cem.FileSystemService 19 | import fr.janalyse.cem.tools.* 20 | import fr.janalyse.cem.tools.Hashes.sha1 21 | import zio.* 22 | import zio.lmdb.*, zio.lmdb.json.* 23 | import zio.nio.charset.Charset 24 | import zio.nio.file.* 25 | import zio.json.* 26 | 27 | import java.io.File 28 | import java.time.{Instant, OffsetDateTime, ZoneId} 29 | import java.util.UUID 30 | 31 | sealed trait ExampleIssue { 32 | def filepath: Path 33 | } 34 | case class ExampleContentIssue(filepath: Path, throwable: Throwable) extends ExampleIssue 35 | case class ExampleNoParentDirectory(filepath: Path) extends ExampleIssue 36 | case class ExampleFilenameIssue(filepath: Path, throwable: Throwable) extends ExampleIssue 37 | case class ExampleIOIssue(filepath: Path, throwable: Throwable) extends ExampleIssue 38 | case class ExampleIdentifierNotFoundIssue(filepath: Path) extends ExampleIssue 39 | case class ExampleUUIDIdentifierIssue(filepath: Path, id: String, throwable: Throwable) extends ExampleIssue 40 | case class ExampleCreatedOnDateFormatIssue(filepath: Path, throwable: Throwable) extends ExampleIssue 41 | case class ExampleGitIssue(filepath: Path, throwable: Throwable) extends ExampleIssue 42 | case class ExampleInvalidAttachmentFilename(filepath: Path, attachFilename: String) extends ExampleIssue 43 | case class ExampleAttachmentContentIssue(filepath: Path, attachmentFilename: String, throwable: Throwable) extends ExampleIssue 44 | case class ExampleStorageIssue(filepath: Path, message: String) extends ExampleIssue 45 | 46 | case class CodeExample( 47 | filepath: Option[Path], 48 | filename: String, 49 | content: String, 50 | hash: String, 51 | uuid: UUID, // embedded 52 | category: Option[String] = None, // optionally embedded - default value is containing directory 53 | createdOn: Option[OffsetDateTime] = None, // embedded 54 | lastUpdated: Option[OffsetDateTime] = None, // computed from file 55 | summary: Option[String] = None, // embedded 56 | keywords: Set[String] = Set.empty, // embedded 57 | publish: List[String] = Nil, // embedded 58 | authors: List[String] = Nil, // embedded 59 | runWith: Option[String] = None, // embedded 60 | testWith: Option[String] = None, // embedded 61 | managedBy: Option[String] = None, // embedded 62 | license: Option[String] = None, // embedded 63 | updatedCount: Option[Int] = None, // computed from GIT history 64 | attachments: Map[String, String] = Map.empty, // embedded 65 | lastSeen: Option[OffsetDateTime] = None // last seen/used date, useful for database garbage collection purposes 66 | ) derives LMDBCodecJson { 67 | def fileExtension: String = filename.split("[.]", 2).drop(1).headOption.getOrElse("") 68 | def isTestable: Boolean = keywords.contains("@testable") 69 | def isExclusive: Boolean = keywords.contains("@exclusive") // exclusive examples are executed sequentially 70 | def shouldFail: Boolean = keywords.contains("@fail") 71 | def isPublishable: Boolean = publish.nonEmpty 72 | override def toString: String = s"$category $filename $uuid $summary" 73 | } 74 | 75 | object CodeExample { 76 | given JsonEncoder[Path] = JsonEncoder[String].contramap(p => p.toString) 77 | given JsonDecoder[Path] = JsonDecoder[String].map(p => Path(p)) 78 | //given JsonDecoder[CodeExample] = DeriveJsonDecoder.gen 79 | //given JsonEncoder[CodeExample] = DeriveJsonEncoder.gen 80 | 81 | def exampleContentExtractValue(from: String, key: String): Option[String] = { 82 | val RE = ("""(?m)(?i)^(?:(?:// )|(?:## )|(?:- )|(?:-- )) *""" + key + """ *: *(.*)$""").r 83 | RE.findFirstIn(from).collect { case RE(value) => value.trim }.filter(_.size > 0) 84 | } 85 | 86 | def exampleContentExtractValueList(from: String, key: String): List[String] = { 87 | exampleContentExtractValue(from, key).map(_.split("""\s*[,;]\s*""").toList).getOrElse(Nil) 88 | } 89 | 90 | def filenameFromFilepath(filepath: String): String = { 91 | new File(filepath).getName 92 | } 93 | 94 | def exampleCategoryFromFilepath(examplePath: Path, searchPath: Path): Option[String] = { 95 | examplePath.parent 96 | .map(parent => searchPath.relativize(parent)) 97 | .map(_.toString) 98 | .filter(_.size > 0) 99 | } 100 | 101 | def fileLastModified(examplePath: Path) = { 102 | OffsetDateTime.ofInstant(Instant.ofEpochMilli(examplePath.toFile.lastModified), ZoneId.systemDefault()) 103 | } 104 | 105 | def getAttachmentContent(examplePath: Path, attachmentFilename: String) = { 106 | val attachmentFilenameRE = "(?i)[-a-z0-9_][-a-z0-9_.]+" 107 | for { 108 | _ <- ZIO.cond(attachmentFilename.matches(attachmentFilenameRE), (), ExampleInvalidAttachmentFilename(examplePath, attachmentFilename)) 109 | exampleDirectory <- ZIO.fromOption(examplePath.parent).mapError(_ => ExampleNoParentDirectory(examplePath)) 110 | attachmentPath = exampleDirectory / attachmentFilename 111 | content <- FileSystemService.readFileContent(attachmentPath).mapError(th => ExampleAttachmentContentIssue(examplePath, attachmentFilename, th)) 112 | } yield content 113 | } 114 | 115 | def getGitMetaData(examplePath: Path, content: String): ZIO[LMDB, ExampleIssue, Option[GitMetaData]] = { 116 | val collectionName = "code-examples-metadata" 117 | val exampleKey = sha1(examplePath.toString) 118 | val contentHash = sha1(content) 119 | 120 | val usingGitLogic = 121 | ZIO 122 | .attempt(GitOps.getGitFileMetaData(examplePath.toFile.toPath)) 123 | .mapError(th => ExampleGitIssue(examplePath, th)) 124 | 125 | for { 126 | collection <- LMDB 127 | .collectionGet[CodeExampleMetaData](collectionName) 128 | .orElse(LMDB.collectionCreate[CodeExampleMetaData](collectionName)) 129 | .mapError(th => ExampleStorageIssue(examplePath, s"Storage issue with collection $collectionName")) 130 | foundMetaData <- collection 131 | .fetch(exampleKey) 132 | .map(_.filter(_.metaDataFileContentHash == contentHash)) 133 | .mapError(th => ExampleStorageIssue(examplePath, s"Couldn't fetch anything from $collectionName")) 134 | gitMetaData <- ZIO.from(foundMetaData.map(_.gitMetaData)).orElse(usingGitLogic) 135 | currentDateTime <- Clock.currentDateTime 136 | updatedFoundMetaData = CodeExampleMetaData( 137 | gitMetaData = gitMetaData, 138 | metaDataFileContentHash = contentHash, 139 | metaDataLastUsed = currentDateTime 140 | ) 141 | _ <- collection 142 | .upsertOverwrite(exampleKey, updatedFoundMetaData) 143 | .mapError(th => ExampleStorageIssue(examplePath, s"Couldn't upsert anything in collection $collectionName")) 144 | } yield gitMetaData 145 | } 146 | 147 | def upsertExample(examplePath: Path, example: CodeExample) = { 148 | val collectionName = "code-examples" 149 | val exampleKey = example.uuid.toString 150 | for { 151 | collection <- LMDB 152 | .collectionGet[CodeExample](collectionName) 153 | .orElse(LMDB.collectionCreate[CodeExample](collectionName)) 154 | .mapError(th => ExampleStorageIssue(examplePath, s"Storage issue with collection $collectionName")) 155 | _ <- collection 156 | .upsertOverwrite(exampleKey, example) 157 | .tapError(th => ZIO.logError(th.toString)) 158 | .mapError(th => ExampleStorageIssue(examplePath, s"Couldn't upsert anything in collection $collectionName")) 159 | } yield () 160 | } 161 | 162 | def build( 163 | filepath: Option[Path], 164 | filename: String, 165 | content: String, 166 | uuid: UUID, // embedded 167 | category: Option[String] = None, // optionally embedded - default value is containing directory 168 | createdOn: Option[OffsetDateTime] = None, // embedded 169 | lastUpdated: Option[OffsetDateTime] = None, // computed from file 170 | summary: Option[String] = None, // embedded 171 | keywords: Set[String] = Set.empty, // embedded 172 | publish: List[String] = Nil, // embedded 173 | authors: List[String] = Nil, // embedded 174 | runWith: Option[String] = None, // embedded 175 | testWith: Option[String] = None, // embedded 176 | managedBy: Option[String] = None, // embedded 177 | license: Option[String] = None, // embedded 178 | updatedCount: Option[Int] = None, // computed from GIT history 179 | attachments: Map[String, String] = Map.empty, // embedded 180 | lastSeen: Option[OffsetDateTime] = None // last seen/used date, useful for database garbage collection purposes 181 | ): CodeExample = { 182 | val hash = sha1(content + filename + category.getOrElse("") + attachments.keys.mkString + attachments.values.mkString) 183 | CodeExample( 184 | uuid = uuid, 185 | hash = hash, 186 | content = content, 187 | filename = filename, 188 | filepath = filepath, 189 | category = category, 190 | createdOn = createdOn, 191 | lastUpdated = lastUpdated, 192 | updatedCount = updatedCount, 193 | summary = summary, 194 | keywords = keywords, 195 | publish = publish, 196 | authors = authors, 197 | runWith = runWith, 198 | testWith = testWith, 199 | managedBy = managedBy, 200 | license = license, 201 | attachments = attachments, 202 | lastSeen = lastSeen 203 | ) 204 | } 205 | 206 | def buildFromFile( 207 | examplePath: Path, 208 | fromSearchPath: Path 209 | ): ZIO[FileSystemService & LMDB, ExampleIssue, CodeExample] = { 210 | for { 211 | filename <- ZIO 212 | .getOrFail(Option(examplePath.filename).map(_.toString)) 213 | .mapError(th => ExampleFilenameIssue(examplePath, th)) 214 | givenContent <- FileSystemService.readFileContent(examplePath).mapError(th => ExampleContentIssue(examplePath, th)) 215 | content = givenContent.replaceAll("\r", "") 216 | category = exampleContentExtractValue(content, "category").orElse(exampleCategoryFromFilepath(examplePath, fromSearchPath)) 217 | foundId = exampleContentExtractValue(content, "id") 218 | foundCreatedOn = exampleContentExtractValue(content, "created-on") 219 | id <- ZIO 220 | .getOrFail(foundId) 221 | .mapError(th => ExampleIdentifierNotFoundIssue(examplePath)) 222 | uuid <- ZIO 223 | .attempt(UUID.fromString(id)) 224 | .mapError(th => ExampleUUIDIdentifierIssue(examplePath, id, th)) 225 | gitMetaData <- getGitMetaData(examplePath, givenContent) 226 | createdOn <- ZIO 227 | .attempt(foundCreatedOn.map(OffsetDateTime.parse)) 228 | .mapAttempt(_.orElse(gitMetaData.map(_.createdOn))) 229 | .mapError(th => ExampleCreatedOnDateFormatIssue(examplePath, th)) 230 | lastUpdated <- ZIO 231 | .attempt(gitMetaData.map(_.lastUpdated)) 232 | .mapAttempt(_.getOrElse(fileLastModified(examplePath))) 233 | .mapError(th => ExampleIOIssue(examplePath, th)) 234 | updatedCount = gitMetaData.map(_.changesCount) 235 | attachmentsNames = exampleContentExtractValueList(content, "attachments") 236 | attachments <- ZIO.foreach(attachmentsNames)(name => getAttachmentContent(examplePath, name).map(content => name -> content)).map(_.toMap) 237 | currentDate <- Clock.currentDateTime 238 | example = build( 239 | uuid = uuid, 240 | content = content, 241 | filename = filename, 242 | filepath = Some(examplePath), 243 | category = category, 244 | createdOn = createdOn, 245 | lastUpdated = Some(lastUpdated), 246 | updatedCount = updatedCount, 247 | summary = exampleContentExtractValue(content, "summary"), 248 | keywords = exampleContentExtractValueList(content, "keywords").map(_.trim).filter(_.size > 0).toSet, 249 | publish = exampleContentExtractValueList(content, "publish"), 250 | authors = exampleContentExtractValueList(content, "authors"), 251 | runWith = exampleContentExtractValue(content, "run-with"), 252 | testWith = exampleContentExtractValue(content, "test-with"), 253 | managedBy = exampleContentExtractValue(content, "managed-by"), 254 | license = exampleContentExtractValue(content, "license"), 255 | attachments = attachments, 256 | lastSeen = Some(currentDate) 257 | ) 258 | _ <- upsertExample(examplePath, example) 259 | } yield example 260 | } 261 | 262 | } 263 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/model/CodeExampleMetaData.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem.model 2 | 3 | import fr.janalyse.cem.tools.GitMetaData 4 | import zio.json.* 5 | import zio.lmdb.json.LMDBCodecJson 6 | 7 | import java.time.OffsetDateTime 8 | 9 | case class CodeExampleMetaData( 10 | gitMetaData: Option[GitMetaData], 11 | metaDataFileContentHash: String, 12 | metaDataLastUsed: OffsetDateTime // last seen/used date, useful for database garbage collection purposes 13 | ) derives LMDBCodecJson 14 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/model/Overview.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.model 17 | 18 | import fr.janalyse.cem.templates.txt.* 19 | import fr.janalyse.cem.{ApplicationConfig, CodeExampleManagerConfig, PublishAdapterConfig} 20 | import zio.config.* 21 | import zio.* 22 | 23 | import java.util.UUID 24 | import java.time.Instant 25 | 26 | case class OverviewContext( 27 | title: String, 28 | examplesCount: Int, 29 | examples: List[ExampleContext], 30 | examplesByCategory: List[ExamplesForCategoryContext], 31 | projectName: String, 32 | projectURL: String, 33 | version: String, 34 | lastUpdated: String 35 | ) 36 | case class ExampleContext(category: String, filename: String, summary: String, url: String) 37 | case class ExamplesForCategoryContext(category: String, categoryExamples: Seq[ExampleContext]) 38 | 39 | object Overview { 40 | 41 | def makeOverview(publishedExamples: Iterable[RemoteExample], adapter: PublishAdapterConfig): Task[Option[CodeExample]] = { 42 | if (publishedExamples.isEmpty) ZIO.none 43 | else { 44 | import fr.janalyse.tools.NaturalSort.ord 45 | val exampleContexts = for { 46 | publishedExample <- publishedExamples.toSeq 47 | category = publishedExample.example.category.getOrElse("Without category") 48 | filename = publishedExample.example.filename 49 | summary = publishedExample.example.summary.getOrElse("") 50 | url = publishedExample.state.url 51 | } yield { 52 | ExampleContext(category = category, filename = filename, summary = summary, url = url) 53 | } 54 | val examplesContextByCategory = 55 | exampleContexts 56 | .groupBy(_.category) 57 | .toList 58 | .map { case (category, examplesByCategory) => ExamplesForCategoryContext(category, examplesByCategory.sortBy(_.filename)) } 59 | .sortBy(_.category) 60 | 61 | val templateLogic = for { 62 | config <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig) 63 | overviewContext = OverviewContext( 64 | title = config.summary.title, 65 | examplesCount = exampleContexts.size, 66 | examples = exampleContexts.sortBy(_.summary).toList, 67 | examplesByCategory = examplesContextByCategory, 68 | projectName = config.metaInfo.name, 69 | projectURL = config.metaInfo.projectURL, 70 | version = config.metaInfo.version, 71 | lastUpdated = Instant.now().toString 72 | ) 73 | overviewContent <- ZIO.attempt(ExamplesOverviewTemplate.render(overviewContext).body) 74 | filename = "index.md" 75 | category = Option.empty[String] 76 | attachments = Map.empty[String, String] 77 | } yield { 78 | CodeExample.build( 79 | filepath = None, 80 | filename = filename, 81 | category = category, 82 | summary = Some(config.summary.title), 83 | keywords = Set.empty, 84 | publish = List(adapter.activationKeyword), 85 | authors = Nil, 86 | uuid = UUID.fromString(adapter.overviewUUID), 87 | content = overviewContent, 88 | attachments = attachments 89 | ) 90 | } 91 | templateLogic.asSome 92 | } 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/model/RemoteExample.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.model 17 | 18 | case class RemoteExample( 19 | example: CodeExample, 20 | state: RemoteExampleState 21 | ) 22 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/model/RemoteExampleState.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.model 17 | 18 | import java.util.UUID 19 | 20 | case class RemoteExampleState( 21 | remoteId: String, 22 | description: String, 23 | url: String, 24 | files: List[String], 25 | uuid: UUID, 26 | hash: String 27 | ) 28 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/model/RunStatus.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem.model 2 | 3 | import zio.json.* 4 | import zio.lmdb.json.LMDBCodecJson 5 | 6 | import java.util.UUID 7 | import java.time.OffsetDateTime 8 | 9 | case class RunStatus( 10 | example: CodeExample, 11 | exitCodeOption: Option[Int], 12 | stdout: String, 13 | startedTimestamp: OffsetDateTime, 14 | duration: Long, 15 | runSessionDate: OffsetDateTime, 16 | runSessionUUID: UUID, 17 | success: Boolean, 18 | timeout: Boolean, 19 | runState: String 20 | ) derives LMDBCodecJson 21 | 22 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/tools/DescriptionTools.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.tools 17 | 18 | import fr.janalyse.cem.PublishAdapterConfig 19 | import fr.janalyse.cem.model.CodeExample 20 | 21 | object DescriptionTools { 22 | private val metaDataRE = """#\s*([-0-9a-f]+)\s*/\s*([0-9a-f]+)\s*$""".r.unanchored 23 | 24 | def extractMetaDataFromDescription(description: String): Option[(String, String)] = { 25 | metaDataRE 26 | .findFirstMatchIn(description) 27 | .filter(_.groupCount == 2) 28 | .map(m => (m.group(1), m.group(2))) 29 | } 30 | 31 | def makeDescription(example: CodeExample): Option[String] = { 32 | for { 33 | summary <- example.summary.orElse(Some("")) 34 | uuid = example.uuid 35 | chksum = example.hash 36 | cemURL = "https://github.com/dacr/code-examples-manager" 37 | } yield s"$summary / published by $cemURL #$uuid/$chksum" 38 | } 39 | 40 | /** rename file only on the remote publish site in order to take benefit of colorization feature 41 | * @param filename 42 | * @param config 43 | * @return 44 | */ 45 | def remoteExampleFileRename(filename: String, config: PublishAdapterConfig): String = { 46 | config.filenameRenameRules.values.foldLeft(filename) { (current, rule) => rule.rename(current) } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/tools/GitMetaData.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem.tools 2 | 3 | import zio.json.* 4 | import java.time.{Instant, OffsetDateTime, ZoneId} 5 | 6 | case class GitMetaData( 7 | changesCount: Int, 8 | createdOn: OffsetDateTime, 9 | lastUpdated: OffsetDateTime, 10 | ) derives JsonCodec 11 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/tools/GitOps.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem.tools 2 | 3 | import java.io.File 4 | import java.nio.file.Path 5 | import java.time.{Instant, OffsetDateTime, ZoneId} 6 | import scala.util.Properties 7 | 8 | object GitOps { 9 | import org.eclipse.jgit.storage.file.FileRepositoryBuilder 10 | import org.eclipse.jgit.api.Git 11 | import org.eclipse.jgit.lib.Repository 12 | import org.eclipse.jgit.lib.Constants 13 | import org.eclipse.jgit.revwalk.RevCommit 14 | import scala.jdk.CollectionConverters.* 15 | 16 | def getLocalRepositoryFromFile(fromFile: File, ceilingDirectoryOption: Option[File] = None): Option[Repository] = 17 | val builder = new FileRepositoryBuilder() 18 | builder.setMustExist(true) 19 | ceilingDirectoryOption.foreach(ceilingDirectory => builder.addCeilingDirectory(ceilingDirectory)) 20 | builder.findGitDir(fromFile) 21 | Option(builder.getGitDir()).map(_ => builder.build()) 22 | 23 | def getFileLog(git: Git, file: File, revision: String = Constants.HEAD): List[RevCommit] = 24 | val repository = git.getRepository 25 | val repoHomeDir = repository.getDirectory.toPath.getParent 26 | val revisionId = repository.resolve(Constants.HEAD) 27 | val targetFile = repoHomeDir.relativize(file.toPath) 28 | val targetFileLogs = git.log().add(revisionId).addPath(targetFile.toString).call() 29 | targetFileLogs.asScala.toList 30 | 31 | def commitTimeInstant(revCommit: RevCommit): OffsetDateTime = 32 | OffsetDateTime.ofInstant(Instant.ofEpochSecond(revCommit.getCommitTime), ZoneId.systemDefault()) 33 | 34 | def getGitFileMetaData(git: Git, filePath: Path): Option[GitMetaData] = 35 | val targetFileLogs = getFileLog(git, filePath.toFile) 36 | val changesCount = targetFileLogs.size 37 | for { 38 | createdOn <- targetFileLogs.lastOption.map(commitTimeInstant) 39 | lastUpdated <- targetFileLogs.headOption.map(commitTimeInstant) 40 | } yield { 41 | GitMetaData(changesCount = changesCount, createdOn = createdOn, lastUpdated = lastUpdated) 42 | } 43 | 44 | def getGitFileMetaData(filePath: Path): Option[GitMetaData] = 45 | for { 46 | homeFile <- Properties.envOrNone("HOME").map(new File(_)).filter(_.exists()) 47 | fromFile <- Option(filePath).map(_.toFile).filter(_.exists()) 48 | repo <- getLocalRepositoryFromFile(fromFile, Some(homeFile)) 49 | git = Git(repo) // TODO add close() call 50 | metadata <- getGitFileMetaData(git, filePath) 51 | } yield metadata 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/tools/Hashes.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.tools 17 | 18 | object Hashes { 19 | // TODO Migrate to an effect 20 | def sha1(that: String): String = { 21 | import java.math.BigInteger 22 | import java.security.MessageDigest 23 | // Inspired from https://alvinalexander.com/source-code/scala-method-create-md5-hash-of-string 24 | val content = that 25 | val md = MessageDigest.getInstance("SHA-1") 26 | val digest = md.digest(content.getBytes) 27 | val bigInt = new BigInteger(1, digest) 28 | val hashedString = bigInt.toString(16) 29 | hashedString 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/scala/fr/janalyse/cem/tools/HttpTools.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.tools 17 | 18 | import sttp.model.Uri 19 | import zio.* 20 | 21 | object HttpTools { 22 | def uriParse(link: String): Task[Uri] = { 23 | ZIO.from(Uri.parse(link)).mapError(msg => new Error(msg)) 24 | } 25 | 26 | def webLinkingExtractNext(link: String): Option[String] = { 27 | // Using Web Linking to get large amount of results : https://tools.ietf.org/html/rfc5988 28 | val nextLinkRE = """.*<([^>]+)>; rel="next".*""".r 29 | nextLinkRE.findFirstMatchIn(link).map(_.group(1)) 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/twirl/fr.janalyse.cem.templates/ExamplesOverviewTemplate.scala.txt: -------------------------------------------------------------------------------- 1 | @(context:OverviewContext) 2 | # @context.title 3 | - Generated by [@context.projectName](@context.projectURL) release @context.version 4 | - @context.examplesCount published examples 5 | 6 | @for(exs4cat <- context.examplesByCategory) {## @exs4cat.category 7 | @for(ex <- exs4cat.categoryExamples) {- [@ex.filename](@ex.url) : @ex.summary 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/test/scala/fr/janalyse/cem/FileSystemServiceStub.scala: -------------------------------------------------------------------------------- 1 | package fr.janalyse.cem 2 | 3 | import zio.* 4 | import zio.nio.file.Path 5 | 6 | import scala.util.matching.Regex 7 | 8 | class FileSystemServiceStub(contents: Map[Path, String] = Map.empty, files: Map[Path, List[Path]] = Map.empty) extends FileSystemService: 9 | 10 | override def searchFiles(searchRoot: Path, searchOnlyRegex: Option[Regex], ignoreMaskRegex: Option[Regex]): Task[List[Path]] = 11 | ZIO.getOrFail(files.get(searchRoot)) 12 | 13 | override def readFileLines(inputPath: Path, maxLines: Option[Int]): Task[List[String]] = 14 | for 15 | content <- ZIO.getOrFail(contents.get(inputPath)) 16 | lines = content.split("\r?\n").toList 17 | selectedLines = maxLines.map(n => lines.take(n)).getOrElse(lines) 18 | yield lines 19 | 20 | override def readFileContent(inputPath: Path): Task[String] = 21 | ZIO.getOrFail(contents.get(inputPath)) 22 | 23 | object FileSystemServiceStub: 24 | def stubWithContents(contents: Map[Path, String]): TaskLayer[FileSystemService] = ZLayer.succeed( 25 | FileSystemServiceStub(contents) 26 | ) 27 | -------------------------------------------------------------------------------- /src/test/scala/fr/janalyse/cem/RemoteGithubOperationsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem 17 | 18 | import zio.* 19 | import zio.test.* 20 | import zio.test.Assertion.* 21 | import zio.logging.* 22 | import fr.janalyse.cem.model.* 23 | import fr.janalyse.cem.model.WhatToDo.* 24 | import fr.janalyse.cem.tools.DescriptionTools.* 25 | import org.junit.runner.RunWith 26 | import sttp.client3.asynchttpclient.zio.AsyncHttpClientZioBackend 27 | import sttp.capabilities.zio.ZioStreams 28 | import sttp.capabilities.WebSockets 29 | import sttp.client3.testing.SttpBackendStub 30 | import zio.nio.file.Path 31 | import zio.lmdb.LMDB 32 | 33 | @RunWith(classOf[zio.test.junit.ZTestJUnitRunner]) 34 | class RemoteGithubOperationsSpec extends ZIOSpecDefault { 35 | 36 | import RemoteGithubOperations.* 37 | 38 | val zexample1 = { 39 | val filename = Path("test-data/sample1/fake-testing-pi.sc") 40 | val searchRoot = Path("test-data/sample1") 41 | val content = 42 | """// summary : Simplest scalatest test framework usage. 43 | |// keywords : scalatest, pi, @testable 44 | |// publish : gist, snippet 45 | |// authors : David Crosson 46 | |// license : GPL 47 | |// id : 8f2e14ba-9856-4500-80ab-3b9ba2234ce2 48 | |// execution : scala ammonite script (http://ammonite.io/) - run as follow 'amm scriptname.sc' 49 | | 50 | |import $ivy.`org.scalatest::scalatest:3.2.0` 51 | |import org.scalatest._,matchers.should.Matchers._ 52 | | 53 | |math.Pi shouldBe 3.14d +- 0.01d""".stripMargin 54 | CodeExample 55 | .buildFromFile(filename, searchRoot) 56 | .provide( 57 | FileSystemServiceStub.stubWithContents(Map(filename -> content)), 58 | Scope.default, 59 | LMDB.live // TODO - Replace with TestLMDB when available 60 | ) 61 | } 62 | 63 | // ---------------------------------------------------------------------------------------------- 64 | val t1 = test("apply changes") { 65 | val config = PublishAdapterConfig( 66 | enabled = true, 67 | kind = "github", 68 | activationKeyword = "gist", 69 | apiEndPoint = "https://api.github.com", 70 | overviewUUID = "cafacafe-cafecafe", 71 | token = Some("aaa-aa"), 72 | defaultVisibility = None, 73 | filenameRenameRules = Map.empty 74 | ) 75 | 76 | val logic = for { 77 | example1 <- zexample1 78 | uuid1 = example1.uuid 79 | state1 = RemoteExampleState( 80 | remoteId = "6e40f8239fa6828ab45a064b8131fdfc", // // MDQ6R2lzdDQ1NTk5OTQ= --> 04:Gist4559994 --> 4559994 81 | description = "desc", 82 | url = "https://truc/aa-bb", 83 | files = List(example1.filename), 84 | uuid = uuid1, 85 | hash = example1.hash 86 | ) 87 | todos = List(UpdateRemoteExample(uuid1, example1, state1)) 88 | results <- githubRemoteExamplesChangesApply(config, todos) 89 | } yield results 90 | 91 | val stub: SttpBackendStub[Task, Any] = AsyncHttpClientZioBackend.stub 92 | .whenRequestMatches(_.uri.toString() == "https://api.github.com/gists/6e40f8239fa6828ab45a064b8131fdfc") 93 | .thenRespond("""{"id":"aa-bb", "html_url":"https://truc/aa-bb"}""") 94 | 95 | val stubLayer = ZLayer.succeed(stub) 96 | 97 | logic.provide(stubLayer).map(result => assertTrue(true)) 98 | } 99 | 100 | // ---------------------------------------------------------------------------------------------- 101 | override def spec = { 102 | suite("RemoteGithubOperationsTools tests")(t1) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/test/scala/fr/janalyse/cem/SynchronizeSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem 17 | 18 | import zio.* 19 | import zio.test.* 20 | import zio.test.Assertion.* 21 | import org.junit.runner.RunWith 22 | import fr.janalyse.cem.model.{CodeExample, ExampleIssue} 23 | 24 | import java.util.UUID 25 | import scala.util.Success 26 | 27 | @RunWith(classOf[zio.test.junit.ZTestJUnitRunner]) 28 | class SynchronizeSpec extends ZIOSpecDefault { 29 | // ---------------------------------------------------------------------------------------------- 30 | val t1 = test("check examples coherency success with valid examples") { 31 | val examplesWithIssues: List[Either[ExampleIssue, CodeExample]] = List( 32 | Right(CodeExample.build(filepath = None, filename = "pi-1.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd"))), 33 | Right(CodeExample.build(filepath = None, filename = "pi-2.sc", content = "42", uuid = UUID.fromString("a49b0c53-3ec3-4404-bd7d-c249a4868a2b"))) 34 | ) 35 | assertZIO(Synchronize.examplesCheckCoherency(examplesWithIssues))(isUnit) 36 | } 37 | // ---------------------------------------------------------------------------------------------- 38 | val t2 = test("check examples coherency should fail on duplicates UUID") { 39 | val examplesWithIssues: List[Either[ExampleIssue, CodeExample]] = List( 40 | Right(CodeExample.build(filepath = None, filename = "pi-1.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd"))), 41 | Right(CodeExample.build(filepath = None, filename = "pi-2.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd"))) 42 | ) 43 | //assertZIO(Synchronize.examplesCheckCoherency(examplesWithIssues).exit)(fails(isSubtype[Exception](anything))) 44 | assertZIO(Synchronize.examplesCheckCoherency(examplesWithIssues).exit)(fails(hasMessage(containsString("Duplicated UUIDs")))) 45 | } 46 | 47 | // ---------------------------------------------------------------------------------------------- 48 | override def spec = { 49 | suite("CodeExample tests")(t1, t2) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/scala/fr/janalyse/cem/model/CodeExampleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.model 17 | 18 | import fr.janalyse.cem.FileSystemServiceStub 19 | import zio.* 20 | import zio.test.* 21 | import zio.test.Assertion.* 22 | import zio.nio.file.Path 23 | import org.junit.runner.RunWith 24 | import zio.lmdb.LMDB 25 | 26 | @RunWith(classOf[zio.test.junit.ZTestJUnitRunner]) 27 | class CodeExampleSpec extends ZIOSpecDefault { 28 | 29 | val exampleFakeTestingFilename = Path("test-data/sample1/fake-testing-pi.sc") 30 | val exampleFakeTestingSearchRoot = Path("test-data/sample1") 31 | val exampleFakeTestingPiContent = 32 | """// summary : Simplest scalatest test framework usage. 33 | |// keywords : scalatest, pi, @testable 34 | |// publish : gist, snippet 35 | |// authors : David Crosson 36 | |// license : GPL 37 | |// id : 8f2e14ba-9856-4500-80ab-3b9ba2234ce2 38 | |// execution : scala ammonite script (http://ammonite.io/) - run as follow 'amm scriptname.sc' 39 | | 40 | |import $ivy.`org.scalatest::scalatest:3.2.0` 41 | |import org.scalatest._,matchers.should.Matchers._ 42 | | 43 | |math.Pi shouldBe 3.14d +- 0.01d""".stripMargin 44 | 45 | // ---------------------------------------------------------------------------------------------- 46 | val t1 = test("make an example") { 47 | for { 48 | example <- CodeExample 49 | .buildFromFile(exampleFakeTestingFilename, exampleFakeTestingSearchRoot) 50 | .provide( 51 | FileSystemServiceStub.stubWithContents(Map(exampleFakeTestingFilename -> exampleFakeTestingPiContent)), 52 | Scope.default, 53 | LMDB.live // TODO - Replace with TestLMDB when available 54 | ) 55 | } yield assertTrue(example.filename == "fake-testing-pi.sc") && 56 | assertTrue(example.category.isEmpty) && 57 | assertTrue(example.summary.contains("Simplest scalatest test framework usage.")) && 58 | assertTrue(example.fileExtension == "sc") && 59 | assertTrue(example.publish == List("gist", "snippet")) && 60 | assertTrue(example.authors == List("David Crosson")) && 61 | assertTrue(example.keywords == Set("scalatest", "pi", "@testable")) && 62 | assertTrue(example.uuid.toString == "8f2e14ba-9856-4500-80ab-3b9ba2234ce2") && 63 | assert(example.content)(matchesRegex("(?s).*id [:] 8f2e14ba-9856-4500-80ab-3b9ba2234ce2.*")) && 64 | assertTrue(example.hash == "5f6dd8a2d2f813ee946542161503d61cb9a8074e") 65 | } 66 | 67 | // ---------------------------------------------------------------------------------------------- 68 | val t2 = test("automatically recognize categories from sub-directory name") { 69 | val inputsAndExpectedResults: List[((String, String), Option[String])] = List( 70 | ("test-data/fake-testing-pi.sc", "test-data/") -> None, 71 | ("test-data/fake-testing-pi.sc", "") -> Some("test-data"), 72 | ("test-data/sample1/fake-testing-pi.sc", "test-data/sample1") -> None, 73 | ("test-data/sample1/fake-testing-pi.sc", "test-data") -> Some("sample1"), 74 | ("test-data/sample1/fake-testing-pi.sc", "test-data/") -> Some("sample1"), 75 | ("test-data/sample1/fake-testing-pi.sc", "") -> Some("test-data/sample1") 76 | ) 77 | inputsAndExpectedResults 78 | .map { case ((filename, searchRoot), expectedResult) => 79 | assertTrue(CodeExample.exampleCategoryFromFilepath(Path(filename), Path(searchRoot)) == expectedResult) 80 | } 81 | .reduce(_ && _) 82 | } 83 | 84 | // ---------------------------------------------------------------------------------------------- 85 | val t3 = test("meta data single value extraction") { 86 | import CodeExample.{exampleContentExtractValue => extractor} 87 | assertTrue(extractor("// summary : hello", "summary") == Option("hello")) && 88 | assert(extractor("// summary :", "summary"))(isNone) && 89 | assert(extractor("// summary : ", "summary"))(isNone) && 90 | assert(extractor("// truc : \n// summary : \n// machin : \n", "summary"))(isNone) && 91 | assert(extractor("// truc : \n// summary :\n// machin : \n", "summary"))(isNone) 92 | } 93 | // ---------------------------------------------------------------------------------------------- 94 | val t4 = test("meta data list value extraction") { 95 | import CodeExample.{exampleContentExtractValueList => extractor} 96 | assertTrue(extractor("// publish : toto", "publish") == List("toto")) && 97 | assertTrue(extractor("// publish :", "publish").isEmpty) && 98 | assertTrue(extractor("// publish : ", "publish").isEmpty) 99 | } 100 | // ---------------------------------------------------------------------------------------------- 101 | override def spec = { 102 | suite("CodeExample tests")(t1, t2, t3, t4) 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/test/scala/fr/janalyse/cem/tools/DescriptionToolsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.tools 17 | 18 | import zio.test.* 19 | import zio.test.Assertion.* 20 | import fr.janalyse.cem.model.CodeExample 21 | import fr.janalyse.cem.tools.DescriptionTools.* 22 | import java.util.UUID 23 | import org.junit.runner.RunWith 24 | 25 | @RunWith(classOf[zio.test.junit.ZTestJUnitRunner]) 26 | class DescriptionToolsSpec extends ZIOSpecDefault { 27 | 28 | override def spec = { 29 | suite("RemoteOperationsTools tests")( 30 | // ---------------------------------------------------------------------------------------------- 31 | test("extractMetaDataFromDescription can return example uuid and checksum from the description") { 32 | val description = "Blah example / published by https://github.com/dacr/code-examples-manager #7135b214-5b48-47d0-afd7-c7f64c0a31c3/5ec6b73c57561e0cc578dea654eeddce09433252" 33 | assertTrue(extractMetaDataFromDescription(description).contains("7135b214-5b48-47d0-afd7-c7f64c0a31c3" -> "5ec6b73c57561e0cc578dea654eeddce09433252")) 34 | }, 35 | // ---------------------------------------------------------------------------------------------- 36 | test("extractMetaDataFromDescription should return none if no uuid checksum is encoded in the description") { 37 | assertTrue(extractMetaDataFromDescription("").isEmpty) && 38 | assertTrue(extractMetaDataFromDescription("blah bouh").isEmpty) 39 | }, 40 | // ---------------------------------------------------------------------------------------------- 41 | test("makeDescription should return a description for ready to publish code examples") { 42 | val example = CodeExample.build(filepath = None, filename = "truc.sc", uuid = UUID.fromString("049e6849-0c93-4b96-a914-f694f6982f5e"), content = "blah") 43 | assert(makeDescription(example))( 44 | isSome( 45 | endsWithString(s"#049e6849-0c93-4b96-a914-f694f6982f5e/${example.hash}") 46 | ) 47 | ) 48 | } 49 | ) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/scala/fr/janalyse/cem/tools/HashesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 David Crosson 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package fr.janalyse.cem.tools 17 | 18 | import zio.* 19 | import zio.test.* 20 | import zio.test.TestAspect.* 21 | import zio.test.Gen.* 22 | import zio.test.Assertion.* 23 | import org.junit.runner.RunWith 24 | import fr.janalyse.cem.tools.Hashes.* 25 | 26 | @RunWith(classOf[zio.test.junit.ZTestJUnitRunner]) 27 | class HashesSpec extends ZIOSpecDefault { 28 | 29 | def spec = { 30 | suite("Hash function tests")( 31 | // ---------------------------------------------------------------------------------------------- 32 | test("sha1 compute the right hash value") { 33 | val example = "Please hash me !" 34 | assertTrue(sha1(example) == "4031d74d6a72919da236a388bdf3b966126b80f2") 35 | }, 36 | // ---------------------------------------------------------------------------------------------- 37 | test("sha1 should not fail") { 38 | assertTrue(sha1("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709") 39 | }, 40 | // ---------------------------------------------------------------------------------------------- 41 | test("sha1 hashes are never empty") { 42 | check(Gen.string) { content => 43 | assert(sha1(content))(isNonEmptyString) 44 | } 45 | }, 46 | test("sha1 hashes are different if their content are differents") { 47 | check(Gen.string, Gen.string) { (content1, content2) => 48 | assertTrue(content1 != content2 && sha1(content1) != sha1(content2)) || 49 | assertTrue(content1 == content2 && sha1(content1) == sha1(content2)) 50 | } 51 | } 52 | ) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /tracing.sbt: -------------------------------------------------------------------------------- 1 | Compile / resourceGenerators += Def.task { 2 | val dir = (Compile / sourceManaged).value 3 | val projectName = name.value 4 | val projectGroup = organization.value 5 | val projectPage = homepage.value.map(_.toString).getOrElse("https://github.com/dacr") 6 | val projectVersion = version.value 7 | val buildDateTime = java.time.Instant.now().toString 8 | val buildUUID = java.util.UUID.randomUUID.toString 9 | val file = dir / "cem-meta.conf" 10 | IO.write( 11 | file, 12 | s"""code-examples-manager-config { 13 | | meta-info { 14 | | project-name = "$projectName" 15 | | project-code = "cem" 16 | | project-group = "$projectGroup" 17 | | project-page = "$projectPage" 18 | | build-version = "$projectVersion" 19 | | build-date-time = "$buildDateTime" 20 | | build-uuid = "$buildUUID" 21 | | contact-email = "crosson.david@gmail.com" 22 | | } 23 | |}""".stripMargin 24 | ) 25 | Seq(file) 26 | }.taskValue 27 | -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | ThisBuild / version := "2.4.9-SNAPSHOT" 2 | --------------------------------------------------------------------------------