20 |
21 |
23 |
24 |
25 |
31 |
34 |
35 |
36 |
37 |
38 |
39 |
41 |
42 | {}
43 |
44 |
45 |
49 |
50 |
51 |
55 |
56 |
61 |
64 |
66 |
67 | Login failed
68 |
69 |
70 |
76 |
79 | loaded
80 |
82 |
83 |
84 | loaded
85 |
87 |
88 |
89 |
94 |
95 |
96 | fetching data...
97 | progress-wrapper
98 |
99 |
100 | Data fetched
101 |
102 |
105 |
106 |
107 |
108 |
114 |
115 |
116 | Data deployed to database
117 | No expath-pkg.xml or repo.xml in repo
118 |
120 |
121 |
122 |
123 |
124 |
129 |
132 | progress-wrapper
133 |
134 |
135 |
136 | incremental update done
137 |
140 |
141 | Update Failed!
142 |
144 |
145 |
146 |
147 | return btoa(encodeURI($input));
148 |
149 |
150 |
151 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
178 |
179 |
180 |
181 |
182 |
183 | Documentation
184 |
187 |
188 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
--------------------------------------------------------------------------------
/src/modules/gitlab.xqm:
--------------------------------------------------------------------------------
1 | xquery version "3.1";
2 |
3 | module namespace gitlab="http://e-editiones.org/tuttle/gitlab";
4 |
5 | import module namespace app="http://e-editiones.org/tuttle/app" at "app.xqm";
6 | import module namespace config="http://e-editiones.org/tuttle/config" at "config.xqm";
7 |
8 | declare namespace http="http://expath.org/ns/http-client";
9 |
10 | declare function gitlab:repo-url($config as map(*)) as xs:string {
11 | ``[`{$config?baseurl}`projects/`{$config?project-id}`/repository]``
12 | };
13 |
14 | declare function gitlab:commit-by-ref-url($config as map(*), $ref as xs:string) as xs:string {
15 | gitlab:repo-url($config) || "/commits/" || $ref
16 | };
17 |
18 | (:
19 | the `commits` API endpoint is _paged_ and will only return _20_ commits by default
20 | ?ref_name={ref}&per_page=100
21 | will always return 100 commits, which might result in to much overhead
22 | :)
23 | declare function gitlab:commit-ref-url($config as map(*)) as xs:string {
24 | gitlab:repo-url($config) || "/commits/?ref_name=" || $config?ref
25 | };
26 |
27 | declare function gitlab:commit-ref-url($config as map(*), $per-page as xs:integer) as xs:string {
28 | gitlab:commit-ref-url($config) || "&per_page=" || $per-page
29 | };
30 |
31 | (:
32 | The Gitlab API allows to specify a revision range as the value for ref_name
33 | ?ref_name={ref}...{deployed_hash}
34 | This will only retrieve commits since deployed_hash but the result is paged as well.
35 |
36 | ?ref_name={ref}...{deployed_hash}&per_page=100
37 | will return _up to_ 100 commits until deployed_hash is reached.
38 | :)
39 | declare function gitlab:newer-commits-url($config as map(*), $base as xs:string, $per-page as xs:integer) as xs:string {
40 | gitlab:repo-url($config) || "/commits?ref_name=" || $config?ref || "..." || $base || "&per_page=" || $per-page
41 | };
42 |
43 | (:~
44 | : clone defines Version repo
45 | :)
46 | declare function gitlab:get-archive($config as map(*), $sha as xs:string) {
47 | gitlab:request(
48 | gitlab:repo-url($config) || "/archive.zip?sha=" || $sha, $config?token)
49 | };
50 |
51 | (:~
52 | : Get commit info for a specific sha
53 | :)
54 | declare function gitlab:get-specific-commit($config as map(*), $ref as xs:string) as map(*) {
55 | let $commit :=
56 | gitlab:request-json(
57 | gitlab:commit-by-ref-url($config, $ref), $config?token)
58 |
59 | return
60 | map {
61 | "sha" : $commit?id,
62 | "date": $commit?committed_date
63 | }
64 | };
65 |
66 | (:~
67 | : Get the last commit
68 | :)
69 | declare function gitlab:get-last-commit($config as map(*)) {
70 | let $commit :=
71 | array:head(
72 | gitlab:request-json(
73 | gitlab:commit-ref-url($config, 1), $config?token))
74 |
75 | return
76 | map {
77 | "sha" : $commit?id,
78 | "date": $commit?committed_date
79 | }
80 | };
81 |
82 | (:~
83 | : Get all commits
84 | :)
85 | declare function gitlab:get-commits($config as map(*)) as array(*)* {
86 | gitlab:get-commits($config, 100)
87 | };
88 |
89 | (:~
90 | : Get N commits
91 | :)
92 | declare function gitlab:get-commits($config as map(*), $count as xs:integer) as array(*)* {
93 | if ($count <= 0)
94 | then error(xs:QName("gitlab:illegal-argument"), "$count must be greater than zero in gitlab:get-commits")
95 | else
96 | let $json := gitlab:get-raw-commits($config, $count)
97 | let $commits :=
98 | if (empty($json))
99 | then []
100 | else if ($count >= array:size($json)) (: return everything :)
101 | then $json
102 | else array:subarray($json, 1, $count)
103 |
104 | return
105 | array:for-each($commits, gitlab:short-commit-info#1)
106 | };
107 |
108 | declare %private function gitlab:short-commit-info ($commit-info as map(*)) as array(*) {
109 | [
110 | $commit-info?id,
111 | $commit-info?message
112 | ]
113 | };
114 |
115 | (:~
116 | : Get commits in full
117 | :)
118 | declare function gitlab:get-raw-commits($config as map(*), $count as xs:integer) as array(*)? {
119 | gitlab:request-json(
120 | gitlab:commit-ref-url($config, $count), $config?token)
121 | };
122 |
123 | (:~
124 | : Get diff between production collection and gitlab-newest
125 | :)
126 | declare function gitlab:get-newest-commits($config as map(*)) {
127 | reverse(
128 | gitlab:request-json(
129 | gitlab:newer-commits-url($config, $config?deployed, 100), $config?token)
130 | ?*)
131 | };
132 |
133 | (:~
134 | : Check if sha exist
135 | :)
136 | declare function gitlab:available-sha($config as map(*), $sha as xs:string) as xs:boolean {
137 | $sha = gitlab:get-commits($config)?*?1
138 | };
139 |
140 | (:~
141 | : Get files removed and added from commit
142 | :)
143 | declare function gitlab:get-commit-files($config as map(*), $sha as xs:string) as array(*) {
144 | gitlab:request-json(
145 | gitlab:repo-url($config) || "/commits/" || $sha || "/diff?per_page=100", $config?token)
146 | };
147 |
148 | (:~
149 | : Get blob of a file
150 | :)
151 | declare function gitlab:get-blob($config as map(*), $filename as xs:string, $sha as xs:string) {
152 | let $file := escape-html-uri(replace($filename,"/", "%2f"))
153 | let $file-url := gitlab:repo-url($config) || "/files/" || $file || "?ref=" || $sha
154 | let $json := gitlab:request-json($file-url, $config?token)
155 |
156 | return
157 | util:base64-decode($json?content)
158 | };
159 |
160 | (:~
161 | : Get HTTP-URL
162 | :)
163 | declare function gitlab:get-url($config as map(*)) {
164 | let $info := gitlab:request-json($config?baseurl || "/projects/" || $config?project-id, $config?token)
165 |
166 | return $info?http_url_to_repo
167 | };
168 |
169 | (:~
170 | : Handle edge case where a file created in this changeset is also removed
171 | :
172 | : So, in order to not fire useless and potentially harmful side-effects like
173 | : triggers or indexing we filter out all of these documents as if they were
174 | : never there.
175 | :)
176 | declare function gitlab:remove-or-ignore ($changes as map(*), $filename as xs:string) as map(*) {
177 | if ($filename = $changes?new)
178 | then map:put($changes, "new", $changes?new[. ne $filename]) (: filter document from new :)
179 | else map:put($changes, "del", ($changes?del, $filename)) (: add document to be removed :)
180 | };
181 |
182 | declare %private function gitlab:aggregate-filechanges($changes as map(*), $next as map(*)) as map(*) {
183 | if ($next?renamed_file) then
184 | gitlab:remove-or-ignore($changes, $next?old_path)
185 | => map:put("new", ($changes?new, $next?new_path))
186 | else if ($next?deleted_file) then
187 | gitlab:remove-or-ignore($changes, $next?new_path)
188 | (: added or modified :)
189 | else if ($next?new_file or $next?new_path = $next?old_path) then
190 | map:put($changes, "new", ($changes?new, $next?new_path))
191 | else $changes
192 | };
193 |
194 | declare function gitlab:get-changes ($collection-config as map(*)) as map(*) {
195 | let $changes :=
196 | for $commit in gitlab:get-newest-commits($collection-config)
197 | return gitlab:get-commit-files($collection-config, $commit?short_id)?*
198 |
199 | let $aggregated := fold-left($changes, map{}, gitlab:aggregate-filechanges#2)
200 | let $filtered := fold-left($aggregated?new, map{}, app:ignore-reducer#2)
201 | return map {
202 | "del": $aggregated?del,
203 | "new": $filtered?new,
204 | "ignored": $filtered?ignored
205 | }
206 | };
207 |
208 | (:~
209 | : Run incremental update on collection in dry mode
210 | :)
211 | declare function gitlab:incremental-dry($config as map(*)) as map(*) {
212 | let $changes := gitlab:get-changes($config)
213 | return map {
214 | 'new': array{ $changes?new },
215 | 'del': array{ $changes?del },
216 | 'ignored': array{ $changes?ignored }
217 | }
218 | };
219 |
220 | (:~
221 | : Run incremental update on collection
222 | :)
223 | declare function gitlab:incremental($config as map(*)) as map(*) {
224 | let $last-commit := gitlab:get-last-commit($config)
225 | let $changes := gitlab:get-changes($config)
226 | let $new := gitlab:incremental-add($config, $changes?new, $last-commit?sha)
227 | let $del := gitlab:incremental-delete($config, $changes?del)
228 | let $writesha := app:write-commit-info($config?path, $last-commit)
229 | return map {
230 | 'new': array{ $new },
231 | 'del': array{ $del },
232 | 'ignored': array{ $changes?ignored }
233 | }
234 | };
235 |
236 | declare function gitlab:check-signature ($collection as xs:string, $apikey as xs:string) as xs:boolean {
237 | request:get-header("X-Gitlab-Token") = $apikey
238 | };
239 |
240 | (:~
241 | : Incremental updates delete files
242 | :)
243 | declare %private function gitlab:incremental-delete($config as map(*), $files as xs:string*) as array(*)* {
244 | for $filepath in $files
245 | return
246 | try {
247 | [ $filepath, app:delete-resource($config, $filepath) ]
248 | }
249 | catch * {
250 | [ $filepath, false(), map{
251 | "code": $err:code, "description": $err:description, "value": $err:value,
252 | "line": $err:line-number, "column": $err:column-number, "module": $err:module
253 | }]
254 | }
255 | };
256 |
257 | (:~
258 | : Incremental update fetch and add files from git
259 | :)
260 | declare %private function gitlab:incremental-add($config as map(*), $files as xs:string*, $sha as xs:string) as array(*)* {
261 | for $filepath in $files
262 | return
263 | try {
264 | [ $filepath,
265 | app:add-resource($config, $filepath,
266 | gitlab:get-blob($config, $filepath, $sha))]
267 | }
268 | catch * {
269 | [ $filepath, false(), map{
270 | "code": $err:code, "description": $err:description, "value": $err:value,
271 | "line": $err:line-number, "column": $err:column-number, "module": $err:module
272 | }]
273 | }
274 | };
275 |
276 | (:~
277 | : Gitlab request
278 | :)
279 |
280 | (: If the response header `x-next-page` has a value, there are commits missing. :)
281 | declare %private function gitlab:has-next-page($response as element(http:response)) as xs:boolean {
282 | let $x-next-page := $response//http:header[@name="x-next-page"]
283 | return exists($x-next-page/@value) and $x-next-page/@value/string() ne ''
284 | };
285 |
286 | declare %private function gitlab:request-json($url as xs:string, $token as xs:string?) {
287 | let $response := app:request-json(gitlab:build-request($url, $token))
288 |
289 | return (
290 | if (gitlab:has-next-page($response[1]))
291 | then util:log("warn", ('Paged gitlab request has next page! URL:', $url))
292 | else (),
293 | $response[2]
294 | )
295 | };
296 |
297 | declare %private function gitlab:request($url as xs:string, $token as xs:string?) {
298 | app:request(gitlab:build-request($url, $token))[2]
299 | };
300 |
301 | declare %private function gitlab:build-request($url as xs:string, $token as xs:string) as element(http:request) {
302 | 
22 | Tuttle Git DashboardAPI
23 |
24 |
152 |
162 |
153 | 
154 | 
160 |
161 |
154 |
155 |
156 |
157 |
158 |
159 |
160 | Git to DBimport data from Git into eXist-db
185 |Git Repositories
186 |Error [{$error?code}]
519 |{ 520 | if (map:contains($error, "module")) 521 | then ``[An error occurred in `{$error?module}` at line `{$error?line}`, column `{$error?column}`]`` 522 | else "An error occurred!" 523 | }
524 |Description
525 |{$error?description}
526 | 527 | 528 | }; 529 | 530 | declare %private 531 | function api:catch-error($function, $error as map(*)) { 532 | util:log('error', ($function, ': [', $error?code, '] ', $error?description)), 533 | roaster:response(500, "application/json", map { 534 | "message": $error?description, "error": $error 535 | }) 536 | }; 537 | 538 | declare %private function api:get-default-collection-config() as map(*)? { 539 | config:collections(config:default-collection()) 540 | }; 541 | 542 | declare %private function api:get-collection-config($collection as xs:string?) as map(*)? { 543 | let $git-collection := 544 | if (exists($collection) and $collection ne '') 545 | then xmldb:decode-uri($collection) 546 | else config:default-collection() 547 | 548 | let $collection-config := config:collections($git-collection) 549 | 550 | return 551 | if (empty($git-collection)) 552 | then error((), "git collection not found!") 553 | else if (empty($collection-config)) 554 | then error((), "collection config " || $git-collection || " not found!") 555 | else $collection-config 556 | }; 557 | 558 | (: end of route handlers :) 559 | 560 | (:~ 561 | : This function "knows" all modules and their functions 562 | : that are imported here 563 | : You can leave it as it is, but it has to be here 564 | :) 565 | declare function api:lookup ($name as xs:string) { 566 | function-lookup(xs:QName($name), 1) 567 | }; 568 | 569 | roaster:route($api:definitions, api:lookup#1) 570 | --------------------------------------------------------------------------------