31 |
32 |
33 |
34 |
35 |
36 |
37 |
43 |
44 |
49 |
50 |
51 |
57 |
58 |
59 |
60 |
61 |
Snapshots list
62 |
63 |
64 |
Attention! The SNAPSHOT-2020.05.06 contains the index for the previous day.
65 |
66 |
67 | -
68 | Sort by Name
69 |
70 | -
71 | Sort by Time
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
93 |
94 |
95 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
499 |
500 |
501 |
--------------------------------------------------------------------------------
/modules/router/router.go:
--------------------------------------------------------------------------------
1 | // Copyright © 2020 Uzhinskiy Boris
2 | // Licensed under the Apache License, Version 2.0 (the "License");
3 | // you may not use this file except in compliance with the License.
4 | // You may obtain a copy of the License at
5 | //
6 | // http://www.apache.org/licenses/LICENSE-2.0
7 | //
8 | // Unless required by applicable law or agreed to in writing, software
9 | // distributed under the License is distributed on an "AS IS" BASIS,
10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | // See the License for the specific language governing permissions and
12 | // limitations under the License.
13 |
14 | package router
15 |
16 | import (
17 | "encoding/json"
18 | "fmt"
19 | "log"
20 | "mime"
21 | "net/http"
22 | "os"
23 | "path"
24 | "reflect"
25 | "regexp"
26 | "sort"
27 | "strings"
28 | "text/template"
29 |
30 | "time"
31 |
32 | "github.com/flant/elasticsearch-extractor/modules/config"
33 | "github.com/flant/elasticsearch-extractor/modules/front"
34 | "github.com/flant/elasticsearch-extractor/modules/version"
35 | "github.com/uzhinskiy/lib.go/helpers"
36 | )
37 |
38 | type Filter struct {
39 | Field string `json:"field,omitempty"`
40 | Operation string `json:"operation,omitempty"`
41 | Value string `json:"value,omitempty"`
42 | }
43 |
44 | type Router struct {
45 | conf config.Config
46 | nc map[string]*http.Client
47 | nodes nodesArray
48 | sl []snapItem
49 | }
50 |
51 | type apiRequest struct {
52 | Action string `json:"action,omitempty"` // Имя вызываемого метода*
53 | Values struct {
54 | Indices []string `json:"indices,omitempty"`
55 | Repo string `json:"repo,omitempty"`
56 | OrderDir string `json:"odir,omitempty"`
57 | OrderType string `json:"otype,omitempty"`
58 | Snapshot string `json:"snapshot,omitempty"`
59 | Index string `json:"index,omitempty"`
60 | } `json:"values,omitempty"`
61 | Search struct {
62 | Index string `json:"index,omitempty"`
63 | Cluster string `json:"cluster,omitempty"`
64 | Xql string `json:"xql,omitempty"`
65 | Fields []string `json:"fields,omitempty"`
66 | Filters map[string]Filter `json:"filters,omitempty"`
67 | Mapping []string `json:"mapping,omitempty"`
68 | Timefields []string `json:"timefields,omitempty"`
69 | DateStart string `json:"date_start,omitempty"`
70 | DateEnd string `json:"date_end,omitempty"`
71 | SearchAfter string `json:"search_after,omitempty"`
72 | Count bool `json:"count,omitempty"`
73 | Fname string `json:"fname,omitempty"`
74 | } `json:"search,omitempty"`
75 | }
76 |
77 | type snapStatus struct {
78 | Snapshots []struct {
79 | Snapshot string `json:"snapshot,omitempty"`
80 | State string `json:"state,omitempty"`
81 | Indices map[string]struct {
82 | ShardsStats struct {
83 | Total int `json:"total,omitempty"`
84 | } `json:"shards_stats,omitempty"`
85 | Stats struct {
86 | Total struct {
87 | Size int `json:"size_in_bytes,omitempty"`
88 | } `json:"total,omitempty"`
89 | } `json:"stats,omitempty"`
90 | Shards map[string]struct {
91 | Stats struct {
92 | Total struct {
93 | Size int `json:"size_in_bytes,omitempty"`
94 | } `json:"total,omitempty"`
95 | } `json:"stats,omitempty"`
96 | } `json:"shards,omitempty"`
97 | } `json:"indices,omitempty"`
98 | } `json:"snapshots,omitempty"`
99 | }
100 |
101 | type singleNode struct {
102 | Ip string `json:"ip,omitempty"`
103 | Name string `json:"name,omitempty"`
104 | Dt string `json:"dt,omitempty"`
105 | Dtb int
106 | Du string `json:"du,omitempty"`
107 | Dup string `json:"dup,omitempty"`
108 | D string `json:"d,omitempty"`
109 | DiskFree int
110 | }
111 |
112 | type nodesStatus struct {
113 | nlist []singleNode
114 | }
115 |
116 | type nodesArray struct {
117 | //sync.RWMutex
118 | list []int
119 | max int
120 | sum int
121 | }
122 |
123 | type IndexInSnap struct {
124 | Name string
125 | Size int
126 | Shards []int
127 | }
128 |
129 | type indexGroup struct {
130 | Index string `json:"index,omitempty"`
131 | }
132 |
133 | type Cluster struct {
134 | Name string
135 | Host string
136 | Type string
137 | }
138 |
139 | type snapResponse struct {
140 | Snapshots []snapItem `json:"snapshots"`
141 | }
142 | type snapItem struct {
143 | Snapshot string `json:"snapshot,omitempty"`
144 | Uuid string `json:"uuid,omitempty"`
145 | State string `json:"state,omitempty"`
146 | CreateEpoch int64
147 | CreateDate string
148 | }
149 |
150 | type scrollResponse struct {
151 | ScrollID string `json:"_scroll_id,omitempty"`
152 | HitsRoot Hits `json:"hits"`
153 | }
154 |
155 | type HitsTotal struct {
156 | Value int64 `json:"value"`
157 | }
158 |
159 | type Hits struct {
160 | Total HitsTotal `json:"total"`
161 | Hits []Hit `json:"hits"`
162 | MaxScore float64 `json:"max_score"`
163 | }
164 |
165 | type Hit struct {
166 | Source map[string]interface{} `json:"_source,omitempty"`
167 | Fields map[string]interface{} `json:"fields,omitempty"`
168 | }
169 |
170 | type IndicesInSnap map[string]*IndexInSnap
171 |
172 | type ClusterHealth struct {
173 | ClusterName string `json:"cluster_name,omitempty"`
174 | Status string `json:"status,omitempty"`
175 | InitializingShards int `json:"initializingShards,omitempty"`
176 | UnassignedShards int `json:"unassigned_shards,omitempty"`
177 | }
178 |
179 | type JSONRow map[string]interface{}
180 |
181 | func Run(cnf config.Config) {
182 | rt := Router{}
183 | rt.conf = cnf
184 | rt.nc = make(map[string]*http.Client)
185 | rt.netClientPrepare()
186 | _, err := rt.getNodes()
187 | if err != nil {
188 | log.Println(err)
189 | }
190 |
191 | http.HandleFunc("/", rt.FrontHandler)
192 | http.HandleFunc("/api/", rt.ApiHandler)
193 | http.ListenAndServe(cnf.App.Bind+":"+cnf.App.Port, nil)
194 | }
195 |
196 | // web-ui
197 | func (rt *Router) FrontHandler(w http.ResponseWriter, r *http.Request) {
198 | file := r.URL.Path
199 |
200 | remoteIP := helpers.GetIP(r.RemoteAddr, r.Header.Get("X-Real-IP"), r.Header.Get("X-Forwarded-For"))
201 | if file == "/" {
202 | file = "/index.html"
203 | }
204 | if file == "/search/" {
205 | file = "/search.html"
206 | }
207 |
208 | if strings.Contains(file, "/data/") {
209 | /*wdir, err := os.Getwd()
210 | if err != nil {
211 | log.Println(err)
212 | }*/
213 |
214 | fi, err := os.Lstat("/tmp" + file)
215 | if err != nil {
216 | http.Error(w, err.Error(), 404)
217 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", 404, "\t", err.Error(), "\t", r.UserAgent())
218 | return
219 | }
220 |
221 | bytes, err := getFile("/tmp"+file, fi.Size())
222 | if err != nil {
223 | http.Error(w, err.Error(), 404)
224 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", 404, "\t", err.Error(), "\t", r.UserAgent())
225 | return
226 | }
227 |
228 | contentType := mime.TypeByExtension(path.Ext("/tmp" + file))
229 | if contentType == "application/json" {
230 | w.Header().Set("Content-Type", "application/octet-stream")
231 | } else {
232 | w.Header().Set("Content-Type", contentType)
233 | }
234 |
235 | w.Header().Set("Access-Control-Allow-Origin", "*")
236 | w.Header().Set("X-Server", version.Version)
237 |
238 | w.Write(bytes)
239 | return
240 | }
241 |
242 | cFile := strings.Replace(file, "/", "", 1)
243 | data, err := front.Asset(cFile)
244 | if err != nil {
245 | http.Error(w, err.Error(), 404)
246 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", 404, "\t", err.Error(), "\t", r.UserAgent())
247 | return
248 | }
249 |
250 | /* отправить его клиенту */
251 | contentType := mime.TypeByExtension(path.Ext(cFile))
252 | w.Header().Set("Content-Type", contentType)
253 | w.Header().Set("Access-Control-Allow-Origin", "*")
254 | w.Header().Set("X-Server", version.Version)
255 | if strings.Contains(file, ".html") {
256 | t := template.Must(template.New("index").Parse(string(data)))
257 | t.Execute(w, rt.conf.App.Kibana)
258 | } else {
259 | w.Write(data)
260 | }
261 | }
262 |
263 | func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) {
264 | var request apiRequest
265 |
266 | defer r.Body.Close()
267 | remoteIP := helpers.GetIP(r.RemoteAddr, r.Header.Get("X-Real-IP"), r.Header.Get("X-Forwarded-For"))
268 |
269 | w.Header().Add("Access-Control-Allow-Origin", "*")
270 | w.Header().Add("Access-Control-Allow-Methods", "POST,OPTIONS")
271 | w.Header().Add("Access-Control-Allow-Credentials", "true")
272 | w.Header().Add("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
273 | w.Header().Add("Content-Type", "application/json; charset=utf-8")
274 | w.Header().Set("X-Server", version.Version)
275 |
276 | if r.Method == "OPTIONS" {
277 | return
278 | }
279 |
280 | if r.Method != http.MethodPost {
281 | http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
282 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusMethodNotAllowed, "\t", "Invalid request method ")
283 | return
284 | }
285 | err := json.NewDecoder(r.Body).Decode(&request)
286 | if err != nil {
287 | http.Error(w, err.Error(), http.StatusInternalServerError)
288 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
289 | return
290 | }
291 | switch request.Action {
292 | case "get_repositories":
293 | {
294 | response, err := rt.doGet(rt.conf.Snapshot.Host+"_cat/repositories?format=json", "Snapshot")
295 | if err != nil {
296 | http.Error(w, err.Error(), http.StatusInternalServerError)
297 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
298 | return
299 | }
300 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
301 | w.Write(response)
302 | }
303 | case "get_nodes":
304 | {
305 | nresp, err := rt.getNodes()
306 | if err != nil {
307 | http.Error(w, err.Error(), http.StatusInternalServerError)
308 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
309 | return
310 | }
311 |
312 | j, _ := json.Marshal(nresp)
313 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
314 | w.Write(j)
315 | }
316 |
317 | case "get_indices":
318 | {
319 | response, err := rt.doGet(rt.conf.Snapshot.Host+"extracted*/_recovery/", "Snapshot")
320 | if err != nil {
321 | http.Error(w, err.Error(), http.StatusInternalServerError)
322 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
323 | return
324 | }
325 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
326 | w.Write(response)
327 | }
328 |
329 | case "del_index":
330 | {
331 | if request.Values.Index == "" {
332 | msg := `{"error":"Required parameter Values.Index is missed"}`
333 | http.Error(w, msg, http.StatusBadRequest)
334 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusBadRequest, "\t", msg)
335 | return
336 | }
337 | response, err := rt.doDel(rt.conf.Snapshot.Host+request.Values.Index, "Snapshot")
338 | if err != nil {
339 | http.Error(w, err.Error(), http.StatusInternalServerError)
340 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
341 | return
342 | }
343 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
344 | w.Write(response)
345 | }
346 |
347 | case "get_snapshots":
348 | {
349 | if request.Values.Repo == "" {
350 | msg := `{"error":"Required parameter Values.Repo is missed"}`
351 | http.Error(w, msg, http.StatusBadRequest)
352 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusBadRequest, "\t", msg)
353 | return
354 | }
355 |
356 | response, err := rt.doGet(rt.conf.Snapshot.Host+"_snapshot/"+request.Values.Repo+"/*?verbose=false&format=json", "Snapshot")
357 | if err != nil {
358 | http.Error(w, err.Error(), http.StatusInternalServerError)
359 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
360 | return
361 | }
362 |
363 | var snap_resp snapResponse
364 | var snap_items []snapItem
365 | err = json.Unmarshal(response, &snap_resp)
366 | if err != nil {
367 | http.Error(w, err.Error(), http.StatusInternalServerError)
368 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
369 | return
370 | }
371 | re := regexp.MustCompile(`^(.*)-(\d{4}\.\d{2}\.\d{2})`)
372 |
373 | if !rt.conf.Snapshot.Include {
374 | for _, n := range snap_resp.Snapshots {
375 | matched, err := regexp.MatchString(`^[\.]\S+`, n.Snapshot)
376 | if err != nil {
377 | log.Println("Regex error for ", n.Snapshot)
378 | }
379 | if !matched {
380 | match := re.FindStringSubmatch(n.Snapshot)
381 | if len(match) < 3 {
382 | log.Println("Skip snapshot with unexpected name:", n.Snapshot)
383 | continue
384 | }
385 | n.CreateDate = match[2]
386 | d, err := time.Parse("2006.01.02", n.CreateDate)
387 | n.CreateEpoch = d.Unix()
388 | if err != nil {
389 | http.Error(w, err.Error(), http.StatusInternalServerError)
390 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
391 | return
392 | }
393 | snap_items = append(snap_items, n)
394 | }
395 | }
396 | } else {
397 | for _, n := range snap_resp.Snapshots {
398 | match := re.FindStringSubmatch(n.Snapshot)
399 | if len(match) < 3 {
400 | log.Println("Skip snapshot with unexpected name:", n.Snapshot)
401 | continue
402 | }
403 | n.CreateDate = match[2]
404 | d, err := time.Parse("2006.01.02", n.CreateDate)
405 | n.CreateEpoch = d.Unix()
406 | if err != nil {
407 | http.Error(w, err.Error(), http.StatusInternalServerError)
408 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
409 | return
410 | }
411 | snap_items = append(snap_items, n)
412 | }
413 | }
414 |
415 | if request.Values.OrderType == "time" {
416 |
417 | if request.Values.OrderDir == "asc" {
418 | sort.Slice(snap_items[:], func(i, j int) bool {
419 | return snap_items[i].CreateEpoch < snap_items[j].CreateEpoch
420 | })
421 | } else {
422 | sort.Slice(snap_items[:], func(i, j int) bool {
423 | return snap_items[i].CreateEpoch > snap_items[j].CreateEpoch
424 | })
425 | }
426 |
427 | } else if request.Values.OrderType == "name" {
428 |
429 | if request.Values.OrderDir == "asc" {
430 | sort.Slice(snap_items[:], func(i, j int) bool {
431 | return snap_items[i].Snapshot < snap_items[j].Snapshot
432 | })
433 |
434 | } else {
435 | sort.Slice(snap_items[:], func(i, j int) bool {
436 | return snap_items[i].Snapshot > snap_items[j].Snapshot
437 | })
438 | }
439 |
440 | }
441 | rt.sl = snap_items
442 | j, _ := json.Marshal(snap_items)
443 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
444 | w.Write(j)
445 | }
446 |
447 | case "get_snapshots_sorted":
448 | {
449 | if request.Values.OrderType == "time" {
450 |
451 | if request.Values.OrderDir == "asc" {
452 | sort.Slice(rt.sl[:], func(i, j int) bool {
453 | return rt.sl[i].CreateEpoch < rt.sl[j].CreateEpoch
454 | })
455 | } else {
456 | sort.Slice(rt.sl[:], func(i, j int) bool {
457 | return rt.sl[i].CreateEpoch > rt.sl[j].CreateEpoch
458 | })
459 | }
460 |
461 | } else if request.Values.OrderType == "name" {
462 |
463 | if request.Values.OrderDir == "asc" {
464 | sort.Slice(rt.sl[:], func(i, j int) bool {
465 | return rt.sl[i].Snapshot < rt.sl[j].Snapshot
466 | })
467 |
468 | } else {
469 | sort.Slice(rt.sl[:], func(i, j int) bool {
470 | return rt.sl[i].Snapshot > rt.sl[j].Snapshot
471 | })
472 | }
473 |
474 | }
475 |
476 | j, _ := json.Marshal(rt.sl)
477 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent(), "\t", "Get Snapshots from cache")
478 | w.Write(j)
479 | }
480 |
481 | case "get_snapshot":
482 | {
483 |
484 | if request.Values.Repo == "" {
485 | msg := `{"error":"Required parameter Values.Repo is missed"}`
486 | http.Error(w, msg, http.StatusBadRequest)
487 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusBadRequest, "\t", msg)
488 | return
489 | }
490 |
491 | if request.Values.Snapshot == "" {
492 | msg := `{"error":"Required parameter Values.Snapshot is missed"}`
493 | http.Error(w, msg, http.StatusBadRequest)
494 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusBadRequest, "\t", msg)
495 | return
496 | }
497 |
498 | status_response, err := rt.doGet(rt.conf.Snapshot.Host+"_snapshot/"+request.Values.Repo+"/"+request.Values.Snapshot+"/_status", "Snapshot")
499 | if err != nil {
500 | http.Error(w, err.Error(), http.StatusInternalServerError)
501 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
502 | return
503 | }
504 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
505 | w.Write(status_response)
506 | }
507 |
508 | case "restore":
509 | {
510 | if request.Values.Repo == "" {
511 | msg := `{"error":"Required parameter Values.Repo is missed"}`
512 | http.Error(w, msg, http.StatusBadRequest)
513 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusBadRequest, "\t", msg)
514 | return
515 | }
516 |
517 | if request.Values.Snapshot == "" {
518 | msg := `{"error":"Required parameter Values.Snapshot is missed"}`
519 | http.Error(w, msg, http.StatusBadRequest)
520 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusBadRequest, "\t", msg)
521 | return
522 | }
523 |
524 | status_response, err := rt.doGet(rt.conf.Snapshot.Host+"_snapshot/"+request.Values.Repo+"/"+request.Values.Snapshot+"/_status", "Snapshot")
525 | if err != nil {
526 | http.Error(w, err.Error(), http.StatusInternalServerError)
527 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
528 | return
529 | }
530 | var snap_status snapStatus
531 | err = json.Unmarshal(status_response, &snap_status)
532 | if err != nil {
533 | http.Error(w, err.Error(), http.StatusInternalServerError)
534 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
535 | return
536 | }
537 |
538 | ch_response, err := rt.doGet(rt.conf.Snapshot.Host+"_cluster/health/extracted*", "Snapshot")
539 | if err != nil {
540 | http.Error(w, err.Error(), http.StatusInternalServerError)
541 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t_cluster/health/extracted*\t", http.StatusInternalServerError, "\t", err.Error())
542 | return
543 | }
544 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t_cluster/health/extracted*\t", r.UserAgent())
545 |
546 | var ch_status ClusterHealth
547 | err = json.Unmarshal(ch_response, &ch_status)
548 |
549 | if err != nil {
550 | http.Error(w, err.Error(), http.StatusInternalServerError)
551 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t_cluster/health/extracted*\t", http.StatusInternalServerError, "\t", err.Error())
552 | return
553 | }
554 |
555 | // Если в кластере есть недовосстановленные индексы - прерываем
556 | if ch_status.InitializingShards > 5 || ch_status.UnassignedShards > 5 {
557 | msg := `{"error":"Indices will not be restored at now. Please wait"}`
558 | http.Error(w, msg, http.StatusTooManyRequests)
559 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusTooManyRequests, "\t", msg)
560 | return
561 | }
562 |
563 | indices := make(IndicesInSnap)
564 |
565 | for _, iname := range request.Values.Indices {
566 | ind := snap_status.Snapshots[0].Indices[iname]
567 | indices[iname] = &IndexInSnap{}
568 | indices[iname].Size = ind.Stats.Total.Size
569 | if ind.ShardsStats.Total > 0 {
570 | for s := range snap_status.Snapshots[0].Indices[iname].Shards {
571 | indices[iname].Shards = append(indices[iname].Shards, snap_status.Snapshots[0].Indices[iname].Shards[s].Stats.Total.Size)
572 | }
573 | }
574 | }
575 |
576 | index_list_for_restore, index_list_not_restore := rt.Barrel(indices, rt.conf.Snapshot.IsS3)
577 |
578 | t := time.Now()
579 | req := map[string]interface{}{
580 | "ignore_unavailable": false,
581 | "include_global_state": false,
582 | "include_aliases": false,
583 | "rename_pattern": "(.+)",
584 | "rename_replacement": fmt.Sprintf("extracted_$1-%s", t.Format("02-01-2006")),
585 | "indices": index_list_for_restore,
586 | "index_settings": map[string]interface{}{"index.number_of_replicas": 0},
587 | }
588 |
589 | response, err := rt.doPost(rt.conf.Snapshot.Host+"_snapshot/"+request.Values.Repo+"/"+request.Values.Snapshot+"/_restore?wait_for_completion=false", req, "Snapshot")
590 | if err != nil {
591 | msg := fmt.Sprintf(`{"error":"%s"}`, err)
592 | http.Error(w, msg, 500)
593 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", 500, "\t", err.Error(), "\t", response)
594 | return
595 | }
596 |
597 | if len(index_list_not_restore) > 0 {
598 | msg := fmt.Sprintf(`{"message":"Indices '%v' will not be restored: Not enough space", "error":1}`, index_list_not_restore)
599 | w.Write([]byte(msg))
600 | }
601 |
602 | /* Не создаем паттерны для восстановленных индексов
603 | for _, iname := range index_list_for_restore {
604 | if strings.Contains(iname, "v3") {
605 | ip_req := map[string]interface{}{
606 | "type": "index-pattern",
607 | "index-pattern": map[string]interface{}{
608 | "title": "extracted_v3-*",
609 | "timeFieldName": "timestamp"}}
610 |
611 | ip_resp, err := rt.doPost(rt.conf.Snapshot.Host+".kibana/_doc/index-pattern:v3-080", ip_req, "Snapshot")
612 | if err != nil {
613 | msg := fmt.Sprintf(`{"error":"%s"}`, err)
614 | http.Error(w, msg, 500)
615 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\tcreate index-pattern\t", 500, "\t", err.Error(), "\t", ip_resp)
616 | }
617 | } else {
618 | ip_req := map[string]interface{}{
619 | "type": "index-pattern",
620 | "index-pattern": map[string]interface{}{
621 | "title": "extracted_*",
622 | "timeFieldName": "@timestamp"}}
623 |
624 | ip_resp, err := rt.doPost(rt.conf.Snapshot.Host+".kibana/_doc/index-pattern:080", ip_req, "Snapshot")
625 | if err != nil {
626 | msg := fmt.Sprintf(`{"error":"%s"}`, err)
627 | http.Error(w, msg, 500)
628 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\tcreate index-pattern\t", 500, "\t", err.Error(), "\t", ip_resp)
629 | }
630 |
631 | }
632 | }
633 | */
634 |
635 | msg := fmt.Sprintf(`{"message":"Indices '%v' will be restored", "error":0}`, index_list_for_restore)
636 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
637 | w.Write([]byte(msg))
638 |
639 | }
640 | /* ---- search --- */
641 | case "get_clusters":
642 | {
643 | var cl []Cluster
644 | cl = append(cl, Cluster{rt.conf.Snapshot.Name, rt.conf.Snapshot.Host, "Snapshot"})
645 | cl = append(cl, Cluster{rt.conf.Search.Name, rt.conf.Search.Host, "Search"})
646 | j, _ := json.Marshal(cl)
647 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
648 | w.Write(j)
649 | }
650 | case "get_index_groups":
651 | {
652 | response, err := rt.getIndexGroups(request.Search.Cluster)
653 | if err != nil {
654 | http.Error(w, err.Error(), http.StatusInternalServerError)
655 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
656 | return
657 | }
658 | j, _ := json.Marshal(response)
659 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent())
660 | w.Write(j)
661 | }
662 |
663 | case "get_mapping":
664 | {
665 | t := time.Now()
666 | var (
667 | fullm map[string]interface{}
668 | m map[string]interface{}
669 | host string
670 | )
671 | if request.Search.Cluster == "Snapshot" {
672 | host = rt.conf.Snapshot.Host
673 | } else if request.Search.Cluster == "Search" {
674 | host = rt.conf.Search.Host
675 | }
676 | flatMap := make(map[string]string)
677 | response, err := rt.doGet(host+request.Search.Index+"*"+t.Format("2006.01.02")+"*,"+request.Search.Index+"*"+t.Format("02-01-2006")+"*/_mapping", "Search")
678 | if err != nil {
679 | http.Error(w, err.Error(), http.StatusInternalServerError)
680 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
681 | return
682 | }
683 |
684 | err = json.Unmarshal(response, &fullm)
685 | if err != nil {
686 | http.Error(w, err.Error(), http.StatusInternalServerError)
687 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
688 | return
689 | }
690 | for _, v := range fullm {
691 | m = v.(map[string]interface{})
692 | }
693 |
694 | if mapping, hasMap := m["mappings"]; hasMap {
695 | rt.flattenMap("", mapping.(map[string]interface{})["properties"].(map[string]interface{}), flatMap)
696 | }
697 |
698 | j, _ := json.Marshal(flatMap)
699 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent(), "\t", host+request.Search.Index)
700 | w.Write(j)
701 | }
702 |
703 | case "search":
704 | {
705 | var (
706 | use_source string
707 | query string
708 | filters string
709 | must_not string
710 | xql string
711 | full_query string
712 | timefield string
713 | sort string
714 | tf string
715 | fields string
716 | req map[string]interface{}
717 | host string
718 | )
719 | if request.Search.Cluster == "Snapshot" {
720 | host = rt.conf.Snapshot.Host
721 | } else if request.Search.Cluster == "Search" {
722 | host = rt.conf.Search.Host
723 | }
724 |
725 | ds, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Search.DateStart+" (MSK)")
726 | de, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Search.DateEnd+" (MSK)")
727 |
728 | if len(request.Search.Fields) == 0 {
729 | use_source = `"_source": true`
730 | } else {
731 | use_source = `"_source": false`
732 | }
733 |
734 | for _, f := range request.Search.Filters {
735 |
736 | if f.Operation == "is" {
737 | filters += `{ "wildcard": {"` + f.Field + `.keyword": {"value": "` + f.Value + `" } } },`
738 | } else if f.Operation == "exists" {
739 | filters += `{ "exists": {"field":"` + f.Field + `" } },`
740 | } else if f.Operation == "is_not" {
741 | must_not += `{ "match_phrase": {"` + f.Field + `":"` + f.Value + `" } },`
742 | } else if f.Operation == "does_not_exists" {
743 | must_not += `{ "exists": {"field":"` + f.Field + `" } },`
744 | }
745 | }
746 | filters += `{"match_all": {}}`
747 | must_not, _ = strings.CutSuffix(must_not, ",")
748 |
749 | if request.Search.Xql != "" {
750 | xql = `{ "simple_query_string": { "query": "` + request.Search.Xql + `" } }`
751 | }
752 | if len(request.Search.Timefields) > 0 {
753 | timefield = request.Search.Timefields[0]
754 | fields = `"fields": ["` + timefield + `", "` + strings.Join(request.Search.Fields, "\", \"") + `" ]`
755 | sort = `"sort": [ {"` + timefield + `": "desc" } ]`
756 | tf = `{ "range": { "` + timefield + `": {
757 | "gte": "` + ds.Format("2006-01-02T15:04:05.000Z") + `",
758 | "lte": "` + de.Format("2006-01-02T15:04:05.000Z") + `",
759 | "format": "strict_date_optional_time" } } },`
760 | } else {
761 | sort = ""
762 | tf = ""
763 | fields = `"fields": ["` + strings.Join(request.Search.Fields, "\", \"") + `" ]`
764 | }
765 | query = fmt.Sprintf(`"query": { "bool": { "must": [ %s ],"filter": [ %s %s ], "should": [],"must_not": [ %s ] }}`, xql, tf, filters, must_not)
766 |
767 | full_query = fmt.Sprintf(`{"size": 500, %s, %s, %s, %s }`, sort, use_source, fields, query)
768 | if request.Search.Count {
769 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent(), "\t", host+request.Search.Index, "\t", "action: Count", "\tquery: ", "{"+query+"}")
770 | _ = json.Unmarshal([]byte("{"+query+"}"), &req)
771 | cresponse, err := rt.doPost(host+request.Search.Index+"/_count", req, "Search")
772 | if err != nil {
773 | http.Error(w, err.Error(), http.StatusInternalServerError)
774 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
775 | return
776 | }
777 |
778 | w.Write(cresponse)
779 | } else {
780 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent(), "\t", host+request.Search.Index, "\t", "action: Search", "\tquery: ", full_query)
781 | _ = json.Unmarshal([]byte(full_query), &req)
782 | sresponse, err := rt.doPost(host+request.Search.Index+"/_search", req, "Search")
783 | if err != nil {
784 | http.Error(w, err.Error(), http.StatusInternalServerError)
785 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
786 | return
787 | }
788 | w.Write(sresponse)
789 | }
790 |
791 | }
792 |
793 | case "prepare_csv":
794 | {
795 | //allocateSpaceForFile("/tmp/data/"+request.Search.Fname+".csv", 100)
796 | var (
797 | use_source string
798 | query string
799 | filters string
800 | must_not string
801 | xql string
802 | full_query string
803 | timefield string
804 | sort string
805 | tf string
806 | fields string
807 | req map[string]interface{}
808 | scrollresponse scrollResponse
809 | fields_list []string
810 | host string
811 | request_batch int64
812 | )
813 |
814 | request_batch = rt.conf.Search.RequestBatch
815 |
816 | if request.Search.Cluster == "Snapshot" {
817 | host = rt.conf.Snapshot.Host
818 | } else if request.Search.Cluster == "Search" {
819 | host = rt.conf.Search.Host
820 | }
821 |
822 | ds, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Search.DateStart+" (MSK)")
823 | de, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Search.DateEnd+" (MSK)")
824 |
825 | if len(request.Search.Fields) == 0 {
826 | use_source = `"_source": true`
827 | fields_list = request.Search.Mapping
828 | } else {
829 | use_source = `"_source": false`
830 | fields_list = request.Search.Fields
831 | }
832 |
833 | for _, f := range request.Search.Filters {
834 |
835 | if f.Operation == "is" {
836 | filters += `{ "wildcard": {"` + f.Field + `.keyword": {"value": "` + f.Value + `" } } },`
837 | } else if f.Operation == "exists" {
838 | filters += `{ "exists": {"field":"` + f.Field + `" } },`
839 | } else if f.Operation == "is_not" {
840 | must_not += `{ "match_phrase": {"` + f.Field + `":"` + f.Value + `" } },`
841 | } else if f.Operation == "does_not_exists" {
842 | must_not += `{ "exists": {"field":"` + f.Field + `" } },`
843 | }
844 | }
845 | filters += `{"match_all": {}}`
846 | must_not, _ = strings.CutSuffix(must_not, ",")
847 |
848 | if request.Search.Xql != "" {
849 | xql = `{ "simple_query_string": { "query": "` + request.Search.Xql + `" } }`
850 | }
851 | if len(request.Search.Timefields) > 0 {
852 | timefield = request.Search.Timefields[0]
853 | fields = `"fields": ["` + timefield + `", "` + strings.Join(request.Search.Fields, "\", \"") + `" ]`
854 | sort = `"sort": [ {"` + timefield + `": "desc" } ]`
855 | tf = `{ "range": { "` + timefield + `": {
856 | "gte": "` + ds.Format("2006-01-02T15:04:05.000Z") + `",
857 | "lte": "` + de.Format("2006-01-02T15:04:05.000Z") + `",
858 | "format": "strict_date_optional_time" } } },`
859 | } else {
860 | sort = ""
861 | tf = ""
862 | fields = `"fields": ["` + strings.Join(request.Search.Fields, "\", \"") + `" ]`
863 | }
864 | query = fmt.Sprintf(`"query": { "bool": { "must": [ %s ],"filter": [ %s %s ], "should": [],"must_not": [ %s ] }}`, xql, tf, filters, must_not)
865 |
866 | full_query = fmt.Sprintf(`{"size": %d, %s, %s, %s, %s }`, request_batch, sort, use_source, fields, query)
867 |
868 | err = json.Unmarshal([]byte(full_query), &req)
869 | if err != nil {
870 | http.Error(w, err.Error(), http.StatusInternalServerError)
871 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
872 | return
873 | }
874 | sresponse, err := rt.doPost(host+request.Search.Index+"/_search?scroll=10m", req, "Search")
875 | if err != nil {
876 | http.Error(w, err.Error(), http.StatusInternalServerError)
877 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
878 | return
879 | }
880 |
881 | f, err := os.OpenFile("/tmp/data/"+request.Search.Fname+".csv", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
882 | if err != nil {
883 | log.Fatal(err)
884 | }
885 | defer f.Close()
886 |
887 | if len(request.Search.Timefields) > 0 {
888 | f.WriteString(request.Search.Timefields[0] + `;` + strings.Join(fields_list, ";") + "\n")
889 | } else {
890 | f.WriteString(strings.Join(fields_list, ";") + "\n")
891 | }
892 |
893 | err = json.Unmarshal(sresponse, &scrollresponse)
894 | if err != nil {
895 | http.Error(w, err.Error(), http.StatusInternalServerError)
896 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
897 | return
898 | }
899 |
900 | if len(scrollresponse.HitsRoot.Hits) > 0 {
901 | var data any
902 | for _, row := range scrollresponse.HitsRoot.Hits {
903 | fileInfo, err := os.Stat("/tmp/data/" + request.Search.Fname + ".csv")
904 | if err != nil {
905 | log.Println(err)
906 | return
907 | }
908 |
909 | if fileInfo.Size() > rt.conf.Search.FileLimit.Size {
910 | return
911 | }
912 | if len(request.Search.Fields) == 0 {
913 | f.WriteString(fmt.Sprintf("%v;", row.Source[request.Search.Timefields[0]]))
914 | } else {
915 | f.WriteString(fmt.Sprintf("%v;", row.Fields[request.Search.Timefields[0]]))
916 | }
917 | for _, fm := range fields_list {
918 | if len(request.Search.Fields) == 0 {
919 | data = row.Source[fm]
920 | } else {
921 | data = row.Fields[fm]
922 | }
923 |
924 | if data == nil {
925 | f.WriteString(fmt.Sprintf("%s;", "--"))
926 | } else {
927 | switch reflect.TypeOf(data).Kind() {
928 | case reflect.Slice:
929 | {
930 | s := reflect.ValueOf(data)
931 | var ss string
932 | for i := 0; i < s.Len(); i++ {
933 | ss = ss + fmt.Sprintf("%v, ", s.Index(i))
934 | }
935 | ss = strings.TrimSuffix(ss, ", ")
936 | ss = strings.Replace(ss, "\n", "", -1)
937 | ss = strings.Replace(ss, "\"", "\"\"", -1)
938 | f.WriteString(fmt.Sprintf(`"%s";`, ss))
939 |
940 | }
941 | case reflect.String:
942 | {
943 | f.WriteString(fmt.Sprintf(`"%v";`, strings.Replace(strings.Replace(data.(string), "\n", "", -1), "\"", "\"\"", -1)))
944 | }
945 | default:
946 | {
947 | f.WriteString(fmt.Sprintf("%v;", data))
948 | }
949 | }
950 |
951 | }
952 |
953 | }
954 | f.WriteString("\n")
955 | }
956 | }
957 |
958 | if scrollresponse.ScrollID != "" {
959 | scroll := map[string]interface{}{"scroll": "10m", "scroll_id": scrollresponse.ScrollID}
960 | for i := 0; i < rt.conf.Search.FileLimit.Rows/10000; i++ {
961 | sresponse, err := rt.doPost(host+"_search/scroll", scroll, "Search")
962 | if err != nil {
963 | log.Println("Failed to get scroll batch: ", err)
964 | return
965 | }
966 | _ = json.Unmarshal(sresponse, &scrollresponse)
967 | if len(scrollresponse.HitsRoot.Hits) == 0 {
968 | _, _ = rt.doDel(request.Search.Cluster+"_search/scroll/"+scrollresponse.ScrollID, "Search")
969 | break
970 | }
971 | if len(scrollresponse.HitsRoot.Hits) > 0 {
972 |
973 | var data any
974 | for _, row := range scrollresponse.HitsRoot.Hits {
975 | fileInfo, err := os.Stat("/tmp/data/" + request.Search.Fname + ".csv")
976 | if err != nil {
977 | log.Println(err)
978 | return
979 | }
980 | if fileInfo.Size() > rt.conf.Search.FileLimit.Size {
981 | return
982 | }
983 |
984 | if len(request.Search.Fields) == 0 {
985 | f.WriteString(fmt.Sprintf("%v;", row.Source[request.Search.Timefields[0]]))
986 | } else {
987 | f.WriteString(fmt.Sprintf("%v;", row.Fields[request.Search.Timefields[0]]))
988 | }
989 | for _, fm := range fields_list {
990 | if len(request.Search.Fields) == 0 {
991 | data = row.Source[fm]
992 | } else {
993 | data = row.Fields[fm]
994 | }
995 |
996 | if data == nil {
997 | f.WriteString(fmt.Sprintf("%s;", "--"))
998 | } else {
999 | switch reflect.TypeOf(data).Kind() {
1000 | case reflect.Slice:
1001 | {
1002 | s := reflect.ValueOf(data)
1003 | var ss string
1004 | for i := 0; i < s.Len(); i++ {
1005 | ss = ss + fmt.Sprintf("%v, ", s.Index(i))
1006 | }
1007 | ss = strings.TrimSuffix(ss, ", ")
1008 | ss = strings.Replace(ss, "\n", "", -1)
1009 | ss = strings.Replace(ss, "\"", "\"\"", -1)
1010 | f.WriteString(fmt.Sprintf(`"%s";`, ss))
1011 | }
1012 | case reflect.String:
1013 | {
1014 | f.WriteString(fmt.Sprintf(`"%v";`, strings.Replace(strings.Replace(data.(string), "\n", "", -1), "\"", "\"\"", -1)))
1015 | }
1016 | default:
1017 | {
1018 | f.WriteString(fmt.Sprintf("%v;", data))
1019 | }
1020 | }
1021 | }
1022 | }
1023 | f.WriteString("\n")
1024 | }
1025 | }
1026 | }
1027 |
1028 | }
1029 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent(), "\t", host+request.Search.Index, "\t", "action: CSV", "\tquery: ", full_query, "\tfile: ", request.Search.Fname)
1030 | w.Write([]byte("Done"))
1031 | }
1032 | case "prepare_json":
1033 | {
1034 | err := rt.saveHintsToJsonFile(request)
1035 | if err != nil {
1036 | http.Error(w, err.Error(), http.StatusInternalServerError)
1037 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
1038 | return
1039 | }
1040 |
1041 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", r.UserAgent(), "\t", "action: JSON")
1042 |
1043 | w.Write([]byte("Done"))
1044 | }
1045 |
1046 | default:
1047 | {
1048 | http.Error(w, "Service Unavailable", http.StatusServiceUnavailable)
1049 | log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", http.StatusServiceUnavailable, "\t", "Invalid request method ")
1050 | return
1051 |
1052 | }
1053 |
1054 | }
1055 | }
1056 |
--------------------------------------------------------------------------------