├── Changelog.md ├── VERSION ├── pkg ├── ioc │ ├── README.md │ └── factory.go ├── openmetric │ ├── pod_one.s │ └── manager.go ├── util │ ├── gopsutil_hack.s │ ├── strategy │ │ ├── strategy.go │ │ ├── backoff_test.go │ │ └── options.go │ ├── signal.go │ ├── cast.go │ ├── os_nolinux.go │ ├── net_test.go │ ├── os_linux.go │ ├── stats │ │ └── util.go │ ├── rand.go │ ├── gob.go │ ├── strings_sort.go │ ├── gopsutil_hack.go │ ├── strings_test.go │ ├── throttle │ │ └── throttle.go │ ├── eventcollector │ │ └── collect_test.go │ ├── grpc.go │ ├── http.go │ ├── json.go │ ├── time_test.go │ ├── batch │ │ ├── type.go │ │ └── options.go │ ├── context.go │ ├── recoverutils │ │ └── goroutine.go │ ├── stop.go │ ├── fs2 │ │ └── stat.go │ └── singleton │ │ └── singleton.go ├── appconfig │ ├── README.md │ └── http.go ├── plugin │ ├── input │ │ ├── cpu │ │ │ ├── gopsutil_hack.s │ │ │ ├── gopsutil_hack.go │ │ │ └── init.go │ │ ├── dialcheck │ │ │ ├── mock │ │ │ │ ├── collectRange.json │ │ │ │ ├── normal-config.json │ │ │ │ └── normal.md │ │ │ └── dialcheck_test.go │ │ ├── httpcheck │ │ │ ├── mock │ │ │ │ ├── collectRange.json │ │ │ │ ├── normal-config.json │ │ │ │ └── normal.md │ │ │ └── httpcheck_test.go │ │ ├── jvm │ │ │ ├── init.go │ │ │ └── convert.go │ │ ├── processperf │ │ │ └── README.md │ │ ├── mem │ │ │ ├── init.go │ │ │ └── mem.go │ │ ├── tcp │ │ │ └── init.go │ │ ├── disk │ │ │ └── init.go │ │ ├── load │ │ │ ├── init.go │ │ │ └── load.go │ │ ├── swap │ │ │ ├── init.go │ │ │ └── swap.go │ │ ├── process │ │ │ ├── init.go │ │ │ ├── cpu_stat.go │ │ │ └── process.go │ │ ├── traffic │ │ │ └── init.go │ │ ├── standard │ │ │ ├── mysqlw │ │ │ │ └── mysql.go │ │ │ └── wrap.go │ │ ├── thread │ │ │ └── init.go │ │ ├── springboot │ │ │ └── springboot_test.go │ │ ├── nvidia_smi │ │ │ └── README.md │ │ ├── inputproxy │ │ │ └── providers.go │ │ └── all │ │ │ └── all.go │ ├── api │ │ ├── model.go │ │ ├── utils.go │ │ ├── pipeline.go │ │ └── input.go │ ├── output │ │ ├── sls │ │ │ └── init.go │ │ ├── gateway │ │ │ └── init.go │ │ ├── console │ │ │ └── init.go │ │ ├── all │ │ │ └── all.go │ │ └── factory.go │ └── integration │ │ └── alibabacloud │ │ ├── aliyun.go │ │ ├── README.md │ │ └── aliyun_redis_test.go ├── k8s │ ├── README.md │ ├── k8ssync │ │ └── README.md │ ├── k8ssysmetrics │ │ ├── cadvisor │ │ │ ├── README.md │ │ │ └── cadvisor_pipeline.go │ │ ├── factory.go │ │ └── common │ │ │ └── api.go │ ├── k8smeta │ │ ├── README.md │ │ ├── endpoints.go │ │ └── service.go │ ├── listwatchext │ │ └── README.md │ ├── silence │ │ ├── controller.go │ │ ├── pod_update_listener_component.go │ │ └── utils.go │ ├── k8sutils │ │ └── node.go │ ├── resources │ │ └── resources.go │ └── nodehostname │ │ └── nodehostname.go ├── server │ ├── pb │ │ └── README.md │ ├── registry │ │ ├── README.md │ │ ├── bistream │ │ │ └── README.md │ │ └── code.go │ ├── fordev │ │ └── fortest.go │ └── gen-pb.sh ├── bistream │ ├── biztypes │ │ ├── README.md │ │ └── biztypes.go │ └── rpccodes │ │ └── rpccodes.go ├── collecttask │ ├── README.md │ ├── test.go │ └── collecttaskcri │ │ └── utils.go ├── cri │ ├── pouch │ │ ├── README.md │ │ └── tools.go │ ├── README.md │ ├── dockerutils │ │ ├── consts.go │ │ ├── events.go │ │ ├── sock.go │ │ └── jsonlog.go │ ├── utils.go │ ├── containerdutils │ │ └── containerd_k3s.go │ ├── criutils │ │ ├── types.go │ │ ├── container.go │ │ ├── md5sum.go │ │ ├── zombies.go │ │ ├── glob.go │ │ ├── processes.go │ │ └── helper.go │ ├── cricore │ │ └── nsenter.go │ ├── mount.go │ └── impl │ │ ├── netproxy │ │ └── common.go │ │ ├── metastore_pod.go │ │ └── engine │ │ └── docker_oom_recoder.go ├── collectconfig │ ├── executor │ │ ├── transforms │ │ │ ├── cleanurl.yaml │ │ │ ├── substring_1.yaml │ │ │ ├── substring_2.yaml │ │ │ ├── substring_3.yaml │ │ │ ├── regexp1.yaml │ │ │ ├── substring_error.yaml │ │ │ ├── mapping1.yaml │ │ │ ├── regexp2.yaml │ │ │ └── const.json │ │ ├── examples │ │ │ ├── loganalysis │ │ │ │ └── 1.demolog │ │ │ ├── simple │ │ │ │ └── 1.demolog │ │ │ ├── transform1 │ │ │ │ └── 1.demolog │ │ │ ├── timestamp │ │ │ │ └── 1.demolog │ │ │ ├── json │ │ │ │ └── 1.demolog │ │ │ ├── regexp_path │ │ │ │ └── 1.demolog │ │ │ └── multiline │ │ │ │ └── 1.json │ │ ├── from.go │ │ ├── from_log.go │ │ ├── testutils │ │ │ └── log_generator.go │ │ ├── pool.go │ │ ├── window.go │ │ ├── delay.go │ │ ├── utils │ │ │ ├── file_linux.go │ │ │ └── file_darwin.go │ │ ├── slice_test.go │ │ ├── storage │ │ │ ├── types.go │ │ │ └── node_hll.go │ │ ├── transform_discard.go │ │ ├── time_processtime.go │ │ ├── logstream │ │ │ ├── utils_test.go │ │ │ └── logstream_reader.go │ │ ├── where_alwaystrue.go │ │ ├── filematch │ │ │ ├── filematch_regexp_test.go │ │ │ ├── filematch_test.go │ │ │ ├── filematch_abs.go │ │ │ ├── filewatch_format.go │ │ │ ├── filematch_sls.go │ │ │ ├── filematch_format2.go │ │ │ ├── filematch_glob.go │ │ │ ├── regexp_test.go │ │ │ ├── filematch_format2_test.go │ │ │ └── filematch.go │ │ ├── logparser_json.go │ │ ├── transform_regexp_v1_test.go │ │ ├── transform_const.go │ │ ├── consumer_log_sub.go │ │ ├── v2.go │ │ ├── timeparser │ │ │ ├── timebinarysearch │ │ │ │ └── tsb.go │ │ │ └── timeparser.go │ │ ├── log-parse-cpu.json │ │ ├── time_elect_unix.go │ │ ├── time_elect_unix_milli.go │ │ ├── logparser_grok.go │ │ ├── testv2_test.go │ │ ├── time_test.go │ │ ├── elect_refmeta.go │ │ ├── split_test.go │ │ ├── elect_refvar_test.go │ │ ├── transform_append_v1.go │ │ ├── elect_pathvar.go │ │ ├── logcontext_test.go │ │ ├── logparser_regexp.go │ │ ├── elect_refindex.go │ │ ├── transform_cleanurl_v1.go │ │ ├── elect_context.go │ │ ├── agg │ │ │ └── agg.go │ │ ├── regexp_test.go │ │ ├── log-where-regexp.json │ │ ├── elect_refvar.go │ │ ├── v1.go │ │ ├── transform_mapping_v1.go │ │ ├── elect_line.go │ │ ├── time_elect_golanglayout.go │ │ ├── transform_mapping_v1_test.go │ │ ├── where1.json │ │ ├── elect_regexp.go │ │ ├── time_elect_auto.go │ │ ├── logparser_regexp_test.go │ │ ├── transform_append_v1_test.go │ │ ├── logparser_json_test.go │ │ ├── log-where-contains.json │ │ └── elect_refname_test.go │ └── type_loganlysis.go ├── logger │ ├── log_test.go │ └── http.go ├── accumulator │ ├── accumulator.go │ └── memory.go ├── transfer │ ├── pb │ │ ├── gen.sh │ │ └── transfer.proto │ ├── manager │ │ └── utils.go │ └── api.go ├── meta │ ├── type.go │ ├── ref_label_test.go │ └── ref_label.go ├── telegraf │ └── utils.go ├── pipeline │ ├── manager_builtin.go │ ├── standard │ │ └── outputs.go │ └── integration │ │ └── alibabacloud │ │ ├── frequency_test.go │ │ ├── openapi.go │ │ └── timecache.go ├── core │ ├── helper.go │ ├── agent_mode.go │ └── hostfs.go ├── agent │ └── server │ │ └── api.go └── clusteragent │ └── clusteragent.go ├── scripts ├── api │ ├── basic │ │ ├── help │ │ └── version │ ├── lsm │ │ └── state │ ├── agent │ │ └── syncConfig │ ├── docker │ │ └── help │ ├── log │ │ ├── debug │ │ │ ├── start │ │ │ └── stop │ │ └── trace │ │ │ ├── start │ │ │ └── stop │ ├── meta │ │ ├── k8s │ │ │ ├── nodes │ │ │ │ └── list │ │ │ └── pods │ │ │ │ ├── local │ │ │ │ ├── byIp │ │ │ │ ├── byApp │ │ │ │ └── get │ │ └── local │ │ │ ├── list │ │ │ ├── hackErrorContainers │ │ │ ├── getByCid │ │ │ ├── list-simple │ │ │ └── get │ ├── pprof │ │ ├── allocs │ │ ├── goroutine │ │ ├── heap │ │ └── profile │ └── cri │ │ ├── findByIp │ │ ├── findByCid │ │ ├── findByName │ │ └── netproxy │ │ └── portforward ├── build │ ├── README.md │ ├── detect-goproxy.sh │ ├── agent-builder │ │ ├── README.md │ │ ├── build.sh │ │ └── Dockerfile │ └── build-using-docker.sh ├── docker │ ├── bin │ │ ├── docker.sh │ │ ├── start.sh │ │ ├── restart.sh │ │ ├── app.sh │ │ ├── stop.sh │ │ ├── init_bashrc.sh │ │ └── app.ini │ ├── init.sh │ ├── agent-base │ │ ├── README.md │ │ ├── build.sh │ │ └── Dockerfile │ ├── sc │ ├── entrypoint.sh │ ├── ensure_supervisord.sh │ ├── buildx.sh │ └── build.sh ├── vm │ ├── supervisord │ ├── initd_holoinsight-agent.sh │ ├── agent.sh │ ├── uninstall.sh │ ├── README.md │ ├── agent.ini │ ├── supervisord.conf │ ├── upload.sh │ └── build-lite.sh ├── pre-commit.sh ├── git │ └── rebase-main-and-reset-author.sh ├── gen-git-info.sh └── deploy │ └── build-and-replace-to-pod.sh ├── HEADER ├── cmd ├── containerhelper │ ├── handlers │ │ ├── httpcheck.json │ │ ├── fixout │ │ │ ├── common.go │ │ │ └── decode.go │ │ ├── hello.go │ │ ├── countZombies.sh │ │ ├── glob.go │ │ ├── cat.go │ │ ├── httpcheck.go │ │ ├── inspect.go │ │ ├── processes.go │ │ ├── listfiles.go │ │ ├── previewLog.go │ │ ├── all.go │ │ ├── inputproxy.go │ │ ├── countZombies.go │ │ └── httpproxy_test.go │ └── model │ │ ├── api.go │ │ ├── types.go │ │ └── factory.go ├── agent │ └── main.go └── plugintest │ ├── telegraf_ping.json │ ├── telegraf_nginx.json │ ├── telegraf_redis.json │ ├── telegraf_mongodb.json │ ├── telegraf_mysql.json │ └── main.go ├── .gitignore ├── .github ├── ISSUE_TEMPLATE │ ├── feature_request.md │ ├── bug_report.md │ ├── refactor-request.md │ ├── feature_request.yml │ └── bug_report.yml ├── workflows │ ├── build.yaml │ ├── upload-docker-image.yaml │ ├── upload-release-docker-image.yaml │ └── pr-merge-gate.yaml └── PULL_REQUEST_TEMPLATE.md └── .pre-commit-config.yaml /Changelog.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.0-SNAPSHOT -------------------------------------------------------------------------------- /pkg/ioc/README.md: -------------------------------------------------------------------------------- 1 | 依赖注入的包 2 | -------------------------------------------------------------------------------- /pkg/openmetric/pod_one.s: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/util/gopsutil_hack.s: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/appconfig/README.md: -------------------------------------------------------------------------------- 1 | # agent本身配置 -------------------------------------------------------------------------------- /pkg/plugin/input/cpu/gopsutil_hack.s: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/k8s/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | k8s 相关的模块. 3 | -------------------------------------------------------------------------------- /pkg/server/pb/README.md: -------------------------------------------------------------------------------- 1 | proto 包统一放在某个地方 2 | -------------------------------------------------------------------------------- /pkg/bistream/biztypes/README.md: -------------------------------------------------------------------------------- 1 | # 防止循环依赖 2 | -------------------------------------------------------------------------------- /pkg/collecttask/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 管理采集配置 3 | -------------------------------------------------------------------------------- /scripts/api/basic/help: -------------------------------------------------------------------------------- 1 | curl localhost:9117 2 | -------------------------------------------------------------------------------- /pkg/plugin/input/dialcheck/mock/collectRange.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /pkg/plugin/input/httpcheck/mock/collectRange.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /scripts/api/basic/version: -------------------------------------------------------------------------------- 1 | curl localhost:9117/version 2 | -------------------------------------------------------------------------------- /scripts/api/lsm/state: -------------------------------------------------------------------------------- 1 | curl localhost:9117/api/lsm/state 2 | -------------------------------------------------------------------------------- /pkg/cri/pouch/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | pouch 是阿里巴巴的开源容器运行时, 大部分跟docker兼容. 3 | -------------------------------------------------------------------------------- /pkg/k8s/k8ssync/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | clusteragent 同步 k8s 元数据到 registry. 3 | -------------------------------------------------------------------------------- /pkg/server/registry/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 与 registry 的交互. 3 | 内置了CA证书. 4 | -------------------------------------------------------------------------------- /scripts/api/agent/syncConfig: -------------------------------------------------------------------------------- 1 | curl localhost:9117/api/agent/syncConfig 2 | -------------------------------------------------------------------------------- /scripts/api/docker/help: -------------------------------------------------------------------------------- 1 | curl localhost:9117/api/docker/containers 2 | -------------------------------------------------------------------------------- /scripts/api/log/debug/start: -------------------------------------------------------------------------------- 1 | curl localhost:9117/api/log/debug/start 2 | -------------------------------------------------------------------------------- /scripts/api/log/debug/stop: -------------------------------------------------------------------------------- 1 | curl localhost:9117/api/log/debug/stop 2 | -------------------------------------------------------------------------------- /scripts/api/log/trace/start: -------------------------------------------------------------------------------- 1 | curl localhost:9117/api/log/trace/start 2 | -------------------------------------------------------------------------------- /scripts/api/log/trace/stop: -------------------------------------------------------------------------------- 1 | curl localhost:9117/api/log/trace/stop 2 | -------------------------------------------------------------------------------- /HEADER: -------------------------------------------------------------------------------- 1 | Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. -------------------------------------------------------------------------------- /scripts/api/meta/k8s/nodes/list: -------------------------------------------------------------------------------- 1 | curl -s localhost:9117/api/meta/k8s/nodes 2 | -------------------------------------------------------------------------------- /scripts/api/meta/local/list: -------------------------------------------------------------------------------- 1 | curl -s localhost:9117/api/meta/local/list 2 | -------------------------------------------------------------------------------- /scripts/api/pprof/allocs: -------------------------------------------------------------------------------- 1 | curl localhost:9117/debug/pprof/allocs > allocs 2 | -------------------------------------------------------------------------------- /scripts/api/meta/k8s/pods/local: -------------------------------------------------------------------------------- 1 | curl -s localhost:9117/api/meta/k8s/pods/local 2 | -------------------------------------------------------------------------------- /scripts/api/pprof/goroutine: -------------------------------------------------------------------------------- 1 | curl localhost:9117/debug/pprof/goroutine?debug=1 2 | -------------------------------------------------------------------------------- /cmd/containerhelper/handlers/httpcheck.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "http://www.baidu.com" 3 | } -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/cleanurl.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - cleanUrlV1: {} 3 | -------------------------------------------------------------------------------- /pkg/k8s/k8ssysmetrics/cadvisor/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 基于 cadvisor 的 pod/container 系统指标采集 3 | -------------------------------------------------------------------------------- /pkg/k8s/k8smeta/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 负责同步整个k8s集群的元数据到本地内存里并建立好相关索引. 3 | 这里用的模型都是k8s的模型. 4 | -------------------------------------------------------------------------------- /scripts/build/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 这里存放 agent bin 的构建. 3 | docker 镜像, 以及 vm 模式的安装包则在另外的目录. 4 | -------------------------------------------------------------------------------- /scripts/docker/bin/docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | chroot $HOSTFS docker $@ 5 | -------------------------------------------------------------------------------- /scripts/api/cri/findByIp: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "ip=$1" \ 3 | localhost:9117/api/cri/get 4 | -------------------------------------------------------------------------------- /scripts/api/cri/findByCid: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "cid=$1" \ 3 | localhost:9117/api/cri/get 4 | -------------------------------------------------------------------------------- /scripts/api/meta/local/hackErrorContainers: -------------------------------------------------------------------------------- 1 | curl -s localhost:9117/api/meta/local/hackErrorContainers 2 | -------------------------------------------------------------------------------- /scripts/vm/supervisord: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/traas-stack/holoinsight-agent/HEAD/scripts/vm/supervisord -------------------------------------------------------------------------------- /scripts/docker/bin/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | ensure_supervisord.sh 5 | 6 | sc start app 7 | -------------------------------------------------------------------------------- /scripts/api/meta/k8s/pods/byIp: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "ip=$1" \ 3 | localhost:9117/api/meta/k8s/pods/byIp 4 | -------------------------------------------------------------------------------- /scripts/api/meta/local/getByCid: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "cid=$1" \ 3 | localhost:9117/api/meta/local/getByCid 4 | -------------------------------------------------------------------------------- /scripts/docker/bin/restart.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | ensure_supervisord.sh 5 | 6 | sc restart app 7 | -------------------------------------------------------------------------------- /scripts/pre-commit.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | cd `dirname $0`/.. 5 | 6 | pre-commit run --all-files 7 | -------------------------------------------------------------------------------- /scripts/api/meta/k8s/pods/byApp: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "app=$1" \ 3 | localhost:9117/api/meta/k8s/pods/byAPp 4 | -------------------------------------------------------------------------------- /scripts/api/meta/local/list-simple: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "type=simple" \ 3 | localhost:9117/api/meta/local/list 4 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/examples/loganalysis/1.demolog: -------------------------------------------------------------------------------- 1 | 2022-02-03 08:07:09 ERROR [main] fail to connect to 127.0.0.1 IOException2 -------------------------------------------------------------------------------- /pkg/cri/pouch/tools.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package pouch 6 | -------------------------------------------------------------------------------- /pkg/plugin/api/model.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package api 6 | -------------------------------------------------------------------------------- /scripts/api/cri/findByName: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "ns=$1" \ 3 | --data-urlencode "pod=$2" \ 4 | localhost:9117/api/cri/get 5 | -------------------------------------------------------------------------------- /scripts/api/cri/netproxy/portforward: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "addr=$1" \ 3 | localhost:9117/api/cri/netproxy/portforward 4 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/substring_1.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - substringV1: 3 | begin: 4 4 | end: -1 5 | emptyIfError: true 6 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/substring_2.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - substringV1: 3 | begin: 4 4 | end: 6 5 | emptyIfError: true 6 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/substring_3.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - substringV1: 3 | begin: 100 4 | end: -1 5 | emptyIfError: true 6 | -------------------------------------------------------------------------------- /pkg/k8s/k8smeta/endpoints.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package k8smeta 6 | -------------------------------------------------------------------------------- /pkg/k8s/k8smeta/service.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package k8smeta 6 | -------------------------------------------------------------------------------- /pkg/openmetric/manager.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package openmetric 6 | -------------------------------------------------------------------------------- /pkg/plugin/input/jvm/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package jvm 6 | 7 | -------------------------------------------------------------------------------- /pkg/plugin/input/processperf/README.md: -------------------------------------------------------------------------------- 1 | 采集进程的性能信息 2 | 3 | 1. 进程基本信息 4 | 1. pid/名字/cmdline 5 | 2. 进程性能信息 6 | 1. cpu 7 | 2. 内存 8 | 3. 网络 -------------------------------------------------------------------------------- /scripts/api/meta/local/get: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "ns=$1" \ 3 | --data-urlencode "pod=$2" \ 4 | localhost:9117/api/meta/local/get 5 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/examples/simple/1.demolog: -------------------------------------------------------------------------------- 1 | 1668516572374[2022-11-15_20:49:32.374][INFO][V1][I] metric: atstbase rmcMeta atstbase-00-1-0 keys 0 -------------------------------------------------------------------------------- /pkg/collectconfig/executor/examples/transform1/1.demolog: -------------------------------------------------------------------------------- 1 | 2022-11-1520:49:32.374 [INFO] [main] cost=1 2 | 2022-11-1520:49:32.374 [ERROR] [main] cost=2 3 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/from.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/regexp1.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - regexpReplaceV1: 3 | expression: holo(.*) 4 | replacement: "Holo${1}XXX" 5 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/substring_error.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - substringV1: 3 | begin: 100 4 | end: -1 5 | emptyIfError: false 6 | -------------------------------------------------------------------------------- /scripts/vm/initd_holoinsight-agent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # 开机自启动时会调用这个脚本 5 | exec /usr/local/holoinsight/agent/bin/ctl.sh "$@" 6 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/examples/timestamp/1.demolog: -------------------------------------------------------------------------------- 1 | 1668516572374[2022-11-15_20:49:32.374][INFO][V1][I] metric: atstbase rmcMeta atstbase-00-1-0 keys 0 -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/mapping1.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - mappingV1: 3 | mappings: 4 | a: aa 5 | b: bb 6 | defaultValue: xx 7 | -------------------------------------------------------------------------------- /scripts/api/meta/k8s/pods/get: -------------------------------------------------------------------------------- 1 | curl -sG \ 2 | --data-urlencode "ns=$1" \ 3 | --data-urlencode "name=$2" \ 4 | localhost:9117/api/meta/k8s/pods/get 5 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/from_log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/examples/json/1.demolog: -------------------------------------------------------------------------------- 1 | {"time":"2023-03-23 16:19:01","level":"INFO","cost":1} 2 | {"time":"2023-03-23 16:19:01","level":"ERROR","cost":2} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea 3 | .git 4 | *.iml 5 | *.log 6 | .DS_Store 7 | /build 8 | *.db 9 | ignoreme 10 | /gitcommit 11 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/testutils/log_generator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package testutils 6 | -------------------------------------------------------------------------------- /pkg/server/registry/bistream/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 这是一个通用的基于grpc的双向流. 3 | 4 | 1. 依赖 registry.Service 5 | 2. 不依赖业务实现, 业务实现是可以定制的 6 | 1. 握手 handler 7 | 2. 各种 rpc handler 8 | -------------------------------------------------------------------------------- /scripts/docker/bin/app.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export GOTRACEBACK=all 4 | export GODEBUG=gctrace=1,madvdontneed=1 5 | 6 | exec /usr/local/holoinsight/agent/bin/agent 7 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/pool.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | // TODO 其他几个地方也有pool 跟他们合并 8 | -------------------------------------------------------------------------------- /pkg/server/registry/code.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package registry 6 | 7 | const ( 8 | CodeOk int32 = 0 9 | ) 10 | -------------------------------------------------------------------------------- /pkg/plugin/input/dialcheck/mock/normal-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "executeRule": { 3 | "type": "fixedRate", 4 | "fixedRate": 5000 5 | }, 6 | "network": "tcp", 7 | "port": 8080, 8 | "networkMode": "POD" 9 | } -------------------------------------------------------------------------------- /scripts/docker/bin/stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | if ! ps aux | grep /usr/bin/supervisord | grep -v grep >/dev/null 2>&1; then 5 | # 进程不存在, 先启动一下 6 | return 7 | fi 8 | 9 | sc stop app 10 | -------------------------------------------------------------------------------- /scripts/docker/init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | f=/usr/local/holoinsight/agent/bin/init_bashrc.sh 5 | echo "[ -e $f ] && source $f" >> /root/.bashrc 6 | 7 | rm /usr/local/holoinsight/agent/bin/init.sh 8 | -------------------------------------------------------------------------------- /cmd/containerhelper/model/api.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package model 6 | 7 | type ( 8 | Handler func(action string, resp *Resp) error 9 | ) 10 | -------------------------------------------------------------------------------- /pkg/util/strategy/strategy.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package strategy 6 | 7 | type RetryStrategy interface { 8 | GetNext() int 9 | Reset() 10 | } 11 | -------------------------------------------------------------------------------- /scripts/docker/agent-base/README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | Agent base image. 3 | 4 | # Changelog 5 | #### 1.0.1 6 | 1. Remove hardcode /etc/timezone 7 | 2. Remove hardcode LANG=zh_CN.UTF-8 8 | 9 | #### 1.0.0 10 | 1. First Commit 11 | -------------------------------------------------------------------------------- /pkg/bistream/rpccodes/rpccodes.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package rpccodes 6 | 7 | const ( 8 | OK = 0 9 | ILLEGAL_ARGUMENTS = 400 10 | ) 11 | -------------------------------------------------------------------------------- /scripts/docker/sc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | bin=/usr/bin/supervisorctl 5 | conf=/etc/supervisord.conf 6 | 7 | if [ "$#" = "0" ]; then 8 | $bin -c $conf status 9 | exit 0 10 | fi 11 | 12 | $bin -c $conf "$@" 13 | -------------------------------------------------------------------------------- /cmd/containerhelper/handlers/fixout/common.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package fixout 6 | 7 | const ( 8 | StdoutFd = 1 9 | StderrFd = 2 10 | bufSize = 4096 11 | ) 12 | -------------------------------------------------------------------------------- /scripts/git/rebase-main-and-reset-author.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # doc: rebase origin/main and reset author of commits to current author 4 | 5 | git fetch origin 6 | git rebase origin/main --exec 'git commit --amend --reset-author --no-edit' 7 | -------------------------------------------------------------------------------- /pkg/logger/log_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package logger 6 | 7 | import "testing" 8 | 9 | func TestLog(t *testing.T) { 10 | Infof("AAAA") 11 | Errorf("BBBB") 12 | } 13 | -------------------------------------------------------------------------------- /pkg/util/signal.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | type ( 8 | Signal struct { 9 | } 10 | ) 11 | 12 | func NewSignal() *Signal { 13 | return &Signal{} 14 | } 15 | -------------------------------------------------------------------------------- /pkg/cri/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 这个包存放 容器运行时(cri) 的接口/模型/辅助函数. 子目录是一些具体的 cri 实现. 3 | 4 | docker/ 标准docker 5 | pouch/ alibaba pouch 6 | 7 | # 元数据定制 8 | 1. app 的来源: 用户自定义方式(labels/ENV) > 标准 app 标签 9 | 2. hostname 的来源: 用户自定义方式(labels/ENV) > pod.spec.Hostname 10 | -------------------------------------------------------------------------------- /pkg/plugin/api/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package api 6 | 7 | func BoolToFloat64(b bool) float64 { 8 | if b { 9 | return 1 10 | } else { 11 | return 0 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /scripts/build/detect-goproxy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | if [ -n "$GOPROXY" ]; then 5 | echo use GOPROXY=$GOPROXY 6 | elif go env GOPROXY >/dev/null 2>&1; then 7 | export GOPROXY=`go env GOPROXY` 8 | echo use GOPROXY=$GOPROXY 9 | fi 10 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/window.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import "time" 8 | 9 | type ( 10 | XWindow struct { 11 | Interval time.Duration 12 | } 13 | ) 14 | -------------------------------------------------------------------------------- /pkg/cri/dockerutils/consts.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package dockerutils 6 | 7 | const ( 8 | MergedDir = "MergedDir" 9 | AttrIsDockerJsonLog = "isDockerJsonLog" 10 | ) 11 | -------------------------------------------------------------------------------- /pkg/cri/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package cri 6 | 7 | func ShortContainerId(cid string) string { 8 | if len(cid) <= 12 { 9 | return cid 10 | } 11 | return cid[:12] 12 | } 13 | -------------------------------------------------------------------------------- /pkg/util/cast.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | // BoolToInt64 converts bool to int64 8 | func BoolToInt64(b bool) int64 { 9 | if b { 10 | return 1 11 | } 12 | return 0 13 | } 14 | -------------------------------------------------------------------------------- /scripts/api/pprof/heap: -------------------------------------------------------------------------------- 1 | curl localhost:9117/debug/pprof/heap > /tmp/heap 2 | echo 3 | 4 | echo Use the following command to download the heap and view the flame graph. 5 | echo kubectl -n $POD_NAMESPACE cp $POD_NAME:/tmp/heap /tmp/heap 6 | echo go tool pprof -http=":$RANDOM" /tmp/heap 7 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/delay.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | type ( 8 | DelayCalculator struct{} 9 | ) 10 | 11 | func NewDelayCalculator() *DelayCalculator { 12 | return &DelayCalculator{} 13 | } 14 | -------------------------------------------------------------------------------- /scripts/docker/bin/init_bashrc.sh: -------------------------------------------------------------------------------- 1 | alias h='cd /usr/local/holoinsight/agent' 2 | alias g='cd /usr/local/holoinsight/agent/logs' 3 | 4 | if [ "$HI_AGENT_MODE" = "daemonset" ]; then 5 | alias tohost='chroot /$HOSTFS bash' 6 | alias tohostns='nsenter -t 1 -m -u -i -n -F bash --restricted -c bash' 7 | fi 8 | -------------------------------------------------------------------------------- /scripts/vm/agent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export GOTRACEBACK=all 4 | export GODEBUG=gctrace=1,madvdontneed=1 5 | 6 | script_dir=`dirname $0` 7 | 8 | # 如果文件存在就从它加载一些环境变量 9 | if [ -e "$script_dir/env.sh" ]; then 10 | source $script_dir/env.sh 11 | fi 12 | 13 | exec $script_dir/agent 14 | 15 | -------------------------------------------------------------------------------- /pkg/util/os_nolinux.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | /* 4 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 5 | */ 6 | 7 | package util 8 | 9 | import "os" 10 | 11 | func OpenFileReadonly(file string) (*os.File, error) { 12 | return os.OpenFile(file, os.O_RDONLY, 0) 13 | } 14 | -------------------------------------------------------------------------------- /scripts/api/pprof/profile: -------------------------------------------------------------------------------- 1 | curl localhost:9117/debug/pprof/profile?seconds=60 > /tmp/profile 2 | echo 3 | 4 | echo Use the following command to download the profile and view the flame graph. 5 | echo kubectl -n $POD_NAMESPACE cp $POD_NAME:/tmp/profile /tmp/profile 6 | echo go tool pprof -http=":$RANDOM" /tmp/profile 7 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/utils/file_linux.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package utils 6 | 7 | import ( 8 | "os" 9 | "syscall" 10 | ) 11 | 12 | func GetInode(stat os.FileInfo) uint64 { 13 | return stat.Sys().(*syscall.Stat_t).Ino 14 | } 15 | -------------------------------------------------------------------------------- /scripts/build/agent-builder/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | Agent builder image. 3 | 4 | This image is used to build agent bin. 5 | 6 | # Changelog 7 | #### 1.0.2 8 | - Update Golang to 1.22 9 | 10 | #### 1.0.1 11 | - Use docker buildx 12 | 13 | #### 1.0.0 14 | - First version, supports linux/amd64 & linux/arm64/v8 15 | -------------------------------------------------------------------------------- /pkg/plugin/input/httpcheck/mock/normal-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "executeRule": { 3 | "type": "fixedRate", 4 | "fixedRate": 5000 5 | }, 6 | "schema": "http", 7 | "port": 8080, 8 | "path": "/path", 9 | "timeout": 3000, 10 | "method": "GET", 11 | "successCodes": [ 12 | 200 13 | ], 14 | "networkMode": "POD" 15 | } -------------------------------------------------------------------------------- /pkg/collectconfig/executor/utils/file_darwin.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package utils 6 | 7 | import ( 8 | "os" 9 | "syscall" 10 | ) 11 | 12 | func GetInode(stat os.FileInfo) uint64 { 13 | // darwin 也有 Ino 14 | return stat.Sys().(*syscall.Stat_t).Ino 15 | } 16 | -------------------------------------------------------------------------------- /pkg/util/net_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | import ( 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | func TestReplaceHost(t *testing.T) { 13 | assert.Equal(t, "2.2.2.2:2222", ReplaceHost("1.1.1.1:2222", "2.2.2.2")) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/slice_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | func TestSlice(t *testing.T) { 13 | var a []string = nil 14 | fmt.Println(a) 15 | fmt.Println(a[:0]) 16 | fmt.Println(a[:]) 17 | } 18 | -------------------------------------------------------------------------------- /scripts/docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$USESUPERVISOR" = "true" ]; then 4 | echo 'use supervisord to manage agent process' 5 | exec /usr/bin/supervisord -n 6 | fi 7 | 8 | # TODO stdout log rotate 9 | exec >>/usr/local/holoinsight/agent/logs/stdout.log 10 | exec 2>&1 11 | 12 | exec /usr/local/holoinsight/agent/bin/app.sh 13 | -------------------------------------------------------------------------------- /pkg/util/os_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | /* 4 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 5 | */ 6 | 7 | package util 8 | 9 | import ( 10 | "golang.org/x/sys/unix" 11 | "os" 12 | ) 13 | 14 | func OpenFileReadonly(file string) (*os.File, error) { 15 | return os.OpenFile(file, os.O_RDONLY|unix.O_NOATIME, 0) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/k8s/k8ssysmetrics/factory.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package k8ssysmetrics 6 | 7 | import ( 8 | "os" 9 | ) 10 | 11 | func GetNewPodSystemResourceCollector() string { 12 | if s := os.Getenv("HI_K8S_SYS_COLLECTOR"); s != "" { 13 | return s 14 | } 15 | return "cadvisor" 16 | } 17 | -------------------------------------------------------------------------------- /pkg/accumulator/accumulator.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package accumulator 6 | 7 | import "github.com/traas-stack/holoinsight-agent/pkg/model" 8 | 9 | type ( 10 | // Like Telegraf.Accumulator 11 | Accumulator interface { 12 | AddMetric(*model.Metric) 13 | AddError(error) 14 | } 15 | ) 16 | -------------------------------------------------------------------------------- /pkg/k8s/listwatchext/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 3 | 这个包是对 listwatch 的一个补充让它更加适应我们的场景. 4 | 5 | 1. 初始化是时进行一次全量list, 然后同步到服务端 6 | 2. 然后开始监听增量, 并进行适当合并(比如key相同的只保留最后一条元数据), 然后**批量**同步到服务端 7 | 3. 一旦同步到服务端发生不可知错误, 需要重新开始listwatch, 这样才能保证元数据是不丢失的 8 | 9 | 直接使用k8s的cache包可以做到类似的效果, 但缺点是我们拦截不到第一次的全量list的结果. 它的第一次全量list对我们来说只是n次add. 10 | 而我们会使用全量list的结果来做全量数据同步, 包含删除已经不存在的pod. 11 | -------------------------------------------------------------------------------- /pkg/plugin/output/sls/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package sls 6 | 7 | import "github.com/traas-stack/holoinsight-agent/pkg/plugin/output" 8 | 9 | func init() { 10 | output.Register("sls", func(config output.Config) (output.Output, error) { 11 | return NewSLSOutput() 12 | }) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/transfer/pb/gen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | script_dir=$( dirname $0 ) 5 | 6 | SRC_DIR=$script_dir 7 | DST_DIR=$script_dir 8 | 9 | protoc-3.14.0 \ 10 | -I=$SRC_DIR \ 11 | -I=$SRC_DIR/include \ 12 | --go_out=paths=source_relative:$DST_DIR \ 13 | --go-grpc_out=$DST_DIR \ 14 | --go-grpc_opt=paths=source_relative \ 15 | $SRC_DIR/*.proto 16 | -------------------------------------------------------------------------------- /pkg/util/stats/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package stats 6 | 7 | func SingleExporter(f func(tableX string, metricsX map[string]uint64)) TableStatsExporter { 8 | return func(metrics map[string]map[string]uint64) { 9 | for k, v := range metrics { 10 | f(k, v) 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /cmd/containerhelper/handlers/hello.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package handlers 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/cmd/containerhelper/model" 9 | ) 10 | 11 | func helloHandler(inputType string, resp *model.Resp) error { 12 | resp.Data = "world" 13 | return nil 14 | } 15 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/storage/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package storage 6 | 7 | type ( 8 | // TODO 这里似乎将 agg 的落沉淀到底层的datanode上 9 | // 这是我不太想的, 但如果不这么做, 底层就要存储 interface{} 10 | DataNode interface { 11 | AddCount() 12 | MergeNumber(float64) 13 | MergeHll(string) 14 | } 15 | ) 16 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transforms/regexp2.yaml: -------------------------------------------------------------------------------- 1 | filters: 2 | - switchCaseV1: 3 | cases: 4 | - caseWhere: 5 | regexp: 6 | expression: 'holo(.*)' 7 | catchGroups: true 8 | action: 9 | regexpReplaceV1: 10 | replacement: "Holo${1}XXX" 11 | defaultAction: 12 | constV1: 13 | value: "defaultValue" 14 | -------------------------------------------------------------------------------- /pkg/plugin/output/gateway/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package gateway 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/output" 9 | ) 10 | 11 | func init() { 12 | output.Register("cmgateway", newGatewayOutput) 13 | output.Register("gateway", newGatewayOutput) 14 | } 15 | -------------------------------------------------------------------------------- /scripts/vm/uninstall.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | bin_dir=`dirname $0 | xargs realpath` 5 | echo bin dir $bin_dir 6 | 7 | agent_home=` realpath $bin_dir/.. ` 8 | echo agent home $agent_home 9 | 10 | if [ -e "$bin_dir/ctl.sh" ]; then 11 | echo "find $bin_dir/ctl.sh, $bin_dir/ctl.sh stop first" 12 | $bin_dir/ctl.sh stop 13 | fi 14 | 15 | rm -rf $agent_home 16 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/examples/regexp_path/1.demolog: -------------------------------------------------------------------------------- 1 | |||0a374dd916617622522722575e006d|INFO |DubboServerHandler-10.55.77.217:20880-thread-197|DUBBO-MONITOR-PROVIDER-DIGEST-LOG|superlink|PROVIDER|-|gloandecision|10.55.78.187:38588 -> 10.55.77.217:20880|com.antbank.superlink.facade.MessageSendService|send|Y|2475|-|{"com.antbank.fc.common.lang.event.EventContext":"{\"tntInstId\":\"XHBKC1MO\"}"} -------------------------------------------------------------------------------- /pkg/plugin/output/console/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package console 6 | 7 | import "github.com/traas-stack/holoinsight-agent/pkg/plugin/output" 8 | 9 | func init() { 10 | output.Register("console", func(config output.Config) (output.Output, error) { 11 | return NewConsoleOutput() 12 | }) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/cri/containerdutils/containerd_k3s.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package containerdutils 6 | 7 | // K3s is a commonly used distribution of K8s and we have built-in support for it. 8 | const ( 9 | K3sDefaultStateDir = "/run/k3s/containerd" 10 | K3sDefaultAddress = "/run/k3s/containerd/containerd.sock" 11 | ) 12 | -------------------------------------------------------------------------------- /pkg/plugin/output/all/all.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package all 6 | 7 | import ( 8 | _ "github.com/traas-stack/holoinsight-agent/pkg/plugin/output/console" 9 | _ "github.com/traas-stack/holoinsight-agent/pkg/plugin/output/gateway" 10 | _ "github.com/traas-stack/holoinsight-agent/pkg/plugin/output/sls" 11 | ) 12 | -------------------------------------------------------------------------------- /pkg/meta/type.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package meta 6 | 7 | type ( 8 | Data struct { 9 | } 10 | 11 | Syncer interface { 12 | Start() error 13 | 14 | Stop() error 15 | 16 | AddEventHandler(EventHandler) 17 | } 18 | 19 | EventHandler interface { 20 | OnUpdate(oldObj, newObj []*Data) 21 | } 22 | ) 23 | -------------------------------------------------------------------------------- /pkg/util/rand.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | import "math/rand" 8 | 9 | func RandBetween(min, max int) int { 10 | return min + rand.Intn(max-min) 11 | } 12 | 13 | func RandBetween64(min, max int64) int64 { 14 | if min == max { 15 | return 0 16 | } 17 | return min + rand.Int63n(max-min) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transform_discard.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | type ( 8 | xDiscardFilter struct { 9 | } 10 | ) 11 | 12 | func (x *xDiscardFilter) Init() error { 13 | return nil 14 | } 15 | 16 | func (x *xDiscardFilter) Filter(ctx *LogContext) (interface{}, error) { 17 | return nil, nil 18 | } 19 | -------------------------------------------------------------------------------- /pkg/plugin/input/cpu/gopsutil_hack.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package cpu 6 | 7 | import ( 8 | _ "unsafe" 9 | ) 10 | 11 | // 该error在internal包里, 默认我们无法引用, 通过 go:linkename 的方式将其暴露出来 12 | //go:linkname errNotImplementedError github.com/shirou/gopsutil/v3/internal/common.ErrNotImplementedError 13 | var errNotImplementedError error 14 | -------------------------------------------------------------------------------- /scripts/gen-git-info.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | script_dir=`dirname $0` 5 | project_root=`realpath $script_dir/..` 6 | 7 | if command -v git >/dev/null 2>&1 && [ -e ".git" ]; then 8 | echo '[gen-git-info] gen git info' 9 | git rev-parse HEAD > $project_root/gitcommit 10 | else 11 | echo '[gen-git-info] no git command or .git' 12 | echo "unknown" > $project_root/gitcommit 13 | fi 14 | -------------------------------------------------------------------------------- /scripts/docker/ensure_supervisord.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | if ! ps aux | grep /usr/bin/supervisord | grep -v grep >/dev/null 2>&1; then 5 | # Run as root with current envs visible 6 | sudo -E supervisord 7 | sleep 1 8 | pid_file=/var/run/supervisord.pid 9 | pid=`cat $pid_file` 10 | # see https://linux.die.net/man/5/proc 11 | sudo bash -c "echo -17 > /proc/$pid/oom_adj" 12 | fi 13 | -------------------------------------------------------------------------------- /cmd/agent/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "github.com/traas-stack/holoinsight-agent/pkg/bootstrap" 10 | "os" 11 | ) 12 | 13 | // agent entry 14 | func main() { 15 | if err := bootstrap.App.Bootstrap(); err != nil { 16 | fmt.Printf("bootstrap error %+v\n", err) 17 | os.Exit(1) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pkg/k8s/silence/controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package silence 6 | 7 | import ( 8 | v1 "k8s.io/api/core/v1" 9 | "k8s.io/client-go/tools/cache" 10 | ) 11 | 12 | type ( 13 | ControllerManager interface { 14 | PodHandler() cache.ResourceEventHandler 15 | Run(stopCh chan struct{}) 16 | IsUnderControl(pod *v1.Pod) bool 17 | } 18 | ) 19 | -------------------------------------------------------------------------------- /scripts/vm/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | VM模式下的相关脚本. 3 | 该脚本可以在 Mac m1 或 Linux 上执行, 要求机器上装有 docker. Mac m1 特别慢, 建议在 Linux 上执行, 可以通过 rsync 将代码复制到 Linux 上再构建. 4 | 5 | # 使用 6 | ```bash 7 | ./scripts/vm/build.sh 1.0.0 8 | ``` 9 | 10 | 构建的结果在 `/root/workspace/remote/cloudmonitor-agent/scripts/vm/holoinsight-agent_linux-amd64_1.0.0.tar.gz` 11 | 12 | 之后我们把它上传到 OSS, 给用户下载安装即可. 13 | ```bash 14 | ./scripts/vm/upload.sh 1.0.0 15 | ``` 16 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/time_processtime.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import "github.com/traas-stack/holoinsight-agent/pkg/util" 8 | 9 | type ( 10 | timeProcessTimeParser struct { 11 | } 12 | ) 13 | 14 | func (a *timeProcessTimeParser) Parse(context *LogContext) (int64, error) { 15 | return util.CurrentMS(), nil 16 | } 17 | -------------------------------------------------------------------------------- /pkg/util/strategy/backoff_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package strategy 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | func TestBackoffStrategy(t *testing.T) { 13 | reconnectStrategy := NewBackOffStrategy(0.3) 14 | 15 | for i := 0; i < 1000; i++ { 16 | fmt.Println(reconnectStrategy.GetNext()) 17 | } 18 | 19 | t.Log("success") 20 | } 21 | -------------------------------------------------------------------------------- /scripts/docker/agent-base/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # docs: Run thi script to build base image for agent and push images to Docker Hub 5 | 6 | cd `dirname $0` 7 | 8 | tag=$1 9 | if [ -z "$tag" ]; then 10 | echo 'usage: build.sh ' 11 | exit 1 12 | fi 13 | 14 | 15 | docker buildx build \ 16 | --platform=linux/amd64,linux/arm64/v8 \ 17 | -t holoinsight/agent-base:$tag \ 18 | --pull --push . 19 | -------------------------------------------------------------------------------- /pkg/plugin/input/cpu/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package cpu 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("cpu", func(config input.Config) (api.Input, error) { 14 | return &cpuInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/plugin/input/mem/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package mem 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("mem", func(config input.Config) (api.Input, error) { 14 | return &memInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/plugin/input/tcp/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package tcp 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("tcp", func(config input.Config) (api.Input, error) { 14 | return &tcpInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/collecttask/test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package collecttask 6 | 7 | type ( 8 | One struct { 9 | One *CollectTask 10 | } 11 | ) 12 | 13 | func (f *One) GetAll() []*CollectTask { 14 | return []*CollectTask{f.One} 15 | } 16 | 17 | func (f *One) Listen(listener ChangeListener) { 18 | } 19 | 20 | func (f *One) RemoveListen(listener ChangeListener) { 21 | } 22 | -------------------------------------------------------------------------------- /pkg/plugin/input/disk/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package disk 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("disk", func(config input.Config) (api.Input, error) { 14 | return &diskInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/plugin/input/load/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package load 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("load", func(config input.Config) (api.Input, error) { 14 | return &loadInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/plugin/input/swap/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package mem 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("swap", func(config input.Config) (api.Input, error) { 14 | return &swapInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/plugin/input/process/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package process 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("process", func(config input.Config) (api.Input, error) { 14 | return &processInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/plugin/input/traffic/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package traffic 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input" 10 | ) 11 | 12 | func init() { 13 | input.Register("traffic", func(config input.Config) (api.Input, error) { 14 | return &trafficInput{}, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/logstream/utils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package logstream 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | func TestName(t *testing.T) { 13 | lines, err := ReadLastLines("/Users/xzchaoo/logs/a.log", 13) 14 | if err != nil { 15 | panic(err) 16 | } 17 | for _, line := range lines { 18 | fmt.Println(line, len(line)) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /pkg/cri/dockerutils/events.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package dockerutils 6 | 7 | import "strings" 8 | 9 | // action 形如 'dir' 'exec_create: bash -c sc stop app', 对于后者我们提取出 exec_create 10 | func ExtractEventAction(rawAction string) string { 11 | if strings.Contains(rawAction, ":") { 12 | rawAction = rawAction[:strings.Index(rawAction, ":")] 13 | } 14 | return rawAction 15 | } 16 | -------------------------------------------------------------------------------- /scripts/build/agent-builder/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # doc: Run this script to build agent-builder image for multi arch. 4 | 5 | cd `dirname $0` 6 | 7 | set -e 8 | script_dir=`dirname $0` 9 | 10 | tag=$1 11 | 12 | if [ -z "$tag" ]; then 13 | echo 'usage: build.sh ' 14 | exit 1 15 | fi 16 | 17 | docker buildx build \ 18 | --platform linux/amd64,linux/arm64/v8 \ 19 | -t holoinsight/agent-builder:$tag \ 20 | --pull --push . 21 | -------------------------------------------------------------------------------- /cmd/containerhelper/handlers/countZombies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # docs: count zombies processes 5 | COUNT_LIMIT=100 6 | 7 | count=0 8 | for i in /proc/*; do 9 | b=`basename $i` 10 | if [ -e "$i/status" ]; then 11 | if grep 'State: Z (zombie)' "$i/status" >/dev/null 2>&1; then 12 | count=`expr $count + 1` 13 | if [ "$count" -gt "$COUNT_LIMIT" ]; then 14 | break 15 | fi 16 | fi 17 | fi 18 | done 19 | 20 | echo $count 21 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/where_alwaystrue.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | type ( 8 | xWhereAlwaysTrue struct{} 9 | ) 10 | 11 | func (x xWhereAlwaysTrue) Test(ctx *LogContext) (ret bool, _ error) { 12 | we := ctx.whereEvent 13 | if we != nil { 14 | we.Name = "alwaysTrue" 15 | defer func() { 16 | we.Result = ret 17 | }() 18 | } 19 | 20 | return true, nil 21 | } 22 | -------------------------------------------------------------------------------- /pkg/plugin/input/standard/mysqlw/mysql.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package mysqlw 6 | 7 | import "github.com/traas-stack/holoinsight-agent/pkg/pipeline/integration/base" 8 | 9 | type ( 10 | MySQLConf struct { 11 | base.Conf 12 | Confs []*MySQLConf_Conf 13 | } 14 | MySQLConf_Conf struct { 15 | Host string 16 | Port int 17 | User string 18 | Password string 19 | } 20 | ) 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Thanks for taking the time to fill this! 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Describe This Problem 11 | A clear and concise description of what the problem is. 12 | 13 | ## Proposal 14 | Maybe you have considered some ideas or solutions about this feature. 15 | 16 | ## Additional Context 17 | Add any other context about the feature request here. 18 | -------------------------------------------------------------------------------- /pkg/util/gob.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | import ( 8 | "bytes" 9 | "encoding/gob" 10 | ) 11 | 12 | func GobEncode(obj interface{}) ([]byte, error) { 13 | buf := bytes.NewBuffer(nil) 14 | err := gob.NewEncoder(buf).Encode(obj) 15 | return buf.Bytes(), err 16 | } 17 | 18 | func GobDecode(b []byte, obj interface{}) error { 19 | return gob.NewDecoder(bytes.NewBuffer(b)).Decode(obj) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/util/strings_sort.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | type ( 8 | SortStringsByLength []string 9 | ) 10 | 11 | func (s SortStringsByLength) Len() int { 12 | return len(s) 13 | } 14 | 15 | func (s SortStringsByLength) Less(i, j int) bool { 16 | return len(s[i]) < len(s[j]) 17 | } 18 | 19 | func (s SortStringsByLength) Swap(i, j int) { 20 | t := s[i] 21 | s[i] = s[j] 22 | s[j] = t 23 | } 24 | -------------------------------------------------------------------------------- /pkg/plugin/integration/alibabacloud/aliyun.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package alibabacloud 6 | 7 | type ( 8 | AliyunMetricDesc struct { 9 | MetricName string 10 | AliyunMetricName string 11 | Dimensions []string 12 | Average bool 13 | Minimum bool 14 | Maximum bool 15 | Value bool 16 | Sum bool 17 | Max bool 18 | } 19 | ) 20 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/filematch/filematch_regexp_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package filematch 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | func TestRegexpFileMatcher(t *testing.T) { 13 | m, err := NewRegexpFileMatcher("/Users/xzchaoo/logs", "^/Users/xzchaoo/logs/(?P[^/]+)/(?P[^/]+).log$", -1, 10) 14 | if err != nil { 15 | panic(err) 16 | } 17 | fmt.Println(m.Find()) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/filematch/filematch_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package filematch 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | func TestName(t *testing.T) { 13 | o := "/home/admin/logs/foo/{time:yyyy}/{time:MM}/{time:dd}/{time:HH}/foo.log" 14 | f := newFilePathFormatter() 15 | if f.ContainsFormat(o) { 16 | fmt.Println(f.Format(o)) 17 | } else { 18 | fmt.Println(o) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /pkg/util/gopsutil_hack.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | import ( 8 | _ "unsafe" 9 | ) 10 | 11 | // 该error在internal包里, 默认我们无法引用, 通过 go:linkename 的方式将其暴露出来 12 | //go:linkname errNotImplementedError github.com/shirou/gopsutil/v3/internal/common.ErrNotImplementedError 13 | var errNotImplementedError error 14 | 15 | func IsNotImplemented(err error) bool { 16 | return err == errNotImplementedError 17 | } 18 | -------------------------------------------------------------------------------- /pkg/plugin/input/dialcheck/mock/normal.md: -------------------------------------------------------------------------------- 1 | insert into gaea_collect_config(gmt_create, gmt_modified, tenant, table_name, json, version, collect_range, executor_selector, `type`, deleted) values(now(), now(), 'dev', 'dialcheck_foobar_normal', 2 | '{"executeRule":{"type":"fixedRate","fixedRate":5000},"network":"tcp","port":8080,"networkMode":"POD"}', 3 | 1, 4 | '{"type":"cloudmonitor","cloudmonitor":{"table":"dev_server","condition":[{"app":["registry"]}]}}', 5 | '{"sidecar":{},"type":"sidecar"}', 'dialcheck', 0) 6 | 7 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/logparser_json.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "encoding/json" 9 | "strings" 10 | ) 11 | 12 | type ( 13 | jsonParser struct { 14 | } 15 | ) 16 | 17 | func (j *jsonParser) Parse(ctx *LogContext) error { 18 | if len(ctx.columnMap) == 0 && strings.HasPrefix(ctx.GetLine(), "{") { 19 | return json.Unmarshal([]byte(ctx.GetLine()), &ctx.columnMap) 20 | } 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/plugin/integration/alibabacloud/README.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 同步 aliyun cloudmonitor 的数据到 HoloInsight. 3 | 实现原理就是定时轮询 aliyun openapi 查数据, 然后写到我们, 效率很低但没有其他办法. 4 | 5 | # 遇到的问题 6 | 1. aliyun 的 openapi 一次只能查一个 metric, 一个时间区间 7 | 2. aliyun 的 openapi 有限流, 普通用户每个月只有几百万次的免费额度; TODO 有限速吗? 8 | 3. aliyun 的 指标非常多, 我们需要精挑细选一些有用的指标 9 | 4. aliyun 的 数据 自身就有一定的延迟, 我们同步数据的时候最好按区间去同步 10 | 11 | # 优化 12 | 1. 区间查询 + 降频 13 | 2. 减少指标 14 | 3. metrics endpoints 用 aliyun 内网域名 15 | 4. 查一下用户有没有购买对应的云产品, 如果没买就不查云产品指标了 (依赖更多阿里云api, 以及需要更高权限) 16 | -------------------------------------------------------------------------------- /pkg/cri/criutils/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package criutils 6 | 7 | import "errors" 8 | 9 | type ( 10 | ProcessInfo struct { 11 | User string `json:"user"` 12 | Name string `json:"name"` 13 | CmdlineSlice []string `json:"cmdlineSlice"` 14 | Tgid int32 `json:"tgid"` 15 | } 16 | ) 17 | 18 | var ( 19 | ErrUnsupported = errors.New("unsupported") 20 | ErrDisabled = errors.New("disabled") 21 | ) 22 | -------------------------------------------------------------------------------- /cmd/containerhelper/model/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package model 6 | 7 | type ( 8 | RespBase struct { 9 | Success bool `json:"success,omitempty"` 10 | Message string `json:"message,omitempty"` 11 | DebuLogs []string `json:"debugLogs,omitempty"` 12 | } 13 | Resp struct { 14 | RespBase 15 | Data interface{} `json:"data"` 16 | } 17 | RespDataBytes struct { 18 | Base RespBase 19 | Data []byte `json:"data"` 20 | } 21 | ) 22 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transform_regexp_v1_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | func TestStrReplacer(t *testing.T) { 13 | sr, err := parseStrReplacer("a-$1x-x${2}x,x${3}x,$4abc") 14 | if err != nil { 15 | panic(err) 16 | } 17 | str := sr.replace([]string{"_", "aaa", "bbb", "ccc"}) 18 | assert.Equal(t, "a-aaax-xbbbx,xcccx,$4abc", str) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/k8s/k8ssysmetrics/common/api.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package common 6 | 7 | import "time" 8 | 9 | type ( 10 | // 系统指标采集器 11 | // 这里并没有像 xflush-agent 一样 是很多细粒度采集方法, 而是只有一个很简单的 start 12 | // 因此如果实现不一样(cadvisor/asar) 那么总体的执行流程就很不一样 13 | // 不像 asar/vsar 只有细节的差异 14 | SysCollector interface { 15 | Name() string 16 | Start() 17 | Stop() 18 | } 19 | SysCollectorProvider func(suffix string, interval time.Duration) (SysCollector, error) 20 | ) 21 | -------------------------------------------------------------------------------- /cmd/containerhelper/handlers/glob.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package handlers 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/cmd/containerhelper/model" 9 | "os" 10 | "path/filepath" 11 | ) 12 | 13 | func globHandler(action string, resp *model.Resp) error { 14 | pattern := os.Args[2] 15 | files, err := filepath.Glob(pattern) 16 | if err != nil { 17 | return err 18 | } 19 | resp.Data = files 20 | resp.Success = true 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transform_const.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | type ( 8 | // xConstFilter always returns a const value 9 | xConstFilter struct { 10 | value string 11 | } 12 | ) 13 | 14 | func (x *xConstFilter) Init() error { 15 | return nil 16 | } 17 | 18 | func (x *xConstFilter) Filter(ctx *LogContext) (interface{}, error) { 19 | if x.value != "" { 20 | return x.value, nil 21 | } 22 | return ctx.contextValue, nil 23 | } 24 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/consumer_log_sub.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | type ( 8 | // 子消费者, 是一种实际的日志处理 9 | SubConsumer interface { 10 | setParent(parent *Consumer) 11 | Update(f func()) 12 | ProcessGroup(iw *inputWrapper, ctx *LogContext, maxTs *int64) 13 | // Emit emits data with timestamp equals to expectedTs 14 | // Returns true if data is not empty 15 | Emit(expectedTs int64) bool 16 | init() 17 | MaybeFlush() 18 | } 19 | ) 20 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/v2.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | type ( 8 | LogSource interface { 9 | Start() 10 | Stop() 11 | } 12 | // 拉模式的logsource 13 | PullLogSource interface { 14 | LogSource 15 | } 16 | LogConsumer interface{} 17 | LogConsumerManager struct { 18 | consumers map[string]LogConsumer 19 | } 20 | LogTaskPipeline struct { 21 | Source LogSource 22 | ConsumerManager LogConsumerManager 23 | } 24 | ) 25 | -------------------------------------------------------------------------------- /pkg/telegraf/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package telegraf 6 | 7 | import ( 8 | telegraf2 "github.com/influxdata/telegraf" 9 | "github.com/traas-stack/holoinsight-agent/pkg/logger" 10 | "reflect" 11 | ) 12 | 13 | func MaybeInitLog(input telegraf2.Input) { 14 | if input == nil { 15 | return 16 | } 17 | if logField := reflect.ValueOf(input).Elem().FieldByName("Log"); logField.IsNil() { 18 | logField.Set(reflect.ValueOf(logger.ZapLogger.InfoS)) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /cmd/containerhelper/model/factory.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package model 6 | 7 | import "sync" 8 | 9 | var mutex sync.Mutex 10 | var handlers = make(map[string]Handler) 11 | 12 | func RegisterHandler(inputType string, handler Handler) { 13 | mutex.Lock() 14 | defer mutex.Unlock() 15 | handlers[inputType] = handler 16 | } 17 | 18 | func GetHandler(inputType string) (Handler, bool) { 19 | mutex.Lock() 20 | defer mutex.Unlock() 21 | h, ok := handlers[inputType] 22 | return h, ok 23 | } 24 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/timeparser/timebinarysearch/tsb.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package timebinarysearch 6 | 7 | // 前提: 8 | // 1. 日志时间戳非递减 9 | // 2. 几乎每行都有日志时间戳, 允许存在一些行解不出时间戳, 但这样行不能太多 10 | // 3. 允许自定义时间戳解析方式 11 | 12 | // 时间二分搜索 13 | // 1. 读首行: 行首可能非法, 可以找前n行的第一个合法行 14 | // 2. 读尾行: 行尾可能非法, 可以找后n行的第一个合法行 15 | // 3. 二分法算出一个offset, 从该offset向前找到第一个最近的行 16 | // 4. 重复流程 17 | // 5. 找出>=指定时间的第一行 18 | // 6. 找出<指定时间的最后一行 19 | // 20 | //func search(f *os.File, ts int64) { 21 | // // f.Name() 22 | //} 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Thanks for taking the time to fill out this bug report! 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Describe this problem 11 | What this problem is and what happened. 12 | 13 | ## To Reproduce 14 | How to reproduce this problem. 15 | 16 | ## Expected behavior 17 | What is your expected behavior. 18 | 19 | ## Additional Information 20 | If possible, please attach other context you think may relate to this problem. Like runtime environment, modified config, error log etc. 21 | -------------------------------------------------------------------------------- /pkg/plugin/input/httpcheck/mock/normal.md: -------------------------------------------------------------------------------- 1 | insert into gaea_collect_config(gmt_create, gmt_modified, tenant, table_name, json, version, collect_range, executor_selector, `type`, deleted) values(now(), now(), 'dev', 'httpcheck_foobar_normal', 2 | '{"executeRule":{"type":"fixedRate","fixedRate":5000},"schema":"http","port":8080,"path":"/path","timeout":3000,"method":"GET","successCodes":[200],"networkMode":"POD"}', 3 | 1, 4 | '{"type":"cloudmonitor","cloudmonitor":{"table":"dev_server","condition":[{"app":["cloudmonitor-registry"]}]}}', 5 | '{"sidecar":{},"type":"sidecar"}', 'httpcheck', 0) 6 | 7 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/filematch/filematch_abs.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package filematch 6 | 7 | type ( 8 | AbsFileMatcher struct { 9 | Path string 10 | } 11 | ) 12 | 13 | func (m *AbsFileMatcher) IsDynamicMultiFiles() bool { 14 | return false 15 | } 16 | 17 | func (m *AbsFileMatcher) Find() ([]FatPath, int, error) { 18 | return []FatPath{{Path: m.Path}}, 0, nil 19 | } 20 | 21 | func NewAbsFileMatcher(path string) *AbsFileMatcher { 22 | return &AbsFileMatcher{ 23 | Path: path, 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/log-parse-cpu.json: -------------------------------------------------------------------------------- 1 | { 2 | "from": { 3 | "type": "cpu" 4 | }, 5 | "groupBy": { 6 | "groups": [ 7 | { 8 | "name": "ip", 9 | "elect": { 10 | "type": "refMeta", 11 | "refMeta": { 12 | "name": "ip" 13 | } 14 | } 15 | } 16 | ] 17 | }, 18 | "executeRule": { 19 | "interval": 5000 20 | }, 21 | "output": { 22 | "type": "cmgateway", 23 | "cmgateway": { 24 | "_doc": "如果强制指定了, 则会写数据到这个metric下", 25 | "metricName": "foobar" 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/timeparser/timeparser.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package timeparser 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/util" 9 | "time" 10 | ) 11 | 12 | type ( 13 | TimeParser struct { 14 | } 15 | ) 16 | 17 | // 解析时间 18 | func ParseTime(line string) int64 { 19 | layout := "2006-01-02 15:04:05" 20 | t, err := time.ParseInLocation(layout, line[:len(layout)], time.Local) 21 | if err != nil { 22 | return -1 23 | } else { 24 | return util.ToMS(t) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pkg/plugin/input/thread/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package thread 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/collecttask" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 10 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input/standard/providers" 11 | ) 12 | 13 | func init() { 14 | providers.RegisterInputProvider("countthreadtask", func(task *collecttask.CollectTask) (api.Input, error) { 15 | return &input{ 16 | task: task, 17 | }, nil 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/storage/node_hll.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package storage 6 | 7 | type ( 8 | HllDataNode struct { 9 | } 10 | ) 11 | 12 | func NewHllDataNode() DataNode { 13 | return &HllDataNode{} 14 | } 15 | 16 | func (h *HllDataNode) AddCount() { 17 | panic("HllDataNode doesn't support AddCount") 18 | } 19 | 20 | func (h *HllDataNode) MergeNumber(f float64) { 21 | panic("HllDataNode doesn't support MergeNumber") 22 | } 23 | 24 | func (h *HllDataNode) MergeHll(str string) { 25 | // TODO 待实现 26 | } 27 | -------------------------------------------------------------------------------- /pkg/pipeline/manager_builtin.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package pipeline 6 | 7 | // builtinConfigPrefix is the config prefix of all builtin tasks 8 | const builtinConfigPrefix = "BUILTIN_" 9 | 10 | // commonSysTaskTypes are the types of common sys tasks 11 | var commonSysTaskTypes = []string{"cpu", "mem", "load", "traffic", "tcp", "process", "swap", "disk"} 12 | 13 | // commonSysTaskTags contain tags which need to be added into datum generated by common sys tasks 14 | var commonSysTaskTags = []string{"hostname", "ip"} 15 | -------------------------------------------------------------------------------- /pkg/util/strings_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | import ( 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | func Test_ConvertStringSliceToHashSet(t *testing.T) { 13 | assert.Nil(t, ConvertStringSliceToHashSet(nil)) 14 | 15 | x := ConvertStringSliceToHashSet([]string{}) 16 | assert.NotNil(t, x) 17 | assert.Equal(t, 0, len(x)) 18 | 19 | x = ConvertStringSliceToHashSet([]string{"a"}) 20 | assert.Equal(t, 1, len(x)) 21 | 22 | _, ok := x["a"] 23 | assert.True(t, ok) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/cri/cricore/nsenter.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | /* 4 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 5 | */ 6 | 7 | package cricore 8 | 9 | import ( 10 | "context" 11 | "errors" 12 | "github.com/traas-stack/holoinsight-agent/pkg/cri" 13 | "net" 14 | "time" 15 | ) 16 | 17 | func NsEnterAndRunCodes(nsFile string, callback func()) error { 18 | return errors.New("unsupported") 19 | } 20 | 21 | func NsEnterDial(ctx context.Context, c *cri.Container, network, addr string, timeout time.Duration) (net.Conn, error) { 22 | return nil, errors.New("unsupported") 23 | } 24 | -------------------------------------------------------------------------------- /cmd/containerhelper/handlers/cat.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package handlers 6 | 7 | import ( 8 | "errors" 9 | "github.com/traas-stack/holoinsight-agent/cmd/containerhelper/model" 10 | "os" 11 | ) 12 | 13 | func catHandler(action string, resp *model.Resp) error { 14 | if len(os.Args) != 3 { 15 | return errors.New("miss args") 16 | } 17 | file := os.Args[2] 18 | content, err := os.ReadFile(file) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | resp.Data = content 24 | resp.Success = true 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /pkg/k8s/k8sutils/node.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package k8sutils 6 | 7 | import v1 "k8s.io/api/core/v1" 8 | 9 | func GetNodeIP(node *v1.Node) string { 10 | for _, address := range node.Status.Addresses { 11 | if address.Type == "InternalIP" { 12 | return address.Address 13 | } 14 | } 15 | return "" 16 | } 17 | 18 | func GetNodeHostname(node *v1.Node) string { 19 | for _, address := range node.Status.Addresses { 20 | if address.Type == "Hostname" { 21 | return address.Address 22 | } 23 | } 24 | return "" 25 | } 26 | -------------------------------------------------------------------------------- /scripts/build/agent-builder/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22 as golang 2 | 3 | FROM centos:7 4 | # This image is used to build agent bin. We use old version centos:7 to avoid GLIBC error in some older version OS. 5 | # GLIBC error example: 6 | # /usr/local/holoinsight/agent/bin/agent: /lib64/libc.so.6: version `GLIBC_2.28' not found (required by /usr/local/holoinsight/agent/bin/agent) 7 | 8 | RUN yum install -y make gcc && \ 9 | yum -q -y clean all && \ 10 | rm -rf /var/cache/yum 11 | 12 | COPY --from=golang /usr/local/go /usr/local/go 13 | 14 | ENV GOROOT=/usr/local/go 15 | ENV PATH=$PATH:$GOROOT/bin 16 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/time_elect_unix.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "github.com/spf13/cast" 9 | ) 10 | 11 | type ( 12 | timeElectUnixParser struct { 13 | elect XElect 14 | } 15 | ) 16 | 17 | func (e *timeElectUnixParser) Parse(ctx *LogContext) (int64, error) { 18 | s, err := e.elect.ElectString(ctx) 19 | if err != nil { 20 | return TimeParseError, err 21 | } 22 | i64, err := cast.ToInt64E(s) 23 | if err != nil { 24 | return TimeParseError, err 25 | } 26 | return i64 * 1000, nil 27 | } 28 | -------------------------------------------------------------------------------- /pkg/plugin/input/jvm/convert.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package jvm 6 | 7 | import "github.com/spf13/cast" 8 | 9 | // 1k 1m 1g 10 | func convertToBytes(s string) int64 { 11 | if s == "" { 12 | return 0 13 | } 14 | i64, err := cast.ToInt64E(s[0 : len(s)-1]) 15 | if err != nil { 16 | return 0 17 | } 18 | 19 | switch s[len(s)-1] { 20 | case 'k': 21 | return i64 * 1024 22 | case 'm': 23 | return i64 * 1024 * 1024 24 | case 'g': 25 | return i64 * 1024 * 1024 * 1024 26 | default: 27 | // unknown unit 28 | return 0 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/time_elect_unix_milli.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "github.com/spf13/cast" 9 | ) 10 | 11 | type ( 12 | timeElectUnixMilliParser struct { 13 | elect XElect 14 | } 15 | ) 16 | 17 | func (e *timeElectUnixMilliParser) Parse(ctx *LogContext) (int64, error) { 18 | s, err := e.elect.ElectString(ctx) 19 | if err != nil { 20 | return TimeParseError, err 21 | } 22 | i64, err := cast.ToInt64E(s) 23 | if err != nil { 24 | return TimeParseError, err 25 | } 26 | return i64, nil 27 | } 28 | -------------------------------------------------------------------------------- /pkg/pipeline/standard/outputs.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package standard 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/model" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/output" 10 | ) 11 | 12 | type ( 13 | Output struct { 14 | Tenant string 15 | O output.Output 16 | ConfigKey string 17 | } 18 | ) 19 | 20 | func (o *Output) Write(metrics []*model.Metric) { 21 | oe := output.Extension{ 22 | Tenant: o.Tenant, 23 | ConfigKey: o.ConfigKey, 24 | } 25 | o.O.WriteMetricsV1(metrics, oe) 26 | } 27 | -------------------------------------------------------------------------------- /scripts/docker/agent-base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | 3 | # install common softwares 4 | RUN yum -q -y install epel-release && yum install -q -y sudo net-tools iproute dstat which supervisor stress unzip jq screen wget telnet less && \ 5 | yum -y clean all && \ 6 | rm -rf /var/cache/yum 7 | 8 | # setup bash 9 | RUN echo 'PS1="\n\e[1;37m[\e[m\e[1;32m\u\e[m\e[1;33m@\e[m\e[1;35m\h\e[m \e[1;35m`hostname`\e[m \e[4m\`pwd\`\e[m\e[1;37m]\e[m\e[1;36m\e[m\n\\$ "' >> /etc/bashrc && \ 10 | echo 'alias vim="vi"' >> /etc/bashrc && \ 11 | echo 'alias ll="ls -laF"' >> /etc/bashrc && \ 12 | echo 'shell /bin/bash' >> /root/.screenrc 13 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/logparser_grok.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "github.com/vjeantet/grok" 9 | _ "github.com/vjeantet/grok" 10 | ) 11 | 12 | type ( 13 | grokParser struct { 14 | g *grok.Grok 15 | expression string 16 | } 17 | ) 18 | 19 | func (g *grokParser) Parse(ctx *LogContext) error { 20 | m, err := g.g.ParseTyped(g.expression, ctx.GetLine()) 21 | if err != nil { 22 | return err 23 | } 24 | if len(m) == 0 { 25 | return LogParseNotMatched 26 | } 27 | ctx.columnMap = m 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/testv2_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "encoding/json" 9 | "fmt" 10 | "github.com/traas-stack/holoinsight-agent/pkg/collectconfig" 11 | "io" 12 | "os" 13 | "testing" 14 | ) 15 | 16 | func TestName(t *testing.T) { 17 | f, err := os.Open("demo1.json") 18 | if err != nil { 19 | panic(err) 20 | } 21 | defer f.Close() 22 | b, err := io.ReadAll(f) 23 | if err != nil { 24 | panic(err) 25 | } 26 | sql := &collectconfig.SQLTask{} 27 | json.Unmarshal(b, sql) 28 | fmt.Printf("%+v\n", sql) 29 | } 30 | -------------------------------------------------------------------------------- /scripts/vm/agent.ini: -------------------------------------------------------------------------------- 1 | [program:agent] 2 | command=/usr/local/holoinsight/agent/bin/agent.sh 3 | process_name=%(program_name)s 4 | autostart=true 5 | # 启动后持续3秒处于RUNNING状态则认为启动成功 6 | #startsecs=3 7 | # 重试 8 | startretries=3 9 | autorestart=unexpected 10 | exitcodes=0,7 11 | stopsignal=TERM 12 | stopwaitsecs=3 13 | stopasgroup=true 14 | killasgroup=true 15 | redirect_stderr=true 16 | stdout_logfile=/usr/local/holoinsight/agent/logs/stdout.log 17 | stdout_logfile_maxbytes=50MB 18 | stdout_logfile_backups=10 19 | stdout_capture_maxbytes=0 20 | stdout_events_enabled=true 21 | #environment=KEY="val",KEY2="val2" 22 | directory=/usr/local/holoinsight/agent 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/refactor-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Refactor Request 3 | about: Suggest something that needs to be refactored. 4 | title: '' 5 | labels: refactor 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Refactor Request 11 | 12 | 16 | 17 | ### Description of the proposed solution 18 | 19 | 20 | 21 | ### Additional context 22 | 23 | 24 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/time_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "fmt" 9 | "github.com/traas-stack/holoinsight-agent/pkg/collectconfig" 10 | "testing" 11 | ) 12 | 13 | func TestTimeParse(t *testing.T) { 14 | parser, err := parseTimeParser(&collectconfig.TimeConf{ 15 | Type: "auto", 16 | }) 17 | if err != nil { 18 | panic(err) 19 | } 20 | ts, err := parser.Parse(&LogContext{log: &LogGroup{ 21 | Line: "2022-04-22 14:50:31.000 process_lines,160", 22 | }}) 23 | if err != nil { 24 | panic(err) 25 | } 26 | fmt.Println(ts) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/plugin/input/springboot/springboot_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package springboot 6 | 7 | import ( 8 | "encoding/json" 9 | "fmt" 10 | "github.com/traas-stack/holoinsight-agent/pkg/accumulator" 11 | telegraf2 "github.com/traas-stack/holoinsight-agent/pkg/telegraf" 12 | "os" 13 | "testing" 14 | ) 15 | 16 | func TestSpringBoot(t *testing.T) { 17 | b := &SpringBoot{BaseUrl: "http://11.166.169.151:8080/actuator"} 18 | m := &accumulator.Memory{} 19 | err := b.Gather(telegraf2.Adapt(m, "_")) 20 | fmt.Println(err) 21 | json.NewEncoder(os.Stdout).Encode(m.Metrics) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/cri/mount.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package cri 6 | 7 | import "sort" 8 | 9 | type ( 10 | mountPointSortHelper []*MountPoint 11 | ) 12 | 13 | func (m mountPointSortHelper) Len() int { 14 | return len(m) 15 | } 16 | 17 | func (m mountPointSortHelper) Less(i, j int) bool { 18 | // 降序排列 19 | return len(m[i].Source) > len(m[j].Source) 20 | } 21 | 22 | func (m mountPointSortHelper) Swap(i, j int) { 23 | t := m[i] 24 | m[i] = m[j] 25 | m[j] = t 26 | } 27 | 28 | func SortMountPointsByLongSourceFirst(mounts []*MountPoint) { 29 | sort.Sort(mountPointSortHelper(mounts)) 30 | } 31 | -------------------------------------------------------------------------------- /scripts/docker/bin/app.ini: -------------------------------------------------------------------------------- 1 | [program:app] 2 | command=/usr/local/holoinsight/agent/bin/app.sh 3 | process_name=%(program_name)s 4 | autostart=true 5 | #startsecs=3 6 | startretries=3 7 | autorestart=unexpected 8 | # exitcode=7 is for lossless redeploy 9 | exitcodes=0 10 | stopsignal=TERM 11 | stopwaitsecs=10 12 | stopasgroup=true 13 | killasgroup=true 14 | user=root 15 | redirect_stderr=true 16 | stdout_logfile=/usr/local/holoinsight/agent/logs/stdout.log 17 | stdout_logfile_maxbytes=50MB 18 | stdout_logfile_backups=10 19 | stdout_capture_maxbytes=0 20 | stdout_events_enabled=true 21 | #environment=KEY="val",KEY2="val2" 22 | directory=/usr/local/holoinsight/agent 23 | -------------------------------------------------------------------------------- /pkg/appconfig/http.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package appconfig 6 | 7 | import ( 8 | "encoding/json" 9 | "net/http" 10 | "runtime" 11 | "time" 12 | ) 13 | 14 | var uptime = time.Now() 15 | 16 | func init() { 17 | http.HandleFunc("/version", func(writer http.ResponseWriter, request *http.Request) { 18 | r := map[string]interface{}{ 19 | "goversion": runtime.Version(), 20 | "version": agentVersion, 21 | "buildTime": agentBuildTime, 22 | "commit": gitcommit, 23 | "uptime": uptime.Format(time.RFC3339), 24 | } 25 | json.NewEncoder(writer).Encode(r) 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/elect_refmeta.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import "errors" 8 | 9 | type ( 10 | xElectRefMeta struct { 11 | value string 12 | } 13 | ) 14 | 15 | func (x *xElectRefMeta) Init() { 16 | } 17 | 18 | func (x *xElectRefMeta) Elect(ctx *LogContext) (interface{}, error) { 19 | return x.value, nil 20 | } 21 | 22 | func (x *xElectRefMeta) ElectString(ctx *LogContext) (string, error) { 23 | return x.value, nil 24 | } 25 | 26 | func (x *xElectRefMeta) ElectNumber(ctx *LogContext) (float64, error) { 27 | return 0, errors.New("unsupported") 28 | } 29 | -------------------------------------------------------------------------------- /pkg/util/throttle/throttle.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package throttle 6 | 7 | import ( 8 | "sync/atomic" 9 | "time" 10 | ) 11 | 12 | func ThrottleFirst(d time.Duration) func(f func()) { 13 | wip := int32(0) 14 | 15 | return func(f func()) { 16 | if atomic.AddInt32(&wip, 1) == 1 { 17 | go drainLoop(d, &wip, f) 18 | } 19 | } 20 | } 21 | 22 | func drainLoop(d time.Duration, wip *int32, f func()) { 23 | delta := atomic.LoadInt32(wip) 24 | 25 | f() 26 | 27 | time.AfterFunc(d, func() { 28 | if atomic.AddInt32(wip, -delta) != 0 { 29 | drainLoop(d, wip, f) 30 | } 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/filematch/filewatch_format.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package filematch 6 | 7 | type ( 8 | FormatFileMatcher struct { 9 | pattern string 10 | } 11 | ) 12 | 13 | func (m *FormatFileMatcher) IsDynamicMultiFiles() bool { 14 | return false 15 | } 16 | 17 | func NewFormatFileMatcher(pattern string) *FormatFileMatcher { 18 | return &FormatFileMatcher{ 19 | pattern: pattern, 20 | } 21 | } 22 | 23 | func (m *FormatFileMatcher) Find() ([]FatPath, int, error) { 24 | return []FatPath{ 25 | { 26 | Path: stdFilePathFormatter.Format(m.pattern), 27 | }, 28 | }, 0, nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/plugin/input/dialcheck/dialcheck_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package dialcheck 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/api" 9 | "os" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | func TestDialCheck(t *testing.T) { 15 | input := &Input{ 16 | Config: &Config{ 17 | Network: "tcp", 18 | Host: "127.0.0.1", 19 | Ports: []int{22}, 20 | Timeout: 3 * time.Second, 21 | Times: 3, 22 | }, 23 | } 24 | 25 | ma := api.NewMemoryAccumulator() 26 | err := input.Collect(ma) 27 | if err != nil { 28 | panic(err) 29 | } 30 | ma.PrintTo(os.Stdout) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/util/eventcollector/collect_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package eventcollector 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | 11 | "github.com/traas-stack/holoinsight-agent/pkg/appconfig" 12 | "github.com/traas-stack/holoinsight-agent/pkg/logger" 13 | ) 14 | 15 | func TestEventCollector(t *testing.T) { 16 | appconfig.SetDev(true) 17 | logger.SetupZapLogger() 18 | 19 | ec := New(time.Second, func(events []*Event) { 20 | for _, v := range events { 21 | t.Log(v) 22 | } 23 | }) 24 | 25 | ec.Collect("cfgInitError", "slice index out of range!") 26 | 27 | time.Sleep(time.Second * 5) 28 | } 29 | -------------------------------------------------------------------------------- /cmd/plugintest/telegraf_ping.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": "demo/1", 3 | "config": { 4 | "key": "demo", 5 | "type": "telegraf_ping", 6 | "contentObj": { 7 | "executeRule": { 8 | "type": "fixedRate", 9 | "fixedRate": 5000, 10 | "offset": 0 11 | }, 12 | "refMetas": { 13 | "a": { 14 | "name": "a1" 15 | } 16 | }, 17 | "transform": { 18 | "metricFormat": "yyy_%s" 19 | }, 20 | "Port": 6379, 21 | "Password": "" 22 | } 23 | }, 24 | "target": { 25 | "key": "1", 26 | "type": "ip", 27 | "meta": { 28 | "a1": "b", 29 | "ip": "127.0.0.1" 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/split_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestSplit(t *testing.T) { 13 | str := "2022-08-29 16:37:34,747|||0a374dd916617622522722575e006d|INFO |DubboServerHandler-10.55.77.217:20880-thread-197|DUBBO-MONITOR-PROVIDER-DIGEST-LOG|superlink|PROVIDER|-|gloandecision|10.55.78.187:38588 -> 10.55.77.217:20880|com.antbank.superlink.facade.MessageSendService|send|Y|2475|-|{\"com.antbank.fc.common.lang.event.EventContext\":\"{\\\"tntInstId\\\":\\\"XHBKC1MO\\\"}\"}" 14 | println(len(strings.Split(str, "|"))) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/core/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package core 6 | 7 | import ( 8 | "os" 9 | "path/filepath" 10 | ) 11 | 12 | var HelperToolLocalPath = "/usr/local/holoinsight/agent/bin/helper" 13 | var BusyboxLocalPath = "/usr/local/holoinsight/agent/bin/busybox" 14 | var HelperToolPath = "/tmp/holoinsight/helper" 15 | var BusyboxPath = "/tmp/holoinsight/busybox" 16 | 17 | type ( 18 | HelperBaseResp struct { 19 | Success bool `json:"success"` 20 | Message string `json:"message"` 21 | } 22 | ) 23 | 24 | func init() { 25 | wd, _ := os.Getwd() 26 | HelperToolLocalPath = filepath.Join(wd, "/bin/helper") 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | pull_request_target: 5 | branches: [ "main" ] 6 | workflow_dispatch: { } 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | with: 14 | ref: ${{ github.event.pull_request.head.sha }} 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v3 18 | with: 19 | go-version: 1.22 20 | cache: true 21 | - name: Build HoloInsight Agent 22 | shell: bash 23 | run: | 24 | export GOOS=linux 25 | export GOARCH=amd64 26 | ./scripts/gen-git-info.sh 27 | ./scripts/build/build-using-go.sh 28 | -------------------------------------------------------------------------------- /cmd/plugintest/telegraf_nginx.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": "demo/1", 3 | "config": { 4 | "key": "demo", 5 | "type": "telegraf_nginx", 6 | "contentObj": { 7 | "executeRule": { 8 | "type": "fixedRate", 9 | "fixedRate": 5000, 10 | "offset": 0 11 | }, 12 | "refMetas": { 13 | "a": { 14 | "name": "a1" 15 | } 16 | }, 17 | "transform": { 18 | "metricFormat": "yyy_%s" 19 | }, 20 | "port": 9090, 21 | "path": "/status" 22 | } 23 | }, 24 | "target": { 25 | "key": "1", 26 | "type": "ip", 27 | "meta": { 28 | "a1": "b", 29 | "ip": "127.0.0.1" 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cmd/plugintest/telegraf_redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": "demo/1", 3 | "config": { 4 | "key": "demo", 5 | "type": "telegraf_redis", 6 | "contentObj": { 7 | "executeRule": { 8 | "type": "fixedRate", 9 | "fixedRate": 5000, 10 | "offset": 0 11 | }, 12 | "refMetas": { 13 | "a": { 14 | "name": "a1" 15 | } 16 | }, 17 | "transform": { 18 | "metricFormat": "yyy_%s" 19 | }, 20 | "Port": 6379, 21 | "Password": "" 22 | } 23 | }, 24 | "target": { 25 | "key": "1", 26 | "type": "ip", 27 | "meta": { 28 | "a1": "b", 29 | "ip": "127.0.0.1" 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/k8s/silence/pod_update_listener_component.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package silence 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/ioc" 9 | ) 10 | 11 | type ( 12 | PodUpdateListenerMasterComponent struct { 13 | podDeployListener *PodUpdateListener 14 | } 15 | ) 16 | 17 | func (c *PodUpdateListenerMasterComponent) Start() { 18 | c.podDeployListener = NewPodOperationListener(ioc.K8sClientset) 19 | c.podDeployListener.Start() 20 | } 21 | func (c *PodUpdateListenerMasterComponent) Stop() { 22 | x := c.podDeployListener 23 | c.podDeployListener = nil 24 | if x != nil { 25 | x.Stop() 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /cmd/plugintest/telegraf_mongodb.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": "demo/1", 3 | "config": { 4 | "key": "demo", 5 | "type": "telegraf_mongodb", 6 | "contentObj": { 7 | "executeRule": { 8 | "type": "fixedRate", 9 | "fixedRate": 5000, 10 | "offset": 0 11 | }, 12 | "refMetas": { 13 | "a": { 14 | "name": "a1" 15 | } 16 | }, 17 | "transform": { 18 | "metricFormat": "yyy_%s" 19 | }, 20 | "servers": "mongodb://127.0.0.1:27017/?connect=direct" 21 | } 22 | }, 23 | "target": { 24 | "key": "1", 25 | "type": "ip", 26 | "meta": { 27 | "a1": "b", 28 | "ip": "127.0.0.1" 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /scripts/docker/buildx.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # docs: Build agent docker image for multi arch and push images to Docker Hub 5 | 6 | cd `dirname $0`/../.. 7 | 8 | if [ -z "$tag" ]; then 9 | tag=latest 10 | fi 11 | 12 | # If user defines GOPROXY env, then pass it to docker build using --build-arg. 13 | build_opts="" 14 | source ./scripts/build/detect-goproxy.sh 15 | build_opts="" 16 | if [ -n "$GOPROXY" ]; then 17 | build_opts="--build-arg GOPROXY=$GOPROXY" 18 | fi 19 | ./scripts/gen-git-info.sh 20 | 21 | docker buildx build \ 22 | --platform linux/amd64,linux/arm64/v8 \ 23 | $build_opts \ 24 | -f ./scripts/docker/Dockerfile \ 25 | --pull --push \ 26 | -t holoinsight/agent:$tag . 27 | -------------------------------------------------------------------------------- /pkg/pipeline/integration/alibabacloud/frequency_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package alibabacloud 6 | 7 | import ( 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | func TestFrequency(t *testing.T) { 13 | ilf := newIntelligentFrequency(3, 5) 14 | 15 | ilf.set(false) 16 | assert.False(t, ilf.isDown()) 17 | 18 | ilf.set(false) 19 | assert.False(t, ilf.isDown()) 20 | 21 | ilf.set(false) 22 | assert.True(t, ilf.isDown()) 23 | 24 | ilf.set(false) 25 | assert.True(t, ilf.isDown()) 26 | 27 | ilf.set(false) 28 | assert.False(t, ilf.isDown()) 29 | 30 | ilf.set(false) 31 | assert.True(t, ilf.isDown()) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/server/fordev/fortest.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package fordev 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/appconfig" 9 | "github.com/traas-stack/holoinsight-agent/pkg/server/gateway" 10 | "strings" 11 | ) 12 | 13 | func NewDevGateway() (*gateway.Service, error) { 14 | g := appconfig.StdAgentConfig.Gateway 15 | addr := g.Addr 16 | if !strings.Contains(addr, ":") { 17 | addr += ":19610" 18 | } 19 | return gateway.New(gateway.Config{ 20 | Addr: addr, 21 | Secure: g.Secure, 22 | Apikey: appconfig.StdAgentConfig.ApiKey, 23 | ServerName: g.ServiceName, 24 | CaCertBase64: g.CaCert, 25 | }) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/bistream/biztypes/biztypes.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package biztypes 6 | 7 | const ( 8 | BizTypeEcho = 0 9 | BizTypeListDir = 10 10 | BizTypeListDirResp = 11 11 | BizTypePreviewFile = 20 12 | BizTypePreviewFileResp = 21 13 | BizTypeSplitLog = 30 14 | BizTypeSplitLogResp = 31 15 | BizTypeInspect = 40 16 | BizTypeInspectResp = 41 17 | BizTypeMatchFiles = 50 18 | BizTypeMatchFilesResp = 51 19 | BizTypeDryRun = 60 20 | BizTypeDryRunResp = 61 21 | BizTypeHttpProxy = 70 22 | BizTypeHttpProxyResp = 71 23 | BizCheckTask = 80 24 | BizCheckTaskResp = 81 25 | ) 26 | -------------------------------------------------------------------------------- /pkg/cri/criutils/container.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package criutils 6 | 7 | import "github.com/traas-stack/holoinsight-agent/pkg/cri" 8 | 9 | // GetMainBizContainerE get main biz container for pod 10 | func GetMainBizContainerE(i cri.Interface, ns string, pod string) (*cri.Container, error) { 11 | p, err := i.GetPod(ns, pod) 12 | if err != nil { 13 | return nil, err 14 | } 15 | return p.MainBizE() 16 | } 17 | 18 | func GetContainerE(i cri.Interface, ns, pod, k8sContainerName string) (*cri.Container, error) { 19 | p, err := i.GetPod(ns, pod) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | return p.GetContainer(k8sContainerName) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/logger/http.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package logger 6 | 7 | import ( 8 | "net/http" 9 | "time" 10 | ) 11 | 12 | func registerHttpHandler() { 13 | http.HandleFunc("/api/log/debug/start", func(writer http.ResponseWriter, request *http.Request) { 14 | DebugEnabled = true 15 | writer.Write([]byte("OK")) 16 | }) 17 | http.HandleFunc("/api/log/debug/stop", func(writer http.ResponseWriter, request *http.Request) { 18 | DebugEnabled = false 19 | writer.Write([]byte("OK")) 20 | }) 21 | go func() { 22 | // 自动关闭 23 | timer := time.NewTimer(10 * time.Hour) 24 | defer timer.Stop() 25 | for range timer.C { 26 | DebugEnabled = false 27 | } 28 | }() 29 | } 30 | -------------------------------------------------------------------------------- /pkg/plugin/api/pipeline.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package api 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/pkg/collectconfig" 9 | "github.com/traas-stack/holoinsight-agent/pkg/collecttask" 10 | ) 11 | 12 | type ( 13 | Pipeline interface { 14 | // Key returns the unique identifier of the pipeline instance 15 | Key() string 16 | Start() error 17 | Stop() 18 | SetupConsumer(st *SubTask) error 19 | Update(func(Pipeline)) 20 | View(func(Pipeline)) 21 | } 22 | PipelineUpdateAware interface { 23 | UpdateFrom(Pipeline) 24 | } 25 | SubTask struct { 26 | CT *collecttask.CollectTask 27 | SqlTask *collectconfig.SQLTask 28 | } 29 | ) 30 | -------------------------------------------------------------------------------- /pkg/transfer/pb/transfer.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package io.holoinsight.agent.transfer; 4 | 5 | option go_package = "github.com/traas-stack/holoinsight-agent/pkg/transfer/pb"; 6 | 7 | message PingRequest {} 8 | message PingResponse {} 9 | 10 | message StopAndSaveSaveRequest {} 11 | message StopAndSaveSaveResponse { 12 | bytes state = 1; 13 | } 14 | 15 | message TransferDoneRequest {} 16 | message TransferDoneResponse {} 17 | 18 | service TransferSrvice { 19 | rpc prepare(PingRequest)returns(PingResponse){} 20 | rpc ping(PingRequest)returns(PingResponse){} 21 | rpc stop_and_save_state(StopAndSaveSaveRequest) returns(StopAndSaveSaveResponse){} 22 | rpc transfer_done(TransferDoneRequest) returns(TransferDoneResponse){} 23 | } 24 | -------------------------------------------------------------------------------- /pkg/util/grpc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package util 6 | 7 | import ( 8 | "crypto/tls" 9 | "crypto/x509" 10 | "encoding/base64" 11 | "fmt" 12 | "google.golang.org/grpc/credentials" 13 | ) 14 | 15 | func NewClientTLSFromBase64(certBase64, serverNameOverride string) (credentials.TransportCredentials, error) { 16 | b, err := base64.StdEncoding.DecodeString(certBase64) 17 | if err != nil { 18 | return nil, err 19 | } 20 | cp := x509.NewCertPool() 21 | if !cp.AppendCertsFromPEM(b) { 22 | return nil, fmt.Errorf("credentials: failed to append certificates") 23 | } 24 | return credentials.NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil 25 | } 26 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/elect_refvar_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | func TestRefVar(t *testing.T) { 13 | e := &xRefVar{name: "a"} 14 | 15 | s, err := e.ElectString(&LogContext{}) 16 | assert.NoError(t, err) 17 | assert.Equal(t, "", s) 18 | 19 | s, err = e.ElectString(&LogContext{vars: map[string]interface{}{"a": "holoinsight"}}) 20 | assert.NoError(t, err) 21 | assert.Equal(t, "holoinsight", s) 22 | 23 | s, err = e.ElectString(&LogContext{vars: map[string]interface{}{"b": "holoinsight"}}) 24 | assert.NoError(t, err) 25 | assert.Equal(t, "", s) 26 | 27 | } 28 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/transform_append_v1.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "github.com/spf13/cast" 9 | "github.com/traas-stack/holoinsight-agent/pkg/collectconfig" 10 | "strings" 11 | ) 12 | 13 | type ( 14 | xAppendV1Filter struct { 15 | conf *collectconfig.TransformFilterAppendV1 16 | } 17 | ) 18 | 19 | func (x *xAppendV1Filter) Init() error { 20 | return nil 21 | } 22 | 23 | func (x *xAppendV1Filter) Filter(ctx *LogContext) (interface{}, error) { 24 | str := cast.ToString(ctx.contextValue) 25 | if !x.conf.AppendIfMissing || !strings.HasSuffix(str, x.conf.Value) { 26 | str += x.conf.Value 27 | } 28 | return str, nil 29 | } 30 | -------------------------------------------------------------------------------- /cmd/containerhelper/handlers/httpcheck.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package handlers 6 | 7 | import ( 8 | "github.com/traas-stack/holoinsight-agent/cmd/containerhelper/model" 9 | "github.com/traas-stack/holoinsight-agent/pkg/plugin/input/httpcheck" 10 | "io" 11 | "os" 12 | ) 13 | 14 | func httpCheckHandler(action string, resp *model.Resp) error { 15 | configBytes, err := io.ReadAll(os.Stdin) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | input := &httpcheck.Input{} 21 | responseBytes, err := input.ExecuteRequest(configBytes) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | resp.Success = true 27 | resp.Data = responseBytes 28 | 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/elect_pathvar.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import "github.com/spf13/cast" 8 | 9 | type ( 10 | xPathVar struct { 11 | name string 12 | } 13 | ) 14 | 15 | func (x *xPathVar) Init() { 16 | } 17 | 18 | func (x *xPathVar) Elect(ctx *LogContext) (interface{}, error) { 19 | return x.ElectString(ctx) 20 | } 21 | 22 | func (x *xPathVar) ElectString(ctx *LogContext) (string, error) { 23 | if len(ctx.pathTags) > 0 { 24 | return ctx.pathTags[x.name], nil 25 | } 26 | return "", nil 27 | } 28 | 29 | func (x *xPathVar) ElectNumber(ctx *LogContext) (float64, error) { 30 | i, _ := x.Elect(ctx) 31 | return cast.ToFloat64(i), nil 32 | } 33 | -------------------------------------------------------------------------------- /pkg/collectconfig/executor/logcontext_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. 3 | */ 4 | 5 | package executor 6 | 7 | import ( 8 | "fmt" 9 | "regexp" 10 | "testing" 11 | ) 12 | 13 | func TestLogContextRegex(t *testing.T) { 14 | // r := regexp.MustCompile("^(?P