├── utils ├── clock.go └── id.go ├── Docker ├── .localstack │ ├── data │ │ └── .gitignore │ └── template │ │ └── recorded_api_calls.json ├── build │ ├── .gitignore │ └── Dockerfile ├── opensearch-config │ └── opensearch.yml ├── config │ ├── local_override.json │ └── client.config.yaml ├── README.md └── Makefile ├── ingestion ├── clock.go ├── testdata │ ├── assets.go │ ├── System.VFS.DownloadFile │ │ ├── Msg_05.json │ │ ├── Msg_08.json │ │ ├── Msg_07.json │ │ └── Msg_06.json │ ├── Server.Internal.ClientInfo │ │ ├── Client.Info.Updates_01.json │ │ └── Client.Info.Updates_02.json │ ├── System.VFS.ListDirectory │ │ ├── Msg_02.json │ │ ├── Msg_01.json │ │ ├── Msg_05.json │ │ ├── Msg_03.json │ │ └── Msg_04.json │ ├── Generic.Client.Stats │ │ ├── Generic.Client.Stats_02.json │ │ └── Generic.Client.Stats_01.json │ └── Enrollment │ │ └── Enrollment_01.json ├── fixtures │ ├── TestEnrollment.golden │ ├── TestErrorLogs.golden │ ├── TestVFSDownload.golden │ └── TestClientEventMonitoring.golden ├── ping.go ├── logs.go ├── enrolment.go ├── flow_stats.go ├── registration.go ├── monitoring_logs.go ├── uploads.go ├── ingestor.go └── hunts.go ├── scripts ├── dlv.init └── gh_download_run.sh ├── vql ├── uploads │ ├── upload.go │ ├── factory.go │ ├── fixtures │ │ └── TestSparseUploader.golden │ ├── api.go │ ├── buffer.go │ └── sparse.go └── server │ └── notebook │ └── delete.go ├── services ├── hunt_dispatcher.go ├── repository.go ├── hunt_dispatcher │ ├── delete.go │ ├── list.go │ ├── downloads.go │ ├── mutations.go │ ├── modify.go │ └── index.go ├── users │ ├── delete.go │ └── users.go ├── notebook │ ├── shared.go │ ├── delete.go │ ├── paths.go │ ├── notebook.go │ └── annotator.go ├── client_info.go ├── repository │ └── forwards.go ├── retry.go ├── orgs │ ├── ids.go │ └── delete.go ├── vfs_service │ └── vfs_service_test.go ├── launcher.go ├── indexing │ ├── utils.go │ ├── mru.go │ └── names.go ├── frontend │ └── frontend.go ├── launcher │ ├── launcher.go │ └── flows.go ├── inventory │ └── inventory.go ├── instrument.go ├── sanity │ ├── sanity.go │ └── users.go ├── client_info │ └── metadata.go └── notifier │ └── notifier.go ├── .gitmodules ├── artifact_definitions ├── assets.go ├── Linux.Remediation.Quarantine.yaml ├── MacOS.System.QuarantineEvents.yaml ├── Windows.Remediation.Quarantine.yaml ├── Client.Info.Updates.yaml ├── Server.Utils.DeleteClient.yaml ├── Alert │ ├── EventLogModifications.yaml │ └── PowerPickHostVersion.yaml ├── System.VFS.DownloadFile.yaml └── Windows.EventLogs.Bitsadmin.yaml ├── schema ├── scripts │ ├── increment-hunt-errored-script.json │ ├── increment-hunt-completed-script.json │ ├── increment-hunt-started-script.json │ ├── run.sh │ ├── create_scripts.sh │ ├── create_indexes.sh │ └── create_policies.sh ├── api │ ├── notifications.go │ ├── repository.go │ └── collections.go ├── templates │ ├── error.json │ └── transient.json ├── docker │ └── Dockerfile └── policies │ └── rollover_strategy.json ├── constants └── constants.go ├── vql_plugins └── vfs.go ├── datastore ├── wrapper.go ├── elastic.go └── datastoretest │ └── datastore_test.go ├── bin ├── logging.go ├── foreman.go ├── gui.go ├── config.go ├── client.go ├── utils.go ├── version.go ├── server.go ├── users.go ├── debug.go └── query.go ├── .github └── workflows │ ├── remove-old.yml │ ├── build.yml │ └── test.yml ├── server ├── dummy.go ├── api.go ├── mock.go └── elastic.go ├── filestore ├── utils.go ├── fileinfo.go ├── instrument.go ├── name_mapping.go ├── reader.go └── s3filestore_test.go ├── .gitignore ├── TODO.md ├── make.go ├── startup ├── foreman.go ├── tools.go ├── pool.go ├── communicator.go └── client.go ├── result_sets ├── simple │ ├── simple.go │ └── metadata.go └── timed │ ├── factory.go │ ├── timed.go │ └── reader.go ├── README.md ├── Makefile ├── crypto └── server │ └── manager.go ├── foreman └── event_monitoring.go └── testsuite └── testsuite.go /utils/clock.go: -------------------------------------------------------------------------------- 1 | package utils 2 | -------------------------------------------------------------------------------- /Docker/.localstack/data/.gitignore: -------------------------------------------------------------------------------- 1 | * -------------------------------------------------------------------------------- /Docker/build/.gitignore: -------------------------------------------------------------------------------- 1 | cvelociraptor -------------------------------------------------------------------------------- /ingestion/clock.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | -------------------------------------------------------------------------------- /scripts/dlv.init: -------------------------------------------------------------------------------- 1 | source scripts/dlv.star -------------------------------------------------------------------------------- /vql/uploads/upload.go: -------------------------------------------------------------------------------- 1 | package uploads 2 | -------------------------------------------------------------------------------- /services/hunt_dispatcher.go: -------------------------------------------------------------------------------- 1 | package services 2 | -------------------------------------------------------------------------------- /ingestion/testdata/assets.go: -------------------------------------------------------------------------------- 1 | package testdata 2 | 3 | import "embed" 4 | 5 | //go:embed * 6 | var FS embed.FS 7 | -------------------------------------------------------------------------------- /services/repository.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | const ( 4 | Sync bool = true 5 | NoSync bool = false 6 | ) 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "velociraptor"] 2 | path = velociraptor 3 | url = https://github.com/Velocidex/velociraptor.git 4 | -------------------------------------------------------------------------------- /artifact_definitions/assets.go: -------------------------------------------------------------------------------- 1 | package artifact_definitions 2 | 3 | import "embed" 4 | 5 | //go:embed *.yaml 6 | var FS embed.FS 7 | -------------------------------------------------------------------------------- /schema/scripts/increment-hunt-errored-script.json: -------------------------------------------------------------------------------- 1 | { 2 | "script": { 3 | "lang": "painless", 4 | "source": "ctx._source.errors++;" 5 | } 6 | } -------------------------------------------------------------------------------- /schema/scripts/increment-hunt-completed-script.json: -------------------------------------------------------------------------------- 1 | { 2 | "script": { 3 | "lang": "painless", 4 | "source": "ctx._source.completed++;" 5 | } 6 | } -------------------------------------------------------------------------------- /schema/scripts/increment-hunt-started-script.json: -------------------------------------------------------------------------------- 1 | { 2 | "script": { 3 | "lang": "painless", 4 | "source": "ctx._source.scheduled++;" 5 | } 6 | } -------------------------------------------------------------------------------- /Docker/opensearch-config/opensearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | cluster.name: docker-cluster 3 | network.host: 0.0.0.0 4 | discovery.type: single-node 5 | plugins.security.disabled: true 6 | -------------------------------------------------------------------------------- /constants/constants.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import ( 4 | "www.velocidex.com/golang/velociraptor/constants" 5 | ) 6 | 7 | var ( 8 | VERSION = constants.VERSION 9 | ) 10 | -------------------------------------------------------------------------------- /utils/id.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/hex" 5 | 6 | "github.com/google/uuid" 7 | ) 8 | 9 | func GetId() string { 10 | u := uuid.New() 11 | return hex.EncodeToString(u[0:8]) 12 | } 13 | -------------------------------------------------------------------------------- /Docker/build/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM alpine:latest 4 | 5 | WORKDIR / 6 | 7 | COPY /cvelociraptor /cvelociraptor 8 | 9 | EXPOSE 8080 10 | 11 | ENTRYPOINT ["/cvelociraptor"] 12 | -------------------------------------------------------------------------------- /schema/api/notifications.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type NotificationRecord struct { 4 | Key string `json:"key,omitempty"` 5 | Timestamp int64 `json:"timestamp,omitempty"` 6 | DocType string `json:"doc_type"` 7 | } 8 | -------------------------------------------------------------------------------- /schema/api/repository.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type RepositoryEntry struct { 4 | Name string `json:"name,omitempty"` 5 | Definition string `json:"definition,omitempty"` 6 | DocType string `json:"doc_type"` 7 | } 8 | -------------------------------------------------------------------------------- /Docker/config/local_override.json: -------------------------------------------------------------------------------- 1 | {"Client": { 2 | "server_urls": ["https://localhost:8000/"] 3 | }, 4 | "Cloud": { 5 | "addresses": [ 6 | "http://localhost:9200/" 7 | ], 8 | "endpoint": "http://localhost:4566/" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /vql_plugins/vfs.go: -------------------------------------------------------------------------------- 1 | package vql_plugins 2 | 3 | import ( 4 | _ "www.velocidex.com/golang/cloudvelo/vql/server/clients" 5 | _ "www.velocidex.com/golang/cloudvelo/vql/server/hunts" 6 | _ "www.velocidex.com/golang/cloudvelo/vql/server/notebook" 7 | _ "www.velocidex.com/golang/cloudvelo/vql/uploads" 8 | ) 9 | -------------------------------------------------------------------------------- /services/hunt_dispatcher/delete.go: -------------------------------------------------------------------------------- 1 | package hunt_dispatcher 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/velociraptor/utils" 7 | ) 8 | 9 | func (self *HuntStorageManagerImpl) DeleteHunt( 10 | ctx context.Context, hunt_id string) error { 11 | return utils.NotImplementedError 12 | } 13 | -------------------------------------------------------------------------------- /schema/scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | STORED_SCRIPT_DIR=$1 4 | MAPPINGS_DIR=$2 5 | OPENSEARCH_POLICY_DIR=$3 6 | OPENSEARCH_HOST=$4 7 | 8 | ./create_scripts.sh $STORED_SCRIPT_DIR $OPENSEARCH_HOST 9 | ./create_policies.sh $OPENSEARCH_POLICY_DIR $OPENSEARCH_HOST 10 | ./create_indexes.sh $MAPPINGS_DIR $OPENSEARCH_HOST 11 | -------------------------------------------------------------------------------- /vql/uploads/factory.go: -------------------------------------------------------------------------------- 1 | package uploads 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | var ( 8 | mu sync.Mutex 9 | gUploaderFactory CloudUploader 10 | ) 11 | 12 | func SetUploaderService(uploader CloudUploader) error { 13 | mu.Lock() 14 | defer mu.Unlock() 15 | 16 | gUploaderFactory = uploader 17 | return nil 18 | } 19 | -------------------------------------------------------------------------------- /datastore/wrapper.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/config" 7 | "www.velocidex.com/golang/velociraptor/datastore" 8 | ) 9 | 10 | func NewElasticDatastore( 11 | ctx context.Context, 12 | config_obj *config.Config) datastore.DataStore { 13 | 14 | return &ElasticDatastore{ 15 | ctx: ctx, 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /services/users/delete.go: -------------------------------------------------------------------------------- 1 | package users 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var ( 9 | deleteUserCounter = promauto.NewCounter( 10 | prometheus.CounterOpts{ 11 | Name: "velociraptor_delete_user_count", 12 | Help: "Count of users deleted from the Velociraptor system.", 13 | }) 14 | ) 15 | -------------------------------------------------------------------------------- /artifact_definitions/Linux.Remediation.Quarantine.yaml: -------------------------------------------------------------------------------- 1 | name: Linux.Remediation.Quarantine 2 | description: | 3 | This is a placeholder artifact to remind the user that Quarantine is 4 | not supported on cloud Velociraptor instances. 5 | 6 | sources: 7 | - query: | 8 | SELECT log(message="Quarantine is not supported in this Velociraptor instance.", 9 | level="ERROR") 10 | FROM scope() 11 | -------------------------------------------------------------------------------- /artifact_definitions/MacOS.System.QuarantineEvents.yaml: -------------------------------------------------------------------------------- 1 | name: MacOS.System.QuarantineEvents 2 | description: | 3 | This is a placeholder artifact to remind the user that Quarantine is 4 | not supported on cloud Velociraptor instances. 5 | 6 | sources: 7 | - query: | 8 | SELECT log(message="Quarantine is not supported in this Velociraptor instance.", 9 | level="ERROR") 10 | FROM scope() 11 | -------------------------------------------------------------------------------- /artifact_definitions/Windows.Remediation.Quarantine.yaml: -------------------------------------------------------------------------------- 1 | name: Windows.Remediation.Quarantine 2 | description: | 3 | This is a placeholder artifact to remind the user that Quarantine is 4 | not supported on cloud Velociraptor instances. 5 | 6 | sources: 7 | - query: | 8 | SELECT log(message="Quarantine is not supported in this Velociraptor instance.", 9 | level="ERROR") 10 | FROM scope() 11 | -------------------------------------------------------------------------------- /scripts/gh_download_run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | i=$(gh api /repos/Velocidex/cloudvelo/actions/runs?per_page=10 | jq 'limit(1; .workflow_runs[] | select (.name | contains("Tests")) | .id )') 4 | 5 | rm -f artifact/* 6 | 7 | gh run download $i 8 | cd artifact/ && cp TestClientEventMonitoring.golden TestEnrollment.golden TestErrorLogs.golden TestListDirectory.golden TestVFSDownload.golden ../ingestion/fixtures/ && cd - 9 | -------------------------------------------------------------------------------- /services/notebook/shared.go: -------------------------------------------------------------------------------- 1 | package notebook 2 | 3 | import ( 4 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 5 | "www.velocidex.com/golang/velociraptor/utils" 6 | ) 7 | 8 | func checkNotebookAccess(notebook *api_proto.NotebookMetadata, user string) bool { 9 | if notebook.Public { 10 | return true 11 | } 12 | 13 | return notebook.Creator == user || utils.InString(notebook.Collaborators, user) 14 | } 15 | -------------------------------------------------------------------------------- /services/client_info.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "context" 5 | 6 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 7 | ) 8 | 9 | // Can send same message to multiple clients efficiently 10 | type MultiClientMessageQueuer interface { 11 | QueueMessageForMultipleClients( 12 | ctx context.Context, 13 | client_ids []string, 14 | req *crypto_proto.VeloMessage, 15 | notify bool) error 16 | } 17 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.DownloadFile/Msg_05.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.CEV7IE8TURDBS", 3 | "source": "C.77ad4285690698d9", 4 | "auth_state": 1, 5 | "FileBuffer": { 6 | "pathspec": { 7 | "path": "/test/hello.txt", 8 | "components": [ 9 | "test", 10 | "hello.txt" 11 | ], 12 | "accessor": "auto" 13 | }, 14 | "size": 6, 15 | "stored_size": 6, 16 | "eof": true, 17 | "mtime": 1673423095 18 | } 19 | } -------------------------------------------------------------------------------- /bin/logging.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 5 | logging "www.velocidex.com/golang/velociraptor/logging" 6 | ) 7 | 8 | type LogWriter struct { 9 | config_obj *config_proto.Config 10 | } 11 | 12 | func (self *LogWriter) Write(b []byte) (int, error) { 13 | logging.GetLogger(self.config_obj, &logging.ClientComponent).Info("%v", string(b)) 14 | return len(b), nil 15 | } 16 | -------------------------------------------------------------------------------- /schema/templates/error.json: -------------------------------------------------------------------------------- 1 | { 2 | "index_patterns": [ 3 | "*_error" 4 | ], 5 | "template": { 6 | "settings": { 7 | "number_of_shards": 1, 8 | "number_of_replicas": 0 9 | }, 10 | "mappings": { 11 | "dynamic": false, 12 | "properties": { 13 | "client_id": { 14 | "type": "keyword" 15 | }, 16 | "data": { 17 | "type": "text" 18 | } 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /services/repository/forwards.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 5 | "www.velocidex.com/golang/velociraptor/services" 6 | "www.velocidex.com/golang/velociraptor/services/repository" 7 | ) 8 | 9 | func LoadArtifactsFromConfig( 10 | repo_manager services.RepositoryManager, 11 | config_obj *config_proto.Config) error { 12 | 13 | return repository.LoadArtifactsFromConfig(repo_manager, config_obj) 14 | } 15 | -------------------------------------------------------------------------------- /ingestion/testdata/Server.Internal.ClientInfo/Client.Info.Updates_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.Monitoring", 3 | "request_id": 980, 4 | "response_id": 1663868441242227354, 5 | "source": "C.1352adc54e292a23", 6 | "auth_state": 1, 7 | "LogMessage": { 8 | "id": 1, 9 | "message": "Time 0: Server.Internal.ClientInfo: Sending response part 0 401 B (1 rows).", 10 | "timestamp": 1663868441242222, 11 | "artifact": "Server.Internal.ClientInfo", 12 | "level": "DEFAULT" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/remove-old.yml: -------------------------------------------------------------------------------- 1 | name: Remove old artifacts 2 | 3 | on: 4 | schedule: 5 | # Every day at 1am 6 | - cron: '0 1 * * *' 7 | 8 | jobs: 9 | remove-old-artifacts: 10 | runs-on: ubuntu-latest 11 | timeout-minutes: 10 12 | 13 | steps: 14 | - name: Remove old artifacts 15 | uses: c-hive/gha-remove-artifacts@v1 16 | with: 17 | age: '1 week' 18 | # Optional inputs 19 | # skip-tags: true 20 | skip-recent: 2 21 | -------------------------------------------------------------------------------- /schema/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM alpine:3 3 | RUN apk add --no-cache bash curl 4 | COPY ./scripts/*.sh / 5 | RUN chmod +x /create_indexes.sh 6 | RUN chmod +x /create_scripts.sh 7 | RUN chmod +x /create_policies.sh 8 | RUN chmod +x /run.sh 9 | COPY ./templates/*.json /index-templates/ 10 | COPY ./scripts/*.json /stored-scripts/ 11 | COPY ./policies/*.json /index-policies/ 12 | CMD ["/run.sh", "/stored-scripts", "/index-templates", "/index-policies", "http://opensearch:9200"] 13 | -------------------------------------------------------------------------------- /services/retry.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "regexp" 5 | "time" 6 | ) 7 | 8 | // Retry calls to the backend 9 | 10 | var ( 11 | retriableErrors = regexp.MustCompile("version conflict") 12 | ) 13 | 14 | func retry(cb func() error) (err error) { 15 | for i := 0; i < 10; i++ { 16 | err = cb() 17 | if err == nil { 18 | return err 19 | } 20 | 21 | if !retriableErrors.MatchString(err.Error()) { 22 | return err 23 | } 24 | 25 | time.Sleep(time.Second) 26 | } 27 | 28 | return err 29 | } 30 | -------------------------------------------------------------------------------- /services/orgs/ids.go: -------------------------------------------------------------------------------- 1 | package orgs 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/base32" 6 | "encoding/base64" 7 | 8 | "www.velocidex.com/golang/velociraptor/constants" 9 | ) 10 | 11 | func NewOrgId() string { 12 | buf := make([]byte, 2) 13 | _, _ = rand.Read(buf) 14 | 15 | result := base32.HexEncoding.EncodeToString(buf)[:4] 16 | return constants.ORG_PREFIX + result 17 | } 18 | 19 | func NewNonce() string { 20 | nonce := make([]byte, 8) 21 | rand.Read(nonce) 22 | return base64.StdEncoding.EncodeToString(nonce) 23 | } 24 | -------------------------------------------------------------------------------- /schema/scripts/create_scripts.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | DIR=$1 7 | OPENSEARCH_HOST=$2 8 | 9 | PATTERN="$DIR/*.json" 10 | echo -e "Reading stored script files from $DIR" 11 | 12 | for FILE in $PATTERN; do 13 | mapping=$(<$FILE) 14 | index=$(basename "$FILE" .json) 15 | url="$OPENSEARCH_HOST/_scripts/$index" 16 | echo -e "\nCreating stored script: $url\n$mapping\n" 17 | 18 | result=$(curl -XPUT "$url" -s -H 'Content-Type: application/json' -d "$mapping") 19 | echo -e "> $result" 20 | done 21 | -------------------------------------------------------------------------------- /services/vfs_service/vfs_service_test.go: -------------------------------------------------------------------------------- 1 | package vfs_service 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 8 | "www.velocidex.com/golang/velociraptor/utils" 9 | ) 10 | 11 | func TestFoo(t *testing.T) { 12 | dir_components := []string{ 13 | "C:", 14 | "28f56b8f3bd7cad47a107c29b62fe066", 15 | "auto", 16 | "C:", 17 | } 18 | 19 | id := cvelo_services.MakeId(utils.JoinComponents(dir_components, "/")) 20 | fmt.Printf("Id should be %v %v\n", id, utils.JoinComponents(dir_components, "/")) 21 | } 22 | -------------------------------------------------------------------------------- /server/dummy.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 8 | ) 9 | 10 | type DummyBackend struct{} 11 | 12 | func (self DummyBackend) Send(messages []*crypto_proto.VeloMessage) error { 13 | fmt.Printf("Hello, world from send!") 14 | return nil 15 | } 16 | 17 | func (self DummyBackend) Receive( 18 | ctx context.Context, client_id, org_id string) ( 19 | message []*crypto_proto.VeloMessage, err error) { 20 | fmt.Printf("Hello, world from receive!") 21 | return []*crypto_proto.VeloMessage{}, nil 22 | } 23 | -------------------------------------------------------------------------------- /services/notebook/delete.go: -------------------------------------------------------------------------------- 1 | package notebook 2 | 3 | import ( 4 | "context" 5 | 6 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 7 | "www.velocidex.com/golang/vfilter" 8 | ) 9 | 10 | func (self *NotebookStoreImpl) DeleteNotebook(ctx context.Context, 11 | notebook_id string, progress chan vfilter.Row, 12 | really_do_it bool) error { 13 | 14 | // TODO - recursively delete all the notebook items. 15 | 16 | if really_do_it { 17 | return cvelo_services.DeleteDocument(ctx, self.config_obj.OrgId, 18 | "persisted", notebook_id, cvelo_services.SyncDelete) 19 | } 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.DownloadFile/Msg_08.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.CEV7IE8TURDBS", 3 | "request_id": 981, 4 | "source": "C.77ad4285690698d9", 5 | "auth_state": 1, 6 | "flow_stats": { 7 | "total_collected_rows": 1, 8 | "total_logs": 4, 9 | "timestamp": 1673427258104354, 10 | "query_status": [ 11 | { 12 | "duration": 211492581, 13 | "names_with_response": [ 14 | "System.VFS.DownloadFile" 15 | ], 16 | "Artifact": "System.VFS.DownloadFile", 17 | "result_rows": 1, 18 | "query_id": 1, 19 | "total_queries": 1 20 | } 21 | ], 22 | "flow_complete": true 23 | } 24 | } -------------------------------------------------------------------------------- /server/api.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "www.velocidex.com/golang/cloudvelo/config" 5 | "www.velocidex.com/golang/cloudvelo/crypto/server" 6 | "www.velocidex.com/golang/cloudvelo/filestore" 7 | ) 8 | 9 | func NewCommunicator( 10 | config_obj *config.Config, 11 | crypto_manager *server.ServerCryptoManager, 12 | backend CommunicatorBackend) (*Communicator, error) { 13 | 14 | sess, err := filestore.GetS3Session(config_obj) 15 | return &Communicator{ 16 | session: sess, 17 | config_obj: config_obj, 18 | backend: backend, 19 | crypto_manager: crypto_manager, 20 | }, err 21 | } 22 | -------------------------------------------------------------------------------- /filestore/utils.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "www.velocidex.com/golang/cloudvelo/config" 5 | "www.velocidex.com/golang/velociraptor/file_store/api" 6 | ) 7 | 8 | func GetOrgId(file_store_obj api.FileStore) string { 9 | config_obj := GetConfigObj(file_store_obj) 10 | if config_obj == nil { 11 | return "" 12 | } 13 | return config_obj.OrgId 14 | } 15 | 16 | func GetConfigObj(file_store_obj api.FileStore) *config.Config { 17 | switch t := file_store_obj.(type) { 18 | case *S3Filestore: 19 | return t.config_obj 20 | case S3Filestore: 21 | return t.config_obj 22 | default: 23 | return nil 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /services/notebook/paths.go: -------------------------------------------------------------------------------- 1 | package notebook 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/velociraptor/file_store/api" 7 | ) 8 | 9 | type PathManagerWrapper struct { 10 | path api.FSPathSpec 11 | } 12 | 13 | func (self *PathManagerWrapper) GetPathForWriting() (api.FSPathSpec, error) { 14 | return self.path, nil 15 | } 16 | 17 | func (self *PathManagerWrapper) GetQueueName() string { 18 | return self.path.Base() 19 | } 20 | 21 | func (self *PathManagerWrapper) GetAvailableFiles( 22 | ctx context.Context) []*api.ResultSetFileProperties { 23 | return []*api.ResultSetFileProperties{{ 24 | Path: self.path, 25 | }} 26 | } 27 | -------------------------------------------------------------------------------- /ingestion/testdata/Server.Internal.ClientInfo/Client.Info.Updates_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.Monitoring", 3 | "response_id": 1663868441242263337, 4 | "source": "C.1352adc54e292a23", 5 | "auth_state": 1, 6 | "VQLResponse": { 7 | "JSONLResponse": "{\"hostname\":\"devbox\",\"fqdn\":\"devbox\",\"system\":\"linux\",\"release\":\"ubuntu\",\"architecture\":\"amd64\",\"client_version\":\"0.6.8-dev\",\"client_name\":\"velociraptor\",\"build_time\":\"2023-02-02T12:13:03+10:00\"}\n", 8 | "query_id": 2, 9 | "Query": { 10 | "Name": "Server.Internal.ClientInfo", 11 | "VQL": "SELECT * FROM Client_Info_Updates_0_1" 12 | }, 13 | "timestamp": 1663868441242201, 14 | "total_rows": 1 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #Maven 2 | artifact/ 3 | target/ 4 | pom.xml.tag 5 | pom.xml.releaseBackup 6 | pom.xml.versionsBackup 7 | release.properties 8 | 9 | # Eclipse 10 | .project 11 | .classpath 12 | .settings/ 13 | 14 | # IntelliJ 15 | .idea 16 | *.ipr 17 | *.iml 18 | *.iws 19 | 20 | # NetBeans 21 | nb-configuration.xml 22 | 23 | # Visual Studio Code 24 | .vscode 25 | .factorypath 26 | 27 | # OSX 28 | .DS_Store 29 | 30 | # Vim 31 | *.swp 32 | *.swo 33 | 34 | # patch 35 | *.orig 36 | *.rej 37 | 38 | # Local environment 39 | .env 40 | 41 | # Go Output Folder 42 | output/ 43 | testdata/pool_writebacks/* 44 | 45 | __debug_bin 46 | Docker/.localstack/data/* 47 | Docker/.localstack/cache/* 48 | Docker/.localstack/var_libs/* 49 | 50 | pool_client.yaml.* -------------------------------------------------------------------------------- /services/launcher.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "context" 5 | 6 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 7 | flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" 8 | "www.velocidex.com/golang/velociraptor/services" 9 | ) 10 | 11 | // A more efficient launcher 12 | type MultiLauncher interface { 13 | ScheduleVQLCollectorArgsOnMultipleClients( 14 | ctx context.Context, 15 | config_obj *config_proto.Config, 16 | request *flows_proto.ArtifactCollectorArgs, 17 | client_ids []string) error 18 | } 19 | 20 | type Flusher interface { 21 | Flush() 22 | } 23 | 24 | func Flush(launcher services.Launcher) { 25 | l, ok := launcher.Storage().(Flusher) 26 | if ok { 27 | l.Flush() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.DownloadFile/Msg_07.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.CEV7IE8TURDBS", 3 | "request_id": 980, 4 | "source": "C.77ad4285690698d9", 5 | "auth_state": 1, 6 | "LogMessage": { 7 | "number_of_rows": 4, 8 | "jsonl": "{\"client_time\":1673427258,\"level\":\"INFO\",\"message\":\"Starting query execution.\\n\"}\n{\"client_time\":1673427258,\"level\":\"DEFAULT\",\"message\":\"Time 0: System.VFS.DownloadFile: Sending response part 0 221 B (1 rows).\"}\n{\"client_time\":1673427258,\"level\":\"INFO\",\"message\":\"Collection is done after 210.786649ms\\n\"}\n{\"client_time\":1673427258,\"level\":\"DEBUG\",\"message\":\"Query Stats: {\\\"RowsScanned\\\":4,\\\"PluginsCalled\\\":2,\\\"FunctionsCalled\\\":6,\\\"ProtocolSearch\\\":225,\\\"ScopeCopy\\\":11}\\n\"}\n" 9 | } 10 | } -------------------------------------------------------------------------------- /services/hunt_dispatcher/list.go: -------------------------------------------------------------------------------- 1 | package hunt_dispatcher 2 | 3 | import ( 4 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 5 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 6 | "www.velocidex.com/golang/velociraptor/paths" 7 | ) 8 | 9 | // availableHuntDownloadFiles returns the prepared zip downloads available to 10 | // be fetched by the user at this moment. 11 | func availableHuntDownloadFiles(config_obj *config_proto.Config, 12 | hunt_id string) (*api_proto.AvailableDownloads, error) { 13 | 14 | hunt_path_manager := paths.NewHuntPathManager(hunt_id) 15 | download_file := hunt_path_manager.GetHuntDownloadsFile(false, "", false) 16 | download_path := download_file.Dir() 17 | 18 | return GetAvailableDownloadFiles(config_obj, download_path) 19 | } 20 | -------------------------------------------------------------------------------- /services/indexing/utils.go: -------------------------------------------------------------------------------- 1 | package indexing 2 | 3 | import ( 4 | "context" 5 | 6 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 7 | "www.velocidex.com/golang/velociraptor/services" 8 | "www.velocidex.com/golang/velociraptor/utils" 9 | ) 10 | 11 | // Force all the client info to be loaded into the memory cache. 12 | func PopulateClientInfoCache( 13 | ctx context.Context, 14 | config_obj *config_proto.Config) error { 15 | 16 | indexer, err := services.GetIndexer(config_obj) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | output_chan, err := indexer.SearchClientsChan(ctx, 22 | nil, config_obj, "all", 23 | utils.GetSuperuserName(config_obj)) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | for _ = range output_chan { 29 | } 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.ListDirectory/Msg_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "doc_type": "collection", 3 | "session_id": "F.CEV6I8LHAT83O", 4 | "query_id": 1, 5 | "source": "C.77ad4285690698d9", 6 | "auth_state": 1, 7 | "VQLResponse": { 8 | "JSONLResponse": "{\"Components\":[\"test\"],\"Accessor\":\"auto\",\"Stats\":{\"end_idx\":1}}\n", 9 | "Columns": [ 10 | "Components", 11 | "Accessor", 12 | "Stats" 13 | ], 14 | "query_id": 5, 15 | "Query": { 16 | "Name": "System.VFS.ListDirectory/Stats", 17 | "VQL": "SELECT * FROM if(then=System_VFS_ListDirectory_Stats_0_0, condition=precondition_System_VFS_ListDirectory_Stats_0, else={SELECT * FROM scope() WHERE log(message='Query skipped due to precondition') AND FALSE})" 18 | }, 19 | "timestamp": 1673423174784842, 20 | "total_rows": 1 21 | } 22 | } -------------------------------------------------------------------------------- /filestore/fileinfo.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "io/fs" 5 | "time" 6 | 7 | "www.velocidex.com/golang/velociraptor/file_store/api" 8 | ) 9 | 10 | type S3FileInfo struct { 11 | pathspec api.FSPathSpec 12 | size int64 13 | mod_time time.Time 14 | } 15 | 16 | func (self S3FileInfo) Name() string { 17 | return self.pathspec.Base() 18 | } 19 | func (self S3FileInfo) Size() int64 { 20 | return self.size 21 | } 22 | func (self S3FileInfo) Mode() fs.FileMode { 23 | return 0666 24 | } 25 | func (self S3FileInfo) ModTime() time.Time { 26 | return self.mod_time 27 | } 28 | func (self S3FileInfo) IsDir() bool { 29 | return false 30 | } 31 | func (self S3FileInfo) Sys() any { 32 | return nil 33 | } 34 | 35 | func (self S3FileInfo) PathSpec() api.FSPathSpec { 36 | return self.pathspec 37 | } 38 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.DownloadFile/Msg_06.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.CEV7IE8TURDBS", 3 | "source": "C.77ad4285690698d9", 4 | "auth_state": 1, 5 | "VQLResponse": { 6 | "JSONLResponse": "{\"Path\":\"/test/hello.txt\",\"Accessor\":\"auto\",\"Size\":6,\"StoredSize\":6,\"Sha256\":\"5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03\",\"Md5\":\"b1946ac92492d2347c6235b4d2611184\",\"_Components\":[\"test\",\"hello.txt\"]}\n", 7 | "Columns": [ 8 | "Path", 9 | "Accessor", 10 | "Size", 11 | "StoredSize", 12 | "Sha256", 13 | "Md5", 14 | "_Components" 15 | ], 16 | "query_id": 5, 17 | "Query": { 18 | "Name": "System.VFS.DownloadFile", 19 | "VQL": "SELECT * FROM System_VFS_DownloadFile_0_2" 20 | }, 21 | "timestamp": 1673427258315580, 22 | "total_rows": 1 23 | } 24 | } -------------------------------------------------------------------------------- /vql/uploads/fixtures/TestSparseUploader.golden: -------------------------------------------------------------------------------- 1 | { 2 | "UploadResponse": { 3 | "Path": "/sparse.txt", 4 | "Size": 15, 5 | "StoredSize": 10, 6 | "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 7 | "md5": "d41d8cd98f00b204e9800998ecf8427e" 8 | }, 9 | "Sparse File Data": "the lox ju", 10 | "S3 files": [ 11 | "orgs/test/clients/C.1352adc54e292a23/collections/F.1231/uploads/sparse/0be2d1424fd102c85547fbcb46838ebe8617d98bb088c0bce5ab70234da97e90", 12 | "orgs/test/clients/C.1352adc54e292a23/collections/F.1231/uploads/sparse/0be2d1424fd102c85547fbcb46838ebe8617d98bb088c0bce5ab70234da97e90.idx" 13 | ], 14 | "IDX file": "{\"ranges\":[{\"file_length\":5,\"length\":5},{\"file_offset\":5,\"original_offset\":5,\"length\":5},{\"file_offset\":5,\"original_offset\":10,\"file_length\":5,\"length\":5}]}" 15 | } -------------------------------------------------------------------------------- /schema/scripts/create_indexes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | DIR=$1 7 | OPENSEARCH_HOST=$2 8 | 9 | PATTERN="$DIR/*.json" 10 | echo -e "Reading mapping files from $DIR" 11 | 12 | for FILE in $PATTERN; do 13 | template=$(<$FILE) 14 | index=$(basename "$FILE" .json) 15 | url="$OPENSEARCH_HOST/_index_template/$index" 16 | echo -e "\nCreating index template: $url\n$template\n" 17 | 18 | result=$(curl -XPUT "$url" -s -H 'Content-Type: application/json' -d "$template") 19 | echo -e "> $result" 20 | 21 | if [[ $result =~ "resource_already_exists_exception" ]]; then 22 | echo "==> Index template already exists" 23 | elif [[ $result =~ "acknowledged" ]]; then 24 | echo "==> Index template created" 25 | else 26 | echo -e "==> Unknown result: $result" 27 | exit 1 28 | fi 29 | echo -e "\n\n" 30 | 31 | done 32 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | * VFS 4 | * Recursive VFS listing 5 | * Recursive VFS downloads 6 | * Download file from VFS 7 | * VFS refresh directory wipes out download. 8 | 9 | * Ingestor 10 | * Fail collection when Error status 11 | * Write combine results upload using Bulk API 12 | 13 | * Zip export 14 | * Create zip export of hunt/flow 15 | 16 | * GUI 17 | * Update VQL drill down/Dashboard/Welcome screen 18 | 19 | * Event monitoring 20 | * Figure out how to update client event monitoring table 21 | 22 | * Hunts: 23 | * When we stop a hunt we need to cancel all the outstanding tasks. 24 | * Implement hunt deletion 25 | 26 | * Collections 27 | * Implement collection deletion 28 | 29 | * VQL 30 | * Allow list of VQL plugins/function 31 | * Look through all server related VQL plugins for server management 32 | and ensure they work in the new environment. 33 | -------------------------------------------------------------------------------- /bin/foreman.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "www.velocidex.com/golang/cloudvelo/startup" 7 | ) 8 | 9 | var ( 10 | foreman = app.Command("foreman", "Run the foreman batch program") 11 | ) 12 | 13 | func doForeman() error { 14 | config_obj, err := loadConfig(makeDefaultConfigLoader()) 15 | if err != nil { 16 | return fmt.Errorf("loading config file: %w", err) 17 | } 18 | 19 | ctx, cancel := install_sig_handler() 20 | defer cancel() 21 | 22 | sm, err := startup.StartForeman(ctx, config_obj) 23 | defer sm.Close() 24 | if err != nil { 25 | return err 26 | } 27 | 28 | <-ctx.Done() 29 | 30 | return nil 31 | } 32 | 33 | func init() { 34 | command_handlers = append(command_handlers, func(command string) bool { 35 | if command == foreman.FullCommand() { 36 | FatalIfError(foreman, doForeman) 37 | return true 38 | } 39 | return false 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /schema/scripts/create_policies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | DIR=$1 7 | OPENSEARCH_HOST=$2 8 | 9 | PATTERN="$DIR/*.json" 10 | echo -e "Reading policy files from $DIR" 11 | 12 | for FILE in $PATTERN; do 13 | policy_definition=$(<$FILE) 14 | policy=$(basename "$FILE" .json) 15 | url="$OPENSEARCH_HOST/_plugins/_ism/policies/$policy" 16 | echo -e "\nCreating index policy: $url\n$policy_definition\n" 17 | 18 | result=$(curl -XPUT "$url" -s -H 'Content-Type: application/json' -d "$policy_definition") 19 | echo -e "> $result" 20 | 21 | if [[ $result =~ "resource_already_exists_exception" ]]; then 22 | echo "==> Index template already exists" 23 | elif [[ $result =~ "acknowledged" ]]; then 24 | echo "==> Index template created" 25 | else 26 | echo -e "==> Unknown result: $result" 27 | exit 1 28 | fi 29 | echo -e "\n\n" 30 | 31 | done -------------------------------------------------------------------------------- /server/mock.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 9 | "www.velocidex.com/golang/velociraptor/logging" 10 | ) 11 | 12 | type MockBackend struct { 13 | Tasks [][]*crypto_proto.VeloMessage 14 | idx int 15 | Logger *logging.LogContext 16 | } 17 | 18 | func (self MockBackend) Send(messages []*crypto_proto.VeloMessage) error { 19 | fmt.Printf("Sending to backend %v\n", messages) 20 | return nil 21 | } 22 | 23 | func (self MockBackend) Receive(ctx context.Context, client_id, org_id string) ( 24 | message []*crypto_proto.VeloMessage, err error) { 25 | if self.idx > len(self.Tasks)-1 { 26 | return nil, io.EOF 27 | } 28 | 29 | res := self.Tasks[self.idx] 30 | self.idx++ 31 | self.Logger.Info( 32 | "Serving a mock response with %v messages", len(res)) 33 | 34 | return res, nil 35 | } 36 | -------------------------------------------------------------------------------- /artifact_definitions/Client.Info.Updates.yaml: -------------------------------------------------------------------------------- 1 | name: Client.Info.Updates 2 | description: | 3 | An event version of the Generic.Client.Info. This will send the 4 | client information once when starting 5 | 6 | type: CLIENT_EVENT 7 | 8 | sources: 9 | - query: | 10 | LET Interfaces = SELECT format(format='%02x:%02x:%02x:%02x:%02x:%02x', 11 | args=HardwareAddr) AS MAC 12 | FROM interfaces() 13 | WHERE HardwareAddr 14 | 15 | SELECT config.Version.Name AS Name, 16 | config.Version.BuildTime as BuildTime, 17 | config.Version.Version as Version, 18 | config.Version.ci_build_url AS build_url, 19 | config.Labels AS Labels, 20 | Hostname, OS, Architecture, 21 | Platform, PlatformVersion, KernelVersion, Fqdn, 22 | Interfaces.MAC AS MACAddresses 23 | FROM info() 24 | -------------------------------------------------------------------------------- /ingestion/fixtures/TestEnrollment.golden: -------------------------------------------------------------------------------- 1 | { 2 | "Enrollment": { 3 | "pem": "LS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JSUJDZ0tDQVFFQXlVang3VDdaQ0czS0wvS2xuL0tMby8yN0FZWXlFVFJqdnFmaUhjVHBvcTExUk5BZDZqL1oKdFJHYUk2anE3UmhZR0xqWjI4UERXWjQ2bnA5MFM5QkJXbmNxN0JlQUQ1T1dNb1ErTGxkS1dFSWV1eE5mTkVkWQpSQVpIcGxlWkRtcll5U0w4U3ovaUxleFBiRXJUc1g5U2dZSm1xK1M3emNrdlFwVVNtczlNMEVSVHh6R3VGbzdlCi9OMXdra0xvekJkQUxyUXB1R2ZzUDBJY3FxSkR4K3JOSCt0RmJrc2hCMkN1WU5zeHVMN2phb2wwYlJDMmYweXkKQ0JXMnRVZCtrVG5vRUV1dTZRSDh3SVpjbTc2bFBZREtDYllRZm9VZDArWnc5UmxjSmJuUi93dE9lbUZ2NGxySApJcEdDeXZFUXNJR3JiQW84MGRyby8rVzVDUTVzWFZhNFBRSURBUUFCCi0tLS0tRU5EIFJTQSBQVUJMSUMgS0VZLS0tLS0K", 4 | "enroll_time": 1661385600 5 | }, 6 | "ClientRecord": { 7 | "client_id": "C.1352adc54e292a23", 8 | "hostname": "devbox", 9 | "system": "linux", 10 | "first_seen_at": 0, 11 | "type": "main", 12 | "doc_type": "clients", 13 | "timestamp": 1661385600 14 | } 15 | } -------------------------------------------------------------------------------- /services/indexing/mru.go: -------------------------------------------------------------------------------- 1 | package indexing 2 | 3 | import ( 4 | "time" 5 | 6 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 7 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 8 | ) 9 | 10 | // Keeps track of users->client id used so we can populate the MRU 11 | // search 12 | type MRUItem struct { 13 | Username string `json:"username"` 14 | ClientId string `json:"client_id"` 15 | Timestamp int64 `json:"timestamp"` 16 | DocType string `json:"doc_type"` 17 | } 18 | 19 | func (self Indexer) UpdateMRU( 20 | config_obj *config_proto.Config, 21 | user_name string, client_id string) error { 22 | return cvelo_services.SetElasticIndex(self.ctx, 23 | self.config_obj.OrgId, 24 | "persisted", user_name+":"+client_id, 25 | &MRUItem{ 26 | Username: user_name, 27 | ClientId: client_id, 28 | Timestamp: time.Now().Unix(), 29 | DocType: "user_mru", 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /ingestion/fixtures/TestErrorLogs.golden: -------------------------------------------------------------------------------- 1 | { 2 | "System.VFS.ListDirectory after Error Log": [ 3 | { 4 | "client_id": "C.1352adc54e292a23", 5 | "session_id": "F.CCMS0OJQ7LI36", 6 | "context": "", 7 | "create_time": 0, 8 | "start_time": 0, 9 | "last_active_time": 1661391004000000000, 10 | "uploaded_files": 0, 11 | "uploaded_bytes": 0, 12 | "query_stats": null, 13 | "tasks": "", 14 | "errored": 0, 15 | "timestamp": 1661391003000000000 16 | }, 17 | { 18 | "client_id": "C.1352adc54e292a23", 19 | "session_id": "F.CCMS0OJQ7LI36", 20 | "context": "", 21 | "create_time": 0, 22 | "start_time": 0, 23 | "last_active_time": 0, 24 | "uploaded_files": 0, 25 | "uploaded_bytes": 0, 26 | "query_stats": [ 27 | "{\"status\":10,\"error_message\":\"Something went wrong!!!\"}" 28 | ], 29 | "tasks": "", 30 | "errored": 0, 31 | "timestamp": 1661391005000000000 32 | } 33 | ] 34 | } -------------------------------------------------------------------------------- /ingestion/fixtures/TestVFSDownload.golden: -------------------------------------------------------------------------------- 1 | { 2 | "columns": [ 3 | "Download", 4 | "_FullPath", 5 | "_Components", 6 | "_Accessor", 7 | "_Data", 8 | "Name", 9 | "Size", 10 | "Mode", 11 | "mtime", 12 | "atime", 13 | "ctime", 14 | "btime" 15 | ], 16 | "rows": [ 17 | { 18 | "json": "[{\"size\":6,\"mtime\":1661385600000000,\"MD5\":\"b1946ac92492d2347c6235b4d2611184\",\"SHA256\":\"5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03\",\"components\":[\"clients\",\"C.77ad4285690698d9\",\"collections\",\"F.CEV7IE8TURDBS\",\"uploads\",\"auto\",\"5849a9dbfcb36b4e77fbcbcb7229b26a0601df4a9678bab7b48ccedac923b4c0\"],\"flow_id\":\"F.CEV7IE8TURDBS\"},\"/test/hello.txt\",[\"test\",\"hello.txt\"],\"auto\",{\"DevMajor\":253,\"DevMinor\":0},\"hello.txt\",6,\"-rw-r--r--\",\"2023-01-11T07:44:55Z\",\"2023-01-11T07:44:55Z\",\"2023-01-11T07:44:55Z\",\"0001-01-01T00:00:00Z\"]" 19 | } 20 | ], 21 | "total_rows": 1 22 | } -------------------------------------------------------------------------------- /bin/gui.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "www.velocidex.com/golang/cloudvelo/startup" 7 | ) 8 | 9 | var ( 10 | gui = app.Command("gui", "Start the GUI server") 11 | ) 12 | 13 | func doGUI() error { 14 | config_obj, err := loadConfig(makeDefaultConfigLoader()) 15 | if err != nil { 16 | return fmt.Errorf("loading config file: %w", err) 17 | } 18 | 19 | ctx, cancel := install_sig_handler() 20 | defer cancel() 21 | 22 | // Now start the frontend services 23 | sm, err := startup.StartGUIServices(ctx, config_obj) 24 | if err != nil { 25 | return fmt.Errorf("starting frontend: %w", err) 26 | } 27 | defer sm.Close() 28 | 29 | sm.Wg.Wait() 30 | 31 | return nil 32 | } 33 | 34 | func init() { 35 | command_handlers = append(command_handlers, func(command string) bool { 36 | if command == gui.FullCommand() { 37 | FatalIfError(gui, doGUI) 38 | return true 39 | } 40 | return false 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.ListDirectory/Msg_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.CEV6I8LHAT83O", 3 | "query_id": 2, 4 | "request_id": 980, 5 | "source": "C.77ad4285690698d9", 6 | "auth_state": 1, 7 | "doc_type": "collection", 8 | "LogMessage": { 9 | "number_of_rows": 5, 10 | "jsonl": "{\"client_time\":1673423174,\"level\":\"INFO\",\"message\":\"Starting query execution.\\n\"}\n{\"client_time\":1673423174,\"level\":\"INFO\",\"message\":\"Starting query execution.\\n\"}\n{\"client_time\":1673423174,\"level\":\"DEFAULT\",\"message\":\"Skipping query due to preconditions\\n\"}\n{\"client_time\":1673423174,\"level\":\"INFO\",\"message\":\"Collection is done after 5.283543ms\\n\"}\n{\"client_time\":1673423174,\"level\":\"DEBUG\",\"message\":\"Query Stats: {\\\"RowsScanned\\\":1,\\\"PluginsCalled\\\":1,\\\"FunctionsCalled\\\":1,\\\"ProtocolSearch\\\":0,\\\"ScopeCopy\\\":4}\\n\"}\n", 11 | "artifact": "System.VFS.ListDirectory" 12 | } 13 | } -------------------------------------------------------------------------------- /schema/policies/rollover_strategy.json: -------------------------------------------------------------------------------- 1 | { 2 | "policy": { 3 | "description": "Rollover policy responsible for rolling over indexes when the primary shard reaches a set age.", 4 | "default_state": "rollover", 5 | "states": [ 6 | { 7 | "name": "rollover", 8 | "actions": [ 9 | { 10 | "rollover": { 11 | "min_index_age": "5d" 12 | } 13 | } 14 | ], 15 | "transitions": [ 16 | { 17 | "state_name": "delete", 18 | "conditions": { 19 | "min_index_age": "15d" 20 | } 21 | } 22 | ] 23 | }, 24 | { 25 | "name": "delete", 26 | "actions": [ 27 | { 28 | "delete": {} 29 | } 30 | ] 31 | } 32 | ], 33 | "ism_template": { 34 | "index_patterns": ["*_transient*"], 35 | "priority": 300 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /ingestion/testdata/Generic.Client.Stats/Generic.Client.Stats_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.Monitoring", 3 | "request_id": 980, 4 | "source": "C.1352adc54e292a23", 5 | "auth_state": 1, 6 | "LogMessage": { 7 | "number_of_rows": 5, 8 | "jsonl": "{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Starting query execution for Generic.Client.Stats.\\n\"}\n{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Starting query execution for Generic.Client.Stats.\\n\"}\n{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Generic.Client.Stats: Skipping query due to preconditions\\n\"}\n{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Collection Generic.Client.Stats is done after 12.258991ms\\n\"}\n{\"client_time\":1676476473,\"level\":\"DEBUG\",\"message\":\"Query Stats: {\\\"RowsScanned\\\":1,\\\"PluginsCalled\\\":1,\\\"FunctionsCalled\\\":0,\\\"ProtocolSearch\\\":0,\\\"ScopeCopy\\\":4}\\n\"}\n", 9 | "artifact": "Generic.Client.Stats" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /make.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | /* 4 | Velociraptor - Dig Deeper 5 | Copyright (C) 2019-2022 Rapid7 Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU Affero General Public License as published 9 | by the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU Affero General Public License for more details. 16 | 17 | You should have received a copy of the GNU Affero General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | package main 22 | 23 | import ( 24 | "os" 25 | 26 | "github.com/magefile/mage/mage" 27 | ) 28 | 29 | func main() { os.Exit(mage.Main()) } 30 | -------------------------------------------------------------------------------- /Docker/README.md: -------------------------------------------------------------------------------- 1 | # Docker setup for testing 2 | 3 | This docker compose file will start all the components locally 4 | 5 | ``` 6 | docker-compose --profile dev up 7 | ``` 8 | 9 | ## Inspecting the local s3 bucket 10 | 11 | Make a bucket for use by Velociraptor 12 | 13 | ``` 14 | aws s3 --endpoint-url http://localhost:4566/ --no-verify-ssl mb s3://velociraptor 15 | ``` 16 | 17 | You can use the AWS cli to inspect the local buckets created by localstack. 18 | 19 | ``` 20 | aws s3 --endpoint-url http://localhost:4566/ --no-verify-ssl ls s3://velociraptor/ 21 | ``` 22 | 23 | 24 | ## Replace localstack with minio 25 | 26 | Localstack is very limited and does not support large files. For some 27 | testing we need minio. 28 | 29 | ``` 30 | MINIO_ROOT_USER=admin MINIO_ROOT_PASSWORD=password ./minio server /tmp/minio --console-address ":9001" --address ":4566" 31 | ``` 32 | 33 | ``` 34 | ./mc alias set myminio http://192.168.1.11:4566 admin password 35 | ./mc mb myminio/velociraptor 36 | ``` 37 | -------------------------------------------------------------------------------- /services/frontend/frontend.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "context" 5 | "net/url" 6 | 7 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | "www.velocidex.com/golang/velociraptor/services/frontend" 10 | "www.velocidex.com/golang/velociraptor/utils" 11 | ) 12 | 13 | type FrontendService struct{} 14 | 15 | func (self FrontendService) GetMinionCount() int { 16 | return 0 17 | } 18 | 19 | func (self FrontendService) GetMasterAPIClient(ctx context.Context) ( 20 | api_proto.APIClient, func() error, error) { 21 | return nil, nil, utils.NotImplementedError 22 | } 23 | 24 | func (self FrontendService) GetBaseURL( 25 | config_obj *config_proto.Config) (res *url.URL, err error) { 26 | return frontend.GetBaseURL(config_obj) 27 | } 28 | 29 | func (self FrontendService) GetPublicUrl( 30 | config_obj *config_proto.Config) (res *url.URL, err error) { 31 | return frontend.GetPublicUrl(config_obj) 32 | } 33 | -------------------------------------------------------------------------------- /Docker/Makefile: -------------------------------------------------------------------------------- 1 | up: 2 | mkdir -p .localstack/data 3 | cp .localstack/template/* .localstack/data 4 | docker-compose build 5 | docker-compose --profile all up -d 6 | 7 | base: 8 | mkdir -p .localstack/data 9 | cp .localstack/template/* .localstack/data 10 | docker-compose build 11 | docker-compose --profile base up -d 12 | 13 | down: 14 | docker kill localstack opensearch velociraptor-client velociraptor-frontend velociraptor-gui velociraptor-foreman || true 15 | 16 | restart: down 17 | docker-compose build 18 | docker-compose --profile all up -d 19 | 20 | clean: 21 | rm -f .localstack/data/* 22 | docker kill localstack opensearch velociraptor-client velociraptor-frontend velociraptor-gui velociraptor-foreman || true 23 | docker container rm velociraptor-foreman velociraptor-frontend velociraptor-gui localstack opensearch velociraptor-client || true 24 | docker volume rm docker_opensearch-data 25 | 26 | bucket: 27 | aws s3 --endpoint-url http://localhost:4566/ --no-verify-ssl mb s3://velociraptor 28 | -------------------------------------------------------------------------------- /bin/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "www.velocidex.com/golang/cloudvelo/config" 8 | velo_config "www.velocidex.com/golang/velociraptor/config" 9 | ) 10 | 11 | var ( 12 | override_flag = app.Flag("override", "A json object to override the config."). 13 | Short('o').String() 14 | 15 | override_file = app.Flag("override_file", "A json object to override the config."). 16 | String() 17 | ) 18 | 19 | func loadConfig(velo_loader *velo_config.Loader) (*config.Config, error) { 20 | 21 | override := *override_flag 22 | if *override_file != "" { 23 | fd, err := os.Open(*override_file) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | data, err := ioutil.ReadAll(fd) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | override = string(data) 34 | } 35 | 36 | loader := &config.ConfigLoader{ 37 | VelociraptorLoader: velo_loader, 38 | Filename: *config_path, 39 | JSONPatch: override, 40 | } 41 | 42 | return loader.Load() 43 | } 44 | -------------------------------------------------------------------------------- /ingestion/ping.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "www.velocidex.com/golang/cloudvelo/schema/api" 8 | "www.velocidex.com/golang/cloudvelo/services" 9 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 10 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 11 | "www.velocidex.com/golang/velociraptor/utils" 12 | ) 13 | 14 | func (self Ingestor) HandlePing( 15 | ctx context.Context, 16 | config_obj *config_proto.Config, 17 | message *crypto_proto.VeloMessage) error { 18 | 19 | err := services.SetElasticIndex(ctx, 20 | config_obj.OrgId, 21 | "persisted", message.Source+"_ping", 22 | &api.ClientRecord{ 23 | ClientId: message.Source, 24 | Type: "ping", 25 | Ping: uint64(utils.GetTime().Now().UnixNano()), 26 | DocType: "clients", 27 | Timestamp: uint64(utils.GetTime().Now().Unix()), 28 | }) 29 | if err == nil || 30 | strings.Contains(err.Error(), "document_missing_exception") { 31 | return nil 32 | } 33 | return err 34 | } 35 | -------------------------------------------------------------------------------- /filestore/instrument.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/prometheus/client_golang/prometheus/promauto" 8 | ) 9 | 10 | var ( 11 | S3Historgram = promauto.NewHistogramVec( 12 | prometheus.HistogramOpts{ 13 | Name: "s3_filestore_latency", 14 | Help: "Latency to access datastore.", 15 | Buckets: prometheus.LinearBuckets(0.01, 0.05, 10), 16 | }, 17 | []string{"operation"}, 18 | ) 19 | 20 | s3_counter_upload = promauto.NewCounter(prometheus.CounterOpts{ 21 | Name: "s3_bytes_uploaded", 22 | Help: "Total number of bytes send to S3.", 23 | }) 24 | 25 | s3_counter_download = promauto.NewCounter(prometheus.CounterOpts{ 26 | Name: "s3_bytes_downloaded", 27 | Help: "Total number of bytes read from S3.", 28 | }) 29 | ) 30 | 31 | func Instrument(operation string) func() time.Duration { 32 | timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) { 33 | S3Historgram.WithLabelValues(operation).Observe(v) 34 | })) 35 | 36 | return timer.ObserveDuration 37 | } 38 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.ListDirectory/Msg_05.json: -------------------------------------------------------------------------------- 1 | { 2 | "doc_type": "collection", 3 | "session_id": "F.CEV6I8LHAT83O", 4 | "query_id": 1, 5 | "request_id": 981, 6 | "source": "C.77ad4285690698d9", 7 | "auth_state": 1, 8 | "flow_stats": { 9 | "total_collected_rows": 2, 10 | "total_logs": 14, 11 | "timestamp": 1673423174265325, 12 | "query_status": [ 13 | { 14 | "duration": 6370402, 15 | "Artifact": "System.VFS.ListDirectory", 16 | "query_id": 3, 17 | "total_queries": 3 18 | }, 19 | { 20 | "duration": 519121078, 21 | "names_with_response": [ 22 | "System.VFS.ListDirectory/Listing" 23 | ], 24 | "Artifact": "System.VFS.ListDirectory/Listing", 25 | "result_rows": 1, 26 | "query_id": 1, 27 | "total_queries": 3 28 | }, 29 | { 30 | "duration": 518759663, 31 | "names_with_response": [ 32 | "System.VFS.ListDirectory/Stats" 33 | ], 34 | "Artifact": "System.VFS.ListDirectory/Stats", 35 | "result_rows": 1, 36 | "query_id": 2, 37 | "total_queries": 3 38 | } 39 | ], 40 | "flow_complete": true 41 | } 42 | } -------------------------------------------------------------------------------- /artifact_definitions/Server.Utils.DeleteClient.yaml: -------------------------------------------------------------------------------- 1 | name: Server.Utils.DeleteClient 2 | description: | 3 | This artifact completely removes a client from the data store. 4 | 5 | Be careful with this one: there is no way to recover old 6 | data. However, if the client still exists, it will just 7 | automatically re-enrol when it next connects. You will still be able 8 | to talk to it, it is just that old collected data is deleted. 9 | 10 | type: SERVER 11 | 12 | parameters: 13 | - name: ClientIdList 14 | description: A list of client ids to delete. 15 | default: 16 | 17 | - name: ReallyDoIt 18 | description: If you really want to delete the client, check this. 19 | type: bool 20 | 21 | sources: 22 | - query: | 23 | let clients_list = SELECT ClientId 24 | FROM parse_records_with_regex( 25 | accessor="data", file=ClientIdList, 26 | regex="(?P[^,]+)") 27 | WHERE log(message="Deleting client " + ClientId) 28 | 29 | SELECT * FROM foreach(row=clients_list, 30 | query={ 31 | SELECT * FROM client_delete(client_id=ClientId, 32 | really_do_it=ReallyDoIt) 33 | }) 34 | -------------------------------------------------------------------------------- /services/launcher/launcher.go: -------------------------------------------------------------------------------- 1 | package launcher 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/Velocidex/ttlcache/v2" 9 | "www.velocidex.com/golang/cloudvelo/config" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | "www.velocidex.com/golang/velociraptor/services" 12 | "www.velocidex.com/golang/velociraptor/services/launcher" 13 | ) 14 | 15 | type Launcher struct { 16 | services.Launcher 17 | config_obj *config_proto.Config 18 | } 19 | 20 | func NewLauncherService( 21 | ctx context.Context, 22 | wg *sync.WaitGroup, 23 | config_obj *config_proto.Config, 24 | cloud_config *config.ElasticConfiguration) (services.Launcher, error) { 25 | 26 | lru := ttlcache.NewCache() 27 | lru.SetCacheSizeLimit(20000) 28 | lru.SkipTTLExtensionOnHit(true) 29 | lru.SetTTL(5 * time.Minute) 30 | 31 | go func() { 32 | <-ctx.Done() 33 | lru.Close() 34 | }() 35 | 36 | launcher_service := &launcher.Launcher{ 37 | Storage_: &FlowStorageManager{ 38 | cache: lru, 39 | cloud_config: cloud_config, 40 | }, 41 | } 42 | 43 | return &Launcher{ 44 | Launcher: launcher_service, 45 | config_obj: config_obj, 46 | }, nil 47 | } 48 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.ListDirectory/Msg_03.json: -------------------------------------------------------------------------------- 1 | { 2 | "doc_type": "collection", 3 | "orgId": "Org123", 4 | "client_id": "C.123", 5 | "session_id": "F.CEV6I8LHAT83O", 6 | "source": "C.77ad4285690698d9", 7 | "auth_state": 1, 8 | "VQLResponse": { 9 | "JSONLResponse": "{\"_FullPath\":\"/test/hello.txt\",\"_Components\":[\"test\",\"hello.txt\"],\"_Accessor\":\"auto\",\"_Data\":{\"DevMajor\":253,\"DevMinor\":0},\"Name\":\"hello.txt\",\"Size\":6,\"Mode\":\"-rw-r--r--\",\"mtime\":\"2023-01-11T07:44:55Z\",\"atime\":\"2023-01-11T07:44:55Z\",\"ctime\":\"2023-01-11T07:44:55Z\",\"btime\":\"0001-01-01T00:00:00Z\"}\n", 10 | "Columns": [ 11 | "_FullPath", 12 | "_Components", 13 | "_Accessor", 14 | "_Data", 15 | "Name", 16 | "Size", 17 | "Mode", 18 | "mtime", 19 | "atime", 20 | "ctime", 21 | "btime" 22 | ], 23 | "query_id": 5, 24 | "Query": { 25 | "Name": "System.VFS.ListDirectory/Listing", 26 | "VQL": "SELECT * FROM if(then=System_VFS_ListDirectory_Listing_0_0, condition=precondition_System_VFS_ListDirectory_Listing_0, else={SELECT * FROM scope() WHERE log(message='Query skipped due to precondition') AND FALSE})" 27 | }, 28 | "timestamp": 1673423174784896, 29 | "total_rows": 1 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /services/notebook/notebook.go: -------------------------------------------------------------------------------- 1 | package notebook 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 8 | "www.velocidex.com/golang/velociraptor/services" 9 | "www.velocidex.com/golang/velociraptor/services/notebook" 10 | ) 11 | 12 | // The NotebookManager service is the main entry point to the 13 | // notebooks. It is composed by various storage related 14 | // implementations which can be locally overriden for cloud 15 | // environments. 16 | type NotebookManager struct { 17 | *notebook.NotebookManager 18 | config_obj *config_proto.Config 19 | } 20 | 21 | func NewNotebookManagerService( 22 | ctx context.Context, 23 | wg *sync.WaitGroup, 24 | config_obj *config_proto.Config) services.NotebookManager { 25 | 26 | timeline_storer := NewSuperTimelineStorer(config_obj) 27 | store := NewNotebookStore(ctx, wg, config_obj, timeline_storer) 28 | 29 | annotator := NewSuperTimelineAnnotator(config_obj, timeline_storer) 30 | 31 | notebook_service := notebook.NewNotebookManager(config_obj, store, 32 | timeline_storer, &SuperTimelineReader{}, &SuperTimelineWriter{}, 33 | annotator, notebook.NewAttachmentManager(config_obj, store)) 34 | 35 | return notebook_service 36 | } 37 | -------------------------------------------------------------------------------- /services/users/users.go: -------------------------------------------------------------------------------- 1 | package users 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "sync" 7 | 8 | "www.velocidex.com/golang/cloudvelo/config" 9 | "www.velocidex.com/golang/velociraptor/services" 10 | "www.velocidex.com/golang/velociraptor/services/users" 11 | ) 12 | 13 | var ( 14 | validUsernameRegEx = regexp.MustCompile("^[a-zA-Z0-9@.\\-_#+]+$") 15 | ) 16 | 17 | // The record stored in the elastic index 18 | type UserRecord struct { 19 | Username string `json:"username"` 20 | Record string `json:"record"` // An encoded api_proto.VelociraptorUser 21 | DocType string `json:"doc_type"` 22 | } 23 | 24 | type UserGUIOptions struct { 25 | Username string `json:"username"` 26 | GUIOptions string `json:"gui_options"` // An endoded api_proto.SetGUIOptionsRequest 27 | DocType string `json:"doc_type"` 28 | Type string `json:"type"` 29 | } 30 | 31 | func StartUserManager( 32 | ctx context.Context, 33 | wg *sync.WaitGroup, 34 | config_obj *config.Config) error { 35 | 36 | storage, err := NewUserStorageManager(ctx, wg, config_obj) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | service := users.NewUserManager(config_obj.VeloConf(), storage) 42 | services.RegisterUserManager(service) 43 | 44 | return nil 45 | } 46 | -------------------------------------------------------------------------------- /bin/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "www.velocidex.com/golang/cloudvelo/startup" 5 | logging "www.velocidex.com/golang/velociraptor/logging" 6 | ) 7 | 8 | var ( 9 | // Run the client. 10 | client_cmd = app.Command("client", "Run the velociraptor client") 11 | ) 12 | 13 | func doRunClient() error { 14 | ctx, cancel := install_sig_handler() 15 | defer cancel() 16 | 17 | config_obj, err := loadConfig(makeDefaultConfigLoader(). 18 | WithRequiredClient(). 19 | WithRequiredLogging(). 20 | WithWriteback()) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | // Report any errors from this function. 26 | logger := logging.GetLogger(&config_obj.Config, &logging.ClientComponent) 27 | defer func() { 28 | if err != nil { 29 | logger.Error("doRunClient Error: %v", err) 30 | } 31 | }() 32 | 33 | sm, err := startup.StartClientServices(ctx, &config_obj.Config, on_error) 34 | if err != nil { 35 | return err 36 | } 37 | defer sm.Close() 38 | 39 | <-ctx.Done() 40 | return nil 41 | } 42 | 43 | func init() { 44 | command_handlers = append(command_handlers, func(command string) bool { 45 | if command == client_cmd.FullCommand() { 46 | FatalIfError(client_cmd, doRunClient) 47 | return true 48 | } 49 | return false 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /Docker/.localstack/template/recorded_api_calls.json: -------------------------------------------------------------------------------- 1 | {"a": "s3", "m": "PUT", "p": "/velociraptor", "d": "", "h": {"Remote-Addr": "172.18.0.1", "Host": "localhost:4566", "Accept-Encoding": "identity", "User-Agent": "aws-cli/1.22.34 Python/3.10.4 Linux/5.11.0-49-generic botocore/1.23.34", "X-Amz-Date": "20220911T152902Z", "X-Amz-Content-Sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "Authorization": "AWS4-HMAC-SHA256 Credential=test/20220911/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=36071b74cd7ac36dc24c80afb861a30c453946d73947f3e361a8d215632b4e38", "Content-Length": "0", "x-localstack-authorization": "AWS4-HMAC-SHA256 Credential=test/20220911/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=36071b74cd7ac36dc24c80afb861a30c453946d73947f3e361a8d215632b4e38", "X-Forwarded-For": "172.18.0.1, localhost:4566, 127.0.0.1, localhost:4566", "x-localstack-edge": "http://localhost:4566", "x-localstack-tgt-api": "s3", "content-type": "binary/octet-stream", "Connection": "close"}, "rd": "PENyZWF0ZUJ1Y2tldFJlc3BvbnNlIHhtbG5zPSJodHRwOi8vczMuYW1hem9uYXdzLmNvbS9kb2MvMjAwNi0wMy0wMSI+PENyZWF0ZUJ1Y2tldFJlc3BvbnNlPjxCdWNrZXQ+dmVsb2NpcmFwdG9yPC9CdWNrZXQ+PC9DcmVhdGVCdWNrZXRSZXNwb25zZT48L0NyZWF0ZUJ1Y2tldFJlc3BvbnNlPg=="} 2 | -------------------------------------------------------------------------------- /datastore/elastic.go: -------------------------------------------------------------------------------- 1 | /* 2 | Velociraptor has an abstract data store interface. The datastore is 3 | used to store simple records atomically. 4 | 5 | The file based Datastore uses the file path to combine a number of 6 | different entities into a path. Accessing the data means an exact 7 | match on each member of the path. 8 | 9 | In the elastic based datastore we match multiple indexes exactly to 10 | access the record. Therefore we need to map from the DSPathSpec to 11 | an elastic base record. 12 | */ 13 | 14 | package datastore 15 | 16 | import ( 17 | "errors" 18 | 19 | "www.velocidex.com/golang/velociraptor/file_store/api" 20 | ) 21 | 22 | var ( 23 | InvalidPath = errors.New("InvalidPath") 24 | ) 25 | 26 | type DatastoreRecord struct { 27 | Timestamp int64 `json:"timestamp"` 28 | ClientId string `json:"client_id"` 29 | VFSPath string `json:"vfs_path"` 30 | FlowId string `json:"flow_id"` 31 | Artifact string `json:"artifact"` 32 | Type string `json:"type"` 33 | DocType string `json:"doc_type"` 34 | JSONData string `json:"data"` 35 | ID string `json:"id"` 36 | } 37 | 38 | func DSPathSpecToRecord(path api.DSPathSpec) (*DatastoreRecord, error) { 39 | return &DatastoreRecord{ 40 | Type: "Generic", 41 | VFSPath: path.AsClientPath(), 42 | }, nil 43 | } 44 | -------------------------------------------------------------------------------- /services/hunt_dispatcher/downloads.go: -------------------------------------------------------------------------------- 1 | package hunt_dispatcher 2 | 3 | import ( 4 | "strings" 5 | 6 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 7 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 8 | "www.velocidex.com/golang/velociraptor/datastore" 9 | "www.velocidex.com/golang/velociraptor/file_store/api" 10 | ) 11 | 12 | func GetAvailableDownloadFiles(config_obj *config_proto.Config, 13 | download_dir api.FSPathSpec) (*api_proto.AvailableDownloads, error) { 14 | result := &api_proto.AvailableDownloads{} 15 | 16 | db, err := datastore.GetDB(config_obj) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | children, err := db.ListChildren(config_obj, download_dir.AsDatastorePath()) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | for _, child := range children { 27 | stats := &api_proto.ContainerStats{} 28 | err := db.GetSubject(config_obj, child, stats) 29 | if err != nil { 30 | continue 31 | } 32 | 33 | result.Files = append(result.Files, &api_proto.AvailableDownloadFile{ 34 | Name: child.Base(), 35 | Type: stats.Type, 36 | Size: stats.TotalCompressedBytes, 37 | Path: strings.Join(stats.Components, "/"), 38 | Complete: stats.Hash != "", 39 | Stats: stats, 40 | }) 41 | } 42 | 43 | return result, nil 44 | } 45 | -------------------------------------------------------------------------------- /vql/uploads/api.go: -------------------------------------------------------------------------------- 1 | package uploads 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "www.velocidex.com/golang/velociraptor/accessors" 8 | actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" 9 | "www.velocidex.com/golang/velociraptor/uploads" 10 | "www.velocidex.com/golang/vfilter" 11 | ) 12 | 13 | // An object that can do multipart uploading 14 | type CloudUploader interface { 15 | // A constructor. 16 | New(ctx context.Context, 17 | scope vfilter.Scope, 18 | dest *accessors.OSPath, 19 | accessor string, 20 | name *accessors.OSPath, 21 | mtime, atime, ctime, btime time.Time, 22 | size int64, // Expected size. 23 | uploader_type string) (CloudUploader, error) 24 | 25 | // Upload the buffer as a single part upload. This is used for 26 | // files that are smaller than BUFF_SIZE. 27 | PutWhole(buf []byte) error 28 | 29 | // Upload the buffer as a multipart upload. NOTE: This will be 30 | // called with minimum 5mb buffers for each part except for the 31 | // final part. 32 | Put(buf []byte) error 33 | 34 | // Once the upload is successfull this should be called. If not a 35 | // Close will cancel the upload. 36 | Commit() 37 | 38 | // Finalize the upload. 39 | Close() error 40 | 41 | SetIndex(index *actions_proto.Index) 42 | 43 | GetVQLResponse() *uploads.UploadResponse 44 | } 45 | -------------------------------------------------------------------------------- /startup/foreman.go: -------------------------------------------------------------------------------- 1 | package startup 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/config" 7 | "www.velocidex.com/golang/cloudvelo/foreman" 8 | "www.velocidex.com/golang/cloudvelo/services/orgs" 9 | "www.velocidex.com/golang/velociraptor/api" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | "www.velocidex.com/golang/velociraptor/services" 12 | ) 13 | 14 | func StartForeman( 15 | ctx context.Context, 16 | config_obj *config.Config) (*services.Service, error) { 17 | 18 | // Come up with a suitable services plan depending on the frontend 19 | // role. 20 | if config_obj.Frontend == nil { 21 | config_obj.Frontend = &config_proto.FrontendConfig{} 22 | } 23 | if config_obj.Services == nil { 24 | config_obj.Services = &config_proto.ServerServicesConfig{ 25 | ClientInfo: true, 26 | RepositoryManager: true, 27 | Launcher: true, 28 | } 29 | } 30 | 31 | sm := services.NewServiceManager(ctx, config_obj.VeloConf()) 32 | _, err := orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj) 33 | if err != nil { 34 | return sm, err 35 | } 36 | 37 | err = api.StartMonitoringService(sm.Ctx, sm.Wg, config_obj.VeloConf()) 38 | if err != nil { 39 | return sm, err 40 | } 41 | 42 | err = foreman.StartForemanService(sm.Ctx, sm.Wg, config_obj) 43 | return sm, err 44 | } 45 | -------------------------------------------------------------------------------- /ingestion/testdata/Enrollment/Enrollment_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "E:Enrol", 3 | "urgent": true, 4 | "source": "C.1352adc54e292a23", 5 | "CSR": { 6 | "pem": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQ1lqQ0NBVW9DQVFBd0hURWJNQmtHQTFVRUF4TVNReTR4TXpVeVlXUmpOVFJsTWpreVlUSXpNSUlCSWpBTgpCZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF5VWp4N1Q3WkNHM0tML0tsbi9LTG8vMjdBWVl5CkVUUmp2cWZpSGNUcG9xMTFSTkFkNmovWnRSR2FJNmpxN1JoWUdMaloyOFBEV1o0Nm5wOTBTOUJCV25jcTdCZUEKRDVPV01vUStMbGRLV0VJZXV4TmZORWRZUkFaSHBsZVpEbXJZeVNMOFN6L2lMZXhQYkVyVHNYOVNnWUptcStTNwp6Y2t2UXBVU21zOU0wRVJUeHpHdUZvN2UvTjF3a2tMb3pCZEFMclFwdUdmc1AwSWNxcUpEeCtyTkgrdEZia3NoCkIyQ3VZTnN4dUw3amFvbDBiUkMyZjB5eUNCVzJ0VWQra1Rub0VFdXU2UUg4d0laY203NmxQWURLQ2JZUWZvVWQKMCtadzlSbGNKYm5SL3d0T2VtRnY0bHJISXBHQ3l2RVFzSUdyYkFvODBkcm8vK1c1Q1E1c1hWYTRQUUlEQVFBQgpvQUF3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUJKTVNObXNPK0NQeU9jQU9UNEhwenVad3FHWU9oZFU4NzdMCkdlVUNmZ1ZUK014UDBtMjRCSjd1eEQ2OEpkNFpVK2VzMUlMdHA5Qkg5NVhidENrZE9ZTHQ2NWRmeVl5NnQ2WVAKNEgvWTNBT2E1NEI1M2lyZE5rWHNPbUpnaENTU0dDdFE5UWlEdC9LbHE4K3R6VWZpUXdKUTduRnNZanl4YXlpUwplYTR5VjVod0RFUlVkQzg2Z3MxTGJXMkVmcDNpRGQvUG1LOStzTUQwcWhDcTVVSEQrV2FRbGtCblJGbURJT2Z3CjRCWW9lVE1LcHIzU0Jsd0xkWlVWQmtkNUVUY3lZNFd2YXF1TTgrSFRDMXBUN25SUllYSHZLZ1FzNGdkMU5oYlYKclU0K0czemxac2twU3Z5U0JlZElsZmJyV3FTUGNveDM3V0x2K1gzZXRvbEV1Q1NIZmh3PQotLS0tLUVORCBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /startup/tools.go: -------------------------------------------------------------------------------- 1 | package startup 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/config" 7 | cvelo_datastore "www.velocidex.com/golang/cloudvelo/datastore" 8 | "www.velocidex.com/golang/cloudvelo/filestore" 9 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 10 | "www.velocidex.com/golang/cloudvelo/services/orgs" 11 | "www.velocidex.com/golang/velociraptor/datastore" 12 | "www.velocidex.com/golang/velociraptor/file_store" 13 | "www.velocidex.com/golang/velociraptor/services" 14 | ) 15 | 16 | func StartToolServices( 17 | ctx context.Context, config_obj *config.Config) (*services.Service, error) { 18 | sm := services.NewServiceManager(ctx, config_obj.VeloConf()) 19 | _, err := orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj) 20 | if err != nil { 21 | return sm, err 22 | } 23 | 24 | // Install the ElasticDatastore 25 | datastore.OverrideDatastoreImplementation( 26 | cvelo_datastore.NewElasticDatastore(ctx, config_obj)) 27 | 28 | file_store_obj, err := filestore.NewS3Filestore(ctx, config_obj) 29 | if err != nil { 30 | return nil, err 31 | } 32 | file_store.OverrideFilestoreImplementation( 33 | config_obj.VeloConf(), file_store_obj) 34 | 35 | err = cvelo_services.StartElasticSearchService(ctx, config_obj) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | return sm, nil 41 | } 42 | -------------------------------------------------------------------------------- /bin/utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "gopkg.in/alecthomas/kingpin.v2" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | logging "www.velocidex.com/golang/velociraptor/logging" 12 | ) 13 | 14 | func FatalIfError(command *kingpin.CmdClause, cb func() error) { 15 | err := cb() 16 | kingpin.FatalIfError(err, command.FullCommand()) 17 | } 18 | 19 | func install_sig_handler() (context.Context, context.CancelFunc) { 20 | quit := make(chan os.Signal, 1) 21 | signal.Notify(quit, syscall.SIGHUP, 22 | syscall.SIGINT, 23 | syscall.SIGTERM, 24 | syscall.SIGQUIT) 25 | 26 | ctx, cancel := context.WithCancel(context.Background()) 27 | 28 | go func() { 29 | select { 30 | case <-quit: 31 | // Ordered shutdown now. 32 | cancel() 33 | 34 | case <-ctx.Done(): 35 | return 36 | } 37 | }() 38 | 39 | return ctx, cancel 40 | 41 | } 42 | 43 | func on_error(ctx context.Context, config_obj *config_proto.Config) { 44 | select { 45 | 46 | // It's ok we are supposed to exit. 47 | case <-ctx.Done(): 48 | return 49 | 50 | default: 51 | // Log the error. 52 | logger := logging.GetLogger(config_obj, &logging.ClientComponent) 53 | logger.Error("Exiting hard due to bug or KillKillKill! This should not happen!") 54 | 55 | os.Exit(-1) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /result_sets/simple/simple.go: -------------------------------------------------------------------------------- 1 | package simple 2 | 3 | import ( 4 | "www.velocidex.com/golang/velociraptor/file_store/api" 5 | ) 6 | 7 | // This is the record we store in the elastic datastore. Simple 8 | // Results sets are written from collections and contain a table rows. 9 | type SimpleResultSetRecord struct { 10 | ClientId string `json:"client_id"` 11 | FlowId string `json:"flow_id"` 12 | Artifact string `json:"artifact"` 13 | Type string `json:"type"` 14 | StartRow int64 `json:"start_row"` 15 | EndRow int64 `json:"end_row"` 16 | VFSPath string `json:"vfs_path"` 17 | JSONData string `json:"data"` 18 | TotalRows uint64 `json:"total_rows"` 19 | Timestamp int64 `json:"timestamp"` 20 | ID string `json:"id"` 21 | } 22 | 23 | // Examine the pathspec and construct a new Elastic record. Because 24 | // Elastic can index on multiple terms we do not need to build a 25 | // single VFS path - we just split out the path into indexable terms 26 | // then search them directly for a more efficient match. 27 | 28 | // Failing this, we index on vfs path. 29 | 30 | // This code is basically the inverse of the path manager mechanism. 31 | func NewSimpleResultSetRecord( 32 | log_path api.FSPathSpec, 33 | id string) *SimpleResultSetRecord { 34 | return &SimpleResultSetRecord{ 35 | VFSPath: log_path.AsClientPath(), 36 | ID: id, 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /result_sets/timed/factory.go: -------------------------------------------------------------------------------- 1 | package timed 2 | 3 | import ( 4 | "context" 5 | 6 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 7 | "www.velocidex.com/golang/velociraptor/file_store/api" 8 | "www.velocidex.com/golang/velociraptor/json" 9 | "www.velocidex.com/golang/velociraptor/result_sets" 10 | "www.velocidex.com/golang/velociraptor/utils" 11 | ) 12 | 13 | type TimedFactory struct{} 14 | 15 | func (self TimedFactory) NewTimedResultSetWriter( 16 | config_obj *config_proto.Config, 17 | path_manager api.PathManager, 18 | opts *json.EncOpts, 19 | completion func()) (result_sets.TimedResultSetWriter, error) { 20 | return NewTimedResultSetWriter( 21 | config_obj, path_manager, opts, completion) 22 | } 23 | 24 | func (self TimedFactory) NewTimedResultSetWriterWithClock( 25 | config_obj *config_proto.Config, 26 | path_manager api.PathManager, 27 | opts *json.EncOpts, 28 | completion func(), clock utils.Clock) (result_sets.TimedResultSetWriter, error) { 29 | return NewTimedResultSetWriter( 30 | config_obj, path_manager, opts, completion) 31 | } 32 | 33 | func (self TimedFactory) NewTimedResultSetReader( 34 | ctx context.Context, 35 | config_obj *config_proto.Config, 36 | path_manager api.PathManager) (result_sets.TimedResultSetReader, error) { 37 | 38 | return &TimedResultSetReader{ 39 | path_manager: path_manager, 40 | config_obj: config_obj, 41 | }, nil 42 | } 43 | -------------------------------------------------------------------------------- /startup/pool.go: -------------------------------------------------------------------------------- 1 | // +build XXXX 2 | 3 | package startup 4 | 5 | import ( 6 | "context" 7 | 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | "www.velocidex.com/golang/velociraptor/executor" 10 | "www.velocidex.com/golang/velociraptor/http_comms" 11 | "www.velocidex.com/golang/velociraptor/services" 12 | "www.velocidex.com/golang/velociraptor/services/orgs" 13 | ) 14 | 15 | // StartClientServices starts the various services needed by the 16 | // client. 17 | func StartPoolClientServices( 18 | sm *services.Service, 19 | config_obj *config_proto.Config, 20 | exe *executor.PoolClientExecutor) error { 21 | 22 | // Create a suitable service plan. 23 | if config_obj.Frontend == nil { 24 | config_obj.Frontend = &config_proto.FrontendConfig{} 25 | } 26 | 27 | if config_obj.Services == nil { 28 | config_obj.Services = services.ClientServicesSpec() 29 | } 30 | 31 | _, err := services.GetOrgManager() 32 | if err != nil { 33 | _, err = orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj) 34 | if err != nil { 35 | return err 36 | } 37 | } 38 | 39 | _, err = http_comms.StartHttpCommunicatorService( 40 | sm.Ctx, sm.Wg, config_obj, exe, 41 | func(ctx context.Context, config_obj *config_proto.Config) {}) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | err = executor.StartEventTableService( 47 | sm.Ctx, sm.Wg, config_obj, exe.Outbound) 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /startup/communicator.go: -------------------------------------------------------------------------------- 1 | package startup 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/config" 7 | ingestor_services "www.velocidex.com/golang/cloudvelo/ingestion/services" 8 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 9 | "www.velocidex.com/golang/cloudvelo/services/orgs" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | "www.velocidex.com/golang/velociraptor/services" 12 | ) 13 | 14 | // StartFrontendServices starts the binary as a frontend: 15 | func StartCommunicatorServices( 16 | ctx context.Context, 17 | config_obj *config.Config) (*services.Service, error) { 18 | 19 | if config_obj.Frontend == nil { 20 | config_obj.Frontend = &config_proto.FrontendConfig{} 21 | } 22 | if config_obj.Services == nil { 23 | config_obj.Services = &config_proto.ServerServicesConfig{ 24 | ClientInfo: true, 25 | RepositoryManager: true, 26 | Launcher: true, 27 | } 28 | } 29 | 30 | sm := services.NewServiceManager(ctx, config_obj.VeloConf()) 31 | err := cvelo_services.StartElasticSearchService(ctx, config_obj) 32 | if err != nil { 33 | return sm, err 34 | } 35 | 36 | _, err = orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj) 37 | if err != nil { 38 | return sm, err 39 | } 40 | 41 | // Start the ingestion services 42 | err = sm.Start(ingestor_services.StartHuntStatsUpdater) 43 | if err != nil { 44 | return sm, err 45 | } 46 | 47 | return sm, nil 48 | } 49 | -------------------------------------------------------------------------------- /services/inventory/inventory.go: -------------------------------------------------------------------------------- 1 | package inventory 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | 8 | artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" 9 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 10 | "www.velocidex.com/golang/velociraptor/services" 11 | ) 12 | 13 | var ( 14 | notImplementedError = errors.New("Inventory service not supported") 15 | ) 16 | 17 | type Dummy struct{} 18 | 19 | func (self Dummy) Get() *artifacts_proto.ThirdParty { 20 | return nil 21 | } 22 | 23 | func (self Dummy) ProbeToolInfo( 24 | ctx context.Context, config_obj *config_proto.Config, 25 | name, version string) (*artifacts_proto.Tool, error) { 26 | return nil, notImplementedError 27 | } 28 | 29 | func (self Dummy) GetToolInfo(ctx context.Context, config_obj *config_proto.Config, 30 | tool, version string) (*artifacts_proto.Tool, error) { 31 | return nil, notImplementedError 32 | } 33 | 34 | func (self Dummy) AddTool(ctx context.Context, config_obj *config_proto.Config, 35 | tool *artifacts_proto.Tool, opts services.ToolOptions) error { 36 | return notImplementedError 37 | } 38 | 39 | func (self Dummy) RemoveTool( 40 | config_obj *config_proto.Config, tool_name string) error { 41 | return notImplementedError 42 | } 43 | 44 | func NewInventoryDummyService( 45 | ctx context.Context, 46 | wg *sync.WaitGroup, 47 | config_obj *config_proto.Config) (services.Inventory, error) { 48 | 49 | return Dummy{}, nil 50 | } 51 | -------------------------------------------------------------------------------- /services/hunt_dispatcher/mutations.go: -------------------------------------------------------------------------------- 1 | package hunt_dispatcher 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Velocidex/ordereddict" 7 | "www.velocidex.com/golang/cloudvelo/services" 8 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 9 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 10 | "www.velocidex.com/golang/velociraptor/services/hunt_manager" 11 | velo_utils "www.velocidex.com/golang/velociraptor/utils" 12 | ) 13 | 14 | // This is only used by the GUI so we do it inline. 15 | func (self *HuntDispatcher) MutateHunt( 16 | ctx context.Context, config_obj *config_proto.Config, 17 | mutation *api_proto.HuntMutation) error { 18 | 19 | hunt_manager, err := hunt_manager.MakeHuntManager(config_obj) 20 | if err != nil { 21 | return err 22 | } 23 | err = hunt_manager.ProcessMutation(ctx, config_obj, 24 | ordereddict.NewDict(). 25 | Set("hunt_id", mutation.HuntId). 26 | Set("mutation", mutation)) 27 | 28 | if err != nil { 29 | return err 30 | } 31 | 32 | if mutation.Assignment != nil { 33 | hunt_flow_entry := &HuntFlowEntry{ 34 | HuntId: mutation.HuntId, 35 | ClientId: mutation.Assignment.ClientId, 36 | FlowId: mutation.Assignment.FlowId, 37 | Timestamp: velo_utils.GetTime().Now().Unix(), 38 | Status: "started", 39 | DocType: "hunt_flow", 40 | } 41 | return services.SetElasticIndex(ctx, 42 | config_obj.OrgId, 43 | "transient", services.DocIdRandom, 44 | hunt_flow_entry) 45 | } 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /ingestion/testdata/System.VFS.ListDirectory/Msg_04.json: -------------------------------------------------------------------------------- 1 | { 2 | "doc_type": "collection", 3 | "session_id": "F.CEV6I8LHAT83O", 4 | "request_id": 980, 5 | "source": "C.77ad4285690698d9", 6 | "auth_state": 1, 7 | "LogMessage": { 8 | "id": 5, 9 | "number_of_rows": 9, 10 | "jsonl": "{\"client_time\":1673423174,\"level\":\"INFO\",\"message\":\"Starting query execution.\\n\"}\n{\"client_time\":1673423174,\"level\":\"DEFAULT\",\"message\":\"generate: registered new query for vfs\\n\"}\n{\"client_time\":1673423174,\"level\":\"DEFAULT\",\"message\":\"generate: Removing generator vfs\\n\"}\n{\"client_time\":1673423174,\"level\":\"DEFAULT\",\"message\":\"Time 0: System.VFS.ListDirectory/Stats: Sending response part 0 64 B (1 rows).\"}\n{\"client_time\":1673423174,\"level\":\"DEFAULT\",\"message\":\"Time 0: System.VFS.ListDirectory/Listing: Sending response part 0 296 B (1 rows).\"}\n{\"client_time\":1673423174,\"level\":\"INFO\",\"message\":\"Collection is done after 518.654953ms\\n\"}\n{\"client_time\":1673423174,\"level\":\"INFO\",\"message\":\"Collection is done after 512.470235ms\\n\"}\n{\"client_time\":1673423174,\"level\":\"DEBUG\",\"message\":\"Query Stats: {\\\"RowsScanned\\\":7,\\\"PluginsCalled\\\":4,\\\"FunctionsCalled\\\":7,\\\"ProtocolSearch\\\":10,\\\"ScopeCopy\\\":23}\\n\"}\n{\"client_time\":1673423174,\"level\":\"DEBUG\",\"message\":\"Query Stats: {\\\"RowsScanned\\\":5,\\\"PluginsCalled\\\":3,\\\"FunctionsCalled\\\":7,\\\"ProtocolSearch\\\":10,\\\"ScopeCopy\\\":19}\\n\"}\n", 11 | "artifact": "System.VFS.ListDirectory/Listing" 12 | } 13 | } -------------------------------------------------------------------------------- /artifact_definitions/Alert/EventLogModifications.yaml: -------------------------------------------------------------------------------- 1 | name: Alert.Windows.Events.EventLogModifications 2 | description: | 3 | It is possible to disable windows event logs on a per channel or per 4 | provider basis. Attackers may disable ciritcal log sources to 5 | prevent detections. 6 | 7 | This artifact monitors the state of the event log system from the 8 | registry and attempts to detect when event logs were disabled. 9 | 10 | type: CLIENT_EVENT 11 | 12 | precondition: 13 | SELECT * FROM info() WHERE OS =~ "windows" 14 | 15 | parameters: 16 | - name: Period 17 | type: int 18 | default: 60 19 | 20 | sources: 21 | - query: | 22 | LET Publishers = "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\WINEVT\\Publishers\\*\\@" 23 | 24 | LET ProviderNames <= memoize(key="GUID", query={ 25 | SELECT OSPath.Components[-2] AS GUID, 26 | Data.value AS Name 27 | FROM glob(globs=Publishers, accessor="registry") 28 | }) 29 | 30 | LET Key = "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\WINEVT\\Channels\\*" 31 | 32 | LET Query = SELECT Key.Mtime AS Mtime, 33 | Key.OSPath[-1] AS ChannelName, 34 | format(format="%s/%v", args=[Key.OSPath[-1], Enabled]) AS QueryKey , 35 | Key.OSPath AS _Key, 36 | get(item=ProviderNames, field=OwningPublisher).Name AS Publisher, Enabled 37 | FROM read_reg_key(globs=Key) 38 | 39 | SELECT * FROM diff(query=Query, period=Period, key="QueryKey") 40 | WHERE Diff =~ "added" 41 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # Only trigger when a PR is committed. 2 | name: Linux Build All Arches 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | name: Build 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Check out code into the Go module directory 14 | uses: actions/checkout@v2 15 | with: 16 | submodules: true 17 | 18 | - uses: actions/setup-go@v2 19 | with: 20 | go-version: '^1.18' 21 | - run: go version 22 | 23 | - name: Get dependencies 24 | run: | 25 | go get -v -t -d ./... 26 | sudo apt-get install mingw-w64-x86-64-dev gcc-mingw-w64-x86-64 gcc-mingw-w64 musl-tools 27 | 28 | - name: Use Node.js v16 29 | uses: actions/setup-node@v1 30 | with: 31 | node-version: 16 32 | 33 | - name: npm install gui 34 | run: | 35 | cd velociraptor/gui/velociraptor/ 36 | npm ci 37 | npm run build 38 | cd ../../../ 39 | 40 | - name: Build All Architectures 41 | # Uncomment the architectures you want here. NOTE: DarwinBase 42 | # does not include yara or modules with C compilers needed. 43 | run: | 44 | export PATH=$PATH:~/go/bin/ 45 | cd velociraptor/ 46 | make linux 47 | cd ../ 48 | 49 | go run make.go -v LinuxMusl 50 | go run make.go -v Windows 51 | go run make.go -v DarwinBase 52 | 53 | - name: StoreBinaries 54 | uses: actions/upload-artifact@v4 55 | with: 56 | name: Binaries 57 | path: output 58 | -------------------------------------------------------------------------------- /services/hunt_dispatcher/modify.go: -------------------------------------------------------------------------------- 1 | package hunt_dispatcher 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | "www.velocidex.com/golang/velociraptor/services" 10 | ) 11 | 12 | func (self *HuntDispatcher) ModifyHunt( 13 | ctx context.Context, 14 | config_obj *config_proto.Config, 15 | hunt_modification *api_proto.Hunt, 16 | user string) error { 17 | 18 | self.ModifyHuntObject(ctx, hunt_modification.HuntId, 19 | func(hunt *api_proto.Hunt) services.HuntModificationAction { 20 | 21 | // Is the description changed? 22 | if hunt_modification.HuntDescription != "" { 23 | hunt.HuntDescription = hunt_modification.HuntDescription 24 | 25 | } else if hunt_modification.State == api_proto.Hunt_RUNNING { 26 | 27 | // We allow restarting stopped hunts 28 | // but this may not work as intended 29 | // because we still have a hunt index 30 | // - i.e. clients that already 31 | // scheduled the hunt will not 32 | // re-schedule (whether they ran it or 33 | // not). Usually the most reliable way 34 | // to re-do a hunt is to copy it and 35 | // do it again. 36 | hunt.State = api_proto.Hunt_RUNNING 37 | hunt.StartTime = uint64(time.Now().UnixNano() / 1000) 38 | 39 | // We are trying to pause or stop the hunt. 40 | } else if hunt_modification.State == api_proto.Hunt_STOPPED || 41 | hunt_modification.State == api_proto.Hunt_PAUSED { 42 | hunt.State = api_proto.Hunt_STOPPED 43 | } 44 | 45 | return services.HuntPropagateChanges 46 | }) 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /bin/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | Velociraptor - Dig Deeper 3 | Copyright (C) 2019-2022 Rapid7 Inc. 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU Affero General Public License as published 7 | by the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU Affero General Public License for more details. 14 | 15 | You should have received a copy of the GNU Affero General Public License 16 | along with this program. If not, see . 17 | */ 18 | package main 19 | 20 | import ( 21 | "fmt" 22 | "runtime/debug" 23 | 24 | "github.com/Velocidex/yaml/v2" 25 | "gopkg.in/alecthomas/kingpin.v2" 26 | "www.velocidex.com/golang/velociraptor/config" 27 | ) 28 | 29 | var ( 30 | version = app.Command("version", "Report client version.") 31 | ) 32 | 33 | func init() { 34 | command_handlers = append(command_handlers, func(command string) bool { 35 | if command == version.FullCommand() { 36 | res, err := yaml.Marshal(config.GetVersion()) 37 | if err != nil { 38 | kingpin.FatalIfError(err, "Unable to encode version.") 39 | } 40 | 41 | fmt.Printf("%v", string(res)) 42 | 43 | if *verbose_flag { 44 | info, ok := debug.ReadBuildInfo() 45 | if ok { 46 | fmt.Printf("\n\nBuild Info:\n%v\n", info) 47 | } 48 | } 49 | 50 | return true 51 | } 52 | return false 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /services/instrument.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/prometheus/client_golang/prometheus/promauto" 8 | ) 9 | 10 | var ( 11 | OpensearchHistorgram = promauto.NewHistogramVec( 12 | prometheus.HistogramOpts{ 13 | Name: "opensearch_latency", 14 | Help: "Latency to access datastore.", 15 | Buckets: prometheus.LinearBuckets(0.01, 0.05, 10), 16 | }, 17 | []string{"operation"}, 18 | ) 19 | 20 | OpensearchSummary = promauto.NewSummaryVec( 21 | prometheus.SummaryOpts{ 22 | Name: "opensearch_operations", 23 | Help: "Latency to access datastore.", 24 | }, 25 | []string{"operation"}, 26 | ) 27 | 28 | // Watch operations in real time using: 29 | // watch 'curl -s http://localhost:8003/metrics | grep -E "operations{|opensearch_latency_bucket.+Inf"' 30 | OperationCounter = promauto.NewCounterVec( 31 | prometheus.CounterOpts{ 32 | Name: "operations", 33 | Help: "Count of operations.", 34 | }, 35 | []string{"operation"}, 36 | ) 37 | ) 38 | 39 | func Count(operation string) { 40 | OperationCounter.WithLabelValues(operation).Inc() 41 | } 42 | 43 | func Instrument(operation string) func() time.Duration { 44 | timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) { 45 | OpensearchHistorgram.WithLabelValues(operation).Observe(v) 46 | })) 47 | 48 | return timer.ObserveDuration 49 | } 50 | 51 | func Summarize(operation string) func() time.Duration { 52 | timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) { 53 | OpensearchSummary.WithLabelValues(operation).Observe(v) 54 | })) 55 | 56 | return timer.ObserveDuration 57 | } 58 | -------------------------------------------------------------------------------- /ingestion/logs.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "www.velocidex.com/golang/cloudvelo/result_sets/simple" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 10 | "www.velocidex.com/golang/velociraptor/file_store" 11 | "www.velocidex.com/golang/velociraptor/json" 12 | "www.velocidex.com/golang/velociraptor/paths" 13 | "www.velocidex.com/golang/velociraptor/result_sets" 14 | "www.velocidex.com/golang/velociraptor/utils" 15 | ) 16 | 17 | func (self Ingestor) HandleLogs( 18 | ctx context.Context, 19 | config_obj *config_proto.Config, 20 | message *crypto_proto.VeloMessage) error { 21 | 22 | msg := message.LogMessage 23 | if msg.Jsonl == "" { 24 | return errors.New("Invalid log messages response") 25 | } 26 | 27 | log_path_manager := paths.NewFlowPathManager( 28 | message.Source, message.SessionId).Log() 29 | 30 | file_store_factory := file_store.GetFileStore(config_obj) 31 | rs_writer, err := result_sets.NewResultSetWriter( 32 | file_store_factory, log_path_manager, json.DefaultEncOpts(), 33 | utils.BackgroundWriter, result_sets.AppendMode) 34 | if err != nil { 35 | return err 36 | } 37 | defer rs_writer.Close() 38 | 39 | elastic_writer, ok := rs_writer.(*simple.ElasticSimpleResultSetWriter) 40 | if ok { 41 | elastic_writer.SetStartRow(msg.Id) 42 | 43 | // Urgent messages are GUI driven so we need to see logs 44 | // immediately. 45 | if message.Urgent { 46 | elastic_writer.SetSync() 47 | } 48 | } 49 | 50 | payload := msg.Jsonl 51 | rs_writer.WriteJSONL([]byte(payload), uint64(msg.NumberOfRows)) 52 | 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /ingestion/testdata/Generic.Client.Stats/Generic.Client.Stats_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "session_id": "F.Monitoring", 3 | "response_id": 1664246489016657057, 4 | "source": "C.1352adc54e292a23", 5 | "auth_state": 1, 6 | "VQLResponse": { 7 | "JSONLResponse": "{\"Timestamp\":1664246379.037654,\"CPU\":5.56,\"RSS\":83447808,\"CPUPercent\":null}\n{\"Timestamp\":1664246389.0389478,\"CPU\":5.93,\"RSS\":72527872,\"CPUPercent\":0.036995213198136666}\n{\"Timestamp\":1664246399.0391533,\"CPU\":6,\"RSS\":73875456,\"CPUPercent\":0.006999856141185939}\n{\"Timestamp\":1664246409.0397468,\"CPU\":6.12,\"RSS\":72306688,\"CPUPercent\":0.011999287933643665}\n{\"Timestamp\":1664246419.040297,\"CPU\":6.18,\"RSS\":73654272,\"CPUPercent\":0.005999669856118451}\n{\"Timestamp\":1664246429.0415347,\"CPU\":6.26,\"RSS\":65314816,\"CPUPercent\":0.007999010017847958}\n{\"Timestamp\":1664246439.0423741,\"CPU\":6.36,\"RSS\":62390272,\"CPUPercent\":0.009999160598648414}\n{\"Timestamp\":1664246449.0437052,\"CPU\":6.42,\"RSS\":62619648,\"CPUPercent\":0.005999201451737721}\n{\"Timestamp\":1664246459.0448549,\"CPU\":6.5,\"RSS\":64479232,\"CPUPercent\":0.007999080382213527}\n{\"Timestamp\":1664246469.0458136,\"CPU\":6.619999999999999,\"RSS\":61812736,\"CPUPercent\":0.011998849692949804}\n{\"Timestamp\":1664246479.047109,\"CPU\":6.68,\"RSS\":63152128,\"CPUPercent\":0.005999222903775355}\n", 8 | "Columns": [ 9 | "Timestamp", 10 | "CPU", 11 | "RSS", 12 | "CPUPercent" 13 | ], 14 | "query_id": 3, 15 | "Query": { 16 | "Name": "Generic.Client.Stats", 17 | "VQL": "SELECT * FROM if(then=Generic_Client_Stats_0_0, condition=precondition_Generic_Client_Stats_0, else={SELECT * FROM scope() WHERE log(message='Query skipped due to precondition') AND FALSE})" 18 | }, 19 | "timestamp": 1664246489016595, 20 | "total_rows": 11 21 | } 22 | } -------------------------------------------------------------------------------- /services/orgs/delete.go: -------------------------------------------------------------------------------- 1 | package orgs 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/promauto" 9 | "www.velocidex.com/golang/cloudvelo/schema" 10 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 11 | "www.velocidex.com/golang/velociraptor/logging" 12 | "www.velocidex.com/golang/velociraptor/services" 13 | "www.velocidex.com/golang/velociraptor/services/orgs" 14 | "www.velocidex.com/golang/velociraptor/utils" 15 | ) 16 | 17 | var ( 18 | deleteOrgCounter = promauto.NewCounter( 19 | prometheus.CounterOpts{ 20 | Name: "velociraptor_delete_org_count", 21 | Help: "Count of organizations deleted from the Velociraptor system.", 22 | }) 23 | ) 24 | 25 | // Remove the org and all its data. 26 | func (self *OrgManager) DeleteOrg( 27 | ctx context.Context, principal, org_id string) error { 28 | 29 | if utils.IsRootOrg(org_id) { 30 | return errors.New("Can not remove root org.") 31 | } 32 | 33 | logger := logging.GetLogger(self.config_obj, &logging.Audit) 34 | if logger != nil { 35 | logger.Info("Deleted organization: %v", org_id) 36 | } 37 | 38 | err := orgs.RemoveOrgFromUsers(ctx, principal, org_id) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | // Remove the org from the index. 44 | err = cvelo_services.DeleteDocument(ctx, 45 | services.ROOT_ORG_ID, "persisted", 46 | org_id, cvelo_services.SyncDelete) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | self.mu.Lock() 52 | delete(self.orgs, org_id) 53 | delete(self.org_id_by_nonce, org_id) 54 | self.mu.Unlock() 55 | 56 | deleteOrgCounter.Inc() 57 | 58 | // Drop all the org's indexes 59 | return schema.Delete(self.ctx, self.config_obj, 60 | org_id, services.ROOT_ORG_ID) 61 | } 62 | -------------------------------------------------------------------------------- /ingestion/enrolment.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/schema/api" 7 | "www.velocidex.com/golang/cloudvelo/services" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 10 | "www.velocidex.com/golang/velociraptor/logging" 11 | "www.velocidex.com/golang/velociraptor/utils" 12 | ) 13 | 14 | func (self Ingestor) HandleEnrolment( 15 | config_obj *config_proto.Config, 16 | message *crypto_proto.VeloMessage) error { 17 | 18 | csr := message.CSR 19 | if csr == nil { 20 | return nil 21 | } 22 | 23 | client_id, err := self.crypto_manager.AddCertificateRequest(config_obj, csr.Pem) 24 | if err != nil { 25 | logger := logging.GetLogger(config_obj, &logging.FrontendComponent) 26 | logger.Error("While enrolling %v: %v", client_id, err) 27 | return err 28 | } 29 | 30 | return nil 31 | } 32 | 33 | const ( 34 | updateClientInterrogate = ` 35 | { 36 | "script" : { 37 | "source": "ctx._source.last_interrogate = params.last_interrogate", 38 | "lang": "painless", 39 | "params": { 40 | "last_interrogate": %q 41 | } 42 | } 43 | } 44 | ` 45 | ) 46 | 47 | func (self Ingestor) HandleInterrogation( 48 | ctx context.Context, config_obj *config_proto.Config, 49 | message *crypto_proto.VeloMessage) error { 50 | 51 | services.SetElasticIndexAsync( 52 | config_obj.OrgId, 53 | "persisted", message.Source+"_interrogate", 54 | services.BulkUpdateIndex, 55 | &api.ClientRecord{ 56 | ClientId: message.Source, 57 | Type: "interrogation", 58 | LastInterrogate: message.SessionId, 59 | DocType: "clients", 60 | Timestamp: uint64(utils.GetTime().Now().Unix()), 61 | }) 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /artifact_definitions/System.VFS.DownloadFile.yaml: -------------------------------------------------------------------------------- 1 | name: System.VFS.DownloadFile 2 | description: | 3 | This is an internal artifact used by the GUI to populate the 4 | VFS. You may run it manually if you like, but typically it is 5 | launched by the GUI when the user clicks the "Collect from client" 6 | button at the file "Stats" tab. 7 | 8 | If you run it yourself (or via the API) the results will also be 9 | shown in the VFS view. 10 | 11 | parameters: 12 | - name: Path 13 | description: The path of the file to download. 14 | default: / 15 | - name: Components 16 | type: json_array 17 | description: Alternatively, this is an explicit list of components. 18 | - name: Accessor 19 | default: file 20 | - name: Recursively 21 | type: bool 22 | description: | 23 | If specified, Path is interpreted as a directory and 24 | we download all files below it. 25 | 26 | sources: 27 | - query: | 28 | LET download_one_file = 29 | SELECT OSPath AS Path, Accessor, 30 | Size, upload(file=OSPath, accessor=Accessor) AS Upload 31 | FROM stat(filename=Components, accessor=Accessor) 32 | 33 | LET download_recursive = 34 | SELECT OSPath AS Path, Accessor, 35 | Size, upload(file=OSPath, accessor=Accessor) AS Upload 36 | FROM glob(globs="**", root=Components, 37 | accessor=Accessor, nosymlink=TRUE) 38 | WHERE Mode.IsRegular 39 | 40 | SELECT Path, Accessor, 41 | Upload.Size AS Size, 42 | Upload.StoredSize AS StoredSize, 43 | Upload.Sha256 AS Sha256, 44 | Upload.Md5 AS Md5, 45 | Path.Components AS _Components 46 | FROM if(condition=Recursively, 47 | then={ SELECT * FROM download_recursive}, 48 | else={ SELECT * FROM download_one_file}) 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## How to build 4 | 5 | This version of Velociraptor depends on the open source codebase as 6 | used on GitHub. The GitHub repo is included as a git submodule. We use 7 | the same GUI so we need to have the React App built. Therefore we cant 8 | use a simple go get to install the dependency. 9 | 10 | 1. First make sure the git submodule is cloned properly 11 | 12 | ``` 13 | git submodule update --init 14 | ``` 15 | 16 | 2. Next build the Velociraptor GUI 17 | 18 | ``` 19 | make assets 20 | ``` 21 | 22 | 3. Finally we can build the Cloud version by running make in the top level. 23 | 24 | ``` 25 | make linux_musl 26 | ``` 27 | 28 | You will find the binary in `./output/cvelociraptor` 29 | 30 | 31 | ## Try it out with Docker 32 | 33 | Alternatively build the Docker image 34 | 35 | ``` 36 | make docker 37 | ``` 38 | 39 | Start the docker test system 40 | 41 | ``` 42 | cd Docker 43 | make up 44 | ``` 45 | 46 | Clear the docker system 47 | 48 | ``` 49 | make clean 50 | ``` 51 | 52 | The Makefile contains startup commands for all components. 53 | 54 | 55 | ## Notes 56 | 57 | In the codebase and below we use the term Elastic to refer to the 58 | opensource backend database which originally was managed by Elastic 59 | Inc. Recently, the original Elastic database was split into an 60 | opensource project (https://opensearch.org/) and a non-open source 61 | database offered by Elastic Inc. Further, the Elastic maintained Go 62 | client libraries refuse to connect to the open source database. 63 | 64 | As such, we need to decide which flavor of Elastic to support moving 65 | forward. As an open source project we prefer to support open source 66 | dependencies, so this project only supports the opensearch backend. 67 | 68 | Any references to Elastic in the codebase or documentation actually 69 | refer to opensearch and that is the only database that is supported at 70 | this time. 71 | -------------------------------------------------------------------------------- /vql/uploads/buffer.go: -------------------------------------------------------------------------------- 1 | package uploads 2 | 3 | import "io" 4 | 5 | type BufferedWriter struct { 6 | buf []byte 7 | buf_idx uint64 8 | buf_length uint64 9 | 10 | uploader CloudUploader 11 | 12 | // Total amount of data stored 13 | total uint64 14 | 15 | // If the uploader is closed before sending a single part, we send 16 | // the part using a different API. 17 | sent_first_buffer bool 18 | } 19 | 20 | // Flush a buffer into the uploader 21 | func (self *BufferedWriter) Flush() error { 22 | self.sent_first_buffer = true 23 | err := self.uploader.Put(self.buf[:self.buf_idx]) 24 | if err != nil { 25 | return err 26 | } 27 | self.buf_idx = 0 28 | 29 | return nil 30 | } 31 | 32 | func (self *BufferedWriter) Close() error { 33 | if !self.sent_first_buffer { 34 | return self.uploader.PutWhole(self.buf[:self.buf_idx]) 35 | } 36 | 37 | err := self.Flush() 38 | if err != nil { 39 | return err 40 | } 41 | 42 | self.uploader.Commit() 43 | return self.uploader.Close() 44 | } 45 | 46 | func (self *BufferedWriter) Copy(reader io.Reader, length uint64) error { 47 | for length > 0 { 48 | // How much space is left in the buffer 49 | to_read := length 50 | if to_read > self.buf_length-self.buf_idx { 51 | to_read = self.buf_length - self.buf_idx 52 | } 53 | 54 | n, err := reader.Read(self.buf[self.buf_idx : self.buf_idx+to_read]) 55 | if err != nil && err != io.EOF && n == 0 { 56 | return err 57 | } 58 | 59 | if n == 0 { 60 | return nil 61 | } 62 | 63 | self.total += uint64(n) 64 | self.buf_idx += uint64(n) 65 | length -= uint64(n) 66 | 67 | // Buffer is full - flush it 68 | if self.buf_idx >= self.buf_length { 69 | err = self.Flush() 70 | if err != nil { 71 | return err 72 | } 73 | } 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func NewBufferWriter(uploader CloudUploader) *BufferedWriter { 80 | return &BufferedWriter{ 81 | buf: make([]byte, BUFF_SIZE), 82 | buf_length: uint64(BUFF_SIZE), 83 | uploader: uploader, 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /services/sanity/sanity.go: -------------------------------------------------------------------------------- 1 | package sanity 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "www.velocidex.com/golang/cloudvelo/config" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | "www.velocidex.com/golang/velociraptor/logging" 10 | "www.velocidex.com/golang/velociraptor/services" 11 | "www.velocidex.com/golang/velociraptor/utils" 12 | ) 13 | 14 | // This service checks the running server environment for sane 15 | // conditions. 16 | type SanityChecks struct{} 17 | 18 | // Check sanity of general server state - this is only done for the root org. 19 | func (self *SanityChecks) CheckRootOrg( 20 | ctx context.Context, config_obj *config_proto.Config) error { 21 | 22 | // Make sure the initial user accounts are created with the 23 | // administrator roles. 24 | if config_obj.GUI != nil && config_obj.GUI.Authenticator != nil { 25 | // Create initial orgs 26 | org_manager, err := services.GetOrgManager() 27 | if err != nil { 28 | return err 29 | } 30 | 31 | for _, org := range config_obj.GUI.InitialOrgs { 32 | logger := logging.GetLogger(config_obj, &logging.FrontendComponent) 33 | logger.Info("Creating initial org for %v", org.Name) 34 | _, err := org_manager.CreateNewOrg(org.Name, org.OrgId, services.RandomNonce) 35 | if err != nil { 36 | return err 37 | } 38 | } 39 | 40 | err = createInitialUsers(ctx, config_obj, config_obj.GUI.InitialUsers) 41 | if err != nil { 42 | return err 43 | } 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func (self *SanityChecks) Check( 50 | ctx context.Context, config_obj *config_proto.Config) error { 51 | if utils.IsRootOrg(config_obj.OrgId) { 52 | err := self.CheckRootOrg(ctx, config_obj) 53 | if err != nil { 54 | return err 55 | } 56 | } 57 | 58 | return nil 59 | } 60 | 61 | func NewSanityCheckService( 62 | ctx context.Context, 63 | wg *sync.WaitGroup, 64 | config_obj *config.Config) error { 65 | 66 | result := &SanityChecks{} 67 | return result.Check(ctx, config_obj.VeloConf()) 68 | } 69 | -------------------------------------------------------------------------------- /bin/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | crypto_server "www.velocidex.com/golang/cloudvelo/crypto/server" 7 | 8 | "www.velocidex.com/golang/cloudvelo/config" 9 | "www.velocidex.com/golang/cloudvelo/server" 10 | "www.velocidex.com/golang/cloudvelo/startup" 11 | ) 12 | 13 | var ( 14 | communicator = app.Command("frontend", "Run the server frontend") 15 | communicator_mock = communicator.Flag("mock", "Run the mock ingestor").Bool() 16 | ) 17 | 18 | func makeElasticBackend( 19 | config_obj *config.Config, 20 | crypto_manager *crypto_server.ServerCryptoManager) (server.CommunicatorBackend, error) { 21 | 22 | if *communicator_mock { 23 | return server.NewMockElasticBackend(config_obj) 24 | } 25 | return server.NewElasticBackend(config_obj, crypto_manager) 26 | } 27 | 28 | func doCommunicator() error { 29 | config_obj, err := loadConfig(makeDefaultConfigLoader()) 30 | if err != nil { 31 | return fmt.Errorf("loading config file: %w", err) 32 | } 33 | 34 | ctx, cancel := install_sig_handler() 35 | defer cancel() 36 | 37 | sm, err := startup.StartCommunicatorServices(ctx, config_obj) 38 | defer sm.Close() 39 | if err != nil { 40 | return err 41 | } 42 | 43 | var backend server.CommunicatorBackend 44 | crypto_manager, err := crypto_server.NewServerCryptoManager( 45 | sm.Ctx, config_obj.VeloConf(), sm.Wg) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | backend, err = makeElasticBackend(config_obj, crypto_manager) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | server, err := server.NewCommunicator( 56 | config_obj, crypto_manager, backend) 57 | err = server.Start(ctx, config_obj.VeloConf(), sm.Wg) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | <-ctx.Done() 63 | 64 | return nil 65 | } 66 | 67 | func init() { 68 | command_handlers = append(command_handlers, func(command string) bool { 69 | if command == communicator.FullCommand() { 70 | FatalIfError(communicator, doCommunicator) 71 | return true 72 | } 73 | return false 74 | }) 75 | } 76 | -------------------------------------------------------------------------------- /bin/users.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "www.velocidex.com/golang/cloudvelo/services/users" 7 | "www.velocidex.com/golang/cloudvelo/startup" 8 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 9 | "www.velocidex.com/golang/velociraptor/constants" 10 | "www.velocidex.com/golang/velociraptor/services" 11 | ) 12 | 13 | var ( 14 | // Command line interface for VQL commands. 15 | orgs_command = app.Command("orgs", "Manage orgs") 16 | 17 | orgs_user_add = orgs_command.Command("user_add", "Add a user to an org") 18 | orgs_user_add_org = orgs_user_add.Arg("org_id", "Org ID to add user to"). 19 | Required().String() 20 | orgs_user_add_org_name = orgs_user_add.Arg("org_name", "Org ID to add user to"). 21 | Required().String() 22 | orgs_user_add_user = orgs_user_add.Arg("username", "Username to add"). 23 | Required().String() 24 | ) 25 | 26 | func doOrgUserAdd() error { 27 | config_obj, err := loadConfig(makeDefaultConfigLoader(). 28 | WithRequiredFrontend(). 29 | WithRequiredUser(). 30 | WithRequiredLogging()) 31 | if err != nil { 32 | return fmt.Errorf("loading config file: %w", err) 33 | } 34 | 35 | ctx, cancel := install_sig_handler() 36 | defer cancel() 37 | 38 | sm, err := startup.StartToolServices(ctx, config_obj) 39 | defer sm.Close() 40 | 41 | if err != nil { 42 | return err 43 | } 44 | 45 | err = users.StartUserManager(sm.Ctx, sm.Wg, config_obj) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | user_manager := services.GetUserManager() 51 | record, err := user_manager.GetUserWithHashes( 52 | ctx, constants.PinnedServerName, *orgs_user_add_user) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | record.Orgs = append(record.Orgs, &api_proto.OrgRecord{ 58 | Name: *orgs_user_add_org_name, 59 | Id: *orgs_user_add_org, 60 | }) 61 | 62 | return user_manager.SetUser(ctx, record) 63 | } 64 | 65 | func init() { 66 | command_handlers = append(command_handlers, func(command string) bool { 67 | switch command { 68 | case orgs_user_add.FullCommand(): 69 | FatalIfError(orgs_user_add, doOrgUserAdd) 70 | 71 | default: 72 | return false 73 | } 74 | return true 75 | }) 76 | } 77 | -------------------------------------------------------------------------------- /schema/api/collections.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" 5 | "www.velocidex.com/golang/velociraptor/json" 6 | "www.velocidex.com/golang/velociraptor/utils" 7 | ) 8 | 9 | // The source of truth for this record is 10 | // flows_proto.ArtifactCollectorContext but we extract some of the 11 | // fields into the Elastic schema so they can be searched on. 12 | 13 | // We use the database to manipulate exposed fields. 14 | type ArtifactCollectorRecord struct { 15 | ClientId string `json:"client_id"` 16 | SessionId string `json:"session_id"` 17 | Raw string `json:"context,omitempty"` 18 | Tasks string `json:"tasks,omitempty"` 19 | Type string `json:"type"` 20 | Timestamp int64 `json:"timestamp"` 21 | Doc_Type string `json:"doc_type"` 22 | ID string `json:"id"` 23 | } 24 | 25 | func (self *ArtifactCollectorRecord) ToProto() ( 26 | *flows_proto.ArtifactCollectorContext, error) { 27 | 28 | result := &flows_proto.ArtifactCollectorContext{} 29 | err := json.Unmarshal([]byte(self.Raw), result) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | // For mass duplicated flows, client id inside the protobuf is not 35 | // set (since it is the same for all requests). We therefore 36 | // override it from the Elastic record. 37 | if result.ClientId == "" { 38 | result.ClientId = self.ClientId 39 | } 40 | 41 | if result.SessionId == "" { 42 | result.SessionId = self.SessionId 43 | } 44 | 45 | return result, nil 46 | } 47 | 48 | func ArtifactCollectorRecordFromProto( 49 | in *flows_proto.ArtifactCollectorContext, id string) *ArtifactCollectorRecord { 50 | timestamp := utils.GetTime().Now().UnixNano() 51 | self := &ArtifactCollectorRecord{} 52 | self.ClientId = in.ClientId 53 | self.SessionId = in.SessionId 54 | self.Doc_Type = "collection" 55 | self.ID = id 56 | self.Timestamp = timestamp 57 | self.Raw = json.MustMarshalString(in) 58 | 59 | return self 60 | } 61 | 62 | func GetDocumentIdForCollection(session_id, client_id, doc_type string) string { 63 | if doc_type != "" { 64 | return client_id + "_" + session_id + "_" + doc_type 65 | } 66 | return client_id + "_" + session_id 67 | } 68 | -------------------------------------------------------------------------------- /filestore/name_mapping.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "crypto/sha256" 5 | "fmt" 6 | "strings" 7 | 8 | "www.velocidex.com/golang/cloudvelo/config" 9 | "www.velocidex.com/golang/cloudvelo/vql/uploads" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | "www.velocidex.com/golang/velociraptor/file_store/api" 12 | "www.velocidex.com/golang/velociraptor/utils" 13 | ) 14 | 15 | var ( 16 | EmptyConfig = &config_proto.Config{ 17 | Datastore: &config_proto.DatastoreConfig{}, 18 | } 19 | ) 20 | 21 | // This function converts from a filestore pathspect to the S3 22 | // key. Due to limitations in s3 key lengths the key needs to be 23 | // compressed down with a hash. 24 | func PathspecToKey(config_obj *config.Config, 25 | path_spec api.FSPathSpec) string { 26 | 27 | parts := []string{"orgs", utils.NormalizedOrgId(config_obj.OrgId)} 28 | for _, component := range path_spec.Components() { 29 | parts = append(parts, utils.SanitizeString(component)) 30 | } 31 | 32 | return strings.Join(parts, "/") + api.GetExtensionForFilestore(path_spec) 33 | } 34 | 35 | // Build an S3 key from a client upload request. 36 | func S3KeyForClientUpload( 37 | org_id string, request *uploads.UploadRequest) string { 38 | 39 | components := append([]string{"orgs", 40 | utils.NormalizedOrgId(org_id)}, 41 | S3ComponentsForClientUpload(request)...) 42 | 43 | // Support index files. 44 | return strings.Join(components, "/") 45 | } 46 | 47 | func S3ComponentsForClientUpload(request *uploads.UploadRequest) []string { 48 | base := []string{"clients", request.ClientId, "collections", 49 | request.SessionId, "uploads", request.Accessor} 50 | 51 | // Encode the client path in a safe way for s3 paths: 52 | // 1. S3 path are limited to 1024 bytes 53 | // 2. We do not need to go back from an S3 path to a client path 54 | // so we can safely use a one way hash function. 55 | 56 | client_path := strings.Join(request.Components, "\x00") 57 | h := sha256.New() 58 | h.Write([]byte(client_path)) 59 | 60 | file_name := fmt.Sprintf("%02x", h.Sum(nil)) 61 | 62 | // Support index files. 63 | if request.Type == "idx" { 64 | file_name += ".idx" 65 | } 66 | 67 | return append(base, file_name) 68 | } 69 | -------------------------------------------------------------------------------- /server/elastic.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/config" 7 | "www.velocidex.com/golang/cloudvelo/crypto/server" 8 | "www.velocidex.com/golang/cloudvelo/ingestion" 9 | 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 12 | "www.velocidex.com/golang/velociraptor/services" 13 | ) 14 | 15 | // Test with an elastic backend 16 | type ElasticBackend struct { 17 | ingestor ingestion.IngestorInterface 18 | } 19 | 20 | func NewMockElasticBackend(config_obj *config.Config) ( 21 | *ElasticBackend, error) { 22 | ingestor, err := ingestion.NewHTTPIngestor(config_obj) 23 | if err != nil { 24 | return nil, err 25 | } 26 | return &ElasticBackend{ingestor: ingestor}, nil 27 | } 28 | 29 | func NewElasticBackend( 30 | config_obj *config.Config, 31 | crypto_manager *server.ServerCryptoManager) ( 32 | *ElasticBackend, error) { 33 | ingestor, err := ingestion.NewIngestor(config_obj, crypto_manager) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &ElasticBackend{ingestor: ingestor}, nil 38 | } 39 | 40 | // For accepting messages FROM client to SERVER 41 | func (self ElasticBackend) Send( 42 | ctx context.Context, messages []*crypto_proto.VeloMessage) error { 43 | for _, msg := range messages { 44 | err := self.ingestor.Process(ctx, msg) 45 | if err != nil { 46 | return err 47 | } 48 | } 49 | return nil 50 | } 51 | 52 | // For accepting messages FROM server to CLIENT 53 | func (self ElasticBackend) Receive( 54 | ctx context.Context, client_id string, org_id string) ( 55 | message []*crypto_proto.VeloMessage, org_config_obj *config_proto.Config, err error) { 56 | 57 | org_manager, err := services.GetOrgManager() 58 | if err != nil { 59 | return nil, nil, err 60 | } 61 | 62 | org_config_obj, err = org_manager.GetOrgConfig(org_id) 63 | if err != nil { 64 | return nil, nil, err 65 | } 66 | 67 | client_info_manager, err := services.GetClientInfoManager(org_config_obj) 68 | if err != nil { 69 | return nil, nil, err 70 | } 71 | tasks, err := client_info_manager.GetClientTasks(ctx, client_id) 72 | return tasks, org_config_obj, err 73 | } 74 | -------------------------------------------------------------------------------- /filestore/reader.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/awserr" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/s3" 11 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 12 | "www.velocidex.com/golang/velociraptor/file_store/api" 13 | "www.velocidex.com/golang/velociraptor/vtesting" 14 | ) 15 | 16 | type S3Reader struct { 17 | session *session.Session 18 | downloader *s3manager.Downloader 19 | offset int64 20 | bucket string 21 | key string 22 | filename api.FSPathSpec 23 | } 24 | 25 | func (self *S3Reader) Read(buff []byte) (int, error) { 26 | defer Instrument("S3Reader.Read")() 27 | 28 | n, err := self.downloader.Download(aws.NewWriteAtBuffer(buff), 29 | &s3.GetObjectInput{ 30 | Bucket: aws.String(self.bucket), 31 | Key: aws.String(self.key), 32 | Range: aws.String( 33 | fmt.Sprintf("bytes=%d-%d", self.offset, 34 | self.offset+int64(len(buff)-1))), 35 | }) 36 | 37 | if err != nil { 38 | if aerr, ok := err.(awserr.Error); ok { 39 | switch aerr.Code() { 40 | case "InvalidRange": 41 | // Not really an error - this happens at the end of 42 | // the file, just return EOF 43 | return 0, io.EOF 44 | default: 45 | return 0, err 46 | } 47 | } 48 | } 49 | self.offset += n 50 | 51 | s3_counter_download.Add(float64(n)) 52 | 53 | return int(n), nil 54 | } 55 | 56 | func (self *S3Reader) Seek(offset int64, whence int) (int64, error) { 57 | self.offset = offset 58 | return self.offset, nil 59 | } 60 | 61 | func (self *S3Reader) Stat() (api.FileInfo, error) { 62 | defer Instrument("S3Reader.Read")() 63 | 64 | svc := s3.New(self.session) 65 | headObj := s3.HeadObjectInput{ 66 | Bucket: aws.String(self.bucket), 67 | Key: aws.String(self.key), 68 | } 69 | result, err := svc.HeadObject(&headObj) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | return &vtesting.MockFileInfo{ 75 | Name_: self.filename.Base(), 76 | PathSpec_: self.filename, 77 | Size_: aws.Int64Value(result.ContentLength), 78 | }, nil 79 | } 80 | 81 | func (self *S3Reader) Close() error { 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /artifact_definitions/Alert/PowerPickHostVersion.yaml: -------------------------------------------------------------------------------- 1 | name: Alert.Windows.EVTX.PowerPickHostVersion 2 | author: sbattaglia-r7 3 | description: | 4 | 5 | This artifact by itself only indicates that the PowerPick tool may have 6 | been invoked on the client. To capture additional context, ensure that 7 | Powershell script block and module logging are enabled on the clients and 8 | deploy the Windows.ETW.Powershell artifact from the Exchange. 9 | 10 | ----- 11 | 12 | This artifact is based on on PowerPick research by Crowdstrike in 13 | https://www.crowdstrike[.]com/blog/getting-the-bacon-from-cobalt-strike-beacon/ 14 | 15 | As noted in the blog post, when PowerPick tool is run, the PowerShell logs 16 | on the target system may contain an EID 400 event where the 17 | HostVersion and EngineVersion fields in the message have different values. 18 | 19 | In recent puprle team exercises, we observed that the mismatched HostVersion 20 | value was always "1.0", providing a simple way to monitor for this activity 21 | as a backup to other PowerShell or CobaltStrike rules. 22 | 23 | If this artifact generates an event on a client, check the PowerShell Operational 24 | logs for suspicious 410x events (especially 4104). If the Windows.ETW.Powershell 25 | artifact is also enabled on the client and did not fire an event, update that 26 | artifact's IOC list with the new information and redeploy it. 27 | 28 | 29 | # Can be CLIENT, CLIENT_EVENT, SERVER, SERVER_EVENT 30 | type: CLIENT_EVENT 31 | 32 | parameters: 33 | - name: pseventLog 34 | default: 'C:\Windows\System32\winevt\Logs\Windows PowerShell.evtx' 35 | 36 | sources: 37 | - precondition: 38 | SELECT OS From info() where OS = 'windows' 39 | 40 | query: | 41 | SELECT 42 | timestamp(epoch=int(int=System.TimeCreated.SystemTime)) AS EventTime, 43 | System.Computer as Computer, 44 | System.Channel as Channel, 45 | System.Provider.Name as Provider, 46 | System.EventID.Value as EventID, 47 | System.EventRecordID as EventRecordID, 48 | get(field="Message") as Message 49 | FROM watch_evtx(filename=pseventLog) 50 | WHERE EventID = 400 AND Message =~ 'HostVersion=1.0' 51 | -------------------------------------------------------------------------------- /bin/debug.go: -------------------------------------------------------------------------------- 1 | /* 2 | Velociraptor - Hunting Evil 3 | Copyright (C) 2019 Velocidex Innovations. 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU Affero General Public License as published 7 | by the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU Affero General Public License for more details. 14 | 15 | You should have received a copy of the GNU Affero General Public License 16 | along with this program. If not, see . 17 | */ 18 | package main 19 | 20 | import ( 21 | "fmt" 22 | "log" 23 | "net/http" 24 | _ "net/http/pprof" 25 | "regexp" 26 | 27 | "www.velocidex.com/golang/cloudvelo/services" 28 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 29 | logging "www.velocidex.com/golang/velociraptor/logging" 30 | debug_server "www.velocidex.com/golang/velociraptor/services/debug/server" 31 | ) 32 | 33 | var ( 34 | debug_flag = app.Flag("debug", "Enables debug and profile server.").Bool() 35 | debug_regex = app.Flag("debug_filter", "A regex to filter the debug source."). 36 | Default(".").String() 37 | debug_flag_port = app.Flag("debug_port", "Port for the debug server."). 38 | Default("6060").Int64() 39 | ) 40 | 41 | func initDebugServer(config_obj *config_proto.Config) error { 42 | if *debug_flag { 43 | logger := logging.GetLogger(config_obj, &logging.FrontendComponent) 44 | logger.Info("Starting debug server on http://127.0.0.1:%v/debug/pprof", *debug_flag_port) 45 | 46 | re, err := regexp.Compile("(?i)" + *debug_regex) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | services.SetDebugLogger(config_obj, re) 52 | 53 | // Switch off the debug flag so we do not run this again. (The 54 | // GUI runs this function multiple times). 55 | *debug_flag = false 56 | 57 | mux := debug_server.DebugMux(config_obj, "") 58 | 59 | go func() { 60 | log.Println(http.ListenAndServe( 61 | fmt.Sprintf("0.0.0.0:%d", *debug_flag_port), mux)) 62 | }() 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SERVER_CONFIG=./Docker/config/server.config.yaml 2 | CLIENT_CONFIG=./Docker/config/client.config.yaml 3 | OVERRIDE_FILE=./Docker/config/local_override.json 4 | BINARY=./output/cvelociraptor 5 | CONFIG_ARGS= --config $(SERVER_CONFIG) --override_file $(OVERRIDE_FILE) 6 | CLIENT_CONFIG_ARGS= --config $(CLIENT_CONFIG) --override_file $(OVERRIDE_FILE) 7 | DLV=dlv debug --init ./scripts/dlv.init --build-flags="-tags 'server_vql extras'" ./bin/ -- --debug --debug_filter result_set 8 | WRITEBACK_DIR=/tmp/pool_writebacks/ 9 | POOL_NUMBER=20 10 | 11 | all: 12 | go run make.go -v Auto 13 | 14 | debug_client: 15 | $(DLV) client -v $(CLIENT_CONFIG_ARGS) --debug --debug_port 6061 16 | 17 | .PHONY: client 18 | client: 19 | $(BINARY) client -v $(CLIENT_CONFIG_ARGS) 20 | 21 | pool_client: 22 | $(BINARY) pool_client -v $(CLIENT_CONFIG_ARGS) --writeback_dir $(WRITEBACK_DIR) --number $(POOL_NUMBER) 23 | 24 | debug_pool_client: 25 | $(DLV) pool_client -v $(CLIENT_CONFIG_ARGS) --writeback_dir $(WRITEBACK_DIR) --number $(POOL_NUMBER) 26 | 27 | gui: 28 | $(BINARY) $(CONFIG_ARGS) gui -v --debug 29 | 30 | dump: 31 | $(BINARY) $(CONFIG_ARGS) elastic dump -v 32 | 33 | dump_persisted: 34 | $(BINARY) $(CONFIG_ARGS) elastic dump --index="persisted" -v --dump_count 2000 35 | 36 | debug_gui: 37 | $(DLV) $(CONFIG_ARGS) gui -v 38 | 39 | frontend: 40 | $(BINARY) $(CONFIG_ARGS) frontend -v --debug 41 | 42 | frontend-Mock: 43 | $(BINARY) $(CONFIG_ARGS) frontend -v --debug --mock 44 | 45 | .PHONY: foreman 46 | foreman: 47 | $(BINARY) $(CONFIG_ARGS) foreman -v --debug 48 | 49 | debug_foreman: 50 | $(DLV) $(CONFIG_ARGS) foreman -v --debug 51 | 52 | debug_frontend: 53 | $(DLV) $(CONFIG_ARGS) frontend -v --debug 54 | 55 | debug_frontend-Mock: 56 | $(DLV) $(CONFIG_ARGS) frontend -v --debug --mock 57 | 58 | reset_elastic: 59 | $(BINARY) $(CONFIG_ARGS) elastic reset --recreate $(INDEX) 60 | 61 | windows: 62 | go run make.go -v Windows 63 | 64 | linux_m1: 65 | go run make.go -v LinuxM1 66 | 67 | linux_musl: 68 | go run make.go -v LinuxMusl 69 | 70 | docker: 71 | go run make.go -v DockerImage 72 | 73 | assets: 74 | go run make.go -v Assets 75 | 76 | test: 77 | go run make.go -v BareAssets 78 | go test -v ./foreman/ 79 | go test -v ./ingestion/ 80 | go test -v ./filestore/ 81 | go test -v ./services/... 82 | go test -v ./vql/... 83 | -------------------------------------------------------------------------------- /vql/server/notebook/delete.go: -------------------------------------------------------------------------------- 1 | package notebooks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Velocidex/ordereddict" 7 | "www.velocidex.com/golang/cloudvelo/services" 8 | "www.velocidex.com/golang/velociraptor/acls" 9 | "www.velocidex.com/golang/velociraptor/json" 10 | vql_subsystem "www.velocidex.com/golang/velociraptor/vql" 11 | "www.velocidex.com/golang/vfilter" 12 | "www.velocidex.com/golang/vfilter/arg_parser" 13 | 14 | _ "www.velocidex.com/golang/velociraptor/vql/server/notebooks" 15 | ) 16 | 17 | const ( 18 | all_notebook_items = ` 19 | {"query": { 20 | "bool": { 21 | "must": [ 22 | {"match": {"notebook_id": %q}} 23 | ]} 24 | }} 25 | ` 26 | ) 27 | 28 | type DeleteNotebookArgs struct { 29 | NotebookId string `vfilter:"required,field=notebook_id"` 30 | ReallyDoIt bool `vfilter:"optional,field=really_do_it"` 31 | } 32 | 33 | type DeleteNotebookPlugin struct{} 34 | 35 | func (self *DeleteNotebookPlugin) Call(ctx context.Context, 36 | scope vfilter.Scope, 37 | args *ordereddict.Dict) <-chan vfilter.Row { 38 | 39 | output_chan := make(chan vfilter.Row) 40 | 41 | go func() { 42 | defer close(output_chan) 43 | 44 | arg := &DeleteNotebookArgs{} 45 | 46 | err := vql_subsystem.CheckAccess(scope, acls.SERVER_ADMIN) 47 | if err != nil { 48 | scope.Log("notebook_delete: %s", err) 49 | return 50 | } 51 | 52 | err = arg_parser.ExtractArgsWithContext(ctx, scope, args, arg) 53 | if err != nil { 54 | scope.Log("notebook_delete: %s", err.Error()) 55 | return 56 | } 57 | 58 | config_obj, ok := vql_subsystem.GetServerConfig(scope) 59 | if !ok { 60 | scope.Log("notebook_delete: Command can only run on the server") 61 | return 62 | } 63 | 64 | if arg.ReallyDoIt { 65 | err := services.DeleteByQuery( 66 | ctx, config_obj.OrgId, "persisted", 67 | json.Format(all_notebook_items, arg.NotebookId)) 68 | if err != nil { 69 | scope.Log("notebook_delete: %v", err) 70 | } 71 | } 72 | 73 | }() 74 | 75 | return output_chan 76 | } 77 | 78 | func (self DeleteNotebookPlugin) Info( 79 | scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo { 80 | return &vfilter.PluginInfo{ 81 | Name: "notebook_delete", 82 | Doc: "Delete a notebook with all its cells. ", 83 | ArgType: type_map.AddType(scope, &DeleteNotebookArgs{}), 84 | } 85 | } 86 | 87 | func init() { 88 | vql_subsystem.OverridePlugin(&DeleteNotebookPlugin{}) 89 | } 90 | -------------------------------------------------------------------------------- /ingestion/flow_stats.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/schema/api" 7 | "www.velocidex.com/golang/cloudvelo/services" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 10 | flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" 11 | ) 12 | 13 | // When we receive a status we need to modify the collection record. 14 | func (self Ingestor) HandleFlowStats( 15 | ctx context.Context, 16 | config_obj *config_proto.Config, 17 | message *crypto_proto.VeloMessage) error { 18 | 19 | if message == nil || 20 | message.FlowStats == nil || 21 | message.Source == "" || 22 | message.SessionId == "" { 23 | return nil 24 | } 25 | 26 | msg := message.FlowStats 27 | 28 | collector_context := &flows_proto.ArtifactCollectorContext{ 29 | ClientId: message.Source, 30 | SessionId: message.SessionId, 31 | TotalUploadedFiles: msg.TotalUploadedFiles, 32 | TotalExpectedUploadedBytes: msg.TotalExpectedUploadedBytes, 33 | TotalUploadedBytes: msg.TotalUploadedBytes, 34 | TotalCollectedRows: msg.TotalCollectedRows, 35 | TotalLogs: msg.TotalLogs, 36 | ActiveTime: msg.Timestamp, 37 | QueryStats: msg.QueryStatus, 38 | } 39 | 40 | failed, completed := calcFlowOutcome(collector_context) 41 | 42 | // Progress messages are written to this doc id 43 | doc_id := api.GetDocumentIdForCollection( 44 | message.Source, message.SessionId, "stats") 45 | 46 | // Because we can not guarantee the order of messages written we 47 | // will write the final stat message to a different id. This 48 | // ensures it can not be overwritten with an incomplete status 49 | if completed { 50 | doc_id = api.GetDocumentIdForCollection( 51 | message.Source, message.SessionId, "completed") 52 | } 53 | 54 | stats := api.ArtifactCollectorRecordFromProto(collector_context, doc_id) 55 | stats.Type = "stats" 56 | 57 | // The status needs to hit the DB quickly, so the GUI can show 58 | // progress as the collection is received. The bulk data is still 59 | // stored asyncronously. 60 | err := services.SetElasticIndex(ctx, 61 | config_obj.OrgId, 62 | "transient", services.DocIdRandom, 63 | stats) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | return self.maybeHandleHuntFlowStats( 69 | ctx, config_obj, collector_context, failed, completed) 70 | } 71 | -------------------------------------------------------------------------------- /ingestion/registration.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "strings" 7 | 8 | actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" 9 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 10 | "www.velocidex.com/golang/velociraptor/json" 11 | "www.velocidex.com/golang/velociraptor/services" 12 | ) 13 | 14 | type ClientInfoUpdate struct { 15 | ClientId string `json:"client_id"` 16 | Hostname string `json:"hostname"` 17 | Release string `json:"release"` 18 | Architecture string `json:"architecture"` 19 | ClientVersion string `json:"client_version"` 20 | BuildTime string `json:"build_time"` 21 | System string `json:"system"` 22 | InstallTime uint64 `json:"install_time"` 23 | } 24 | 25 | // Register a new client - update the client record and update it's 26 | // client event table. NOTE: This happens automatically every time the 27 | // client starts up so we get to refresh the record each time. This 28 | // way there is no need to run an interrogation flow specifically - it 29 | // just happens automatically. 30 | func (self Ingestor) HandleClientInfoUpdates( 31 | ctx context.Context, 32 | message *crypto_proto.VeloMessage) error { 33 | if message == nil || message.VQLResponse == nil { 34 | return nil 35 | } 36 | 37 | reader := strings.NewReader(message.VQLResponse.JSONLResponse) 38 | scanner := bufio.NewScanner(reader) 39 | buf := make([]byte, len(message.VQLResponse.JSONLResponse)) 40 | scanner.Buffer(buf, len(message.VQLResponse.JSONLResponse)) 41 | 42 | org_manager, err := services.GetOrgManager() 43 | if err != nil { 44 | return err 45 | } 46 | 47 | org_config_obj, err := org_manager.GetOrgConfig(message.OrgId) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | client_info_manager, err := services.GetClientInfoManager(org_config_obj) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | for scanner.Scan() { 58 | serialized := scanner.Text() 59 | row := &ClientInfoUpdate{} 60 | err := json.Unmarshal([]byte(serialized), row) 61 | if err != nil { 62 | return err 63 | } 64 | err = client_info_manager.Set(ctx, 65 | &services.ClientInfo{actions_proto.ClientInfo{ 66 | ClientId: message.Source, 67 | Hostname: row.Hostname, 68 | Fqdn: row.Hostname, 69 | ClientVersion: row.ClientVersion, 70 | BuildTime: row.BuildTime, 71 | System: row.System, 72 | Architecture: row.Architecture, 73 | FirstSeenAt: row.InstallTime, 74 | }}) 75 | if err != nil { 76 | return err 77 | } 78 | } 79 | 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /startup/client.go: -------------------------------------------------------------------------------- 1 | package startup 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "www.velocidex.com/golang/cloudvelo/services/orgs" 8 | "www.velocidex.com/golang/cloudvelo/vql/uploads" 9 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 10 | crypto_utils "www.velocidex.com/golang/velociraptor/crypto/utils" 11 | "www.velocidex.com/golang/velociraptor/executor" 12 | "www.velocidex.com/golang/velociraptor/http_comms" 13 | "www.velocidex.com/golang/velociraptor/services" 14 | "www.velocidex.com/golang/velociraptor/services/writeback" 15 | "www.velocidex.com/golang/velociraptor/utils/tempfile" 16 | ) 17 | 18 | // StartClientServices starts the various services needed by the 19 | // client. 20 | func StartClientServices( 21 | ctx context.Context, 22 | config_obj *config_proto.Config, 23 | on_error func(ctx context.Context, 24 | config_obj *config_proto.Config)) (*services.Service, error) { 25 | 26 | // Create a suitable service plan. 27 | if config_obj.Frontend == nil { 28 | config_obj.Frontend = &config_proto.FrontendConfig{} 29 | } 30 | 31 | if config_obj.Services == nil { 32 | config_obj.Services = services.ClientServicesSpec() 33 | config_obj.Services.Launcher = true 34 | config_obj.Services.RepositoryManager = true 35 | } 36 | 37 | // Make sure the config crypto is ok. 38 | err := crypto_utils.VerifyConfig(config_obj) 39 | if err != nil { 40 | return nil, fmt.Errorf("Invalid config: %w", err) 41 | } 42 | 43 | tempfile.SetTempfile(config_obj) 44 | 45 | writeback_service := writeback.GetWritebackService() 46 | writeback, err := writeback_service.GetWriteback(config_obj) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | // Wait for all services to properly start 52 | // before we begin the comms. 53 | sm := services.NewServiceManager(ctx, config_obj) 54 | 55 | // Start the nanny first so we are covered from here on. 56 | err = sm.Start(executor.StartNannyService) 57 | if err != nil { 58 | return sm, err 59 | } 60 | 61 | _, err = orgs.NewClientOrgManager(sm.Ctx, sm.Wg, sm.Config) 62 | if err != nil { 63 | return sm, err 64 | } 65 | 66 | exe, err := executor.NewClientExecutor(ctx, writeback.ClientId, config_obj) 67 | if err != nil { 68 | return nil, fmt.Errorf("Can not create executor: %w", err) 69 | } 70 | 71 | comm, err := http_comms.StartHttpCommunicatorService( 72 | ctx, sm.Wg, config_obj, exe, on_error) 73 | if err != nil { 74 | return sm, err 75 | } 76 | 77 | err = uploads.InstallVeloCloudUploader( 78 | ctx, config_obj, nil, writeback.ClientId, comm.Manager) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return sm, nil 84 | } 85 | -------------------------------------------------------------------------------- /Docker/config/client.config.yaml: -------------------------------------------------------------------------------- 1 | version: 2 | name: velociraptor 3 | version: 0.6.4-rc4 4 | commit: f3264824 5 | build_time: "2022-04-14T02:23:05+10:00" 6 | Client: 7 | server_urls: 8 | - https://velociraptor-frontend:8000/ 9 | ca_certificate: | 10 | -----BEGIN CERTIFICATE----- 11 | MIIDTDCCAjSgAwIBAgIRAJH2OrT69FpC7IT3ZeZLmXgwDQYJKoZIhvcNAQELBQAw 12 | GjEYMBYGA1UEChMPVmVsb2NpcmFwdG9yIENBMB4XDTIxMDQxMzEwNDY1MVoXDTMx 13 | MDQxMTEwNDY1MVowGjEYMBYGA1UEChMPVmVsb2NpcmFwdG9yIENBMIIBIjANBgkq 14 | hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsLO3/Kq7RAwEhHrbsprrvCsE1rpOMQ6Q 15 | rJHM+0zZbxXchhrYEvi7W+Wae35ptAJehICmbIHwRhgCF2HSkTvNdVzSL9bUQT3Q 16 | XANxxXNrMW0grOJwQjFYBl8Bo+nv1CcJN7IF2vWcFpagfVHX2dPysfCwzzYX+Ai6 17 | OK5MqWwk22TJ5NWtUkH7+bMyS+hQbocr/BwKNWGdRlP/+BuUo6N99bVSXqw3gkz8 18 | FLYHVAKD2K4KaMlgfQtpgYeLKsebjUtKEub9LzJSgEdEFm2bG76LZPbKSGqBLwbv 19 | x+bJcn23vb4VJrWtbtB0GMxB1bHLTkWgD6PV6ejArClJPvDc9rDrOwIDAQABo4GM 20 | MIGJMA4GA1UdDwEB/wQEAwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH 21 | AwIwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUO2IRSDwqgkZt5pkXdScs5Bjo 22 | ULEwKAYDVR0RBCEwH4IdVmVsb2NpcmFwdG9yX2NhLnZlbG9jaWRleC5jb20wDQYJ 23 | KoZIhvcNAQELBQADggEBABRNDOPkGRp/ScFyS+SUY2etd1xLPXbX6R9zxy5AEIp7 24 | xEVSBcVnzGWH8Dqm2e4/3ZiV+IS5blrSQCfULwcBcaiiReyWXONRgnOMXKm/1omX 25 | aP7YUyRKIY+wASKUf4vbi+R1zTpXF4gtFcGDKcsK4uQP84ZtLKHw1qFSQxI7Ptfa 26 | WEhay5yjJwZoyiZh2JCdzUnuDkx2s9SoKi+CL80zRa2rqwYbr0HMepFZ0t83fIzt 27 | zNezVulkexf3I4keCaKkoT6nPqGd7SDOLhOQauesz7ECyr4m0yL4EekAsMceUvGi 28 | xdg66BlldhWSiEBcYmoNn5kmWNhV0AleVItxQkuWwbI= 29 | -----END CERTIFICATE----- 30 | nonce: rKNKAYam310= 31 | #nonce: O123Nonce 32 | writeback_darwin: /etc/velociraptor.writeback.yaml 33 | writeback_linux: /tmp/velociraptor.writeback.yaml 34 | writeback_windows: $ProgramFiles\Velociraptor\velociraptor.writeback.yaml 35 | min_poll: 2 36 | max_poll: 10 37 | windows_installer: 38 | service_name: Velociraptor 39 | install_path: $ProgramFiles\Velociraptor\Velociraptor.exe 40 | service_description: Velociraptor service 41 | darwin_installer: 42 | service_name: com.velocidex.velociraptor 43 | install_path: /usr/local/sbin/velociraptor 44 | version: 45 | name: velociraptor 46 | version: 0.6.4-rc4 47 | commit: f3264824 48 | build_time: "2022-04-14T02:23:05+10:00" 49 | pinned_server_name: VelociraptorServer 50 | max_upload_size: 5242880 51 | use_self_signed_ssl: true 52 | local_buffer: 53 | memory_size: 52428800 54 | disk_size: 1073741824 55 | filename_linux: /var/tmp/Velociraptor_Buffer.bin 56 | filename_windows: $TEMP/Velociraptor_Buffer.bin 57 | filename_darwin: /var/tmp/Velociraptor_Buffer.bin 58 | -------------------------------------------------------------------------------- /services/client_info/metadata.go: -------------------------------------------------------------------------------- 1 | package client_info 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Velocidex/ordereddict" 7 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 8 | "www.velocidex.com/golang/velociraptor/json" 9 | "www.velocidex.com/golang/velociraptor/utils" 10 | "www.velocidex.com/golang/vfilter" 11 | ) 12 | 13 | // We do not support custom metadata 14 | type MetadataEntry struct { 15 | ClientId string `json:"client_id"` 16 | Metadata string `json:"metadata"` 17 | } 18 | 19 | func (self ClientInfoManager) GetMetadata(ctx context.Context, 20 | client_id string) (*ordereddict.Dict, error) { 21 | 22 | result := ordereddict.NewDict() 23 | message, err := cvelo_services.GetElasticRecord(ctx, self.config_obj.OrgId, 24 | "persisted", client_id+"_metadata") 25 | if err != nil || len(message) == 0 { 26 | // Metadata is not there return an empty one 27 | return result, nil 28 | } 29 | 30 | entry := &MetadataEntry{} 31 | err = json.Unmarshal(message, entry) 32 | if err != nil { 33 | return result, nil 34 | } 35 | 36 | err = result.UnmarshalJSON([]byte(entry.Metadata)) 37 | if err != nil { 38 | return result, nil 39 | } 40 | 41 | return result, nil 42 | } 43 | 44 | func valueIsRemove(value vfilter.Any) bool { 45 | if utils.IsNil(value) { 46 | return true 47 | } 48 | 49 | value_str, ok := value.(string) 50 | if ok && value_str == "" { 51 | return true 52 | } 53 | 54 | return false 55 | } 56 | 57 | func (self ClientInfoManager) SetMetadata(ctx context.Context, 58 | client_id string, metadata *ordereddict.Dict, principal string) error { 59 | 60 | // Merge the old values with the new values 61 | old_metadata, err := self.GetMetadata(ctx, client_id) 62 | if err != nil { 63 | old_metadata = ordereddict.NewDict() 64 | } 65 | 66 | for _, k := range old_metadata.Keys() { 67 | new_value, pres := metadata.Get(k) 68 | if pres { 69 | // If the new metadata dict has an empty field then it 70 | // means to remove it from the old metadata. 71 | if valueIsRemove(new_value) { 72 | old_metadata.Delete(k) 73 | } else { 74 | old_metadata.Update(k, new_value) 75 | } 76 | } 77 | } 78 | 79 | // Add any missing fields 80 | for _, k := range metadata.Keys() { 81 | _, pres := old_metadata.Get(k) 82 | if !pres { 83 | value, _ := metadata.Get(k) 84 | old_metadata.Set(k, value) 85 | } 86 | } 87 | 88 | serialized, err := json.Marshal(old_metadata) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | return cvelo_services.SetElasticIndex(ctx, self.config_obj.OrgId, 94 | "persisted", client_id+"_metadata", &MetadataEntry{ 95 | ClientId: client_id, 96 | Metadata: string(serialized), 97 | }) 98 | } 99 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [pull_request] 3 | 4 | jobs: 5 | build: 6 | name: Tests 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Check out code into the Go module directory 10 | uses: actions/checkout@v2 11 | with: 12 | submodules: true 13 | 14 | - uses: actions/setup-go@v2 15 | with: 16 | go-version: '^1.20' 17 | - run: go version 18 | - name: Install Opensearch 19 | uses: ankane/setup-opensearch@v1 20 | with: 21 | opensearch-version: 3 22 | 23 | - name: Start LocalStack 24 | env: 25 | SERVICES: s3 26 | DEBUG: 1 27 | DATA_DIR: /tmp/localstack/data 28 | KINESIS_PROVIDER: kinesalite 29 | 30 | # These have to match up with the config in server.config.yaml 31 | AWS_ACCESS_KEY_ID: test 32 | AWS_SECRET_ACCESS_KEY: test 33 | AWS_DEFAULT_REGION: us-east-1 34 | 35 | run: | 36 | # install LocalStack cli and awslocal 37 | pip3 install cffi==1.15.1 cryptography==41.0.4 pyOpenSSL==23.2.0 pycparser==2.21 38 | pip3 install awscli==1.29.54 awscli-local==0.21 boto3==1.28.54 botocore==1.31.54 cachetools==5.0.0 dill==0.3.2 dnslib==0.9.23 dnspython==2.4.2 docutils==0.16 ecdsa==0.18.0 jmespath==1.0.1 localstack==2.2.0 localstack-client==2.3 localstack-core==2.2.0 localstack-ext==2.2.0 markdown-it-py==3.0.0 mdurl==0.1.2 pbr==5.11.1 plux==1.4.0 psutil==5.9.5 pyaes==1.6.1 pygments==2.16.1 python-dateutil==2.8.2 python-dotenv==1.0.0 python-jose==3.3.0 rich==13.5.3 rsa==4.7.2 s3transfer==0.6.2 semver==3.0.1 stevedore==5.1.0 tabulate==0.9.0 tailer==0.4.1 39 | 40 | # Make sure to pull the latest version of the image 41 | docker pull localstack/localstack:2.2.0 42 | 43 | # Start LocalStack in the background 44 | localstack start -d 45 | # Wait 30 seconds for the LocalStack container to become ready before timing out 46 | echo "Waiting for LocalStack startup..." 47 | localstack wait -t 30 48 | echo "Startup complete" 49 | - name: Initialize localstack 50 | run: | 51 | awslocal s3 mb s3://velociraptor 52 | awslocal s3 ls 53 | echo "Test Execution complete!" 54 | 55 | - name: Run tests 56 | run: | 57 | make test 58 | 59 | - name: Upload Build Artifacts 60 | if: ${{ failure() }} 61 | shell: bash 62 | run: | 63 | mkdir -p artifact_output/ 64 | go test -v ./vql/uploads/ ./foreman/ ./ingestion/ -update 65 | cp -a ./vql/uploads/fixtures/* ./foreman/fixtures/* ./ingestion/fixtures/* artifact_output/ 66 | 67 | - uses: actions/upload-artifact@master 68 | if: ${{ failure() }} 69 | with: 70 | name: artifact 71 | path: artifact_output 72 | -------------------------------------------------------------------------------- /result_sets/simple/metadata.go: -------------------------------------------------------------------------------- 1 | package simple 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/services" 7 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | "www.velocidex.com/golang/velociraptor/file_store/api" 10 | "www.velocidex.com/golang/velociraptor/json" 11 | "www.velocidex.com/golang/velociraptor/utils" 12 | ) 13 | 14 | const ( 15 | // Get the latest metadata record with the most recent timestamp. 16 | result_md_query = ` 17 | { 18 | "query": {"bool": {"must": [ 19 | {"match": {"type": "rs_metadata"}}, 20 | {"match": {"vfs_path": %q}} 21 | ]}}, 22 | "sort": {"timestamp": "desc"}, 23 | "size": 1 24 | } 25 | ` 26 | ) 27 | 28 | type ResultSetMetadataRecord struct { 29 | Timestamp int64 `json:"timestamp"` 30 | VFSPath string `json:"vfs_path"` 31 | ID string `json:"id"` 32 | EndRow int64 `json:"end_row"` 33 | TotalRows int64 `json:"total_rows"` 34 | Type string `json:"type"` 35 | } 36 | 37 | // Because we can not delete result sets in the transient index we 38 | // need to attach a version to each result set. When we need to 39 | // truncate the result set we just increase the version. This makes 40 | // opening and closing result sets a bit slower as it adds one round 41 | // trip. 42 | func GetResultSetMetadata( 43 | ctx context.Context, 44 | config_obj *config_proto.Config, 45 | log_path api.FSPathSpec) (*ResultSetMetadataRecord, error) { 46 | 47 | cvelo_services.Count("GetResultSetMetadata") 48 | cvelo_services.Debug( 49 | cvelo_services.DEBUG_RESULT_SET, 50 | "GetResultSetMetadata: %v", log_path.AsClientPath())() 51 | 52 | base_record := NewSimpleResultSetRecord(log_path, "") 53 | 54 | query := json.Format(result_md_query, base_record.VFSPath) 55 | 56 | hits, _, err := cvelo_services.QueryElasticRaw(ctx, utils.GetOrgId(config_obj), 57 | "transient", query) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | // If there is no metadata record we just create one with an empty 63 | // ID. This should be used to support legacy result sets without a 64 | // metadata record. 65 | if len(hits) == 0 { 66 | return &ResultSetMetadataRecord{ 67 | Timestamp: 0, 68 | VFSPath: base_record.VFSPath, 69 | ID: "", 70 | Type: "rs_metadata", 71 | }, nil 72 | } 73 | 74 | record := &ResultSetMetadataRecord{} 75 | err = json.Unmarshal(hits[0], &record) 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | return record, nil 81 | } 82 | 83 | func SetResultSetMetadata( 84 | ctx context.Context, 85 | config_obj *config_proto.Config, 86 | log_path api.FSPathSpec, md *ResultSetMetadataRecord) error { 87 | 88 | md.Timestamp = utils.GetTime().Now().UnixNano() 89 | return cvelo_services.SetElasticIndex(ctx, utils.GetOrgId(config_obj), 90 | "transient", services.DocIdRandom, md) 91 | } 92 | -------------------------------------------------------------------------------- /ingestion/fixtures/TestClientEventMonitoring.golden: -------------------------------------------------------------------------------- 1 | { 2 | "Generic.Client.Stats Results": [ 3 | { 4 | "client_id": "C.1352adc54e292a23", 5 | "flow_id": "F.Monitoring", 6 | "artifact": "Generic.Client.Stats", 7 | "type": "logs", 8 | "timestamp": 1661385600000000000, 9 | "date": 1661385600, 10 | "vfs_path": "/clients/C.1352adc54e292a23/monitoring_logs/Generic.Client.Stats/2022-08-25.json", 11 | "data": "{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Starting query execution for Generic.Client.Stats.\\n\"}\n{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Starting query execution for Generic.Client.Stats.\\n\"}\n{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Generic.Client.Stats: Skipping query due to preconditions\\n\"}\n{\"client_time\":1676476473,\"level\":\"INFO\",\"message\":\"Collection Generic.Client.Stats is done after 12.258991ms\\n\"}\n{\"client_time\":1676476473,\"level\":\"DEBUG\",\"message\":\"Query Stats: {\\\"RowsScanned\\\":1,\\\"PluginsCalled\\\":1,\\\"FunctionsCalled\\\":0,\\\"ProtocolSearch\\\":0,\\\"ScopeCopy\\\":4}\\n\"}\n" 12 | }, 13 | { 14 | "client_id": "C.1352adc54e292a23", 15 | "flow_id": "F.Monitoring", 16 | "artifact": "Generic.Client.Stats", 17 | "type": "results", 18 | "timestamp": 1661385600000000000, 19 | "date": 1661385600, 20 | "vfs_path": "/clients/C.1352adc54e292a23/monitoring/Generic.Client.Stats/2022-08-25.json", 21 | "data": "{\"Timestamp\":1664246379.037654,\"CPU\":5.56,\"RSS\":83447808,\"CPUPercent\":null,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246389.0389478,\"CPU\":5.93,\"RSS\":72527872,\"CPUPercent\":0.036995213198136666,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246399.0391533,\"CPU\":6,\"RSS\":73875456,\"CPUPercent\":0.006999856141185939,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246409.0397468,\"CPU\":6.12,\"RSS\":72306688,\"CPUPercent\":0.011999287933643665,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246419.040297,\"CPU\":6.18,\"RSS\":73654272,\"CPUPercent\":0.005999669856118451,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246429.0415347,\"CPU\":6.26,\"RSS\":65314816,\"CPUPercent\":0.007999010017847958,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246439.0423741,\"CPU\":6.36,\"RSS\":62390272,\"CPUPercent\":0.009999160598648414,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246449.0437052,\"CPU\":6.42,\"RSS\":62619648,\"CPUPercent\":0.005999201451737721,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246459.0448549,\"CPU\":6.5,\"RSS\":64479232,\"CPUPercent\":0.007999080382213527,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246469.0458136,\"CPU\":6.619999999999999,\"RSS\":61812736,\"CPUPercent\":0.011998849692949804,\"ClientId\":\"C.1352adc54e292a23\"}\n{\"Timestamp\":1664246479.047109,\"CPU\":6.68,\"RSS\":63152128,\"CPUPercent\":0.005999222903775355,\"ClientId\":\"C.1352adc54e292a23\"}\n" 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /schema/templates/transient.json: -------------------------------------------------------------------------------- 1 | { 2 | "index_patterns": [ 3 | "*transient" 4 | ], 5 | "data_stream": { 6 | "timestamp_field": { 7 | "name": "timestamp" 8 | } 9 | }, 10 | "priority": 100, 11 | "template": { 12 | "settings": { 13 | "number_of_shards": 1, 14 | "number_of_replicas": 1 15 | }, 16 | "mappings": { 17 | "dynamic": false, 18 | "properties": { 19 | "session_id": { 20 | "type": "keyword" 21 | }, 22 | "raw": { 23 | "type": "binary" 24 | }, 25 | "tasks": { 26 | "type": "binary" 27 | }, 28 | "id": { 29 | "type": "keyword" 30 | }, 31 | "hunt_id": { 32 | "type": "keyword" 33 | }, 34 | "notebook_id": { 35 | "type": "keyword" 36 | }, 37 | "client_id": { 38 | "type": "keyword" 39 | }, 40 | "components": { 41 | "type": "keyword" 42 | }, 43 | "downloads": { 44 | "type": "binary" 45 | }, 46 | "type": { 47 | "type": "keyword" 48 | }, 49 | "doc_id": { 50 | "type": "keyword" 51 | }, 52 | "doc_type": { 53 | "type": "keyword" 54 | }, 55 | "artifact": { 56 | "type": "keyword" 57 | }, 58 | "flow_id": { 59 | "type": "keyword" 60 | }, 61 | "data": { 62 | "type": "binary" 63 | }, 64 | "start_row": { 65 | "type": "long" 66 | }, 67 | "end_row": { 68 | "type": "long" 69 | }, 70 | "total_rows": { 71 | "type": "long" 72 | }, 73 | "timestamp": { 74 | "type": "long" 75 | }, 76 | "date": { 77 | "type": "long" 78 | }, 79 | "vfs_path": { 80 | "type": "keyword" 81 | }, 82 | "title": { 83 | "type": "keyword" 84 | }, 85 | "correlationId": { 86 | "type": "keyword" 87 | }, 88 | "is_dispatched": { 89 | "type": "boolean" 90 | }, 91 | "key": { 92 | "type": "keyword" 93 | } 94 | } 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /ingestion/monitoring_logs.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "context" 5 | 6 | "www.velocidex.com/golang/cloudvelo/result_sets/timed" 7 | "www.velocidex.com/golang/velociraptor/artifacts" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 10 | "www.velocidex.com/golang/velociraptor/json" 11 | "www.velocidex.com/golang/velociraptor/paths" 12 | artifact_paths "www.velocidex.com/golang/velociraptor/paths/artifacts" 13 | "www.velocidex.com/golang/velociraptor/utils" 14 | ) 15 | 16 | func (self Ingestor) HandleMonitoringLogs( 17 | ctx context.Context, config_obj *config_proto.Config, 18 | message *crypto_proto.VeloMessage) error { 19 | 20 | row := message.LogMessage 21 | artifact_name := artifacts.DeobfuscateString( 22 | config_obj, row.Artifact) 23 | 24 | // Suppress logging of some artifacts 25 | switch artifact_name { 26 | 27 | // Automatically interrogate this client. 28 | case "Client.Info.Updates": 29 | return nil 30 | } 31 | 32 | new_json_response := artifacts.DeobfuscateString(config_obj, row.Jsonl) 33 | 34 | log_path_manager := artifact_paths.NewArtifactLogPathManagerWithMode( 35 | config_obj, message.Source, message.SessionId, artifact_name, 36 | paths.MODE_CLIENT_EVENT) 37 | 38 | rs_writer, err := timed.NewTimedResultSetWriter( 39 | config_obj, log_path_manager, json.DefaultEncOpts(), 40 | utils.BackgroundWriter) 41 | if err != nil { 42 | return err 43 | } 44 | defer rs_writer.Close() 45 | 46 | rs_writer.WriteJSONL([]byte(new_json_response), int(row.NumberOfRows)) 47 | 48 | return nil 49 | } 50 | 51 | func (self Ingestor) HandleMonitoringResponses( 52 | ctx context.Context, config_obj *config_proto.Config, 53 | message *crypto_proto.VeloMessage) error { 54 | 55 | // Ignore messages without a destination Artifact 56 | if message.VQLResponse == nil || message.VQLResponse.Query == nil || 57 | message.VQLResponse.Query.Name == "" { 58 | return nil 59 | } 60 | 61 | response := message.VQLResponse 62 | artifacts.Deobfuscate(config_obj, response) 63 | 64 | // Handle special types of responses 65 | switch message.VQLResponse.Query.Name { 66 | 67 | // Automatically interrogate this client. 68 | case "Server.Internal.ClientInfo": 69 | return self.HandleClientInfoUpdates(ctx, message) 70 | } 71 | 72 | // Add the client id on the end of the record 73 | new_json_response := json.AppendJsonlItem( 74 | []byte(message.VQLResponse.JSONLResponse), "ClientId", message.Source) 75 | 76 | path_manager := artifact_paths.NewArtifactPathManagerWithMode( 77 | config_obj, message.Source, 78 | message.SessionId, message.VQLResponse.Query.Name, 79 | paths.MODE_CLIENT_EVENT) 80 | 81 | rs_writer, err := timed.NewTimedResultSetWriter( 82 | config_obj, path_manager, json.DefaultEncOpts(), 83 | utils.BackgroundWriter) 84 | if err != nil { 85 | return err 86 | } 87 | defer rs_writer.Close() 88 | 89 | rs_writer.WriteJSONL(new_json_response, int(message.VQLResponse.TotalRows)) 90 | 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /services/notebook/annotator.go: -------------------------------------------------------------------------------- 1 | package notebook 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/Velocidex/ordereddict" 8 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 9 | "www.velocidex.com/golang/velociraptor/constants" 10 | "www.velocidex.com/golang/velociraptor/services/notebook" 11 | "www.velocidex.com/golang/velociraptor/timelines" 12 | timelines_proto "www.velocidex.com/golang/velociraptor/timelines/proto" 13 | "www.velocidex.com/golang/velociraptor/utils" 14 | "www.velocidex.com/golang/vfilter" 15 | ) 16 | 17 | type SuperTimelineAnnotator struct { 18 | config_obj *config_proto.Config 19 | SuperTimelineStorer timelines.ISuperTimelineStorer 20 | SuperTimelineWriter timelines.ISuperTimelineWriter 21 | } 22 | 23 | func (self *SuperTimelineAnnotator) AnnotateTimeline( 24 | ctx context.Context, scope vfilter.Scope, 25 | notebook_id string, supertimeline string, 26 | message, principal string, 27 | timestamp time.Time, event *ordereddict.Dict) error { 28 | 29 | timeline, err := self.SuperTimelineStorer.GetTimeline(ctx, notebook_id, 30 | supertimeline, constants.TIMELINE_ANNOTATION) 31 | if err != nil { 32 | timeline = &timelines_proto.Timeline{ 33 | Id: constants.TIMELINE_ANNOTATION, 34 | } 35 | } 36 | 37 | guid, pres := event.GetString(notebook.AnnotationID) 38 | if !pres { 39 | guid = notebook.GetGUID() 40 | } 41 | 42 | writer := &TimelineWriter{ 43 | ctx: ctx, 44 | config_obj: self.config_obj, 45 | stats: timeline, 46 | SuperTimelineStorer: self.SuperTimelineStorer, 47 | notebook_id: notebook_id, 48 | super_timeline: supertimeline, 49 | timeline: timeline.Id, 50 | } 51 | 52 | event.Set("Message", message) 53 | defer writer.Close() 54 | 55 | // An empty timestamp means to delete the event. 56 | if timestamp.Unix() < 10 { 57 | original_timestamp, ok := event.Get(notebook.AnnotationOGTime) 58 | if !ok { 59 | return utils.Wrap(utils.InvalidArgError, "Original Timestamp not provided for deletion event") 60 | } 61 | 62 | timestamp, ok = original_timestamp.(time.Time) 63 | if !ok { 64 | return utils.Wrap(utils.InvalidArgError, "Original Timestamp invalid for deletion event") 65 | } 66 | 67 | // Subtrace 1 sec from the timestamp so we can detect the 68 | // deletion event. 69 | timestamp = timestamp.Add(-time.Second) 70 | event.Set("Deletion", true) 71 | 72 | } else { 73 | event.Update(constants.TIMELINE_DEFAULT_KEY, timestamp). 74 | Set("Notes", message). 75 | Set(notebook.AnnotatedBy, principal). 76 | Set(notebook.AnnotatedAt, utils.GetTime().Now()). 77 | Set(notebook.AnnotationID, guid) 78 | } 79 | 80 | return writer.Write(timestamp, event) 81 | } 82 | 83 | func NewSuperTimelineAnnotator( 84 | config_obj *config_proto.Config, 85 | SuperTimelineStorer timelines.ISuperTimelineStorer) timelines.ISuperTimelineAnnotator { 86 | return &SuperTimelineAnnotator{ 87 | config_obj: config_obj, 88 | SuperTimelineStorer: SuperTimelineStorer, 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /artifact_definitions/Windows.EventLogs.Bitsadmin.yaml: -------------------------------------------------------------------------------- 1 | name: Windows.EventLogs.Bitsadmin 2 | author: "Matt Green - @mgreen27" 3 | description: | 4 | This content will extract BITS Transfer events and enable filtering by URL 5 | and TLD. 6 | 7 | reference: 8 | - https://attack.mitre.org/techniques/T1197/ 9 | - https://mgreen27.github.io/posts/2018/02/18/Sharing_my_BITS.html 10 | 11 | parameters: 12 | - name: EventLog 13 | default: C:\Windows\System32\winevt\Logs\Microsoft-Windows-Bits-Client%4Operational.evtx 14 | - name: TldAllowListRegex 15 | description: TLD allow list regex - anchor TLD - e.g live.com 16 | default: '(office365|dell|live|mozilla|sun|adobe|onenote|microsoft|windowsupdate|google|oracle|hp)\.(net|com|(|\.au))|\.(office\.net|sentinelone\.net|connectwise.net)|(oneclient\.sfx|aka)\.ms|(10|192)\.d{1,3}\.\d{1,3}\.\d{1,3}' 17 | - name: UrlAllowListRegex 18 | description: Secondary whitelist regex. Used for Url 19 | 20 | sources: 21 | - precondition: 22 | SELECT OS From info() where OS = 'windows' 23 | 24 | query: | 25 | -- Find Files in scope 26 | LET files = SELECT * FROM glob(globs=EventLog) 27 | 28 | LET results = SELECT * FROM foreach( 29 | row=files, 30 | query={ 31 | SELECT 32 | timestamp(epoch=int(int=System.TimeCreated.SystemTime)) AS EventTime, 33 | System.Computer as Computer, 34 | System.EventID.Value as EventId, 35 | System.Security.UserID as UserId, 36 | EventData.transferId as TransferId, 37 | EventData.name as Name, 38 | EventData.id as Id, 39 | EventData.url as Url, 40 | url(parse=EventData.url).Host AS TLD, 41 | EventData.peer as Peer, 42 | timestamp(epoch=EventData.fileTime) as FileTime, 43 | EventData.fileLength as fileLength, 44 | EventData.bytesTotal as bytesTotal, 45 | EventData.bytesTransferred as bytesTransferred, 46 | EventData.bytesTransferredFromPeer 47 | FROM parse_evtx(filename=OSPath) 48 | WHERE 49 | EventId = 59 50 | AND NOT if( condition= TldAllowListRegex, 51 | then= TLD =~ TldAllowListRegex, 52 | else= FALSE) 53 | AND NOT if( condition= UrlAllowListRegex, 54 | then= Url =~ UrlAllowListRegex, 55 | else= FALSE) 56 | }) 57 | 58 | SELECT * FROM results 59 | 60 | notebook: 61 | - type: vql_suggestion 62 | name: Stack rank by TLD 63 | template: | 64 | /* 65 | ## TLD stacking - find potential to add to Ignore regex and triage low counts 66 | */ 67 | SELECT TLD,count() as TldTotal, 68 | Url as UrlExample 69 | FROM source(artifact="Windows.EventLogs.Bitsadmin") 70 | GROUP BY TLD 71 | ORDER BY TldTotal 72 | -------------------------------------------------------------------------------- /filestore/s3filestore_test.go: -------------------------------------------------------------------------------- 1 | package filestore_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/aws/aws-sdk-go/service/s3" 10 | "github.com/stretchr/testify/suite" 11 | "www.velocidex.com/golang/cloudvelo/filestore" 12 | "www.velocidex.com/golang/cloudvelo/testsuite" 13 | "www.velocidex.com/golang/velociraptor/file_store" 14 | "www.velocidex.com/golang/velociraptor/file_store/path_specs" 15 | "www.velocidex.com/golang/velociraptor/vtesting/assert" 16 | ) 17 | 18 | type S3FilestoreTest struct { 19 | *testsuite.CloudTestSuite 20 | } 21 | 22 | func (self *S3FilestoreTest) checkForKey() []string { 23 | // Check the actual path in the bucket we are in. 24 | session, err := filestore.GetS3Session(self.ConfigObj) 25 | assert.NoError(self.T(), err) 26 | 27 | svc := s3.New(session) 28 | res, err := svc.ListObjects(&s3.ListObjectsInput{ 29 | Bucket: &self.ConfigObj.Cloud.Bucket, 30 | }) 31 | assert.NoError(self.T(), err) 32 | 33 | result := []string{} 34 | for _, c := range res.Contents { 35 | if c.Key != nil && strings.Contains(*c.Key, "file") { 36 | result = append(result, *c.Key) 37 | } 38 | } 39 | 40 | return result 41 | } 42 | 43 | func (self *S3FilestoreTest) TestS3FileWriting() { 44 | config_obj := self.ConfigObj.VeloConf() 45 | file_store_factory := file_store.GetFileStore(config_obj) 46 | assert.NotNil(self.T(), file_store_factory) 47 | 48 | test_file := path_specs.NewUnsafeFilestorePath("Test", "file") 49 | writer, err := file_store_factory.WriteFile(test_file) 50 | assert.NoError(self.T(), err) 51 | 52 | err = writer.Truncate() 53 | assert.NoError(self.T(), err) 54 | 55 | _, err = writer.Write([]byte("hello")) 56 | assert.NoError(self.T(), err) 57 | 58 | writer.Close() 59 | 60 | // Read the file back 61 | reader, err := file_store_factory.ReadFile(test_file) 62 | assert.NoError(self.T(), err) 63 | 64 | data, err := ioutil.ReadAll(reader) 65 | assert.NoError(self.T(), err) 66 | 67 | assert.Equal(self.T(), "hello", string(data)) 68 | 69 | // Read small ranges 70 | reader.Seek(1, os.SEEK_SET) 71 | buff := make([]byte, 2) 72 | _, err = reader.Read(buff) 73 | assert.NoError(self.T(), err) 74 | 75 | // A partial range read 76 | assert.Equal(self.T(), "el", string(buff)) 77 | 78 | // Make sure the underlying key name reflects the org name in it 79 | keys := self.checkForKey() 80 | assert.Equal(self.T(), 1, len(keys)) 81 | assert.Equal(self.T(), "orgs/test/Test/file.json", keys[0]) 82 | 83 | // Now delete the file. 84 | err = file_store_factory.Delete(test_file) 85 | assert.NoError(self.T(), err) 86 | 87 | reader, err = file_store_factory.ReadFile(test_file) 88 | assert.NoError(self.T(), err) 89 | 90 | // No data is available 91 | data, err = ioutil.ReadAll(reader) 92 | assert.Error(self.T(), err) 93 | 94 | } 95 | 96 | func TestS3Filestore(t *testing.T) { 97 | suite.Run(t, &S3FilestoreTest{ 98 | CloudTestSuite: &testsuite.CloudTestSuite{ 99 | Indexes: []string{"persisted"}, 100 | }, 101 | }) 102 | } 103 | -------------------------------------------------------------------------------- /vql/uploads/sparse.go: -------------------------------------------------------------------------------- 1 | package uploads 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "crypto/sha256" 7 | "encoding/hex" 8 | 9 | "www.velocidex.com/golang/velociraptor/accessors" 10 | actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" 11 | "www.velocidex.com/golang/velociraptor/json" 12 | "www.velocidex.com/golang/velociraptor/uploads" 13 | ) 14 | 15 | func UploadSparse( 16 | ctx context.Context, 17 | ospath *accessors.OSPath, 18 | idx_uploader CloudUploader, 19 | uploader CloudUploader, 20 | range_reader uploads.RangeReader) (*uploads.UploadResponse, error) { 21 | 22 | index := &actions_proto.Index{} 23 | 24 | // This is the response that will be passed into the VQL 25 | // engine. 26 | result := &uploads.UploadResponse{ 27 | Path: ospath.String(), 28 | } 29 | 30 | md5_sum := md5.New() 31 | sha_sum := sha256.New() 32 | 33 | // Does the index contain any sparse runs? 34 | is_sparse := false 35 | 36 | // Adjust the expected size properly to the sum of all 37 | // non-sparse ranges and build the index file. 38 | ranges := range_reader.Ranges() 39 | 40 | // Inspect the ranges and prepare an index. 41 | expected_size := int64(0) 42 | real_size := int64(0) 43 | for _, rng := range ranges { 44 | file_length := rng.Length 45 | if rng.IsSparse { 46 | file_length = 0 47 | } 48 | 49 | index.Ranges = append(index.Ranges, 50 | &actions_proto.Range{ 51 | FileOffset: expected_size, 52 | OriginalOffset: rng.Offset, 53 | FileLength: file_length, 54 | Length: rng.Length, 55 | }) 56 | 57 | if !rng.IsSparse { 58 | expected_size += rng.Length 59 | } else { 60 | is_sparse = true 61 | } 62 | 63 | if real_size < rng.Offset+rng.Length { 64 | real_size = rng.Offset + rng.Length 65 | } 66 | } 67 | 68 | // We need to buffer writes until they reach 5mb before we can 69 | // send them. This is managed by the BufferedWriter object which 70 | // wraps the uploader. 71 | buffer := NewBufferWriter(uploader) 72 | defer buffer.Close() 73 | 74 | for _, rng := range ranges { 75 | if rng.IsSparse { 76 | continue 77 | } 78 | 79 | _, err := range_reader.Seek(rng.Offset, 0) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | err = buffer.Copy(range_reader, uint64(rng.Length)) 85 | if err != nil { 86 | return nil, err 87 | } 88 | } 89 | 90 | // If we are sparse upload the sparse file in one part. 91 | if is_sparse { 92 | serialized, err := json.Marshal(index) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | err = idx_uploader.Put(serialized) 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | idx_uploader.Commit() 103 | 104 | // Set the index on the actual uploader 105 | uploader.SetIndex(index) 106 | } 107 | 108 | result.Size = uint64(real_size) 109 | 110 | // The actual amount of bytes uploaded 111 | result.StoredSize = buffer.total 112 | result.Sha256 = hex.EncodeToString(sha_sum.Sum(nil)) 113 | result.Md5 = hex.EncodeToString(md5_sum.Sum(nil)) 114 | 115 | return result, nil 116 | } 117 | -------------------------------------------------------------------------------- /datastore/datastoretest/datastore_test.go: -------------------------------------------------------------------------------- 1 | package datastoretest 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/stretchr/testify/suite" 7 | "os" 8 | "testing" 9 | "www.velocidex.com/golang/cloudvelo/datastore" 10 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 11 | "www.velocidex.com/golang/cloudvelo/testsuite" 12 | "www.velocidex.com/golang/velociraptor/json" 13 | "www.velocidex.com/golang/velociraptor/utils" 14 | ) 15 | 16 | const ( 17 | get_datastore_doc_query = ` 18 | {"sort": {"timestamp": {"order": "desc"}}, 19 | "size": 1, 20 | "query": { 21 | "bool": { 22 | "must": [ 23 | { 24 | "prefix": { 25 | "id": %q 26 | } 27 | }, 28 | { 29 | "match": { 30 | "doc_type": "datastore" 31 | } 32 | } 33 | ] 34 | } 35 | } 36 | }` 37 | ) 38 | 39 | type DatastoreTest struct { 40 | *testsuite.CloudTestSuite 41 | ctx context.Context 42 | } 43 | 44 | func (self *DatastoreTest) TestDownloadQueriesOnTransientIndex() { 45 | var vfs_path = "/downloads/hunts/H.CP12M5IRRKRUE/H.CP12M5IRRKRUE.json.db" 46 | var serialized = "\"{\"timestamp\":1715614088,\"components\":[\"downloads\",\"hunts\",\"H.CP12M5IRRKRUE\",\"H.CP12M5IRRKRUE.zip\"],\"type\":\"zip\"}\"" 47 | record := datastore.DatastoreRecord{ 48 | ID: cvelo_services.MakeId(vfs_path), 49 | Type: "Generic", 50 | VFSPath: vfs_path, 51 | JSONData: serialized, 52 | DocType: "datastore", 53 | Timestamp: utils.GetTime().Now().UnixNano(), 54 | } 55 | 56 | err := cvelo_services.SetElasticIndex(self.ctx, 57 | "test", "transient", "", record) 58 | 59 | serialized = "{\"timestamp\":1715614088,\"total_uncompressed_bytes\":21375,\"total_compressed_bytes\":20054,\"total_container_files\":18,\"hash\":\"095079e35e17c37bd5ac6f602a15173697f404a2ac9ea4b1f54c653ab25706e2\",\"total_duration\":1,\"components\":[\"downloads\",\"hunts\",\"H.CP12M5IRRKRUE\",\"H.CP12M5IRRKRUE.zip\"],\"type\":\"zip\"}" 60 | record = datastore.DatastoreRecord{ 61 | ID: cvelo_services.MakeId(vfs_path), 62 | Type: "Generic", 63 | VFSPath: vfs_path, 64 | JSONData: serialized, 65 | DocType: "datastore", 66 | Timestamp: utils.GetTime().Now().UnixNano(), 67 | } 68 | err = cvelo_services.SetElasticIndex(self.ctx, 69 | "test", "transient", "", record) 70 | 71 | id := cvelo_services.MakeId(vfs_path) 72 | hit, err := cvelo_services.GetElasticRecord( 73 | self.ctx, "test", "transient", id) 74 | assert.Nil(self.T(), hit) 75 | if assert.Error(self.T(), err) { 76 | assert.Equal(self.T(), os.ErrNotExist, err) 77 | } 78 | result, _, err := cvelo_services.QueryElasticRaw(self.ctx, "test", 79 | "transient", json.Format(get_datastore_doc_query, cvelo_services.MakeId(vfs_path))) 80 | 81 | assert.NoError(self.T(), err) 82 | assert.Equal(self.T(), 1, len(result)) 83 | } 84 | func TestDataStore(t *testing.T) { 85 | suite.Run(t, &DatastoreTest{ 86 | CloudTestSuite: &testsuite.CloudTestSuite{ 87 | Indexes: []string{"transient"}, 88 | }, 89 | }) 90 | } 91 | -------------------------------------------------------------------------------- /services/indexing/names.go: -------------------------------------------------------------------------------- 1 | // Search functions for name only searches (These are used for the 2 | // quick suggestion box). 3 | 4 | package indexing 5 | 6 | import ( 7 | "context" 8 | "strings" 9 | 10 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 11 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 12 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 13 | "www.velocidex.com/golang/velociraptor/json" 14 | ) 15 | 16 | func (self *Indexer) searchClientNameOnly( 17 | ctx context.Context, 18 | config_obj *config_proto.Config, 19 | in *api_proto.SearchClientsRequest) (*api_proto.SearchClientsResponse, error) { 20 | 21 | operator, term := splitIntoOperatorAndTerms(in.Query) 22 | if term == "*" { 23 | term = "" 24 | } 25 | 26 | switch operator { 27 | case "label": 28 | return self.searchWithNames(ctx, config_obj, 29 | "labels", operator, term, in.Offset, in.Limit) 30 | 31 | case "os": 32 | return self.searchWithNames(ctx, config_obj, 33 | "system", operator, term, in.Offset, in.Limit) 34 | 35 | case "client": 36 | return self.searchWithNames(ctx, config_obj, 37 | "client_id", operator, term, in.Offset, in.Limit) 38 | 39 | case "host": 40 | return self.searchWithNames(ctx, config_obj, 41 | "hostname", operator, term, in.Offset, in.Limit) 42 | 43 | case "mac": 44 | return self.searchWithNames(ctx, config_obj, 45 | "mac_addresses", operator, term, in.Offset, in.Limit) 46 | 47 | default: 48 | return self.searchVerbsWithNames(ctx, config_obj, 49 | "hostname", operator, term, in.Offset, in.Limit) 50 | } 51 | } 52 | 53 | // Autocomplete the allowed verbs 54 | func (self *Indexer) searchVerbsWithNames( 55 | ctx context.Context, 56 | config_obj *config_proto.Config, 57 | field, operator, term string, 58 | offset, limit uint64) (*api_proto.SearchClientsResponse, error) { 59 | 60 | res := &api_proto.SearchClientsResponse{} 61 | 62 | // Complete verbs first 63 | term = strings.ToLower(term) 64 | for _, verb := range verbs { 65 | if term == "?" || 66 | strings.HasPrefix(verb, term) { 67 | res.Names = append(res.Names, verb) 68 | } 69 | } 70 | 71 | return res, nil 72 | } 73 | 74 | func (self *Indexer) searchWithNames( 75 | ctx context.Context, 76 | config_obj *config_proto.Config, 77 | field, operator, label string, 78 | offset, limit uint64) (*api_proto.SearchClientsResponse, error) { 79 | 80 | query := json.Format(getAllClientsAgg, field, label, 81 | field, offset, limit+1) 82 | hits, err := cvelo_services.QueryElasticAggregations( 83 | ctx, config_obj.OrgId, "persisted", query) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | prefix, filter := splitSearchTermIntoPrefixAndFilter(label) 89 | result := &api_proto.SearchClientsResponse{} 90 | for _, hit := range hits { 91 | if !strings.HasPrefix(hit, prefix) { 92 | continue 93 | } 94 | 95 | if filter != nil && !filter.MatchString(hit) { 96 | continue 97 | } 98 | 99 | term := hit 100 | if operator != "" { 101 | term = operator + ":" + hit 102 | } 103 | result.Names = append(result.Names, term) 104 | 105 | } 106 | return result, nil 107 | } 108 | -------------------------------------------------------------------------------- /services/hunt_dispatcher/index.go: -------------------------------------------------------------------------------- 1 | package hunt_dispatcher 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/Velocidex/ordereddict" 8 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 9 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 10 | "www.velocidex.com/golang/velociraptor/file_store" 11 | "www.velocidex.com/golang/velociraptor/json" 12 | "www.velocidex.com/golang/velociraptor/paths" 13 | "www.velocidex.com/golang/velociraptor/result_sets" 14 | "www.velocidex.com/golang/velociraptor/utils" 15 | ) 16 | 17 | type HuntIndexEntry struct { 18 | HuntId string `json:"HuntId"` 19 | Description string `json:"Description"` 20 | Created uint64 `json:"Created"` 21 | Started uint64 `json:"Started"` 22 | Expires uint64 `json:"Expires"` 23 | Creator string `json:"Creator"` 24 | 25 | // The hunt object is serialized into JSON here to make it quicker 26 | // to write the index if nothing is changed. 27 | Hunt string `json:"Hunt"` 28 | Tags string `json:"Tags"` 29 | } 30 | 31 | func (self *HuntStorageManagerImpl) FlushIndex( 32 | ctx context.Context) error { 33 | 34 | // Flush the index for the hunts table. 35 | cvelo_services.Count("HuntsFlushIndex") 36 | 37 | start_row := 0 38 | length := 1000 39 | 40 | hits, _, err := cvelo_services.QueryElasticRaw( 41 | ctx, self.config_obj.OrgId, 42 | "persisted", json.Format(getAllHuntsQuery, start_row, length)) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | hunt_path_manager := paths.NewHuntPathManager("") 48 | file_store_factory := file_store.GetFileStore(self.config_obj) 49 | rs_writer, err := result_sets.NewResultSetWriter(file_store_factory, 50 | hunt_path_manager.HuntIndex(), json.DefaultEncOpts(), 51 | 52 | // We need the index to be written immediately so it is 53 | // visible in the GUI. 54 | utils.SyncCompleter, 55 | result_sets.TruncateMode) 56 | if err != nil { 57 | return err 58 | } 59 | defer rs_writer.Close() 60 | 61 | seen_tags := make(map[string]bool) 62 | 63 | for _, hit := range hits { 64 | entry := &HuntEntry{} 65 | err = json.Unmarshal(hit, entry) 66 | if err != nil { 67 | continue 68 | } 69 | 70 | hunt_info, err := entry.GetHunt() 71 | if err != nil { 72 | continue 73 | } 74 | 75 | if hunt_info.State == api_proto.Hunt_ARCHIVED { 76 | continue 77 | } 78 | 79 | for _, tag := range hunt_info.Tags { 80 | seen_tags[tag] = true 81 | } 82 | 83 | rs_writer.Write(ordereddict.NewDict(). 84 | Set("HuntId", hunt_info.HuntId). 85 | Set("Description", hunt_info.HuntDescription). 86 | // Store the tags in the index so we can search for them. 87 | Set("Tags", strings.Join(hunt_info.Tags, "\n")). 88 | Set("Created", hunt_info.CreateTime). 89 | Set("Started", hunt_info.StartTime). 90 | Set("Expires", hunt_info.Expires). 91 | Set("Creator", hunt_info.Creator). 92 | Set("Hunt", entry.Hunt)) 93 | } 94 | 95 | record := &HuntEntry{} 96 | for tag := range seen_tags { 97 | record.Labels = append(record.Labels, tag) 98 | } 99 | 100 | return cvelo_services.SetElasticIndex(ctx, 101 | self.config_obj.OrgId, "persisted", TAGS_ID, record) 102 | } 103 | -------------------------------------------------------------------------------- /services/launcher/flows.go: -------------------------------------------------------------------------------- 1 | package launcher 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 8 | flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" 9 | ) 10 | 11 | var ( 12 | NotFoundError = errors.New("Not found") 13 | ) 14 | 15 | // Get all the flow IDs for this client. 16 | const ( 17 | prefixQuery = `{"prefix": {"id": "%v"}}` 18 | regexQuery = `{"regexp": {"id": "%v[_task|_stats|_stats_completed|_completed]*"}}` 19 | ) 20 | 21 | // Are any queries currenrly running. 22 | func is_running(context *flows_proto.ArtifactCollectorContext) bool { 23 | if context.State == flows_proto.ArtifactCollectorContext_ERROR { 24 | return false 25 | } 26 | 27 | for _, s := range context.QueryStats { 28 | if s.Status == crypto_proto.VeloStatus_PROGRESS { 29 | return true 30 | } 31 | } 32 | return false 33 | } 34 | 35 | func mergeRecords( 36 | collection_context *flows_proto.ArtifactCollectorContext, 37 | stats_context *flows_proto.ArtifactCollectorContext) *flows_proto.ArtifactCollectorContext { 38 | 39 | if stats_context.Request != nil { 40 | collection_context.Request = stats_context.Request 41 | } 42 | 43 | // Copy relevant fields into the main context 44 | if stats_context.TotalUploadedFiles > 0 { 45 | collection_context.TotalUploadedFiles = stats_context.TotalUploadedFiles 46 | } 47 | 48 | if stats_context.TotalExpectedUploadedBytes > 0 { 49 | collection_context.TotalExpectedUploadedBytes = stats_context.TotalExpectedUploadedBytes 50 | } 51 | 52 | if stats_context.TotalUploadedBytes > 0 { 53 | collection_context.TotalUploadedBytes = stats_context.TotalUploadedBytes 54 | } 55 | 56 | if stats_context.TotalCollectedRows > 0 { 57 | collection_context.TotalCollectedRows = stats_context.TotalCollectedRows 58 | } 59 | 60 | if stats_context.TotalLogs > 0 { 61 | collection_context.TotalLogs = stats_context.TotalLogs 62 | } 63 | 64 | if stats_context.ActiveTime > 0 { 65 | collection_context.ActiveTime = stats_context.ActiveTime 66 | } 67 | 68 | if stats_context.CreateTime > 0 { 69 | collection_context.CreateTime = stats_context.CreateTime 70 | } 71 | 72 | // We will encounter two records with QueryStats: the progress 73 | // messages and the completed messages. Make sure that if we see a 74 | // completion message it always replaces the progress message 75 | // regardless which order it appears. 76 | if len(stats_context.QueryStats) > 0 { 77 | if len(collection_context.QueryStats) == 0 || 78 | is_running(collection_context) && !is_running(stats_context) { 79 | collection_context.QueryStats = stats_context.QueryStats 80 | } 81 | } 82 | 83 | // If the final message is errored or cancelled we update this 84 | // message. 85 | if stats_context.State == flows_proto.ArtifactCollectorContext_ERROR { 86 | collection_context.State = stats_context.State 87 | collection_context.Status = stats_context.Status 88 | } 89 | 90 | return collection_context 91 | } 92 | 93 | func processIds(queryTemplate string, ids []string) string { 94 | query := "" 95 | for i, id := range ids { 96 | query += fmt.Sprintf(queryTemplate, id) 97 | if i < len(ids)-1 { 98 | query += "," 99 | } 100 | } 101 | return query 102 | } 103 | -------------------------------------------------------------------------------- /crypto/server/manager.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "crypto/rsa" 6 | "encoding/json" 7 | "errors" 8 | "sync" 9 | 10 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 11 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 12 | "www.velocidex.com/golang/velociraptor/crypto/client" 13 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 14 | "www.velocidex.com/golang/velociraptor/crypto/server" 15 | crypto_utils "www.velocidex.com/golang/velociraptor/crypto/utils" 16 | "www.velocidex.com/golang/velociraptor/logging" 17 | "www.velocidex.com/golang/velociraptor/utils" 18 | ) 19 | 20 | type ServerCryptoManager struct { 21 | *server.ServerCryptoManager 22 | } 23 | 24 | func NewServerCryptoManager( 25 | ctx context.Context, 26 | config_obj *config_proto.Config, 27 | wg *sync.WaitGroup) (*ServerCryptoManager, error) { 28 | if config_obj.Frontend == nil { 29 | return nil, errors.New("No frontend config") 30 | } 31 | 32 | cert, err := crypto_utils.ParseX509CertFromPemStr( 33 | []byte(config_obj.Frontend.Certificate)) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | resolver, err := NewServerPublicKeyResolver(ctx, config_obj, wg) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | base, err := client.NewCryptoManager(ctx, config_obj, 44 | crypto_utils.GetSubjectName(cert), 45 | []byte(config_obj.Frontend.PrivateKey), resolver, 46 | logging.GetLogger(config_obj, &logging.FrontendComponent)) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | server_manager := &ServerCryptoManager{&server.ServerCryptoManager{base}} 52 | return server_manager, nil 53 | } 54 | 55 | type serverPublicKeyResolver struct { 56 | ctx context.Context 57 | } 58 | 59 | func (self *serverPublicKeyResolver) DeleteSubject(client_id string) { 60 | } 61 | 62 | func (self *serverPublicKeyResolver) GetPublicKey( 63 | config_obj *config_proto.Config, 64 | client_id string) (*rsa.PublicKey, bool) { 65 | 66 | record, err := cvelo_services.GetElasticRecord( 67 | context.Background(), config_obj.OrgId, 68 | "persisted", client_id+"_key") 69 | if err != nil { 70 | return nil, false 71 | } 72 | 73 | pem := &crypto_proto.PublicKey{} 74 | err = json.Unmarshal(record, &pem) 75 | if err != nil { 76 | return nil, false 77 | } 78 | 79 | key, err := crypto_utils.PemToPublicKey(pem.Pem) 80 | if err != nil { 81 | return nil, false 82 | } 83 | 84 | return key, true 85 | } 86 | 87 | func (self *serverPublicKeyResolver) SetPublicKey( 88 | config_obj *config_proto.Config, 89 | client_id string, key *rsa.PublicKey) error { 90 | 91 | pem := &crypto_proto.PublicKey{ 92 | Pem: crypto_utils.PublicKeyToPem(key), 93 | EnrollTime: uint64(utils.GetTime().Now().Unix()), 94 | } 95 | return cvelo_services.SetElasticIndex( 96 | self.ctx, config_obj.OrgId, 97 | "persisted", client_id+"_key", 98 | pem) 99 | } 100 | 101 | func (self *serverPublicKeyResolver) Clear() {} 102 | 103 | func NewServerPublicKeyResolver( 104 | ctx context.Context, 105 | config_obj *config_proto.Config, 106 | wg *sync.WaitGroup) (client.PublicKeyResolver, error) { 107 | result := &serverPublicKeyResolver{ 108 | ctx: ctx, 109 | } 110 | 111 | return result, nil 112 | } 113 | -------------------------------------------------------------------------------- /services/sanity/users.go: -------------------------------------------------------------------------------- 1 | package sanity 2 | 3 | import ( 4 | "context" 5 | "crypto/rand" 6 | "encoding/hex" 7 | "strings" 8 | 9 | api_proto "www.velocidex.com/golang/velociraptor/api/proto" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | "www.velocidex.com/golang/velociraptor/logging" 12 | "www.velocidex.com/golang/velociraptor/services" 13 | "www.velocidex.com/golang/velociraptor/services/users" 14 | ) 15 | 16 | func createInitialUsers( 17 | ctx context.Context, 18 | config_obj *config_proto.Config, 19 | user_names []*config_proto.GUIUser) error { 20 | 21 | logger := logging.GetLogger(config_obj, &logging.FrontendComponent) 22 | 23 | superuser := "VelociraptorServer" 24 | if config_obj.Client != nil { 25 | superuser = config_obj.Client.PinnedServerName 26 | } 27 | 28 | // We rely on the orgs to already be existing here. 29 | org_list := []string{"root"} 30 | for _, org := range config_obj.GUI.InitialOrgs { 31 | org_list = append(org_list, org.OrgId) 32 | } 33 | org_manager, err := services.GetOrgManager() 34 | if err != nil { 35 | return err 36 | } 37 | 38 | for _, user := range user_names { 39 | users_manager := services.GetUserManager() 40 | user_record, err := users_manager.GetUser(ctx, superuser, user.Name) 41 | if err != nil || user_record.Name != user.Name { 42 | logger.Info("Initial user %v not present, creating", user.Name) 43 | new_user, err := users.NewUserRecord(config_obj, user.Name) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | // Basic auth requires setting hashed 49 | // password and salt 50 | switch strings.ToLower(config_obj.GUI.Authenticator.Type) { 51 | case "basic": 52 | new_user.PasswordHash, err = hex.DecodeString(user.PasswordHash) 53 | if err != nil { 54 | return err 55 | } 56 | new_user.PasswordSalt, err = hex.DecodeString(user.PasswordSalt) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | // All other auth methods do 62 | // not need a password set, so 63 | // generate a random one 64 | default: 65 | password := make([]byte, 100) 66 | _, err = rand.Read(password) 67 | if err != nil { 68 | return err 69 | } 70 | users.SetPassword(new_user, string(password)) 71 | } 72 | 73 | for _, org_id := range org_list { 74 | org_config_obj, err := org_manager.GetOrgConfig(org_id) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | new_user.Orgs = append(new_user.Orgs, &api_proto.OrgRecord{ 80 | Name: org_config_obj.OrgName, 81 | // For backwards compatibility. 82 | OrgId: org_config_obj.OrgId, 83 | Id: org_config_obj.OrgId, 84 | }) 85 | 86 | // Give them the administrator role in the respective org 87 | err = services.GrantRoles( 88 | org_config_obj, user.Name, []string{"administrator"}) 89 | if err != nil { 90 | return err 91 | } 92 | } 93 | 94 | // Create the new user. 95 | err = users_manager.SetUser(ctx, new_user) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | logger := logging.GetLogger(config_obj, &logging.Audit) 101 | logger.Info("Granting administrator role to %v because they are specified in the config's initial users", 102 | user.Name) 103 | } 104 | } 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /bin/query.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "time" 8 | 9 | "github.com/Velocidex/ordereddict" 10 | kingpin "gopkg.in/alecthomas/kingpin.v2" 11 | "www.velocidex.com/golang/cloudvelo/startup" 12 | "www.velocidex.com/golang/velociraptor/file_store" 13 | "www.velocidex.com/golang/velociraptor/file_store/path_specs" 14 | "www.velocidex.com/golang/velociraptor/file_store/uploader" 15 | "www.velocidex.com/golang/velociraptor/services" 16 | vql_subsystem "www.velocidex.com/golang/velociraptor/vql" 17 | "www.velocidex.com/golang/velociraptor/vql/acl_managers" 18 | "www.velocidex.com/golang/vfilter" 19 | 20 | _ "www.velocidex.com/golang/cloudvelo/vql_plugins" 21 | ) 22 | 23 | var ( 24 | // Command line interface for VQL commands. 25 | query = app.Command("query", "Run a VQL query") 26 | queries = query.Arg("queries", "The VQL Query to run."). 27 | Required().Strings() 28 | max_wait = app.Flag("max_wait", "Maximum time to queue results."). 29 | Default("10").Int() 30 | env_map = query.Flag("env", "Environment for the query."). 31 | StringMap() 32 | file_store_uploader = query.Flag("with_fs_uploader", 33 | "Install a filestore uploader").Bool() 34 | ) 35 | 36 | func doQuery() error { 37 | cloud_config_obj, err := loadConfig( 38 | makeDefaultConfigLoader().WithRequiredLogging()) 39 | if err != nil { 40 | return fmt.Errorf("loading config file: %w", err) 41 | } 42 | 43 | config_obj := cloud_config_obj.VeloConf() 44 | 45 | ctx, cancel := install_sig_handler() 46 | defer cancel() 47 | 48 | sm, err := startup.StartToolServices(ctx, cloud_config_obj) 49 | defer sm.Close() 50 | 51 | if err != nil { 52 | return err 53 | } 54 | 55 | env := ordereddict.NewDict() 56 | for k, v := range *env_map { 57 | env.Set(k, v) 58 | } 59 | 60 | builder := services.ScopeBuilder{ 61 | Config: config_obj, 62 | ACLManager: acl_managers.NullACLManager{}, 63 | Logger: log.New(&LogWriter{config_obj}, "", 0), 64 | Env: env, 65 | } 66 | 67 | if *file_store_uploader { 68 | output_path_spec := path_specs.NewSafeFilestorePath("/") 69 | builder.Uploader = uploader.NewFileStoreUploader( 70 | config_obj, 71 | file_store.GetFileStore(config_obj), 72 | output_path_spec) 73 | } 74 | 75 | manager, err := services.GetRepositoryManager(config_obj) 76 | if err != nil { 77 | return err 78 | } 79 | scope := manager.BuildScope(builder) 80 | defer scope.Close() 81 | 82 | out_fd := os.Stdout 83 | start_time := time.Now() 84 | defer func() { 85 | scope.Log("Completed query in %v", time.Now().Sub(start_time)) 86 | }() 87 | 88 | for _, query := range *queries { 89 | statements, err := vfilter.MultiParse(query) 90 | kingpin.FatalIfError(err, "Unable to parse VQL Query") 91 | 92 | for _, vql := range statements { 93 | for result := range vfilter.GetResponseChannel( 94 | vql, ctx, scope, 95 | vql_subsystem.MarshalJsonl(scope), 96 | 10, *max_wait) { 97 | _, err := out_fd.Write(result.Payload) 98 | if err != nil { 99 | return err 100 | } 101 | } 102 | } 103 | } 104 | return nil 105 | } 106 | 107 | func init() { 108 | command_handlers = append(command_handlers, func(command string) bool { 109 | switch command { 110 | case query.FullCommand(): 111 | FatalIfError(query, doQuery) 112 | 113 | default: 114 | return false 115 | } 116 | return true 117 | }) 118 | } 119 | -------------------------------------------------------------------------------- /ingestion/uploads.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Velocidex/ordereddict" 7 | "www.velocidex.com/golang/cloudvelo/filestore" 8 | "www.velocidex.com/golang/cloudvelo/result_sets/simple" 9 | "www.velocidex.com/golang/cloudvelo/vql/uploads" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 12 | "www.velocidex.com/golang/velociraptor/file_store" 13 | "www.velocidex.com/golang/velociraptor/json" 14 | "www.velocidex.com/golang/velociraptor/paths" 15 | "www.velocidex.com/golang/velociraptor/result_sets" 16 | "www.velocidex.com/golang/velociraptor/utils" 17 | velo_utils "www.velocidex.com/golang/velociraptor/utils" 18 | ) 19 | 20 | // Uploads are being sent separately to the server handler by the 21 | // client. The FileBuffer message only sends metadata about the 22 | // upload. 23 | func (self Ingestor) HandleUploads( 24 | ctx context.Context, 25 | config_obj *config_proto.Config, 26 | message *crypto_proto.VeloMessage) error { 27 | 28 | if message.FileBuffer == nil || message.FileBuffer.Pathspec == nil { 29 | return nil 30 | } 31 | 32 | response := message.FileBuffer 33 | 34 | // We only store the EOF messages 35 | if !response.Eof { 36 | return nil 37 | } 38 | 39 | upload_request := &uploads.UploadRequest{ 40 | ClientId: utils.ClientIdFromSource(message.Source), 41 | SessionId: message.SessionId, 42 | Accessor: message.FileBuffer.Pathspec.Accessor, 43 | Components: message.FileBuffer.Pathspec.Components, 44 | } 45 | 46 | // Figure out where in the filestore the server's 47 | // StartMultipartUpload placed it. 48 | components := filestore.S3ComponentsForClientUpload(upload_request) 49 | 50 | path_manager := paths.NewFlowPathManager(message.Source, message.SessionId) 51 | file_store_factory := file_store.GetFileStore(config_obj) 52 | rs_writer, err := result_sets.NewResultSetWriter( 53 | file_store_factory, path_manager.UploadMetadata(), json.DefaultEncOpts(), 54 | utils.BackgroundWriter, result_sets.AppendMode) 55 | if err != nil { 56 | return err 57 | } 58 | defer rs_writer.Close() 59 | 60 | elastic_writer, ok := rs_writer.(*simple.ElasticSimpleResultSetWriter) 61 | if ok { 62 | elastic_writer.SetStartRow(int64(response.UploadNumber)) 63 | } 64 | 65 | // Write a reference to the main file 66 | rs_writer.Write( 67 | ordereddict.NewDict(). 68 | Set("Timestamp", velo_utils.GetTime().Now().Unix()). 69 | Set("started", velo_utils.GetTime().Now()). 70 | Set("vfs_path", response.Pathspec.Path). 71 | Set("_Components", components). 72 | Set("_Type", ""). 73 | Set("file_size", response.Size). 74 | Set("_accessor", message.FileBuffer.Pathspec.Accessor). 75 | Set("_client_components", message.FileBuffer.Pathspec.Components). 76 | Set("uploaded_size", response.StoredSize)) 77 | 78 | // Write a reference to the index file. 79 | if response.IsSparse { 80 | rs_writer.Write( 81 | ordereddict.NewDict(). 82 | Set("Timestamp", velo_utils.GetTime().Now().Unix()). 83 | Set("started", velo_utils.GetTime().Now()). 84 | Set("vfs_path", response.Pathspec.Path+".idx"). 85 | Set("_Components", components). 86 | Set("_Type", "idx"). 87 | Set("file_size", response.Size). 88 | Set("_accessor", message.FileBuffer.Pathspec.Accessor). 89 | Set("_client_components", message.FileBuffer.Pathspec.Components). 90 | Set("uploaded_size", response.StoredSize)) 91 | } 92 | 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /services/notifier/notifier.go: -------------------------------------------------------------------------------- 1 | package notifier 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "sync" 7 | "time" 8 | 9 | "www.velocidex.com/golang/cloudvelo/schema/api" 10 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 11 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 12 | "www.velocidex.com/golang/velociraptor/logging" 13 | ) 14 | 15 | // A do nothing notifier - we really can not actively notify clients 16 | // at this stage. 17 | 18 | type Nofitier struct { 19 | poll time.Duration 20 | config_obj *config_proto.Config 21 | } 22 | 23 | func (self Nofitier) ListenForNotification(id string) (chan bool, func()) { 24 | output_chan := make(chan bool) 25 | 26 | logger := logging.GetLogger(self.config_obj, &logging.GUIComponent) 27 | ctx, cancel := context.WithCancel(context.Background()) 28 | notify_id := id + "_notify" 29 | go func() { 30 | defer close(output_chan) 31 | 32 | now := time.Now().Unix() 33 | 34 | for { 35 | select { 36 | case <-ctx.Done(): 37 | return 38 | 39 | case <-time.After(self.poll): 40 | serialized, err := cvelo_services.GetElasticRecord( 41 | ctx, self.config_obj.OrgId, "persisted", notify_id) 42 | if err != nil { 43 | continue 44 | } 45 | 46 | notifiction_record := &api.NotificationRecord{} 47 | err = json.Unmarshal(serialized, ¬ifiction_record) 48 | if err != nil { 49 | logger.Error("ListenForNotification: %v", err) 50 | continue 51 | } 52 | 53 | // Notify the caller by closing the channel. 54 | if notifiction_record.Timestamp > now { 55 | return 56 | } 57 | } 58 | } 59 | }() 60 | 61 | return output_chan, cancel 62 | } 63 | 64 | func (self Nofitier) NotifyListener( 65 | ctx context.Context, 66 | config_obj *config_proto.Config, id, tag string) error { 67 | notify_id := id + "_notify" 68 | return cvelo_services.SetElasticIndex( 69 | context.Background(), self.config_obj.OrgId, 70 | "persisted", notify_id, 71 | &api.NotificationRecord{ 72 | Key: notify_id, 73 | Timestamp: time.Now().Unix(), 74 | DocType: "notifications", 75 | }) 76 | } 77 | 78 | // Notify a directly connected listener. 79 | func (self Nofitier) NotifyDirectListener(id string) {} 80 | 81 | func (self Nofitier) CountConnectedClients() uint64 { 82 | return 0 83 | } 84 | 85 | // Notify in the near future - no guarantee of delivery. 86 | func (self Nofitier) NotifyListenerAsync( 87 | ctx context.Context, 88 | config_obj *config_proto.Config, id, tag string) { 89 | } 90 | 91 | // Check if there is someone listening for the specified id. This 92 | // method queries all minion nodes to check if the client is 93 | // connected anywhere - It may take up to 2 seconds to find out. 94 | func (self Nofitier) IsClientConnected(ctx context.Context, 95 | config_obj *config_proto.Config, 96 | client_id string, timeout int) bool { 97 | return false 98 | } 99 | 100 | // Returns a list of all clients directly connected at present. 101 | func (self Nofitier) ListClients() []string { 102 | return nil 103 | } 104 | 105 | // Check only the current node if the client is connected. 106 | func (self Nofitier) IsClientDirectlyConnected(client_id string) bool { 107 | return false 108 | } 109 | 110 | func NewNotificationService( 111 | ctx context.Context, 112 | wg *sync.WaitGroup, 113 | config_obj *config_proto.Config) (*Nofitier, error) { 114 | 115 | return &Nofitier{ 116 | poll: time.Second, 117 | config_obj: config_obj, 118 | }, nil 119 | } 120 | -------------------------------------------------------------------------------- /foreman/event_monitoring.go: -------------------------------------------------------------------------------- 1 | package foreman 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/promauto" 9 | "google.golang.org/protobuf/proto" 10 | actions_proto "www.velocidex.com/golang/velociraptor/actions/proto" 11 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 12 | "www.velocidex.com/golang/velociraptor/constants" 13 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 14 | flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" 15 | "www.velocidex.com/golang/velociraptor/utils" 16 | ) 17 | 18 | var ( 19 | eventsCountGauge = promauto.NewGaugeVec( 20 | prometheus.GaugeOpts{ 21 | Name: "foreman_events_gauge", 22 | Help: "Number of active client events applied to all clients (per organization).", 23 | }, 24 | []string{"orgId"}, 25 | ) 26 | 27 | eventsByLabelCountGauge = promauto.NewGaugeVec( 28 | prometheus.GaugeOpts{ 29 | Name: "foreman_events_by_label_gauge", 30 | Help: "Number of active client events applied to a specific label (per organization).", 31 | }, 32 | []string{"orgId"}, 33 | ) 34 | ) 35 | 36 | const ( 37 | clientsNeedingMonitoringTableUpdate = ` 38 | { 39 | "query": {"bool": { 40 | "filter": [ 41 | {"range": {"last_event_table_version": {"lt": %q}}}, 42 | {"range": {"ping": {"gte": %q}}} 43 | ]} 44 | }, 45 | "_source": { 46 | "includes": ["labels", "client_id", "labels_timestamp", "last_event_table_version", "ping"] 47 | } 48 | } 49 | ` 50 | ) 51 | 52 | // Calculate the event monitoring update message based on the set of 53 | // labels. 54 | func GetClientUpdateEventTableMessage( 55 | ctx context.Context, 56 | config_obj *config_proto.Config, 57 | state *flows_proto.ClientEventTable, 58 | labels []string) *crypto_proto.VeloMessage { 59 | 60 | result := &actions_proto.VQLEventTable{ 61 | Version: uint64(utils.GetTime().Now().UnixNano()), 62 | } 63 | 64 | if state.Artifacts == nil { 65 | state.Artifacts = &flows_proto.ArtifactCollectorArgs{} 66 | } 67 | 68 | for _, event := range state.Artifacts.CompiledCollectorArgs { 69 | result.Event = append(result.Event, 70 | proto.Clone(event).(*actions_proto.VQLCollectorArgs)) 71 | } 72 | 73 | // Now apply any event queries that belong to this client based on 74 | // labels. 75 | for _, table := range state.LabelEvents { 76 | if utils.InString(labels, table.Label) { 77 | for _, event := range table.Artifacts.CompiledCollectorArgs { 78 | result.Event = append(result.Event, 79 | proto.Clone(event).(*actions_proto.VQLCollectorArgs)) 80 | } 81 | } 82 | } 83 | 84 | for _, event := range result.Event { 85 | if event.MaxWait == 0 { 86 | event.MaxWait = config_obj.Defaults.EventMaxWait 87 | } 88 | 89 | if event.MaxWait == 0 { 90 | event.MaxWait = 120 91 | } 92 | 93 | // Event queries never time out 94 | event.Timeout = 99999999 95 | } 96 | 97 | return &crypto_proto.VeloMessage{ 98 | UpdateEventTable: result, 99 | SessionId: constants.MONITORING_WELL_KNOWN_FLOW, 100 | } 101 | } 102 | 103 | // Turn a list of client labels into a unique key. The key is 104 | // constructed from the labels actually in use in the event table. 105 | func labelsKey(labels []string, state *flows_proto.ClientEventTable) string { 106 | result := make([]string, 0, len(labels)) 107 | for _, table := range state.LabelEvents { 108 | if utils.InString(labels, table.Label) { 109 | result = append(result, table.Label) 110 | } 111 | } 112 | 113 | return strings.Join(result, "|") 114 | } 115 | -------------------------------------------------------------------------------- /result_sets/timed/timed.go: -------------------------------------------------------------------------------- 1 | package timed 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/Velocidex/ordereddict" 8 | "www.velocidex.com/golang/cloudvelo/services" 9 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 10 | "www.velocidex.com/golang/velociraptor/file_store/api" 11 | "www.velocidex.com/golang/velociraptor/json" 12 | "www.velocidex.com/golang/velociraptor/paths/artifacts" 13 | "www.velocidex.com/golang/velociraptor/result_sets" 14 | "www.velocidex.com/golang/velociraptor/utils" 15 | ) 16 | 17 | // This is the record we store in the elastic datastore. Timed Results 18 | // are usually written from event artifacts. 19 | type TimedResultSetRecord struct { 20 | ClientId string `json:"client_id"` 21 | FlowId string `json:"flow_id"` 22 | Artifact string `json:"artifact"` 23 | Type string `json:"type"` 24 | Timestamp int64 `json:"timestamp"` 25 | Date int64 `json:"date"` // Timestamp rounded down to the UTC day 26 | VFSPath string `json:"vfs_path"` 27 | JSONData string `json:"data"` 28 | } 29 | 30 | // Examine the pathspec and construct a new Elastic record. 31 | func NewTimedResultSetRecord( 32 | path_manager api.PathManager) *TimedResultSetRecord { 33 | filename, _ := path_manager.GetPathForWriting() 34 | vfs_path := "" 35 | if filename != nil { 36 | vfs_path = filename.AsClientPath() 37 | } 38 | 39 | now := utils.GetTime().Now() 40 | day := now.Truncate(24 * time.Hour).Unix() 41 | 42 | switch t := path_manager.(type) { 43 | case *artifacts.ArtifactPathManager: 44 | return &TimedResultSetRecord{ 45 | ClientId: t.ClientId, 46 | FlowId: t.FlowId, 47 | Artifact: t.FullArtifactName, 48 | Type: "results", 49 | VFSPath: vfs_path, 50 | Timestamp: now.UnixNano(), 51 | Date: day, 52 | } 53 | 54 | case *artifacts.ArtifactLogPathManager: 55 | return &TimedResultSetRecord{ 56 | ClientId: t.ClientId, 57 | FlowId: t.FlowId, 58 | Artifact: t.FullArtifactName, 59 | Type: "logs", 60 | VFSPath: vfs_path, 61 | Timestamp: utils.GetTime().Now().UnixNano(), 62 | Date: day, 63 | } 64 | 65 | default: 66 | return &TimedResultSetRecord{ 67 | Timestamp: utils.GetTime().Now().UnixNano(), 68 | Date: day, 69 | VFSPath: vfs_path, 70 | } 71 | } 72 | } 73 | 74 | type ElasticTimedResultSetWriter struct { 75 | config_obj *config_proto.Config 76 | path_manager api.PathManager 77 | opts *json.EncOpts 78 | ctx context.Context 79 | } 80 | 81 | func (self ElasticTimedResultSetWriter) WriteJSONL( 82 | serialized []byte, total_rows int) { 83 | 84 | record := NewTimedResultSetRecord(self.path_manager) 85 | record.JSONData = string(serialized) 86 | 87 | services.SetElasticIndex(self.ctx, 88 | utils.GetOrgId(self.config_obj), 89 | "transient", services.DocIdRandom, record) 90 | } 91 | 92 | func (self ElasticTimedResultSetWriter) Write(row *ordereddict.Dict) { 93 | serialized, err := json.MarshalWithOptions(row, self.opts) 94 | if err != nil { 95 | return 96 | } 97 | 98 | self.WriteJSONL(serialized, 1) 99 | } 100 | 101 | func (self ElasticTimedResultSetWriter) Flush() { 102 | 103 | } 104 | 105 | func (self ElasticTimedResultSetWriter) Close() { 106 | 107 | } 108 | 109 | func NewTimedResultSetWriter( 110 | config_obj *config_proto.Config, 111 | path_manager api.PathManager, 112 | opts *json.EncOpts, 113 | completion func()) (result_sets.TimedResultSetWriter, error) { 114 | 115 | return &ElasticTimedResultSetWriter{ 116 | config_obj: config_obj, 117 | path_manager: path_manager, 118 | opts: opts, 119 | ctx: context.Background(), 120 | }, nil 121 | } 122 | -------------------------------------------------------------------------------- /testsuite/testsuite.go: -------------------------------------------------------------------------------- 1 | package testsuite 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | "github.com/stretchr/testify/suite" 12 | "www.velocidex.com/golang/cloudvelo/config" 13 | "www.velocidex.com/golang/cloudvelo/schema" 14 | "www.velocidex.com/golang/cloudvelo/services/orgs" 15 | velo_config "www.velocidex.com/golang/velociraptor/config" 16 | "www.velocidex.com/golang/velociraptor/services" 17 | "www.velocidex.com/golang/velociraptor/utils" 18 | "www.velocidex.com/golang/velociraptor/vtesting/assert" 19 | ) 20 | 21 | type CloudTestSuite struct { 22 | suite.Suite 23 | 24 | OrgId string 25 | ConfigObj *config.Config 26 | 27 | Indexes []string 28 | 29 | Sm *services.Service 30 | Ctx context.Context 31 | cancel func() 32 | 33 | writeback_file string 34 | 35 | time_closer func() 36 | } 37 | 38 | // Allow an external file to override the config. This allows us to 39 | // manually test with AWS credentials. 40 | func (self *CloudTestSuite) LoadConfig() *config.Config { 41 | patch := "" 42 | override_filename := os.Getenv("VELOCIRAPTOR_TEST_CONFIG_OVERRIDE") 43 | if override_filename != "" { 44 | data, err := ioutil.ReadFile(override_filename) 45 | require.NoError(self.T(), err) 46 | fmt.Printf("Will override config with %v\n", override_filename) 47 | patch = string(data) 48 | } 49 | 50 | loader := config.ConfigLoader{ 51 | VelociraptorLoader: new(velo_config.Loader). 52 | WithRequiredFrontend(). 53 | WithVerbose(true). 54 | WithEnvLiteralLoader("VELOCONFIG"), 55 | ConfigText: SERVER_CONFIG, 56 | JSONPatch: patch, 57 | } 58 | config_obj, err := loader.Load() 59 | require.NoError(self.T(), err) 60 | 61 | return config_obj 62 | } 63 | 64 | func (self *CloudTestSuite) SetupSuite() { 65 | if self.ConfigObj == nil { 66 | self.ConfigObj = self.LoadConfig() 67 | } 68 | 69 | tempfile, err := ioutil.TempFile("", "test") 70 | assert.NoError(self.T(), err) 71 | 72 | self.writeback_file = tempfile.Name() 73 | 74 | tempfile.Write([]byte(writeback_file)) 75 | tempfile.Close() 76 | 77 | self.ConfigObj.Client.WritebackLinux = tempfile.Name() 78 | self.ConfigObj.Client.WritebackWindows = tempfile.Name() 79 | self.ConfigObj.Client.WritebackDarwin = tempfile.Name() 80 | } 81 | 82 | func (self *CloudTestSuite) TearDownSuite() { 83 | os.Remove(self.writeback_file) 84 | if self.time_closer != nil { 85 | self.time_closer() 86 | } 87 | } 88 | 89 | func (self *CloudTestSuite) TearDownTest() { 90 | self.cancel() 91 | self.Sm.Close() 92 | } 93 | 94 | func (self *CloudTestSuite) SetupTest() { 95 | self.time_closer = utils.MockTime(&utils.IncClock{NowTime: 1744634710}) 96 | 97 | self.Ctx, self.cancel = context.WithTimeout(context.Background(), 98 | time.Second*60) 99 | 100 | config_obj := self.ConfigObj 101 | sm := services.NewServiceManager(self.Ctx, config_obj.VeloConf()) 102 | org_manager, err := orgs.NewOrgManager(sm.Ctx, sm.Wg, self.ConfigObj) 103 | assert.NoError(self.T(), err) 104 | 105 | test_org := self.OrgId 106 | if test_org == "" { 107 | test_org = "test" 108 | } 109 | 110 | // Delete the previous indexes for the org. 111 | err = schema.Delete(self.Ctx, config_obj.VeloConf(), test_org, schema.NO_FILTER) 112 | assert.NoError(self.T(), err) 113 | 114 | _, err = org_manager.CreateNewOrg("test", test_org, services.RandomNonce) 115 | assert.NoError(self.T(), err) 116 | 117 | self.Sm = sm 118 | self.ConfigObj.OrgId = test_org 119 | 120 | // Make sure the index templates are initialized if needed. 121 | err = schema.InstallIndexTemplates(self.Ctx, config_obj) 122 | assert.NoError(self.T(), err) 123 | } 124 | -------------------------------------------------------------------------------- /ingestion/ingestor.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | The ingestor receives VeloMessages from the client and inserts them 4 | into the elastic backend using the correct schema so they may easily 5 | be viewed by the GUI. 6 | 7 | */ 8 | 9 | package ingestion 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | "os" 15 | 16 | "github.com/opensearch-project/opensearch-go/v2" 17 | "www.velocidex.com/golang/cloudvelo/config" 18 | "www.velocidex.com/golang/cloudvelo/crypto/server" 19 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 20 | "www.velocidex.com/golang/velociraptor/constants" 21 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 22 | "www.velocidex.com/golang/velociraptor/json" 23 | "www.velocidex.com/golang/velociraptor/services" 24 | ) 25 | 26 | var ( 27 | idx = 0 28 | ) 29 | 30 | type IngestorInterface interface { 31 | Process(ctx context.Context, message *crypto_proto.VeloMessage) error 32 | } 33 | 34 | // Responsible for inserting VeloMessage objects into elastic. 35 | type Ingestor struct { 36 | client *opensearch.Client 37 | 38 | crypto_manager *server.ServerCryptoManager 39 | 40 | index string 41 | } 42 | 43 | // Log messages to a file - used to generate test data. 44 | func (self Ingestor) LogMessage(message *crypto_proto.VeloMessage) { 45 | filename := fmt.Sprintf("Msg_%02d.json", idx) 46 | idx++ 47 | 48 | fd, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) 49 | if err == nil { 50 | fd.Write([]byte(json.MustMarshalIndent(message))) 51 | } 52 | fd.Close() 53 | } 54 | 55 | func (self Ingestor) Process( 56 | ctx context.Context, message *crypto_proto.VeloMessage) error { 57 | //self.LogMessage(message) 58 | 59 | org_manager, err := services.GetOrgManager() 60 | if err != nil { 61 | return err 62 | } 63 | 64 | config_obj, err := org_manager.GetOrgConfig(message.OrgId) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | // Only accept unauthenticated enrolment requests. Everything 70 | // below is authenticated. 71 | if message.AuthState == crypto_proto.VeloMessage_UNAUTHENTICATED { 72 | return self.HandleEnrolment(config_obj, message) 73 | } 74 | 75 | // Handle the monitoring data - write to timed result set. 76 | if message.SessionId == constants.MONITORING_WELL_KNOWN_FLOW { 77 | if message.LogMessage != nil { 78 | return self.HandleMonitoringLogs(ctx, config_obj, message) 79 | } 80 | 81 | if message.VQLResponse != nil { 82 | return self.HandleMonitoringResponses(ctx, config_obj, message) 83 | } 84 | 85 | return nil 86 | } 87 | 88 | err = self.maybeHandleHuntResponse(ctx, config_obj, message) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | // Handle regular collections - use simple result sets to store 94 | // them. 95 | if message.LogMessage != nil { 96 | return self.HandleLogs(ctx, config_obj, message) 97 | } 98 | 99 | if message.VQLResponse != nil { 100 | return self.HandleResponses(ctx, config_obj, message) 101 | } 102 | 103 | if message.FlowStats != nil { 104 | return self.HandleFlowStats(ctx, config_obj, message) 105 | } 106 | 107 | if message.ForemanCheckin != nil { 108 | return self.HandlePing(ctx, config_obj, message) 109 | } 110 | 111 | if message.FileBuffer != nil { 112 | return self.HandleUploads(ctx, config_obj, message) 113 | } 114 | return nil 115 | } 116 | 117 | func NewIngestor( 118 | config_obj *config.Config, 119 | crypto_manager *server.ServerCryptoManager) (*Ingestor, error) { 120 | 121 | client, err := cvelo_services.GetElasticClientByType( 122 | cvelo_services.PrimaryOpenSearch) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | return &Ingestor{ 128 | client: client, 129 | crypto_manager: crypto_manager, 130 | }, nil 131 | } 132 | -------------------------------------------------------------------------------- /result_sets/timed/reader.go: -------------------------------------------------------------------------------- 1 | package timed 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "strings" 7 | "time" 8 | 9 | "github.com/Velocidex/ordereddict" 10 | cvelo_services "www.velocidex.com/golang/cloudvelo/services" 11 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 12 | "www.velocidex.com/golang/velociraptor/file_store/api" 13 | "www.velocidex.com/golang/velociraptor/json" 14 | "www.velocidex.com/golang/velociraptor/logging" 15 | ) 16 | 17 | type TimedResultSetReader struct { 18 | start, end time.Time 19 | cancel func() 20 | 21 | path_manager api.PathManager 22 | config_obj *config_proto.Config 23 | } 24 | 25 | func (self *TimedResultSetReader) GetAvailableFiles( 26 | ctx context.Context) []*api.ResultSetFileProperties { 27 | return nil 28 | } 29 | 30 | func (self *TimedResultSetReader) Debug() {} 31 | 32 | func (self *TimedResultSetReader) SeekToTime(offset time.Time) error { 33 | self.start = offset 34 | return nil 35 | } 36 | 37 | func (self *TimedResultSetReader) SetMaxTime(end time.Time) { 38 | self.end = end 39 | } 40 | 41 | func (self *TimedResultSetReader) Close() { 42 | if self.cancel != nil { 43 | self.cancel() 44 | } 45 | } 46 | 47 | const getTimedRowsQuery = ` 48 | { 49 | "query": { 50 | "bool": { 51 | "must": [ 52 | {"match": {"client_id": %q}}, 53 | {"match": {"flow_id": %q}}, 54 | {"match": {"artifact": %q}}, 55 | {"match": {"type": %q}}, 56 | {"range": {"timestamp": {"gte": %q}}}, 57 | {"range": {"timestamp": {"lt": %q}}} 58 | ] 59 | } 60 | } 61 | } 62 | ` 63 | 64 | func (self *TimedResultSetReader) Rows( 65 | ctx context.Context) <-chan *ordereddict.Dict { 66 | output_chan := make(chan *ordereddict.Dict) 67 | 68 | go func() { 69 | defer close(output_chan) 70 | 71 | record := NewTimedResultSetRecord(self.path_manager) 72 | end := self.end 73 | if end.IsZero() { 74 | end = time.Unix(3000000000, 0) 75 | } 76 | start := self.start 77 | if start.IsZero() { 78 | start = time.Unix(0, 0) 79 | } 80 | 81 | // Client event artifacts always come from the monitoring 82 | // flow. 83 | if record.ClientId != "server" { 84 | record.FlowId = "F.Monitoring" 85 | } 86 | query := json.Format(getTimedRowsQuery, 87 | record.ClientId, 88 | record.FlowId, 89 | record.Artifact, 90 | record.Type, 91 | start.UnixNano(), 92 | end.UnixNano(), 93 | ) 94 | subctx, cancel := context.WithCancel(ctx) 95 | defer cancel() 96 | 97 | self.cancel = cancel 98 | 99 | hits_chan, err := cvelo_services.QueryChan( 100 | subctx, self.config_obj, 1000, 101 | self.config_obj.OrgId, "transient", query, 102 | "timestamp") 103 | if err != nil { 104 | logger := logging.GetLogger( 105 | self.config_obj, &logging.FrontendComponent) 106 | logger.Error("Reading %v: %v", 107 | self.path_manager, err) 108 | return 109 | } 110 | 111 | for hit := range hits_chan { 112 | record := &TimedResultSetRecord{} 113 | err = json.Unmarshal(hit, record) 114 | if err != nil { 115 | continue 116 | } 117 | 118 | reader := bufio.NewReader(strings.NewReader(record.JSONData)) 119 | for { 120 | row_data, err := reader.ReadBytes('\n') 121 | if err != nil && len(row_data) == 0 { 122 | // Packet is exhausted, go get the next packet 123 | break 124 | } 125 | 126 | row := ordereddict.NewDict() 127 | err = row.UnmarshalJSON(row_data) 128 | if err != nil { 129 | continue 130 | } 131 | 132 | select { 133 | case <-ctx.Done(): 134 | return 135 | 136 | case output_chan <- row.Set( 137 | "_ts", record.Timestamp/1000000): 138 | } 139 | } 140 | } 141 | }() 142 | 143 | return output_chan 144 | } 145 | -------------------------------------------------------------------------------- /ingestion/hunts.go: -------------------------------------------------------------------------------- 1 | package ingestion 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | ingestor_services "www.velocidex.com/golang/cloudvelo/ingestion/services" 8 | "www.velocidex.com/golang/cloudvelo/services" 9 | "www.velocidex.com/golang/cloudvelo/services/hunt_dispatcher" 10 | config_proto "www.velocidex.com/golang/velociraptor/config/proto" 11 | crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto" 12 | flows_proto "www.velocidex.com/golang/velociraptor/flows/proto" 13 | velo_utils "www.velocidex.com/golang/velociraptor/utils" 14 | ) 15 | 16 | func (self Ingestor) maybeHandleHuntResponse( 17 | ctx context.Context, 18 | config_obj *config_proto.Config, 19 | message *crypto_proto.VeloMessage) error { 20 | 21 | // Hunt responses have special SessionId like "F.1234.H" 22 | hunt_id, ok := velo_utils.ExtractHuntId(message.SessionId) 23 | if !ok { 24 | return nil 25 | } 26 | 27 | // All hunt requests start with an initial log message - we use 28 | // this log message to increment the hunt scheduled parts and 29 | // assign the collection to the hunt. 30 | if message.VQLResponse != nil && message.VQLResponse.Query != nil && 31 | strings.Contains(message.VQLResponse.Query.VQL, "Starting Hunt") { 32 | 33 | // Increment the hunt's scheduled count. 34 | ingestor_services.HuntStatsManager.Update(hunt_id).IncScheduled() 35 | hunt_flow_entry := &hunt_dispatcher.HuntFlowEntry{ 36 | HuntId: hunt_id, 37 | ClientId: message.Source, 38 | FlowId: message.SessionId, 39 | Timestamp: velo_utils.GetTime().Now().Unix(), 40 | Status: "started", 41 | DocType: "hunt_flow", 42 | } 43 | return services.SetElasticIndex(ctx, 44 | config_obj.OrgId, 45 | "transient", services.DocIdRandom, 46 | hunt_flow_entry) 47 | } 48 | 49 | return nil 50 | } 51 | 52 | func calcFlowOutcome(collection_context *flows_proto.ArtifactCollectorContext) ( 53 | failed, completed bool) { 54 | 55 | for _, s := range collection_context.QueryStats { 56 | switch s.Status { 57 | 58 | // Flow is not completed as one of the queries is still 59 | // running. 60 | case crypto_proto.VeloStatus_PROGRESS: 61 | return false, false 62 | 63 | // Flow failed by it may still be running. 64 | case crypto_proto.VeloStatus_GENERIC_ERROR: 65 | failed = true 66 | 67 | // This query is ok 68 | case crypto_proto.VeloStatus_OK: 69 | } 70 | } 71 | 72 | return failed, true 73 | } 74 | 75 | // When a collection is completed and the collection is part of the 76 | // hunt we need to update the hunt's collection list and stats. 77 | func (self Ingestor) maybeHandleHuntFlowStats( 78 | ctx context.Context, 79 | config_obj *config_proto.Config, 80 | collection_context *flows_proto.ArtifactCollectorContext, 81 | failed, completed bool) error { 82 | 83 | // Ignore messages for incompleted flows 84 | if !completed { 85 | return nil 86 | } 87 | 88 | // Hunt responses have special SessionId like "F.1234.H" 89 | hunt_id, ok := velo_utils.ExtractHuntId(collection_context.SessionId) 90 | if !ok { 91 | return nil 92 | } 93 | 94 | // Increment the failed flow counter 95 | if failed { 96 | ingestor_services.HuntStatsManager.Update(hunt_id).IncError() 97 | } else { 98 | 99 | // This collection is done, update the hunt status. 100 | ingestor_services.HuntStatsManager.Update(hunt_id).IncCompleted() 101 | } 102 | 103 | hunt_flow_entry := &hunt_dispatcher.HuntFlowEntry{ 104 | HuntId: hunt_id, 105 | ClientId: collection_context.ClientId, 106 | FlowId: collection_context.SessionId, 107 | Timestamp: velo_utils.GetTime().Now().Unix(), 108 | Status: "updated", 109 | DocType: "hunt_flow", 110 | } 111 | return services.SetElasticIndex(ctx, 112 | config_obj.OrgId, 113 | "transient", services.DocIdRandom, 114 | hunt_flow_entry) 115 | } 116 | --------------------------------------------------------------------------------