├── artifacts └── .gitkeep ├── tests └── .gitkeep ├── .prettierrc.js ├── .gitignore ├── jest.config.js ├── .npmignore ├── src ├── types │ ├── raw │ │ ├── fields │ │ │ ├── user.ts │ │ │ ├── container.ts │ │ │ ├── event_outcome.ts │ │ │ ├── timestamp_us.ts │ │ │ ├── kubernetes.ts │ │ │ ├── url.ts │ │ │ ├── page.ts │ │ │ ├── process.ts │ │ │ ├── host.ts │ │ │ ├── http.ts │ │ │ ├── faas.ts │ │ │ ├── observer.ts │ │ │ ├── index.ts │ │ │ ├── user_agent.ts │ │ │ ├── service.ts │ │ │ ├── cloud.ts │ │ │ └── stackframe.ts │ │ ├── apm_base_doc.ts │ │ ├── span_raw.ts │ │ ├── error_raw.ts │ │ ├── transaction_raw.ts │ │ └── metric_raw.ts │ └── index.ts ├── tree.ts ├── cli.ts ├── index.ts └── es_client.ts ├── .eslintrc.js ├── README.md ├── LICENSE ├── tsconfig.json └── package.json /artifacts/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | trailingComma: 'es5', 3 | singleQuote: true, 4 | bracketSpacing: true, 5 | printWidth: 120, 6 | }; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /build/ 3 | /lib/ 4 | /dist/ 5 | /docs/ 6 | /output/ 7 | .idea/* 8 | 9 | .DS_Store 10 | coverage 11 | *.log 12 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('@jest/types').Config.InitialOptions} */ 2 | module.exports = { 3 | preset: 'ts-jest', 4 | testEnvironment: 'node', 5 | testMatch: ["/tests/**.js"] 6 | }; 7 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /.github/ 2 | /.vscode/ 3 | /node_modules/ 4 | /build/ 5 | /tmp/ 6 | .idea/* 7 | /docs/ 8 | 9 | coverage 10 | *.log 11 | .gitlab-ci.yml 12 | 13 | package-lock.json 14 | /*.tgz 15 | /tmp* 16 | /mnt/ 17 | /package/ 18 | -------------------------------------------------------------------------------- /src/types/raw/fields/user.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface User { 9 | id: string; 10 | } 11 | -------------------------------------------------------------------------------- /src/types/raw/fields/container.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Container { 9 | id: string; 10 | } 11 | -------------------------------------------------------------------------------- /src/types/raw/fields/event_outcome.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export type EventOutcome = 'success' | 'failure' | 'unknown'; 9 | -------------------------------------------------------------------------------- /src/types/raw/fields/timestamp_us.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface TimestampUs { 9 | us: number; 10 | } 11 | -------------------------------------------------------------------------------- /src/types/raw/fields/kubernetes.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Kubernetes { 9 | pod?: { uid: string; [key: string]: unknown }; 10 | } 11 | -------------------------------------------------------------------------------- /src/types/raw/fields/url.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Url { 9 | domain?: string; 10 | full: string; 11 | original?: string; 12 | } 13 | -------------------------------------------------------------------------------- /src/types/raw/fields/page.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | // only for RUM agent: shared by error and transaction 9 | export interface Page { 10 | url: string; 11 | } 12 | -------------------------------------------------------------------------------- /src/types/raw/fields/process.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Process { 9 | args?: string[]; 10 | pid: number; 11 | ppid?: number; 12 | title?: string; 13 | } 14 | -------------------------------------------------------------------------------- /src/types/raw/fields/host.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Host { 9 | architecture?: string; 10 | hostname?: string; 11 | name?: string; 12 | ip?: string; 13 | os?: { 14 | platform?: string; 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /src/types/raw/fields/http.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Http { 9 | request?: { method: string; [key: string]: unknown }; 10 | response?: { status_code: number; [key: string]: unknown }; 11 | version?: string; 12 | } 13 | -------------------------------------------------------------------------------- /src/types/raw/fields/faas.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Faas { 9 | id: string; 10 | coldstart?: boolean; 11 | execution?: string; 12 | trigger?: { 13 | type?: string; 14 | request_id?: string; 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /src/types/index.ts: -------------------------------------------------------------------------------- 1 | import { ErrorRaw } from "./raw/error_raw"; 2 | import { MetricRaw } from "./raw/metric_raw"; 3 | import { SpanRaw } from "./raw/span_raw"; 4 | import { TransactionRaw } from "./raw/transaction_raw"; 5 | 6 | 7 | export { TransactionRaw } from "./raw/transaction_raw"; 8 | export { SpanRaw } from "./raw/span_raw"; 9 | export { ErrorRaw } from "./raw/error_raw"; 10 | export { MetricRaw } from "./raw/metric_raw"; 11 | 12 | export type Events = TransactionRaw | SpanRaw | ErrorRaw | MetricRaw -------------------------------------------------------------------------------- /src/types/raw/fields/observer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Observer { 9 | ephemeral_id?: string; 10 | hostname?: string; 11 | id?: string; 12 | name?: string; 13 | type?: string; 14 | version: string; 15 | version_major: number; 16 | } 17 | -------------------------------------------------------------------------------- /src/types/raw/fields/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./cloud"; 2 | export * from "./container"; 3 | export * from "./event_outcome"; 4 | export * from "./faas"; 5 | export * from "./host"; 6 | export * from "./http"; 7 | export * from "./kubernetes"; 8 | export * from "./observer"; 9 | export * from "./page"; 10 | export * from "./process"; 11 | export * from "./service"; 12 | export * from "./stackframe"; 13 | export * from "./timestamp_us"; 14 | export * from "./url"; 15 | export * from "./user"; 16 | export * from "./user_agent"; 17 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 'env': { 3 | 'browser': true, 4 | 'es2021': true, 5 | 'node': true, 6 | }, 7 | 'extends': [ 8 | 'google', 9 | 'prettier' 10 | ], 11 | 'parser': '@typescript-eslint/parser', 12 | 'parserOptions': { 13 | 'ecmaVersion': 12, 14 | 'sourceType': 'module', 15 | }, 16 | 'plugins': [ 17 | '@typescript-eslint', 18 | 'prettier' 19 | ], 20 | 'rules': { 21 | "no-unused-vars": "off", 22 | "@typescript-eslint/no-unused-vars": ["error"] 23 | }, 24 | }; 25 | -------------------------------------------------------------------------------- /src/types/raw/fields/user_agent.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface UserAgent { 9 | device?: { 10 | name: string; 11 | }; 12 | name?: string; 13 | original: string; 14 | os?: { 15 | name: string; 16 | version?: string; 17 | full?: string; 18 | }; 19 | version?: string; 20 | } 21 | -------------------------------------------------------------------------------- /src/types/raw/fields/service.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Service { 9 | name: string; 10 | environment?: string; 11 | framework?: { 12 | name: string; 13 | version?: string; 14 | }; 15 | node?: { 16 | name?: string; 17 | }; 18 | runtime?: { 19 | name: string; 20 | version: string; 21 | }; 22 | language?: { 23 | name: string; 24 | version?: string; 25 | }; 26 | version?: string; 27 | } 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # APM PARSER 2 | 3 | ## Pre-conditions 4 | Makes sure to set `testBuildId` and `journeyName` as APM global labels, it is needed to match appropriate transactions. 5 | 6 | ## Cli usage 7 | ```typescript 8 | yarn cli -u -p -c "https://apm-7-17.es.us-central1.gcp.cloud.es.io:9243" -b "local-90a41a83-90da-487b-a6be-4713a9ad18d8" -n "My cool journey" 9 | ``` 10 | 11 | 12 | ## Library usage 13 | 14 | ```typescript 15 | apmParser({ 16 | param: { journeyName: 'My cool journey', buildId: 'local-90a41a83-90da-487b-a6be-4713a9ad18d8'}, 17 | client: { auth: { username: '', password: '' }, baseURL: '' }, 18 | }) 19 | .then(console.info) 20 | .catch(console.error); 21 | ``` 22 | 23 | -------------------------------------------------------------------------------- /src/types/raw/fields/cloud.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | export interface Cloud { 9 | availability_zone?: string; 10 | instance?: { 11 | name: string; 12 | id: string; 13 | }; 14 | machine?: { 15 | type: string; 16 | }; 17 | project?: { 18 | id: string; 19 | name: string; 20 | }; 21 | provider?: string; 22 | region?: string; 23 | account?: { 24 | id: string; 25 | name: string; 26 | }; 27 | image?: { 28 | id: string; 29 | }; 30 | service?: { 31 | name: string; 32 | }; 33 | } 34 | -------------------------------------------------------------------------------- /src/types/raw/apm_base_doc.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | import { Observer } from './fields/observer'; 9 | 10 | // all documents types extend APMBaseDoc and inherit all properties 11 | export interface APMBaseDoc { 12 | '@timestamp': string; 13 | agent: { 14 | name: string; 15 | version: string; 16 | }; 17 | parent?: { id: string }; // parent ID is not available on root transactions 18 | trace?: { id: string }; 19 | labels?: { 20 | [key: string]: string | number | boolean; 21 | }; 22 | observer?: Observer; 23 | } 24 | -------------------------------------------------------------------------------- /src/tree.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | 3 | type TraceDoc = { 4 | '@timestamp': string; 5 | span_id: string; 6 | transaction_id: string; 7 | trace_id: string; 8 | parent_id: string; 9 | children: TraceDoc; 10 | http: Record 11 | } 12 | 13 | export default (traceDocs: TraceDoc[]) => { 14 | const getId = (el) => (el.span_id ? el.span_id : el.transaction_id); 15 | 16 | const idMapping = traceDocs.reduce((acc, el, i) => { 17 | acc[getId(el)] = i; 18 | return acc; 19 | }, {}); 20 | 21 | let root = {}; 22 | traceDocs.forEach((el) => { 23 | if (!el.parent_id) { 24 | root = el; 25 | return; 26 | } 27 | 28 | const parentEl = traceDocs[idMapping[el.parent_id]]; 29 | 30 | const children = _.concat(_.get(parentEl, 'children', []), el); 31 | _.set(parentEl, 'children', children) 32 | }); 33 | 34 | return root; 35 | }; 36 | -------------------------------------------------------------------------------- /src/types/raw/fields/stackframe.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | interface Line { 9 | column?: number; 10 | number: number; 11 | } 12 | 13 | interface Sourcemap { 14 | error?: string; 15 | updated?: boolean; 16 | } 17 | 18 | interface StackframeBase { 19 | abs_path?: string; 20 | classname?: string; 21 | context?: { 22 | post?: string[]; 23 | pre?: string[]; 24 | }; 25 | exclude_from_grouping?: boolean; 26 | filename?: string; 27 | function?: string; 28 | module?: string; 29 | library_frame?: boolean; 30 | line?: Line; 31 | sourcemap?: Sourcemap; 32 | vars?: { 33 | [key: string]: unknown; 34 | }; 35 | } 36 | 37 | export type StackframeWithLineContext = StackframeBase & { 38 | line: Line & { 39 | context: string; 40 | }; 41 | }; 42 | 43 | export type Stackframe = StackframeBase | StackframeWithLineContext; 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Elasticsearch BV 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "moduleResolution": "Node", 5 | "strict": true, 6 | "noImplicitAny": false 7 | /* Raise error on expressions and declarations with an implied 'any' type. */, 8 | "strictNullChecks": true 9 | /* Enable strict null checks. */, 10 | "strictFunctionTypes": true 11 | /* Enable strict checking of function types. */, 12 | "noUnusedLocals": true 13 | /* Report errors on unused locals. */, 14 | "noUnusedParameters": true 15 | /* Report errors on unused parameters. */, 16 | "noImplicitReturns": true 17 | /* Report error when not all code paths in function return a value. */, 18 | "noFallthroughCasesInSwitch": true 19 | /* Report errors for fallthrough cases in switch statement. */, 20 | "importHelpers": true, 21 | "skipLibCheck": true, 22 | "esModuleInterop": true, 23 | "allowSyntheticDefaultImports": true, 24 | "experimentalDecorators": true, 25 | "resolveJsonModule": true, 26 | "strictPropertyInitialization": false, 27 | "declaration": true, 28 | "sourceMap": true, 29 | "types": [ 30 | "node", 31 | "jest" 32 | ], 33 | "lib": [ 34 | "ES6" 35 | ], 36 | "outDir": "dist" 37 | }, 38 | "include": [ 39 | "src/**/*" 40 | ], 41 | "exclude": [ 42 | "node_modules", 43 | "dist" 44 | ], 45 | } 46 | -------------------------------------------------------------------------------- /src/types/raw/span_raw.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | import { APMBaseDoc } from './apm_base_doc'; 9 | import { EventOutcome } from './fields/event_outcome'; 10 | import { Http } from './fields/http'; 11 | import { Stackframe } from './fields/stackframe'; 12 | import { TimestampUs } from './fields/timestamp_us'; 13 | import { Url } from './fields/url'; 14 | 15 | export interface Processor { 16 | name: 'transaction'; 17 | event: 'span'; 18 | } 19 | 20 | export interface SpanRaw extends APMBaseDoc { 21 | processor: Processor; 22 | trace: { id: string }; // trace is required 23 | event?: { outcome?: EventOutcome }; 24 | service: { 25 | name: string; 26 | environment?: string; 27 | }; 28 | span: { 29 | destination?: { 30 | service: { 31 | resource: string; 32 | }; 33 | }; 34 | action?: string; 35 | duration: { us: number }; 36 | id: string; 37 | name: string; 38 | stacktrace?: Stackframe[]; 39 | subtype?: string; 40 | sync?: boolean; 41 | type: string; 42 | http?: { 43 | url?: { 44 | original?: string; 45 | }; 46 | response: { 47 | status_code: number; 48 | }; 49 | method?: string; 50 | }; 51 | db?: { 52 | statement?: string; 53 | type?: string; 54 | }; 55 | message?: { 56 | queue?: { name: string }; 57 | age?: { ms: number }; 58 | body?: string; 59 | headers?: Record; 60 | }; 61 | composite?: { 62 | count: number; 63 | sum: { us: number }; 64 | compression_strategy: string; 65 | }; 66 | }; 67 | timestamp: TimestampUs; 68 | transaction?: { 69 | id: string; 70 | }; 71 | child?: { id: string[] }; 72 | http?: Http; 73 | url?: Url; 74 | } 75 | -------------------------------------------------------------------------------- /src/types/raw/error_raw.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | import { APMBaseDoc } from './apm_base_doc'; 9 | import { Container } from './fields/container'; 10 | import { Host } from './fields/host'; 11 | import { Http } from './fields/http'; 12 | import { Kubernetes } from './fields/kubernetes'; 13 | import { Page } from './fields/page'; 14 | import { Process } from './fields/process'; 15 | import { Service } from './fields/service'; 16 | import { Stackframe } from './fields/stackframe'; 17 | import { TimestampUs } from './fields/timestamp_us'; 18 | import { Url } from './fields/url'; 19 | import { User } from './fields/user'; 20 | 21 | interface Processor { 22 | name: 'error'; 23 | event: 'error'; 24 | } 25 | 26 | export interface Exception { 27 | attributes?: { 28 | response?: string; 29 | }; 30 | code?: string; 31 | message?: string; // either message or type are given 32 | type?: string; 33 | module?: string; 34 | handled?: boolean; 35 | stacktrace?: Stackframe[]; 36 | } 37 | 38 | interface Log { 39 | message: string; 40 | stacktrace?: Stackframe[]; 41 | } 42 | 43 | export interface ErrorRaw extends APMBaseDoc { 44 | processor: Processor; 45 | timestamp: TimestampUs; 46 | transaction?: { 47 | id: string; 48 | sampled?: boolean; 49 | type: string; 50 | }; 51 | error: { 52 | id: string; 53 | culprit?: string; 54 | grouping_key: string; 55 | // either exception or log are given 56 | exception?: Exception[]; 57 | page?: Page; // special property for RUM: shared by error and transaction 58 | log?: Log; 59 | custom?: Record; 60 | }; 61 | 62 | // Shared by errors and transactions 63 | container?: Container; 64 | host?: Host; 65 | http?: Http; 66 | kubernetes?: Kubernetes; 67 | process?: Process; 68 | service: Service; 69 | url?: Url; 70 | user?: User; 71 | } 72 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import apmParser from './index'; 3 | 4 | import yargs from "yargs"; 5 | import { hideBin } from "yargs/helpers" 6 | 7 | const argv = yargs(hideBin(process.argv)) 8 | .scriptName('performance-testing-dataset-extractor') 9 | .usage('Usage: yarn cli -u -p -c -b -n ') 10 | .options({ 11 | u: { 12 | alias: 'user', 13 | demandOption: true, 14 | describe: 'Username', 15 | type: 'string', 16 | }, 17 | p: { 18 | alias: 'pwd', 19 | demandOption: true, 20 | describe: 'Password', 21 | type: 'string', 22 | }, 23 | c: { 24 | alias: 'cluster', 25 | demandOption: false, 26 | describe: 'ES Cluster', 27 | default: 'https://kibana-ops-e2e-perf.es.us-central1.gcp.cloud.es.io:9243', 28 | type: 'string', 29 | }, 30 | b: { 31 | alias: 'buildId', 32 | demandOption: true, 33 | describe: 'Journey Build Id', 34 | default: '', 35 | type: 'string', 36 | }, 37 | n: { 38 | alias: 'journeyName', 39 | demandOption: true, 40 | describe: 'Journey Name', 41 | default: '', 42 | type: 'string', 43 | }, 44 | tt: { 45 | alias: 'transactionType', 46 | demandOption: false, 47 | describe: 'Transaction Type', 48 | default: ['request'], 49 | type: 'array', 50 | }, 51 | d: { 52 | alias: 'dir', 53 | demandOption: false, 54 | describe: 'Output directory', 55 | default: 'artifacts', 56 | type: 'string', 57 | }, 58 | t: { 59 | alias: 'type', 60 | describe: 'Output type (scalability, esperf)', 61 | default: 'scalability', 62 | type: 'string' 63 | } 64 | }) 65 | .help('h') 66 | .alias('h', 'help').argv; 67 | 68 | apmParser({ 69 | dir: argv.d, 70 | type: argv.t, 71 | param: { journeyName: argv.n, buildId: argv.b, transactionType: argv.tt }, 72 | client: { auth: { username: argv.u, password: argv.p }, baseURL: argv.c }, 73 | }) 74 | .then(() => console.log('Dataset extractor finished successfully')) 75 | .catch((e) => console.error('Dataset extractor failed\n', e)); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@elastic/performance-testing-dataset-extractor", 3 | "version": "0.0.3", 4 | "description": "A library to convert APM traces into JSON format for performance testing.", 5 | "author": "Cavit Baturalp Gurdin ", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/elastic/apm-parser.git" 9 | }, 10 | "license": "MIT", 11 | "main": "./dist/index.js", 12 | "types": "./dist/index.d.ts", 13 | "bin": { 14 | "performance-testing-dataset-extractor": "dist/cli.js" 15 | }, 16 | "scripts": { 17 | "main": "ts-node src/index.ts", 18 | "cli": "ts-node src/cli.ts", 19 | "lint": "npx eslint . --ext .ts,.tsx", 20 | "test": "jest", 21 | "clean": "rm -rf dist build package output", 22 | "docs": "typedoc --entryPoints src/index.ts", 23 | "build": "tsc -p tsconfig.json", 24 | "build-all": "yarn clean && yarn build" 25 | }, 26 | "dependencies": { 27 | "@elastic/elasticsearch": "7.17.0", 28 | "axios": "^0.26.1", 29 | "axios-curlirize": "1.3.7", 30 | "lodash": "^4.17.21", 31 | "qs": "^6.10.3", 32 | "tslib": "^2.3.1", 33 | "yargs": "^17.4.0" 34 | }, 35 | "devDependencies": { 36 | "@types/axios-curlirize": "^1.3.2", 37 | "@types/bluebird": "^3.5.36", 38 | "@types/jest": "^26.0.21", 39 | "@types/lodash": "^4.14.180", 40 | "@types/node": "^15.0.1", 41 | "@types/qs": "^6.9.7", 42 | "@typescript-eslint/eslint-plugin": "^5.15.0", 43 | "@typescript-eslint/parser": "^5.15.0", 44 | "eslint": "^8.11.0", 45 | "eslint-config-google": "^0.14.0", 46 | "eslint-config-prettier": "^8.5.0", 47 | "eslint-plugin-prettier": "^4.0.0", 48 | "jest": "^26.6.3", 49 | "prettier": "^2.6.0", 50 | "ts-jest": "^26.5.4", 51 | "ts-node": "^9.1.1", 52 | "typedoc": "^0.20.35", 53 | "typescript": "^4.2.3" 54 | }, 55 | "engines": { 56 | "node": ">=16" 57 | }, 58 | "files": [ 59 | "dist" 60 | ], 61 | "bugs": { 62 | "url": "https://github.com/elastic/apm-parser/issues" 63 | }, 64 | "homepage": "https://github.com/elastic/apm-parser#readme", 65 | "directories": { 66 | "doc": "docs", 67 | "test": "tests" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/types/raw/transaction_raw.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | import { APMBaseDoc } from './apm_base_doc'; 9 | import { Cloud } from './fields/cloud'; 10 | import { Container } from './fields/container'; 11 | import { EventOutcome } from './fields/event_outcome'; 12 | import { Host } from './fields/host'; 13 | import { Http } from './fields/http'; 14 | import { Kubernetes } from './fields/kubernetes'; 15 | import { Page } from './fields/page'; 16 | import { Process } from './fields/process'; 17 | import { Service } from './fields/service'; 18 | import { TimestampUs } from './fields/timestamp_us'; 19 | import { Url } from './fields/url'; 20 | import { User } from './fields/user'; 21 | import { UserAgent } from './fields/user_agent'; 22 | import { Faas } from './fields/faas'; 23 | 24 | export interface Processor { 25 | name: 'transaction'; 26 | event: 'transaction'; 27 | } 28 | 29 | export interface TransactionRaw extends APMBaseDoc { 30 | processor: Processor; 31 | timestamp: TimestampUs; 32 | trace: { id: string }; // trace is required 33 | event?: { outcome?: EventOutcome }; 34 | transaction: { 35 | duration: { us: number }; 36 | id: string; 37 | marks?: { 38 | // "agent": not defined by APM Server - only sent by RUM agent 39 | agent?: { 40 | [name: string]: number; 41 | }; 42 | }; 43 | name?: string; 44 | page?: Page; // special property for RUM: shared by error and transaction 45 | result?: string; 46 | sampled: boolean; 47 | span_count?: { 48 | started?: number; 49 | dropped?: number; 50 | }; 51 | type: string; 52 | custom?: Record; 53 | message?: { 54 | queue?: { name: string }; 55 | age?: { ms: number }; 56 | body?: string; 57 | headers?: Record; 58 | }; 59 | }; 60 | 61 | // Shared by errors and transactions 62 | container?: Container; 63 | ecs?: { version?: string }; 64 | host?: Host; 65 | http?: Http; 66 | kubernetes?: Kubernetes; 67 | process?: Process; 68 | service: Service; 69 | url?: Url; 70 | user?: User; 71 | user_agent?: UserAgent; 72 | cloud?: Cloud; 73 | faas?: Faas; 74 | } 75 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs/promises'; 2 | import { existsSync } from 'fs'; 3 | import { initClient } from './es_client'; 4 | import path from 'path'; 5 | 6 | type CLIParams = { 7 | dir: string; 8 | type: string; 9 | param: { 10 | journeyName: string; 11 | buildId: string; 12 | transactionType: string[]; 13 | }; 14 | client: { 15 | auth: { 16 | username: string; 17 | password: string; 18 | }; 19 | baseURL: string; 20 | }; 21 | }; 22 | 23 | const apmParser = async ({ param, client }: CLIParams) => { 24 | const authOptions = { 25 | node: client.baseURL, 26 | username: client.auth.username, 27 | password: client.auth.password, 28 | } 29 | const esClient = initClient(authOptions); 30 | const hits = await esClient.getTransactions(param.buildId, param.journeyName); 31 | if (!hits || hits.length === 0) { 32 | console.warn(`No transactions found with 'labels.testBuildId=${param.buildId}' and 'labels.journeyName=${param.journeyName}'\n Output file won't be generated`); 33 | return; 34 | } 35 | 36 | const source = hits[0]._source; 37 | const journeyName = source.labels.journeyName || 'Unknown Journey'; 38 | const kibanaVersion = source.service.version; 39 | const maxUsersCount = source.labels.maxUsersCount || '0'; 40 | 41 | const data = hits.map(hit => hit._source).map(hit => { 42 | return { 43 | processor: hit.processor, 44 | traceId : hit.trace.id, 45 | timestamp: hit["@timestamp"], 46 | environment: hit.environment, 47 | request: { 48 | url: { path: hit.url.path }, 49 | headers: hit.http.request.headers, 50 | method: hit.http.request.method, 51 | body: hit.http.request.body ? JSON.parse(hit.http.request.body.original) : '', 52 | }, 53 | response: {statusCode: hit.http.response.status_code}, 54 | transaction: { 55 | id: hit.transaction.id, 56 | name: hit.transaction.name, 57 | type: hit.transaction.type, 58 | } 59 | } 60 | }); 61 | 62 | const output = { 63 | journeyName, 64 | kibanaVersion, 65 | maxUsersCount, 66 | traceItems: data, 67 | } 68 | 69 | const outputDir = path.resolve('output'); 70 | const fileName = `${output.journeyName.replace(/ /g,'')}-${param.buildId}.json` 71 | const filePath = path.resolve(outputDir, fileName); 72 | 73 | console.log(`Found ${hits.length} transactions, output file: ${filePath}`); 74 | if (!existsSync(outputDir)) { 75 | await fs.mkdir(outputDir); 76 | } 77 | await fs.writeFile(filePath, JSON.stringify(output, null, 2), 'utf8'); 78 | }; 79 | 80 | export default apmParser; 81 | -------------------------------------------------------------------------------- /src/types/raw/metric_raw.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | 8 | import { APMBaseDoc } from './apm_base_doc'; 9 | import { Cloud } from './fields/cloud'; 10 | import { Container } from './fields/container'; 11 | import { Host } from './fields/host'; 12 | import { Kubernetes } from './fields/kubernetes'; 13 | import { Service } from './fields/service'; 14 | 15 | type BaseMetric = APMBaseDoc & { 16 | processor: { 17 | name: 'metric'; 18 | event: 'metric'; 19 | }; 20 | cloud?: Cloud; 21 | container?: Container; 22 | kubernetes?: Kubernetes; 23 | service?: Service; 24 | host?: Host; 25 | }; 26 | 27 | type BaseBreakdownMetric = BaseMetric & { 28 | transaction: { 29 | name: string; 30 | type: string; 31 | }; 32 | span: { 33 | self_time: { 34 | count: number; 35 | sum: { 36 | us: number; 37 | }; 38 | }; 39 | }; 40 | }; 41 | 42 | type TransactionBreakdownMetric = BaseBreakdownMetric & { 43 | transaction: { 44 | duration: { 45 | count: number; 46 | sum: { 47 | us: number; 48 | }; 49 | }; 50 | breakdown: { 51 | count: number; 52 | }; 53 | }; 54 | }; 55 | 56 | type SpanBreakdownMetric = BaseBreakdownMetric & { 57 | span: { 58 | type: string; 59 | subtype?: string; 60 | }; 61 | }; 62 | 63 | type SystemMetric = BaseMetric & { 64 | system: unknown; 65 | service: { 66 | node?: { 67 | name: string; 68 | }; 69 | }; 70 | }; 71 | 72 | type CGroupMetric = SystemMetric; 73 | type JVMMetric = SystemMetric & { 74 | jvm: unknown; 75 | }; 76 | 77 | type TransactionDurationMetric = BaseMetric & { 78 | transaction: { 79 | name: string; 80 | type: string; 81 | result?: string; 82 | duration: { 83 | histogram: { 84 | values: number[]; 85 | counts: number[]; 86 | }; 87 | }; 88 | }; 89 | service: { 90 | name: string; 91 | node?: { 92 | name: string; 93 | }; 94 | environment?: string; 95 | version?: string; 96 | }; 97 | }; 98 | 99 | export type SpanDestinationMetric = BaseMetric & { 100 | span: { 101 | destination: { 102 | service: { 103 | resource: string; 104 | response_time: { 105 | count: number; 106 | sum: { 107 | us: number; 108 | }; 109 | }; 110 | }; 111 | }; 112 | }; 113 | }; 114 | 115 | export type MetricRaw = 116 | | BaseMetric 117 | | TransactionBreakdownMetric 118 | | SpanBreakdownMetric 119 | | TransactionDurationMetric 120 | | SpanDestinationMetric 121 | | SystemMetric 122 | | CGroupMetric 123 | | JVMMetric; 124 | -------------------------------------------------------------------------------- /src/es_client.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable require-jsdoc */ 2 | import {Client} from '@elastic/elasticsearch'; 3 | 4 | interface clientOptions { 5 | node: string, 6 | username: string, 7 | password: string, 8 | } 9 | 10 | export function initClient(options: clientOptions) { 11 | const client = new Client({ 12 | node: options.node, 13 | auth: { 14 | username: options.username, 15 | password: options.password 16 | } 17 | }); 18 | 19 | return { 20 | getTransactions: async function(buildId: string, journeyName: string){ 21 | const result = await client.search({ 22 | "body": { 23 | "track_total_hits": true, 24 | "sort": [ 25 | { 26 | "@timestamp": { 27 | "order": "desc", 28 | "unmapped_type": "boolean" 29 | } 30 | } 31 | ], 32 | "size": 10000, 33 | "stored_fields": [ 34 | "*" 35 | ], 36 | "_source": true, 37 | "query": { 38 | "bool": { 39 | "must": [], 40 | "filter": [ 41 | { 42 | "bool": { 43 | "filter": [ 44 | { 45 | "bool": { 46 | "should": [ 47 | { 48 | "match_phrase": { 49 | "transaction.type": "request" 50 | } 51 | } 52 | ], 53 | "minimum_should_match": 1 54 | } 55 | }, 56 | { 57 | "bool": { 58 | "should": [ 59 | { 60 | "match_phrase": { 61 | "processor.event": "transaction" 62 | } 63 | } 64 | ], 65 | "minimum_should_match": 1 66 | } 67 | }, 68 | { 69 | "bool": { 70 | "should": [ 71 | { 72 | "match_phrase": { 73 | "labels.testBuildId": buildId 74 | } 75 | } 76 | ], 77 | "minimum_should_match": 1 78 | } 79 | }, 80 | { 81 | "bool": { 82 | "should": [ 83 | { 84 | "match_phrase": { 85 | "labels.journeyName": journeyName 86 | } 87 | } 88 | ], 89 | "minimum_should_match": 1 90 | } 91 | } 92 | ] 93 | } 94 | } 95 | ], 96 | "should": [], 97 | "must_not": [] 98 | } 99 | } 100 | } 101 | }) 102 | return result?.body?.hits?.hits; 103 | } 104 | } 105 | } 106 | 107 | --------------------------------------------------------------------------------