├── src ├── main │ ├── webapp │ │ ├── META-INF │ │ │ └── MANIFEST.MF │ │ ├── img │ │ │ ├── info.gif │ │ │ ├── mdb.ico │ │ │ ├── mdb.png │ │ │ ├── sort_asc.png │ │ │ ├── sort_both.png │ │ │ ├── sort_desc.png │ │ │ └── glyphicons-halflings.png │ │ ├── res │ │ │ ├── spinner.gif │ │ │ └── showhide.png │ │ ├── buildInfo.jsp │ │ ├── datetimepicker.html │ │ ├── js │ │ │ ├── jquery.dataTables.sum.js │ │ │ └── jquery.number.min.js │ │ ├── commandResult.jsp │ │ ├── WEB-INF │ │ │ └── web.xml │ │ ├── slowop.jsp │ │ ├── datatables.json │ │ ├── css │ │ │ └── bootstrap-datetimepicker.min.css │ │ └── datatables2.html │ ├── java │ │ └── de │ │ │ └── idealo │ │ │ └── mongodb │ │ │ └── slowops │ │ │ ├── collector │ │ │ ├── Terminable.java │ │ │ ├── StartStopHook.java │ │ │ ├── CollectorManagerInstance.java │ │ │ ├── ProfilingEntry.java │ │ │ └── ProfilingWriter.java │ │ │ ├── jmx │ │ │ └── CollectorManagerMBean.java │ │ │ ├── dto │ │ │ ├── TableDto.java │ │ │ ├── SlowOpsFilterDto.java │ │ │ ├── CommandResultDto.java │ │ │ ├── SlowOpsDto.java │ │ │ ├── CollectorServerDto.java │ │ │ ├── ApplicationStatusDto.java │ │ │ ├── HostInfoDto.java │ │ │ ├── CollectorStatusDto.java │ │ │ └── ProfiledServerDto.java │ │ │ ├── util │ │ │ ├── Terminator.java │ │ │ ├── ProfilingReaderCreator.java │ │ │ ├── Util.java │ │ │ ├── MongoResolver.java │ │ │ └── ConfigReader.java │ │ │ ├── command │ │ │ ├── CmdCurrentOpAll.java │ │ │ ├── ICommand.java │ │ │ ├── CmdCurrentOpNs.java │ │ │ ├── CommandExecutor.java │ │ │ ├── CmdDatabaseStats.java │ │ │ ├── CmdCurrentOp.java │ │ │ ├── CmdHostInfo.java │ │ │ ├── CmdCollectionStats.java │ │ │ └── CmdIdxAccessStats.java │ │ │ ├── grapher │ │ │ ├── AggregatedProfilingGroup.java │ │ │ ├── AggregatedProfilingId.java │ │ │ └── AggregatedProfiling.java │ │ │ ├── servlet │ │ │ ├── SlowOpExample.java │ │ │ ├── ApplicationStatus.java │ │ │ ├── Action.java │ │ │ └── CommandResult.java │ │ │ └── monitor │ │ │ └── MongoDbAccessor.java │ └── resources │ │ ├── config.json │ │ ├── logback.xml │ │ └── config.example.json └── test │ └── java │ └── de │ └── idealo │ └── mongodb │ └── slowops │ ├── dto │ └── ProfiledServerDtoTest.java │ └── util │ └── ConfigReaderTest.java ├── img ├── blog.png ├── video.png ├── slow_operations_app_status.png ├── slow_operations_gui_table.jpg ├── slow_operations_gui_diagram.png ├── slow_operations_gui_diagram_low.png ├── slow_operations_gui_table_low.png └── slow_operations_command_result_page.png ├── .gitignore ├── .github ├── CODEOWNERS └── workflows │ └── add-version-tag.yml ├── catalog-info.yaml ├── Dockerfile ├── docker-compose.yaml ├── renovate.json └── pom.xml /src/main/webapp/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Class-Path: 3 | 4 | -------------------------------------------------------------------------------- /img/blog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/blog.png -------------------------------------------------------------------------------- /img/video.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/video.png -------------------------------------------------------------------------------- /src/main/webapp/img/info.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/img/info.gif -------------------------------------------------------------------------------- /src/main/webapp/img/mdb.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/img/mdb.ico -------------------------------------------------------------------------------- /src/main/webapp/img/mdb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/img/mdb.png -------------------------------------------------------------------------------- /src/main/webapp/res/spinner.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/res/spinner.gif -------------------------------------------------------------------------------- /img/slow_operations_app_status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/slow_operations_app_status.png -------------------------------------------------------------------------------- /img/slow_operations_gui_table.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/slow_operations_gui_table.jpg -------------------------------------------------------------------------------- /src/main/webapp/img/sort_asc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/img/sort_asc.png -------------------------------------------------------------------------------- /src/main/webapp/img/sort_both.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/img/sort_both.png -------------------------------------------------------------------------------- /src/main/webapp/img/sort_desc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/img/sort_desc.png -------------------------------------------------------------------------------- /src/main/webapp/res/showhide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/res/showhide.png -------------------------------------------------------------------------------- /img/slow_operations_gui_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/slow_operations_gui_diagram.png -------------------------------------------------------------------------------- /img/slow_operations_gui_diagram_low.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/slow_operations_gui_diagram_low.png -------------------------------------------------------------------------------- /img/slow_operations_gui_table_low.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/slow_operations_gui_table_low.png -------------------------------------------------------------------------------- /img/slow_operations_command_result_page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/img/slow_operations_command_result_page.png -------------------------------------------------------------------------------- /src/main/webapp/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/idealo/mongodb-slow-operations-profiler/HEAD/src/main/webapp/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | target/ 3 | *.csv 4 | *.log 5 | slowOpsGui*.png 6 | .~lock* 7 | .checkstyle 8 | .classpath 9 | .git_* 10 | .project 11 | .settings 12 | .idealo.yaml 13 | todo.txt 14 | src/main/resources/config.local.json 15 | src/main/resources/config.prod.json 16 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/collector/Terminable.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.collector; 2 | 3 | /** 4 | * Created by kay.agahd on 21.06.17. 5 | */ 6 | public interface Terminable { 7 | 8 | void terminate(); 9 | long getDoneJobs(); 10 | } 11 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-code-owners 2 | 3 | # These owners will be the default owners for everything in 4 | # the repo. Unless a later match takes precedence, 5 | # @global-owner1 and @global-owner2 will be requested for 6 | # review when someone opens a pull request. 7 | * @idealo/cloud-core @kagahd 8 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Component 3 | metadata: 4 | name: mongodb-slow-operations-profiler 5 | description: This java web application collects and stores slow operations from one or more mongoDB system(s) in order to visualize and analyze them. 6 | annotations: 7 | github.com/project-slug: idealo/mongodb-slow-operations-profiler 8 | github.com/team-slug: idealo/cloud-core 9 | spec: 10 | type: service 11 | lifecycle: production 12 | owner: group:cloud-core 13 | system: database-management-toolset 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine/git AS cloner 2 | RUN cd /root && git clone https://github.com/idealo/mongodb-slow-operations-profiler.git 3 | 4 | FROM maven:3.8.7-openjdk-18-slim AS builder 5 | COPY --from=cloner /root/mongodb-slow-operations-profiler/ /usr/src/app/ 6 | WORKDIR /usr/src/app/ 7 | RUN mvn package 8 | 9 | FROM tomcat:9-jdk11-openjdk-slim 10 | COPY --from=builder /usr/src/app/target/mongodb-slow-operations-profiler.war /tmp 11 | WORKDIR /usr/local/tomcat/webapps/mongodb-slow-operations-profiler/ 12 | RUN jar -xfv /tmp/mongodb-slow-operations-profiler.war 13 | RUN chown -R nobody:nogroup /usr/local/tomcat/webapps/ 14 | USER nobody:nogroup -------------------------------------------------------------------------------- /src/main/webapp/buildInfo.jsp: -------------------------------------------------------------------------------- 1 |
2 | <% 3 | java.util.jar.Manifest manifest = new java.util.jar.Manifest(); 4 | manifest.read(pageContext.getServletContext().getResourceAsStream("/META-INF/MANIFEST.MF")); 5 | java.util.jar.Attributes attributes = manifest.getMainAttributes(); 6 | %> 7 | 8 | <%=attributes.getValue("Implementation-Title")%>, opensource, please contribute
9 | Version: <%=attributes.getValue("Implementation-Version")%>
10 | Build-Time: <%=attributes.getValue("Build-Time")%>
11 | Build-Jdk: <%=attributes.getValue("Build-Jdk")%> 12 |
13 | 14 | 15 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/jmx/CollectorManagerMBean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.jmx; 5 | 6 | import java.util.Date; 7 | 8 | /** 9 | * 10 | * 11 | * @author kay.agahd 12 | * @since 22.03.2013 13 | * @version $Id: $ 14 | * @copyright idealo internet GmbH 15 | */ 16 | public interface CollectorManagerMBean { 17 | 18 | long getNumberOfReads(); 19 | 20 | long getNumberOfWrites(); 21 | 22 | long getNumberOfReadsOfRemovedReaders(); 23 | 24 | long getNumberOfWritesOfRemovedWriters(); 25 | 26 | Date getRunningSince(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/TableDto.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.dto; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.collect.Lists; 6 | 7 | public class TableDto { 8 | 9 | private final List> rows; 10 | 11 | public TableDto() { 12 | this.rows = Lists.newArrayList(); 13 | } 14 | 15 | 16 | public void addRow(List row){ 17 | if(row!=null){ 18 | rows.add(row); 19 | } 20 | } 21 | 22 | public void addRows(TableDto tableToAdd){ 23 | if(tableToAdd!=null){ 24 | rows.addAll(tableToAdd.getTableRows()); 25 | } 26 | } 27 | 28 | public List> getTableRows(){ 29 | return rows; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | collector-db: 4 | image: mongo:8.0 5 | container_name: collector-db 6 | test-db: 7 | image: mongo:8.0 8 | container_name: test-db 9 | profiler-webapp: 10 | build: 11 | context: . 12 | dockerfile: Dockerfile 13 | container_name: profiler-webapp 14 | depends_on: 15 | - collector-db 16 | - test-db 17 | ports: 18 | - "8080:8080" 19 | # You may restrict the mapping to the host IP only 20 | # if run on localhost e.g. Docker: 21 | #- "127.0.0.1:8080:8080" 22 | # if run inside VM e.g. Podman or Lima on MacOS: 23 | #- "192.168.127.2:8080:8080" 24 | links: 25 | - "collector-db" 26 | - "test-db" -------------------------------------------------------------------------------- /src/main/resources/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "collector": { 3 | "hosts": [ 4 | "collector-db:27017" 5 | ], 6 | "db": "mongodba", 7 | "collection": "slowops" 8 | }, 9 | "profiled": [ 10 | { 11 | "label": "test-dbs", 12 | "hosts": [ 13 | "test-db:27017" 14 | ], 15 | "ns": [ 16 | "myDB.myCollection", 17 | "anotherDB.*", 18 | "test.*" 19 | ], 20 | "slowMS": 3000, 21 | "adminUser": "", 22 | "adminPw": "", 23 | "collect": true 24 | } 25 | ], 26 | "yAxisScale": "milliseconds", 27 | "adminToken": "mySecureAdminToken", 28 | "defaultSlowMS": 100, 29 | "defaultResponseTimeoutInMs": 3000, 30 | "maxWeblogEntries": 100, 31 | "systemProfileCollectionMaxSizeInMB": 16 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/util/Terminator.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.util; 2 | 3 | 4 | import de.idealo.mongodb.slowops.collector.Terminable; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.concurrent.Callable; 9 | 10 | public class Terminator implements Callable { 11 | 12 | private static final Logger LOG = LoggerFactory.getLogger(Terminator.class); 13 | 14 | private final Terminable terminable; 15 | 16 | public Terminator(Terminable terminable) { 17 | this.terminable = terminable; 18 | } 19 | 20 | @Override 21 | public Long call() throws Exception { 22 | LOG.debug("terminate called"); 23 | terminable.terminate(); 24 | return terminable.getDoneJobs(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | 4 | // Create PRs only for GitHub Security Alerts (Dependabot advisories) 5 | "vulnerabilityAlerts": { 6 | "enabled": true 7 | }, 8 | 9 | // Prevent the extra Dependency Dashboard issue 10 | "extends": [ 11 | ":disableDependencyDashboard" 12 | ], 13 | 14 | // Disable all regular updates (major/minor/patch/digest/pin/etc.) 15 | // Security alert PRs remain allowed. 16 | "packageRules": [ 17 | { 18 | "description": "Disable all non-security updates across all managers", 19 | // Applies to all managers when not specifying matchManagers 20 | "matchUpdateTypes": [ 21 | "major", 22 | "minor", 23 | "patch", 24 | "pin", 25 | "digest", 26 | "rollback", 27 | "bump", 28 | "lockFileMaintenance" 29 | ], 30 | "enabled": false 31 | } 32 | ], 33 | 34 | // Optional: add labels to security PRs 35 | "labels": ["security", "renovate"] 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/SlowOpsFilterDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.dto; 5 | 6 | 7 | /** 8 | * 9 | * 10 | * @author kay.agahd 11 | * @since 20.03.2013 12 | * @version $Id: $ 13 | * @copyright idealo internet GmbH 14 | */ 15 | public class SlowOpsFilterDto { 16 | 17 | private StringBuffer pipeline; 18 | private Object[] parameters; 19 | 20 | /** 21 | * @param pipeline 22 | */ 23 | public void setPipeline(StringBuffer pipeline) { 24 | this.pipeline = pipeline; 25 | 26 | } 27 | 28 | /** 29 | * @param parameters 30 | */ 31 | public void setParameters(Object[] parameters) { 32 | this.parameters = parameters; 33 | } 34 | 35 | /** 36 | * @return 37 | */ 38 | public StringBuffer getPipeline() { 39 | return pipeline; 40 | } 41 | 42 | /** 43 | * @return 44 | */ 45 | public Object[] getParameters() { 46 | return parameters; 47 | } 48 | 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/CmdCurrentOpAll.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.BasicDBObject; 5 | import com.mongodb.DBObject; 6 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 7 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 8 | import de.idealo.mongodb.slowops.dto.TableDto; 9 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 10 | import org.bson.Document; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | 17 | /** 18 | * Created by kay.agahd on 03.04.20. 19 | */ 20 | public class CmdCurrentOpAll extends CmdCurrentOp { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(CmdCurrentOpAll.class); 23 | 24 | 25 | @Override 26 | public boolean isHostCommand(){ 27 | return true; 28 | } 29 | 30 | @Override 31 | public DBObject getQuery(ProfilingReader profilingReader){ 32 | return new BasicDBObject("currentOp", 1); 33 | }; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d [%thread] %-5level %logger{35}:%line - %msg%n 7 | 8 | 9 | 10 | 11 | ${catalina.base}/logs/slow-operations-profiler.log 12 | 13 | ${catalina.base}/logs/slow-operations-profiler.%d{yyyy-MM-dd}.log 14 | 10 15 | 16 | 17 | %d [%thread] %-5level %logger{25}:%line - %msg%n 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/ICommand.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 4 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 5 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 6 | import de.idealo.mongodb.slowops.dto.TableDto; 7 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 8 | 9 | import java.time.format.DateTimeFormatter; 10 | 11 | /** 12 | * Created by kay.agahd on 30.06.17. 13 | */ 14 | public interface ICommand { 15 | 16 | DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); 17 | 18 | TableDto runCommand(ProfilingReader profilingReader, MongoDbAccessor mongoDbAccessor); 19 | 20 | CommandResultDto getCommandResultDto(); 21 | 22 | /** 23 | * Determines whether the command is to be executed once per host or once per database. 24 | * For example, getting the host info is or getting the current operations are both host commands 25 | * because it's not related to an individual database whereas 26 | * listing collections or getting index access stats is related to a specific database. 27 | * @return true if the command is to be executed once per host, else returns false 28 | */ 29 | boolean isHostCommand(); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/CmdCurrentOpNs.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.BasicDBObject; 5 | import com.mongodb.DBObject; 6 | import com.mongodb.client.MongoDatabase; 7 | import com.mongodb.client.model.Filters; 8 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 9 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 10 | import de.idealo.mongodb.slowops.dto.TableDto; 11 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 12 | import org.bson.Document; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.regex.Pattern; 19 | 20 | /** 21 | * Created by kay.agahd on 03.04.20. 22 | */ 23 | public class CmdCurrentOpNs extends CmdCurrentOp { 24 | 25 | private static final Logger LOG = LoggerFactory.getLogger(CmdCurrentOpNs.class); 26 | 27 | @Override 28 | public boolean isHostCommand(){ 29 | return false; 30 | } 31 | 32 | @Override 33 | public DBObject getQuery(ProfilingReader profilingReader){ 34 | return new BasicDBObject("currentOp", 1) 35 | .append("ns", 36 | new BasicDBObject("$regex", "^"+Pattern.quote(profilingReader.getDatabase()+"."))); 37 | }; 38 | 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/CommandExecutor.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | 4 | 5 | import com.mongodb.ServerAddress; 6 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 7 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 8 | import de.idealo.mongodb.slowops.dto.TableDto; 9 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.concurrent.Callable; 14 | 15 | 16 | public class CommandExecutor implements Callable { 17 | 18 | private static final Logger LOG = LoggerFactory.getLogger(CommandExecutor.class); 19 | 20 | private final ProfilingReader profilingReader; 21 | private final ICommand command; 22 | private final ServerAddress[] serverAddresses; 23 | 24 | public CommandExecutor(ProfilingReader profilingReader, ICommand command, ServerAddress ... serverAddresses) { 25 | this.profilingReader = profilingReader; 26 | this.command = command; 27 | this.serverAddresses = serverAddresses; 28 | } 29 | 30 | 31 | @Override 32 | public TableDto call() { 33 | final ProfiledServerDto profiledServerDto = profilingReader.getProfiledServerDto(); 34 | final MongoDbAccessor mongoDbAccessor = new MongoDbAccessor(profiledServerDto.getAdminUser(), profiledServerDto.getAdminPw(), profiledServerDto.getSsl(), serverAddresses); 35 | 36 | return command.runCommand(profilingReader, mongoDbAccessor); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/add-version-tag.yml: -------------------------------------------------------------------------------- 1 | name: Add version tag 2 | 3 | permissions: 4 | contents: write 5 | 6 | on: 7 | workflow_dispatch: 8 | push: 9 | branches: 10 | - master 11 | paths: # only add version tag if pom.xml has changed because it contains the version number 12 | - pom.xml 13 | 14 | 15 | jobs: 16 | add-version-tag: 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v3 22 | 23 | - name: Set up JDK 11 24 | uses: actions/setup-java@v3 25 | with: 26 | distribution: 'adopt' # You can choose other distributions like 'zulu' or 'temurin' 27 | java-version: '11' 28 | 29 | - name: add version tag 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | run: | 33 | VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) 34 | 35 | git config --local user.email "github-actions@idealo.de" 36 | git config --local user.name "github-actions[bot]" 37 | 38 | # Delete the local tag if it exists 39 | git tag -d "v$VERSION" || true # Ignore if the tag doesn't exist 40 | git tag -a "v$VERSION" -m "Tagging version $VERSION" 41 | git push origin :refs/tags/v$VERSION || true # delete remote tag, if exists 42 | git push origin "v$VERSION" # create remote tag 43 | 44 | echo "### Version tag v$VERSION added" >> $GITHUB_STEP_SUMMARY 45 | echo "The repository has been tagged with v$VERSION" >> $GITHUB_STEP_SUMMARY 46 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/collector/StartStopHook.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.collector; 5 | 6 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import javax.servlet.ServletContextEvent; 11 | import javax.servlet.ServletContextListener; 12 | 13 | /** 14 | * 15 | * 16 | * @author kay.agahd 17 | * @since 26.03.2013 18 | * @version $Id: $ 19 | * @copyright idealo internet GmbH 20 | */ 21 | public class StartStopHook implements ServletContextListener{ 22 | 23 | private static final Logger LOG = LoggerFactory.getLogger(StartStopHook.class); 24 | 25 | 26 | /* (non-Javadoc) 27 | * @see javax.servlet.ServletContextListener#contextInitialized(javax.servlet.ServletContextEvent) 28 | */ 29 | @Override 30 | public void contextInitialized(ServletContextEvent arg0) { 31 | LOG.info(">>> contextInitialized"); 32 | CollectorManagerInstance.init(); 33 | LOG.info("<<< contextInitialized"); 34 | 35 | 36 | } 37 | 38 | /* (non-Javadoc) 39 | * @see javax.servlet.ServletContextListener#contextDestroyed(javax.servlet.ServletContextEvent) 40 | */ 41 | @Override 42 | public void contextDestroyed(ServletContextEvent arg0) { 43 | LOG.info(">>> contextDestroyed"); 44 | CollectorManagerInstance.terminate(); 45 | MongoDbAccessor.terminate(); 46 | ExampleSlowOpsCache.terminate(); 47 | LOG.info("<<< contextDestroyed"); 48 | 49 | } 50 | 51 | 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/collector/CollectorManagerInstance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.collector; 5 | 6 | import de.idealo.mongodb.slowops.dto.ApplicationStatusDto; 7 | 8 | import java.util.List; 9 | import java.util.Set; 10 | 11 | /** 12 | * 13 | * 14 | * @author kay.agahd 15 | * @since 02.04.2013 16 | * @version $Id: $ 17 | * @copyright idealo internet GmbH 18 | */ 19 | public final class CollectorManagerInstance { 20 | 21 | private static final CollectorManager INSTANCE = new CollectorManager(); 22 | 23 | private CollectorManagerInstance(){};//no Instances of this! 24 | 25 | public static void init(){ 26 | INSTANCE.startup(); 27 | } 28 | 29 | public static void terminate(){ 30 | INSTANCE.terminate(); 31 | } 32 | 33 | public static ApplicationStatusDto getApplicationStatus(boolean isAuthenticated){ return INSTANCE.getApplicationStatus(isAuthenticated); } 34 | 35 | public static ApplicationStatusDto getApplicationStatus(List idList, boolean isAuthenticated){ return INSTANCE.getApplicationStatus(idList, isAuthenticated); } 36 | 37 | public static void startStopProfilingReaders(List idList, boolean stop){ INSTANCE.startStopProfilingReaders(idList, stop); }; 38 | 39 | public static void setSlowMs(List idList, long ms){ INSTANCE.setSlowMs(idList, ms); }; 40 | 41 | public static void reloadConfig(String cfg){ INSTANCE.reloadConfig(cfg); } 42 | 43 | public static List getProfilingReaders(Set ids){ return INSTANCE.getProfilingReaders(ids); } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/grapher/AggregatedProfilingGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.grapher; 5 | 6 | 7 | /** 8 | * Class to compare parts of AggregatedProfilingId for grouping reasons. 9 | * 10 | * @author kay.agahd 11 | * @since 15.03.2013 12 | * @version $Id: $ 13 | * @copyright idealo internet GmbH 14 | */ 15 | public class AggregatedProfilingGroup extends AggregatedProfilingId { 16 | 17 | /** 18 | * 19 | * @return comma separated String of non-empty fields with their names (commas replaced by semicolon) 20 | */ 21 | public String getLabel() { 22 | final StringBuffer result = new StringBuffer(); 23 | result.append(getField("adr", getAdr())); 24 | result.append(getField("op", getOp())); 25 | result.append(getField("user", getUser())); 26 | result.append(getField("fields", getFields()!=null?getFields().toString():null)); 27 | result.append(getField("sort", getSort()!=null?getSort().toString():null)); 28 | result.append(getField("proj", getProj()!=null?getProj().toString():null)); 29 | 30 | if(result.length() > 0) { 31 | result.deleteCharAt(result.length()-1);//remove last comma 32 | } 33 | 34 | return result.toString(); 35 | } 36 | 37 | private String getField(String name, String field) { 38 | if(field != null && field.length() > 0) { 39 | final StringBuffer result = new StringBuffer(); 40 | result.append(name).append("=").append(field.replace(',', ';')).append(","); 41 | return result.toString(); 42 | } 43 | return ""; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/webapp/datetimepicker.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 20 | 21 | 22 | 28 | 29 |
15 |
16 | 17 | 18 |
19 |
23 |
24 |
25 |
26 | 27 |
30 | 31 | 32 | 33 | 40 | 41 | text text text text text text text text text text text text text text text text text text text text
42 | text text text text text text text text text text text text text text text text text text text text
43 | text text text text text text text text text text text text text text text text text text text text
44 | text text text text text text text text text text text text text text text text text text text text
45 | 46 | 47 | -------------------------------------------------------------------------------- /src/main/resources/config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "collector":{ 3 | "hosts":[ 4 | "myCollectorHost_member1:27017", 5 | "myCollectorHost_member2:27017", 6 | "myCollectorHost_member3:27017"], 7 | "db":"profiling", 8 | "collection":"slowops", 9 | "adminUser":"", 10 | "adminPw":"", 11 | "ssl":false 12 | }, 13 | "profiled":[ 14 | { 15 | "label":"dbs foo", 16 | "hosts":[ 17 | "someHost1:27017", 18 | "someHost2:27017", 19 | "someHost3:27017"], 20 | "ns":[ 21 | "someDatabase.someCollection", 22 | "anotherDatabase.anotherCollection"], 23 | "adminUser":"", 24 | "adminPw":"", 25 | "collect": false, 26 | "ssl":false, 27 | "slowMS":250, 28 | "responseTimeoutInMs": 1500 29 | }, 30 | { 31 | "label":"dbs bar", 32 | "hosts":["someMongoRouter:27017"], 33 | "ns":[ 34 | "someDatabase.someCollection", 35 | "anotherDatabase.*", 36 | "*.myCollection", 37 | "!excludedDb"], 38 | "adminUser":"", 39 | "adminPw":"", 40 | "collect": false, 41 | "ssl":false, 42 | "slowMS":250, 43 | "responseTimeoutInMs": 3000 44 | }, 45 | { 46 | "label":"dbs with many databases to be profiled", 47 | "hosts":["someOtherMongoRouter:27017"], 48 | "ns":["*.*", "!excludedDb"], 49 | "adminUser":"", 50 | "adminPw":"", 51 | "collect": false, 52 | "ssl":true, 53 | "slowMS":250, 54 | "responseTimeoutInMs": 3000, 55 | "systemProfileCollectionMaxSizeInMB": 64 56 | } 57 | ], 58 | "yAxisScale":"milliseconds", 59 | "adminToken": "mySecureAdminToken", 60 | "defaultSlowMS" : 100, 61 | "defaultResponseTimeoutInMs" : 2000, 62 | "defaultExcludedDBs": ["admin", "local", "config"], 63 | "maxWeblogEntries" : 100, 64 | "systemProfileCollectionMaxSizeInMB": 16 65 | } -------------------------------------------------------------------------------- /src/main/webapp/js/jquery.dataTables.sum.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Fairly simply, this plug-in will take the data from an API result set 3 | * and sum it, returning the summed value. The data can come from any data 4 | * source, including column data, cells or rows. 5 | * 6 | * Note that it will attempt to 'deformat' any string based data that is passed 7 | * into it - i.e. it will strip any non-numeric characters in order to make a 8 | * best effort attempt to sum all data types. This can be useful when working 9 | * with formatting numbers such as currency. However the trade-off is that no 10 | * error is thrown if non-numeric data is passed in. You should be aware of this 11 | * in case unexpected values are returned - likely the input data is not what is 12 | * expected. 13 | * 14 | * @name sum() 15 | * @summary Sum the values in a data set. 16 | * @author [Allan Jardine](http://sprymedia.co.uk) 17 | * @requires DataTables 1.10+ 18 | * 19 | * @returns {Number} Summed value 20 | * 21 | * @example 22 | * // Simply get the sum of a column 23 | * var table = $('#example').DataTable(); 24 | * table.column( 3 ).data().sum(); 25 | * 26 | * @example 27 | * // Insert the sum of a column into the columns footer, for the visible 28 | * // data on each draw 29 | * $('#example').DataTable( { 30 | * drawCallback: function () { 31 | * var api = this.api(); 32 | * api.table().footer().to$().html( 33 | * api.column( 4, {page:'current'} ).data().sum() 34 | * ); 35 | * } 36 | * } ); 37 | */ 38 | 39 | jQuery.fn.dataTable.Api.register( 'sum()', function ( ) { 40 | return this.flatten().reduce( function ( a, b ) { 41 | if ( typeof a === 'string' ) { 42 | a = a.replace(/[^\d.-]/g, '') * 1; 43 | } 44 | if ( typeof b === 'string' ) { 45 | b = b.replace(/[^\d.-]/g, '') * 1; 46 | } 47 | 48 | return a + b; 49 | }, 0 ); 50 | } ); 51 | 52 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/CommandResultDto.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.dto; 2 | 3 | import com.google.common.collect.Lists; 4 | import org.bson.Document; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | /** 10 | * Created by kay.agahd on 29.06.17. 11 | */ 12 | public class CommandResultDto { 13 | 14 | 15 | private List tableHeader; 16 | private TableDto tableBody; 17 | private int jsonFormattedColumn = -1; 18 | 19 | public CommandResultDto(){ 20 | tableHeader = Lists.newArrayList(); 21 | tableBody = new TableDto(); 22 | } 23 | 24 | 25 | public List getTableHeader() { 26 | return tableHeader; 27 | } 28 | 29 | public void setTableHeader(List tableHeader) { 30 | this.tableHeader = tableHeader; 31 | } 32 | 33 | public TableDto getTableBody() { 34 | return tableBody; 35 | } 36 | 37 | 38 | public void addTableBody(TableDto tableBodyToAdd) { 39 | this.tableBody.addRows(tableBodyToAdd); 40 | } 41 | 42 | public String getTableHeaderAsDatatableJson() { 43 | final List dataTableHeader = new ArrayList<>(); 44 | 45 | for(String entry : tableHeader){ 46 | dataTableHeader.add(new Document("title", entry)); 47 | } 48 | final Document doc = new Document(); 49 | doc.append("content", dataTableHeader); 50 | return doc.toJson(); 51 | } 52 | 53 | public String getTableBodyAsJson() { 54 | final Document doc = new Document(); 55 | doc.append("content", tableBody.getTableRows()); 56 | //doc.toJson(); 57 | return com.mongodb.util.JSON.serialize(doc); 58 | } 59 | 60 | public void setJsonFormattedColumn(int colNumber){ 61 | jsonFormattedColumn = colNumber; 62 | } 63 | 64 | 65 | public String getJsonFormattedColumnAsDatatableCode(){ 66 | if(jsonFormattedColumn >= 0){ 67 | return ", \"targets\":" + jsonFormattedColumn; 68 | } 69 | return ""; 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/servlet/SlowOpExample.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.servlet; 5 | 6 | import com.google.common.collect.Lists; 7 | import de.idealo.mongodb.slowops.collector.ExampleSlowOpsCache; 8 | import org.bson.Document; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import javax.servlet.RequestDispatcher; 13 | import javax.servlet.ServletException; 14 | import javax.servlet.annotation.WebServlet; 15 | import javax.servlet.http.HttpServlet; 16 | import javax.servlet.http.HttpServletRequest; 17 | import javax.servlet.http.HttpServletResponse; 18 | import java.io.IOException; 19 | import java.util.List; 20 | 21 | 22 | 23 | @WebServlet("/slowop") 24 | public class SlowOpExample extends HttpServlet { 25 | private static final long serialVersionUID = 1L; 26 | 27 | private static final Logger LOG = LoggerFactory.getLogger(SlowOpExample.class); 28 | 29 | /** 30 | * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) 31 | */ 32 | protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 33 | List result = Lists.newLinkedList(); 34 | if(!isEmpty(request, "fp")) { 35 | String fp=request.getParameter("fp"); 36 | 37 | if(!isEmpty(request, "del")) { 38 | boolean deleted = ExampleSlowOpsCache.INSTANCE.remove(fp); 39 | request.setAttribute("deleted", deleted); 40 | } 41 | 42 | result = ExampleSlowOpsCache.INSTANCE.getSlowOp(fp); 43 | } 44 | 45 | request.setAttribute("slowop", result); 46 | 47 | RequestDispatcher view = request.getRequestDispatcher("/slowop.jsp"); 48 | view.forward(request, response); 49 | } 50 | 51 | /** 52 | * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) 53 | */ 54 | protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 55 | doGet(request, response); 56 | } 57 | 58 | 59 | boolean isEmpty(HttpServletRequest request, String param) { 60 | return request.getParameter(param) == null || request.getParameter(param).trim().length() == 0; 61 | } 62 | 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/util/ProfilingReaderCreator.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.util; 2 | 3 | 4 | import com.mongodb.ServerAddress; 5 | import de.idealo.mongodb.slowops.collector.CollectorManager; 6 | import de.idealo.mongodb.slowops.collector.ProfilingEntry; 7 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 8 | import de.idealo.mongodb.slowops.collector.ProfilingWriter; 9 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 10 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.Date; 15 | import java.util.List; 16 | import java.util.concurrent.BlockingQueue; 17 | import java.util.concurrent.Callable; 18 | 19 | 20 | public class ProfilingReaderCreator implements Callable { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(ProfilingReaderCreator.class); 23 | 24 | private final int id; 25 | private final ServerAddress address; 26 | private final ProfiledServerDto dto; 27 | private final String db; 28 | private final List colls; 29 | private final ProfilingWriter writer; 30 | private final BlockingQueue jobQueue; 31 | private final MongoDbAccessor profilingWriterMongo; 32 | 33 | 34 | 35 | public ProfilingReaderCreator(int id, ServerAddress address, ProfiledServerDto dto, String db, List colls, CollectorManager collectorManager, MongoDbAccessor mongo) { 36 | this.id = id; 37 | this.address = address; 38 | this.dto = dto; 39 | this.db = db; 40 | this.colls = colls; 41 | this.writer = collectorManager.getWriter(); 42 | this.jobQueue = collectorManager.getJobQueue(); 43 | this.profilingWriterMongo = mongo; 44 | 45 | } 46 | 47 | public ServerAddress getAddress() { 48 | return address; 49 | } 50 | 51 | public ProfiledServerDto getDto() { 52 | return dto; 53 | } 54 | 55 | public String getDatabase() { 56 | return db; 57 | } 58 | 59 | @Override 60 | public ProfilingReader call() throws Exception { 61 | final Date lastTs = writer.getNewest(profilingWriterMongo, address, db); 62 | return new ProfilingReader(id, jobQueue, address, lastTs, dto, db, colls, !dto.isEnabled(), 0, dto.getSlowMs()); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/webapp/commandResult.jsp: -------------------------------------------------------------------------------- 1 | <%@ page import="de.idealo.mongodb.slowops.dto.CommandResultDto" %> 2 | 3 | 4 | <%@ page language="java" contentType="text/html; charset=ISO-8859-1" pageEncoding="ISO-8859-1" %> 5 | <% 6 | final CommandResultDto commandResult = (CommandResultDto)request.getAttribute("commandResult"); 7 | %> 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | command result page 21 | 47 | 57 | 58 | 59 |

Command result page

60 | 61 | 62 | 63 |
64 | 65 | <%@ include file="buildInfo.jsp" %> 66 | 67 | 68 | -------------------------------------------------------------------------------- /src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | mongodb-slow-operations-profiler 4 | 5 | index.html 6 | 7 | 8 | de.idealo.mongodb.slowops.collector.StartStopHook 9 | 10 | 11 | SlowOps 12 | SlowOps 13 | de.idealo.mongodb.slowops.servlet.SlowOps 14 | 15 | 16 | ApplicationStatus 17 | ApplicationStatus 18 | de.idealo.mongodb.slowops.servlet.ApplicationStatus 19 | 20 | 21 | CommandResult 22 | CommandResult 23 | de.idealo.mongodb.slowops.servlet.CommandResult 24 | 25 | 26 | SlowOpExample 27 | SlowOpExample 28 | de.idealo.mongodb.slowops.servlet.SlowOpExample 29 | 30 | 31 | 32 | Jersey REST Service 33 | com.sun.jersey.spi.container.servlet.ServletContainer 34 | 35 | com.sun.jersey.config.property.packages 36 | de.idealo.mongodb.slowops.servlet 37 | 38 | 39 | com.sun.jersey.api.json.POJOMappingFeature 40 | true 41 | 42 | 1 43 | 44 | 45 | Jersey REST Service 46 | /rest/* 47 | 48 | 49 | 50 | SlowOps 51 | /gui/* 52 | 53 | 54 | ApplicationStatus 55 | /app/* 56 | 57 | 58 | CommandResult 59 | /cmd/* 60 | 61 | 62 | SlowOpExample 63 | /slowop/* 64 | 65 | 66 | -------------------------------------------------------------------------------- /src/main/webapp/slowop.jsp: -------------------------------------------------------------------------------- 1 | <%@ page import="org.bson.Document" %> 2 | <%@ page import="org.bson.json.JsonWriterSettings" %> 3 | <%@ page import="java.util.List" %> 4 | 5 | 6 | <%@ page language="java" contentType="text/html; charset=ISO-8859-1" pageEncoding="ISO-8859-1" %> 7 | <% 8 | final List slowOps = (List)request.getAttribute("slowop"); 9 | final boolean deleted = Boolean.valueOf(String.valueOf(request.getAttribute("deleted"))); 10 | %> 11 | 12 | 13 | 14 | 15 | 16 | 17 | slow operation example 18 | 19 | 20 | 25 | 26 |
27 |

slow operation example

28 | <%if(slowOps.isEmpty()){ 29 | if(deleted){ 30 | out.print("

The slow operation example document has been successfully deleted.
"); 31 | }else{ 32 | out.print("

The slow operation document with this fingerprint could not be found in the database!
You should tick more checkboxes such as 'Label', 'Database', 'Collection', 'Operation', 'Queried fields', 'Sorted fields' and/or 'Projected fields' in the 'Group by' section to get an existing slow operation example document which matches the query shape for a query executed within the same database and collection.
"); 33 | } 34 | 35 | }else{%> 36 | For more complex queries it may help to see an example of the profiled operation to understand better how to interpret the shown fields.
37 | The shown document is a profiled slow operation of the same dbs label, database, and collection which has the same query shape as the clicked one which is defined by: 38 |
    39 |
  • Operation
  • 40 |
  • Queried fields
  • 41 |
  • Sorted fields
  • 42 |
  • Projected fields
  • 43 |
44 | You may &del=1">delete this example document e.g. if indexes have changed and you want to reflect them in this slow operations example document.
45 | A new example document will be re-created as soon as such an operation is collected again. 46 |
47 |
<%JsonWriterSettings.Builder settingsBuilder = JsonWriterSettings.builder().indent(true);
48 | 		for(Document doc : slowOps){
49 | 			out.print(doc.toJson(settingsBuilder.build()));
50 | 		}
51 | 	}%>
52 |
53 | 54 | <%@ include file="buildInfo.jsp" %> 55 | 56 | 57 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/SlowOpsDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.dto; 5 | 6 | import de.idealo.mongodb.slowops.grapher.AggregatedProfiling; 7 | import de.idealo.mongodb.slowops.util.ConfigReader; 8 | import de.idealo.mongodb.slowops.util.Util; 9 | 10 | import java.util.HashMap; 11 | 12 | 13 | /** 14 | * 15 | * 16 | * @author kay.agahd 17 | * @since 19.03.2013 18 | * @version $Id: $ 19 | * @copyright idealo internet GmbH 20 | */ 21 | public class SlowOpsDto { 22 | 23 | private boolean[] visibilityValues; 24 | private StringBuffer labels; 25 | private StringBuffer dataGrid; 26 | private String errorMessage; 27 | private HashMap labelSeries; 28 | private final String scale; 29 | 30 | 31 | public SlowOpsDto() { 32 | final String scaleStr = ConfigReader.getString(ConfigReader.CONFIG, Util.Y_AXIS_SCALE, Util.Y_AXIS_MILLISECONDS); 33 | if(scaleStr.equals(Util.Y_AXIS_MILLISECONDS)){ 34 | scale = Util.Y_AXIS_MILLISECONDS; 35 | }else { 36 | scale = Util.Y_AXIS_SECONDS;//default 37 | } 38 | } 39 | 40 | /** 41 | * @return the visibilityValues 42 | */ 43 | public boolean[] getVisibilityValues() { 44 | return visibilityValues; 45 | } 46 | /** 47 | * @param visibilityValues the visibilityValues to set 48 | */ 49 | public void setVisibilityValues(boolean[] visibilityValues) { 50 | this.visibilityValues = visibilityValues; 51 | } 52 | 53 | public StringBuffer getLabels() {return labels;} 54 | 55 | public void setLabels(StringBuffer labels) {this.labels = labels;} 56 | 57 | /** 58 | * @return the dataGrid 59 | */ 60 | public StringBuffer getDataGrid() { 61 | return dataGrid; 62 | } 63 | /** 64 | * @param dataGrid the dataGrid to set 65 | */ 66 | public void setDataGrid(StringBuffer dataGrid) { 67 | this.dataGrid = dataGrid; 68 | } 69 | /** 70 | * @param errorMessage 71 | */ 72 | public void setErrorMessage(String errorMessage) { 73 | this.errorMessage = errorMessage; 74 | } 75 | /** 76 | * @return the errorMessage 77 | */ 78 | public String getErrorMessage() { 79 | return errorMessage; 80 | } 81 | 82 | /** 83 | * @param labelSeries 84 | */ 85 | public void setLabelSeries(HashMap labelSeries) { 86 | this.labelSeries = labelSeries; 87 | } 88 | 89 | public HashMap getLabelSeries() { 90 | return labelSeries; 91 | } 92 | /** 93 | * @return the scale 94 | */ 95 | public String getScale() { 96 | return scale; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/test/java/de/idealo/mongodb/slowops/dto/ProfiledServerDtoTest.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.dto; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.ServerAddress; 5 | import org.junit.Test; 6 | 7 | import java.util.HashMap; 8 | import java.util.List; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | 12 | /** 13 | * Created by kay.agahd on 26.10.16. 14 | */ 15 | public class ProfiledServerDtoTest { 16 | 17 | @Test 18 | public void getCollectionsPerDatabase() throws Exception { 19 | 20 | ServerAddress[] hosts = {new ServerAddress("127.0.0.1:27017")}; 21 | String[] ns = {"db1.col1", "admin.foo", "local.bar", "db2.firstCollection", "db2.secondCollection", "db3.coll.with.dots"}; 22 | List defaultExcludedDbs = Lists.newArrayList(); 23 | defaultExcludedDbs.add("admin"); 24 | defaultExcludedDbs.add("local"); 25 | ProfiledServerDto dto = new ProfiledServerDto(false, "label", hosts, ns, "adminUser", "adminPw", false, 100, 1000, defaultExcludedDbs, 1); 26 | HashMap> computedCollectionsPerDb = dto.getCollectionsPerDatabase(); 27 | 28 | HashMap> expectedCollectionsPerDb = new HashMap>(); 29 | expectedCollectionsPerDb.put("db1", Lists.newArrayList("col1")); 30 | expectedCollectionsPerDb.put("db2", Lists.newArrayList("firstCollection", "secondCollection")); 31 | expectedCollectionsPerDb.put("db3", Lists.newArrayList("coll.with.dots")); 32 | 33 | assertEquals(expectedCollectionsPerDb, computedCollectionsPerDb); 34 | 35 | //test to exclude default and one specific db 36 | ns = new String[]{"db1.col1", "admin", "dbToBeExcluded.foo", "!dbToBeExcluded"}; 37 | dto = new ProfiledServerDto(false, "label", hosts, ns, "adminUser", "adminPw", false, 100, 1000, defaultExcludedDbs, 1); 38 | computedCollectionsPerDb = dto.getCollectionsPerDatabase(); 39 | 40 | expectedCollectionsPerDb = new HashMap>(); 41 | expectedCollectionsPerDb.put("db1", Lists.newArrayList("col1")); 42 | 43 | assertEquals(expectedCollectionsPerDb, computedCollectionsPerDb); 44 | 45 | //test wrong way to exclude db 46 | ns = new String[]{"db1.col1", "!dbToBeExcluded", "dbToBeExcluded.foo"}; 47 | dto = new ProfiledServerDto(false, "label", hosts, ns, "adminUser", "adminPw", false, 100, 1000, defaultExcludedDbs, 1); 48 | computedCollectionsPerDb = dto.getCollectionsPerDatabase(); 49 | 50 | expectedCollectionsPerDb = new HashMap>(); 51 | expectedCollectionsPerDb.put("db1", Lists.newArrayList("col1")); 52 | expectedCollectionsPerDb.put("dbToBeExcluded", Lists.newArrayList("foo")); 53 | 54 | assertEquals(expectedCollectionsPerDb, computedCollectionsPerDb); 55 | 56 | 57 | 58 | } 59 | 60 | } -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/CmdDatabaseStats.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.BasicDBObject; 5 | import com.mongodb.client.MongoDatabase; 6 | import com.mongodb.client.MongoIterable; 7 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 8 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 9 | import de.idealo.mongodb.slowops.dto.TableDto; 10 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 11 | import de.idealo.mongodb.slowops.util.Util; 12 | import org.bson.Document; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | /** 20 | * Created by kay.agahd on 31.03.20. 21 | */ 22 | public class CmdDatabaseStats implements ICommand { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(CmdDatabaseStats.class); 25 | 26 | 27 | private final CommandResultDto commandResultDto; 28 | 29 | public CmdDatabaseStats() { 30 | commandResultDto = new CommandResultDto(); 31 | commandResultDto.setTableHeader(Lists.newArrayList("dbs label", 32 | "host", 33 | "database", 34 | "objects", 35 | "avgObjectSize", 36 | "dataSize", 37 | "storageSize", 38 | "#indexes", 39 | "indexSize")); 40 | } 41 | 42 | @Override 43 | public boolean isHostCommand(){ 44 | return false; 45 | } 46 | 47 | @Override 48 | public CommandResultDto getCommandResultDto() { 49 | return commandResultDto; 50 | } 51 | 52 | @Override 53 | public TableDto runCommand(ProfilingReader profilingReader, MongoDbAccessor mongoDbAccessor) { 54 | final TableDto table = new TableDto(); 55 | 56 | try{ 57 | final Document commandResultDoc = mongoDbAccessor.runCommand("admin", new BasicDBObject("hostInfo", 1)); 58 | final Object system = commandResultDoc.get("system"); 59 | String hostname = ""; 60 | if (system instanceof Document) { 61 | final Document systemDoc = (Document) system; 62 | hostname = systemDoc.getString("hostname"); 63 | } 64 | 65 | 66 | final MongoDatabase db = mongoDbAccessor.getMongoDatabase(profilingReader.getDatabase()); 67 | final Document dbStats = db.runCommand(new Document("dbStats", 1)); 68 | final List row = Lists.newArrayList(); 69 | row.add(profilingReader.getProfiledServerDto().getLabel()); 70 | row.add(hostname); 71 | row.add(profilingReader.getDatabase()); 72 | row.add(Util.getNumber(dbStats, "objects", 0)); 73 | row.add(Util.getNumber(dbStats, "avgObjSize", 0)); 74 | row.add(Util.getNumber(dbStats, "dataSize",0)); 75 | row.add(Util.getNumber(dbStats, "storageSize",0)); 76 | row.add(Util.getNumber(dbStats, "indexes", 0)); 77 | row.add(Util.getNumber(dbStats, "indexSize",0)); 78 | table.addRow(row); 79 | 80 | }catch (Exception e){ 81 | LOG.warn("Exception while running command", e); 82 | } 83 | 84 | return table; 85 | } 86 | 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/servlet/ApplicationStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.servlet; 5 | 6 | import de.idealo.mongodb.slowops.collector.CollectorManagerInstance; 7 | import de.idealo.mongodb.slowops.util.ConfigReader; 8 | import de.idealo.mongodb.slowops.util.Util; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import javax.servlet.RequestDispatcher; 13 | import javax.servlet.ServletException; 14 | import javax.servlet.annotation.WebServlet; 15 | import javax.servlet.http.HttpServlet; 16 | import javax.servlet.http.HttpServletRequest; 17 | import javax.servlet.http.HttpServletResponse; 18 | import javax.servlet.http.HttpSession; 19 | import java.io.IOException; 20 | 21 | @WebServlet("/app") 22 | public class ApplicationStatus extends HttpServlet { 23 | private static final long serialVersionUID = 1L; 24 | private static final Logger LOG = LoggerFactory.getLogger(ApplicationStatus.class); 25 | 26 | /** 27 | * @see HttpServlet#HttpServlet() 28 | */ 29 | public ApplicationStatus() { 30 | super(); 31 | } 32 | 33 | /** 34 | * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) 35 | */ 36 | protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 37 | LOG.debug(">>> doGet"); 38 | reloadConfig(request); 39 | RequestDispatcher view = request.getRequestDispatcher("/applicationStatus.jsp"); 40 | LOG.debug("doGet"); 41 | view.forward(request, response); 42 | LOG.debug("<<< doGet"); 43 | } 44 | 45 | /** 46 | * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) 47 | */ 48 | protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 49 | doGet(request, response); 50 | } 51 | 52 | private void reloadConfig(HttpServletRequest request){ 53 | if(isAuthenticated(request)) { 54 | final String cfg = request.getParameter("config"); 55 | if (cfg != null && cfg.trim().length() > 0) { 56 | LOG.info("reload application using a new configuration"); 57 | CollectorManagerInstance.reloadConfig(cfg); 58 | } 59 | } 60 | } 61 | 62 | protected static boolean isAuthenticated(HttpServletRequest req) { 63 | final HttpSession session = req.getSession(true); 64 | 65 | final Object isAdminToken = session.getAttribute(Util.ADMIN_TOKEN); 66 | if (isAdminToken instanceof Boolean && (Boolean) isAdminToken) { 67 | LOG.info("authenticated by session"); 68 | return true; 69 | }else{ 70 | final String adminToken = req.getParameter(Util.ADMIN_TOKEN); 71 | if (adminToken != null) { 72 | if (ConfigReader.getString(ConfigReader.CONFIG, Util.ADMIN_TOKEN, "").equals(adminToken)) { 73 | session.setAttribute(Util.ADMIN_TOKEN, true); 74 | LOG.info("authenticated by url param succeeded, loggd in"); 75 | return true; 76 | } 77 | } 78 | } 79 | LOG.info("not authenticated"); 80 | return false; 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/webapp/datatables.json: -------------------------------------------------------------------------------- 1 | { 2 | "collectorStatuses": [ 3 | { 4 | "collections": [ 5 | "offer" 6 | ], 7 | "collectionsAsString": "offer", 8 | "database": "offerStore", 9 | "doneJobsHistory": [ 10 | 8910, 11 | 0, 12 | 1, 13 | 2, 14 | 3, 15 | 4, 16 | 5, 17 | 6 18 | ], 19 | "replSetName": null, 20 | "replSetStatus": "UNKNOWN", 21 | "instanceId": 1, 22 | "label": "localhost-kay", 23 | "lastTs": 1478249811610, 24 | "lastTsFormatted": "2016-11-04 09:56:51", 25 | "profiling": false, 26 | "serverAddress": { 27 | "host": "localhost", 28 | "port": 27017, 29 | "socketAddress": { 30 | "address": "localhost", 31 | "hostName": "localhost", 32 | "hostString": "localhost", 33 | "port": 27017, 34 | "unresolved": false 35 | } 36 | }, 37 | "serverAddressAsString": "localhost:27017", 38 | "slowMs": 3000, 39 | "stopped": true 40 | }, 41 | { 42 | "collections": [ 43 | "first" 44 | ], 45 | "collectionsAsString": "first", 46 | "database": "kay", 47 | "doneJobsHistory": [ 48 | 123, 49 | 0, 50 | 1, 51 | 2, 52 | 3, 53 | 4, 54 | 5, 55 | 6 56 | ], 57 | "replSetName": null, 58 | "replSetStatus": "UNKNOWN", 59 | "instanceId": 1, 60 | "label": "localhost-kay", 61 | "lastTs": 1478249811610, 62 | "lastTsFormatted": "2016-11-04 09:56:51", 63 | "profiling": false, 64 | "serverAddress": { 65 | "host": "localhost", 66 | "port": 27017, 67 | "socketAddress": { 68 | "address": "localhost", 69 | "hostName": "localhost", 70 | "hostString": "localhost", 71 | "port": 27017, 72 | "unresolved": false 73 | } 74 | }, 75 | "serverAddressAsString": "localhost:27017", 76 | "slowMs": 3000, 77 | "stopped": true 78 | }, 79 | { 80 | "collections": [ 81 | "first", "second" 82 | ], 83 | "collectionsAsString": "first", 84 | "database": "kay", 85 | "doneJobsHistory": [ 86 | 123, 87 | 0, 88 | 1, 89 | 2, 90 | 3, 91 | 4, 92 | 5, 93 | 6 94 | ], 95 | "replSetName": null, 96 | "replSetStatus": "UNKNOWN", 97 | "instanceId": 1, 98 | "label": "localhost-test
{key:{sub:doc}}
", 99 | "lastTs": 1478249811610, 100 | "lastTsFormatted": "2016-11-04 09:56:51", 101 | "profiling": false, 102 | "serverAddress": { 103 | "host": "localhost", 104 | "port": 27017, 105 | "socketAddress": { 106 | "address": "localhost", 107 | "hostName": "localhost", 108 | "hostString": "localhost", 109 | "port": 27017, 110 | "unresolved": false 111 | } 112 | }, 113 | "serverAddressAsString": "localhost:27017", 114 | "slowMs": 3000, 115 | "stopped": true 116 | } 117 | ], 118 | "numberOfReads": 0, 119 | "numberOfReadsOfRemovedReaders": 0, 120 | "numberOfWrites": 0 121 | } -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/CollectorServerDto.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.dto; 2 | 3 | import com.mongodb.ServerAddress; 4 | import org.codehaus.jackson.annotate.JsonProperty; 5 | 6 | import java.util.Arrays; 7 | 8 | /** 9 | * Created by kay.agahd on 26.10.16. 10 | */ 11 | public class CollectorServerDto { 12 | 13 | private ServerAddress[] hosts; 14 | private String db; 15 | private String collection; 16 | private String adminUser; 17 | private String adminPw; 18 | private boolean ssl; 19 | 20 | public CollectorServerDto(@JsonProperty("hosts") ServerAddress[] hosts, 21 | @JsonProperty("db") String db, 22 | @JsonProperty("collection") String collection, 23 | @JsonProperty("adminUser") String adminUser, 24 | @JsonProperty("adminPw") String adminPw, 25 | @JsonProperty("ssl") boolean ssl) { 26 | this.hosts = hosts; 27 | this.db = db; 28 | this.collection = collection; 29 | this.adminUser = adminUser; 30 | this.adminPw = adminPw; 31 | this.ssl = ssl; 32 | } 33 | 34 | public ServerAddress[] getHosts() { 35 | return hosts; 36 | } 37 | 38 | public void setHosts(ServerAddress[] hosts) { 39 | this.hosts = hosts; 40 | } 41 | 42 | public String getDb() { 43 | return db; 44 | } 45 | 46 | public void setDb(String db) { 47 | this.db = db; 48 | } 49 | 50 | public String getCollection() { 51 | return collection; 52 | } 53 | 54 | public void setCollection(String collection) { 55 | this.collection = collection; 56 | } 57 | 58 | public String getAdminUser() { 59 | return adminUser; 60 | } 61 | 62 | public void setAdminUser(String adminUser) { 63 | this.adminUser = adminUser; 64 | } 65 | 66 | public String getAdminPw() { 67 | return adminPw; 68 | } 69 | 70 | public void setAdminPw(String adminPw) { 71 | this.adminPw = adminPw; 72 | } 73 | 74 | public boolean getSsl() { 75 | return ssl; 76 | } 77 | 78 | public void setSsl(boolean ssl) { 79 | this.ssl = ssl; 80 | } 81 | 82 | @Override 83 | public boolean equals(Object o) { 84 | if (this == o) return true; 85 | if (o == null || getClass() != o.getClass()) return false; 86 | 87 | CollectorServerDto that = (CollectorServerDto) o; 88 | 89 | // Probably incorrect - comparing Object[] arrays with Arrays.equals 90 | if (!Arrays.equals(hosts, that.hosts)) return false; 91 | if (!db.equals(that.db)) return false; 92 | if (!collection.equals(that.collection)) return false; 93 | if (ssl != that.ssl) return false; 94 | if (adminUser != null ? !adminUser.equals(that.adminUser) : that.adminUser != null) return false; 95 | return adminPw != null ? adminPw.equals(that.adminPw) : that.adminPw == null; 96 | 97 | } 98 | 99 | @Override 100 | public int hashCode() { 101 | int result = Arrays.hashCode(hosts); 102 | result = 31 * result + db.hashCode(); 103 | result = 31 * result + collection.hashCode(); 104 | result = 31 * result + (adminUser != null ? adminUser.hashCode() : 0); 105 | result = 31 * result + (adminPw != null ? adminPw.hashCode() : 0); 106 | return result; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/servlet/Action.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.servlet; 5 | 6 | 7 | import com.google.common.collect.Lists; 8 | import de.idealo.mongodb.slowops.collector.CollectorManagerInstance; 9 | import de.idealo.mongodb.slowops.dto.ApplicationStatusDto; 10 | import org.codehaus.jackson.map.ObjectMapper; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import javax.servlet.http.HttpServletRequest; 15 | import javax.ws.rs.GET; 16 | import javax.ws.rs.Path; 17 | import javax.ws.rs.Produces; 18 | import javax.ws.rs.QueryParam; 19 | import javax.ws.rs.core.Context; 20 | import javax.ws.rs.core.MediaType; 21 | import java.io.IOException; 22 | import java.util.List; 23 | 24 | 25 | @Path("/action") 26 | public class Action { 27 | 28 | private static final Logger LOG = LoggerFactory.getLogger(Action.class); 29 | 30 | private static final int SLOWMS_DEFAULT = 100; 31 | 32 | 33 | @GET 34 | @Produces({MediaType.APPLICATION_JSON}) 35 | public ApplicationStatusDto getApplicationJSON(@QueryParam("cmd") String cmd, @QueryParam("p") String p, @QueryParam("ms") String ms, @Context HttpServletRequest req) { 36 | LOG.info(">>> getApplicationJSON cmd: {} p: {} ms:{}", new Object[]{cmd, p, ms}); 37 | ApplicationStatusDto result = null; 38 | try { 39 | final List pList = Lists.newArrayList(); 40 | final boolean isAuthenticated = ApplicationStatus.isAuthenticated(req); 41 | 42 | if (p != null) { 43 | String[] params = p.split(","); 44 | if (params.length > 0) { 45 | for (String param : params) { 46 | try { 47 | pList.add(Integer.parseInt(param)); 48 | } catch (NumberFormatException e) { 49 | LOG.warn("Parameter should but is not numeric: {}", param); 50 | } 51 | } 52 | } 53 | 54 | if (pList.size() > 0) { 55 | if (isAuthenticated) { 56 | if ("cstart".equals(cmd)) { 57 | CollectorManagerInstance.startStopProfilingReaders(pList, false); 58 | } else if ("cstop".equals(cmd)) { 59 | CollectorManagerInstance.startStopProfilingReaders(pList, true); 60 | } else if ("pstart".equals(cmd)) { 61 | long slowMs = getSlowMs(ms); 62 | CollectorManagerInstance.setSlowMs(pList, Math.abs(slowMs)); 63 | } else if ("pstop".equals(cmd)) { 64 | long slowMs = getSlowMs(ms); 65 | if(slowMs == 0) slowMs = SLOWMS_DEFAULT;//set to default because 0 means profile all ops but we want to stop profiling here 66 | CollectorManagerInstance.setSlowMs(pList, Math.abs(slowMs)*-1); 67 | } 68 | } 69 | result = CollectorManagerInstance.getApplicationStatus(pList, isAuthenticated); 70 | } 71 | } else if ("rc".equals(cmd)) { 72 | result = CollectorManagerInstance.getApplicationStatus(pList, isAuthenticated); 73 | } else { 74 | result = CollectorManagerInstance.getApplicationStatus(isAuthenticated); 75 | } 76 | 77 | 78 | try { 79 | final ObjectMapper mapper = new ObjectMapper(); 80 | LOG.debug("getApplicationJSON result:"); 81 | LOG.debug(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(result)); 82 | } catch (IOException e) { 83 | LOG.error("IOException while logging ApplicationStatusDto", e); 84 | } 85 | 86 | 87 | }catch(Throwable t){ 88 | ApplicationStatusDto.addWebLog(t.getClass().getSimpleName() + " while getting application status."); 89 | LOG.error("Error in getApplicationJSON", t); 90 | } 91 | LOG.info("<<< getApplicationJSON"); 92 | return result; 93 | } 94 | 95 | private long getSlowMs(String ms){ 96 | try { 97 | return Long.parseLong(ms); 98 | } catch (NumberFormatException e) { 99 | ApplicationStatusDto.addWebLog("slowMS must be numeric but was: '" + ms + "' so take default: " + SLOWMS_DEFAULT); 100 | LOG.warn("slowMS must be numeric but was: '{}' so take default: {}", ms, SLOWMS_DEFAULT); 101 | } 102 | return SLOWMS_DEFAULT; 103 | } 104 | 105 | 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/collector/ProfilingEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.collector; 5 | 6 | import com.mongodb.ServerAddress; 7 | import de.idealo.mongodb.slowops.grapher.AggregatedProfilingId; 8 | import org.bson.Document; 9 | 10 | import java.util.Date; 11 | import java.util.LinkedHashSet; 12 | 13 | /** 14 | * 15 | * 16 | * @author kay.agahd 17 | * @since 27.02.2013 18 | * @version $Id: $ 19 | * @copyright idealo internet GmbH 20 | */ 21 | public class ProfilingEntry { 22 | 23 | 24 | final Date ts; 25 | final ServerAddress adr; 26 | final String db; 27 | final String col; 28 | final String op; 29 | final String user; 30 | final LinkedHashSet fields; 31 | final LinkedHashSet sort; 32 | final LinkedHashSet proj; 33 | final Integer nret; 34 | final Integer respLen; 35 | final Integer millis; 36 | final Long cId;//cursorId 37 | String label; 38 | String replSet; 39 | final Integer keys; 40 | final Integer docs; 41 | final Boolean isSort; 42 | final Integer del; 43 | final Integer ins; 44 | final Integer mod; 45 | 46 | 47 | 48 | public ProfilingEntry(Date ts, ServerAddress adr, String db, String col, String op, String user, 49 | LinkedHashSet fields, LinkedHashSet sort, LinkedHashSet proj, Integer nret, Integer respLen, Integer millis, 50 | Long cId, Integer keys, Integer docs, Boolean isSort, Integer del, Integer ins, 51 | Integer mod) { 52 | super(); 53 | this.ts = ts; 54 | this.adr = adr; 55 | this.db = db; 56 | this.col = col; 57 | this.op = op; 58 | this.user = user; 59 | this.fields = fields; 60 | this.sort = sort; 61 | this.proj = proj; 62 | this.nret= nret; 63 | this.respLen = respLen; 64 | this.millis = millis; 65 | this.cId = cId; 66 | this.keys = keys; 67 | this.docs = docs; 68 | this.isSort = isSort; 69 | this.del = del; 70 | this.ins = ins; 71 | this.mod = mod; 72 | } 73 | 74 | public void setLabel(String label){ 75 | this.label = label; 76 | } 77 | 78 | public void setReplSet(String replSet){ 79 | this.replSet = replSet; 80 | } 81 | 82 | 83 | /** 84 | * @return 85 | */ 86 | public Document getDocument() { 87 | final Document obj = new Document(); 88 | if(label!=null && !label.isEmpty()) obj.put("lbl", label); 89 | if(ts!=null) obj.put("ts", ts); 90 | if(adr!=null) obj.put("adr", adr.getHost() + ":" + adr.getPort()); 91 | if(replSet!=null && !replSet.isEmpty()) obj.put("rs", replSet); 92 | if(db!=null && !db.isEmpty()) obj.put("db", db); 93 | if(col!=null && !col.isEmpty()) obj.put("col", col); 94 | if(op!=null && !op.isEmpty()) obj.put("op", op); 95 | if(user!=null && !user.isEmpty()) obj.put("user", user); 96 | if(fields!=null && !fields.isEmpty()) obj.put("fields", fields); 97 | if(sort!=null && !sort.isEmpty()) obj.put("sort", sort); 98 | if(proj!=null && !proj.isEmpty()) obj.put("proj", proj); 99 | if(nret!=null) obj.put("nret", nret); 100 | if(respLen!=null) obj.put("resplen", respLen); 101 | if(millis!=null) obj.put("millis", millis); 102 | if(cId!=null) obj.put("cId", cId); 103 | if(keys!=null) obj.put("keys", keys); 104 | if(docs!=null) obj.put("docs", docs); 105 | if(isSort!=null) obj.put("sortstg", isSort); 106 | if(del!=null) obj.put("del", del); 107 | if(ins!=null) obj.put("ins", ins); 108 | if(mod!=null) obj.put("mod", mod); 109 | 110 | return obj; 111 | } 112 | 113 | public String getFingerprint(){ 114 | final AggregatedProfilingId id = new AggregatedProfilingId(label, db, col, op, fields, sort, proj); 115 | return id.getFingerprint(); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/CmdCurrentOp.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.BasicDBObject; 5 | import com.mongodb.DBObject; 6 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 7 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 8 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 9 | import de.idealo.mongodb.slowops.dto.TableDto; 10 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 11 | import de.idealo.mongodb.slowops.util.Util; 12 | import org.bson.Document; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | /** 20 | * Created by kay.agahd on 29.06.17. 21 | */ 22 | public abstract class CmdCurrentOp implements ICommand { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(CmdCurrentOp.class); 25 | 26 | 27 | private final CommandResultDto commandResultDto; 28 | 29 | public CmdCurrentOp() { 30 | commandResultDto = new CommandResultDto(); 31 | commandResultDto.setTableHeader(Lists.newArrayList("dbs label", 32 | "opid", 33 | "microsecs running", 34 | "secs running", 35 | "op", 36 | "ns", 37 | "command", 38 | "planSummary", 39 | "numYield", 40 | "active")); 41 | 42 | commandResultDto.setJsonFormattedColumn(6); 43 | 44 | } 45 | 46 | public abstract DBObject getQuery(ProfilingReader profilingReader); 47 | 48 | @Override 49 | public CommandResultDto getCommandResultDto() { 50 | return commandResultDto; 51 | } 52 | 53 | 54 | @Override 55 | public TableDto runCommand(ProfilingReader profilingReader, MongoDbAccessor mongoDbAccessor) { 56 | final TableDto table = new TableDto(); 57 | 58 | try{ 59 | final Document commandResultDoc = mongoDbAccessor.runCommand("admin", getQuery(profilingReader)); 60 | 61 | if(commandResultDoc != null){ 62 | Object inprog = commandResultDoc.get("inprog"); 63 | if(inprog instanceof ArrayList) { 64 | final List inprogList = (ArrayList) inprog; 65 | for (Object entry : inprogList) { 66 | if (entry instanceof Document) { 67 | final Document entryDoc = (Document) entry; 68 | final List row = Lists.newArrayList(); 69 | row.add(profilingReader.getProfiledServerDto().getLabel()); 70 | row.add("" + entryDoc.get("opid")); 71 | row.add(Util.getNumber(entryDoc,"microsecs_running", 0)); 72 | row.add(Util.getNumber(entryDoc, "secs_running", 0));//may be Long (v4) or Integer (v3.4) 73 | row.add(entryDoc.getString("op")); 74 | row.add(entryDoc.getString("ns")); 75 | final String originatingCommand = getJson(entryDoc, "originatingCommand"); 76 | row.add(originatingCommand.equals("")?getJson(entryDoc, "command"):originatingCommand); 77 | row.add(entryDoc.getString("planSummary")); 78 | row.add(Util.getNumber(entryDoc, "numYields", 0)); 79 | row.add(entryDoc.getBoolean("active")); 80 | table.addRow(row); 81 | 82 | } 83 | } 84 | } 85 | } 86 | } 87 | catch (Exception e){ 88 | LOG.warn("Exception while running command", e); 89 | } 90 | 91 | return table; 92 | } 93 | 94 | 95 | 96 | private String getJson(Document entryDoc, String key){ 97 | final Object obj = entryDoc.get(key); 98 | if(obj instanceof Document){ 99 | Document doc = (Document) obj; 100 | return doc.toJson(); 101 | } 102 | return ""; 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/CmdHostInfo.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.BasicDBObject; 5 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 6 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 7 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 8 | import de.idealo.mongodb.slowops.dto.TableDto; 9 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 10 | import de.idealo.mongodb.slowops.util.Util; 11 | import org.bson.Document; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.List; 16 | 17 | /** 18 | * Created by kay.agahd on 29.06.17. 19 | */ 20 | public class CmdHostInfo implements ICommand { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(CmdHostInfo.class); 23 | 24 | 25 | private final CommandResultDto commandResultDto; 26 | 27 | public CmdHostInfo() { 28 | commandResultDto = new CommandResultDto(); 29 | commandResultDto.setTableHeader(Lists.newArrayList("Dbs label", 30 | "Hostname", 31 | "Mongodb version", 32 | "CPU arch", 33 | "CPU cores", 34 | "CPU freq in MHz", 35 | "RAM in MB", 36 | "Numa enabled", 37 | "Page size", 38 | "Num pages", 39 | "Max open files", 40 | "OS", 41 | "Kernel version", 42 | "Libc version")); 43 | 44 | } 45 | 46 | @Override 47 | public boolean isHostCommand(){ 48 | return true; 49 | } 50 | 51 | @Override 52 | public CommandResultDto getCommandResultDto() { 53 | return commandResultDto; 54 | } 55 | 56 | 57 | @Override 58 | public TableDto runCommand(ProfilingReader profilingReader, MongoDbAccessor mongoDbAccessor) { 59 | final TableDto table = new TableDto(); 60 | 61 | try{ 62 | final Document commandResultDoc = mongoDbAccessor.runCommand("admin", new BasicDBObject("hostInfo", 1)); 63 | final Document buildInfoDoc = mongoDbAccessor.runCommand("admin", new BasicDBObject("buildInfo", 1)); 64 | 65 | if(commandResultDoc != null) { 66 | Object system = commandResultDoc.get("system"); 67 | Object os = commandResultDoc.get("os"); 68 | Object extra = commandResultDoc.get("extra"); 69 | if (system instanceof Document && os instanceof Document && extra instanceof Document) { 70 | final Document systemDoc = (Document) system; 71 | final Document osDoc = (Document) os; 72 | final Document extraDoc = (Document) extra; 73 | final List row = Lists.newArrayList(); 74 | row.add(profilingReader.getProfiledServerDto().getLabel()); 75 | row.add(systemDoc.getString("hostname")); 76 | if(buildInfoDoc != null) { 77 | row.add(buildInfoDoc.getString("version")); 78 | }else{ 79 | row.add("unknown"); 80 | } 81 | row.add(systemDoc.getString("cpuArch")); 82 | row.add(Util.getNumber(systemDoc, "numCores", 0)); 83 | row.add(Util.getNumber(extraDoc, "cpuFrequencyMHz", 0)); 84 | row.add(Util.getNumber(systemDoc, "memSizeMB", 0)); 85 | row.add(systemDoc.getBoolean("numaEnabled").toString()); 86 | row.add(Util.getNumber(extraDoc, "pageSize", 0)); 87 | row.add(Util.getNumber(extraDoc, "numPages", 0)); 88 | row.add(Util.getNumber(extraDoc, "maxOpenFiles",0)); 89 | row.add(osDoc.getString("name")); 90 | row.add(osDoc.getString("version")); 91 | row.add(extraDoc.getString("libcVersion")); 92 | 93 | table.addRow(row); 94 | } 95 | } 96 | } 97 | catch (Exception e){ 98 | LOG.warn("Exception while running command", e); 99 | } 100 | 101 | return table; 102 | } 103 | 104 | 105 | } 106 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | de.idealo.mongodb 6 | mongodb-slow-operations-profiler 7 | 3.2.5 8 | war 9 | 10 | mongodb-slow-operations-profiler 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | yyyy-MM-dd HH:mm 16 | 17 | 18 | 19 | ${project.name} 20 | 21 | 22 | org.codehaus.mojo 23 | versions-maven-plugin 24 | 2.17.1 25 | 26 | false 27 | 28 | 29 | 30 | org.apache.maven.plugins 31 | maven-compiler-plugin 32 | 3.5.1 33 | 34 | 1.8 35 | 1.8 36 | 37 | 38 | 39 | org.apache.maven.plugins 40 | maven-war-plugin 41 | 3.4.0 42 | 43 | 44 | 45 | true 46 | 47 | 48 | ${maven.build.timestamp} 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.mongodb 59 | mongodb-driver 60 | 3.12.14 61 | 62 | 63 | org.slf4j 64 | slf4j-api 65 | 1.6.6 66 | 67 | 68 | junit 69 | junit 70 | 4.13.2 71 | 72 | 73 | ch.qos.logback 74 | logback-classic 75 | 1.5.10 76 | 77 | 78 | com.google.collections 79 | google-collections 80 | 1.0 81 | 82 | 83 | com.google.guava 84 | guava 85 | 33.3.1-jre 86 | 87 | 88 | org.jongo 89 | jongo 90 | 1.5.1 91 | 92 | 93 | javax.servlet 94 | javax.servlet-api 95 | 4.0.1 96 | provided 97 | 98 | 99 | javax.servlet 100 | jsp-api 101 | 2.0 102 | provided 103 | 104 | 105 | javax.servlet 106 | servlet-api 107 | 2.5 108 | provided 109 | 110 | 111 | 112 | com.sun.jersey 113 | jersey-server 114 | 1.19.4 115 | 116 | 117 | com.sun.jersey 118 | jersey-servlet 119 | 1.19.4 120 | 121 | 122 | com.sun.jersey 123 | jersey-json 124 | 1.19.4 125 | 126 | 127 | javax.ws.rs 128 | jsr311-api 129 | 1.1.1 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/command/CmdCollectionStats.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.command; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.BasicDBObject; 5 | import com.mongodb.client.MongoDatabase; 6 | import com.mongodb.client.MongoIterable; 7 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 8 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 9 | import de.idealo.mongodb.slowops.dto.TableDto; 10 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 11 | import de.idealo.mongodb.slowops.util.Util; 12 | import org.bson.Document; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | /** 20 | * Created by kay.agahd on 29.06.17. 21 | */ 22 | public class CmdCollectionStats implements ICommand { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(CmdCollectionStats.class); 25 | 26 | 27 | private final CommandResultDto commandResultDto; 28 | 29 | public CmdCollectionStats() { 30 | commandResultDto = new CommandResultDto(); 31 | commandResultDto.setTableHeader(Lists.newArrayList("dbs label", 32 | "host", 33 | "database", 34 | "collection", 35 | "sharded", 36 | "size", 37 | "count", 38 | "avgObjectSize", 39 | "storageSize", 40 | "capped", 41 | "#indexes", 42 | "totalIndexSize", 43 | "indexSizes")); 44 | 45 | commandResultDto.setJsonFormattedColumn(12); 46 | 47 | } 48 | 49 | @Override 50 | public boolean isHostCommand(){ 51 | return false; 52 | } 53 | 54 | @Override 55 | public CommandResultDto getCommandResultDto() { 56 | return commandResultDto; 57 | } 58 | 59 | @Override 60 | public TableDto runCommand(ProfilingReader profilingReader, MongoDbAccessor mongoDbAccessor) { 61 | final TableDto table = new TableDto(); 62 | 63 | try{ 64 | final Document commandResultDoc = mongoDbAccessor.runCommand("admin", new BasicDBObject("hostInfo", 1)); 65 | final Object system = commandResultDoc.get("system"); 66 | String hostname = ""; 67 | if (system instanceof Document) { 68 | final Document systemDoc = (Document) system; 69 | hostname = systemDoc.getString("hostname"); 70 | } 71 | 72 | final ArrayList collNames = getCollectionNames(mongoDbAccessor, profilingReader.getDatabase()); 73 | final MongoDatabase db = mongoDbAccessor.getMongoDatabase(profilingReader.getDatabase()); 74 | for(String collName : collNames){ 75 | Document collStats = db.runCommand(new Document("collStats", collName)); 76 | final List row = Lists.newArrayList(); 77 | row.add(profilingReader.getProfiledServerDto().getLabel()); 78 | row.add(hostname); 79 | row.add(profilingReader.getDatabase()); 80 | row.add(collName); 81 | row.add(collStats.getBoolean("sharded")); 82 | row.add(Util.getNumber(collStats, "size",0)); 83 | row.add(Util.getNumber(collStats, "count", 0)); 84 | row.add(Util.getNumber(collStats, "avgObjSize",0)); 85 | row.add(Util.getNumber(collStats, "storageSize",0)); 86 | row.add(collStats.getBoolean("capped")); 87 | row.add(Util.getNumber(collStats, "nindexes", 0)); 88 | row.add(Util.getNumber(collStats, "totalIndexSize",0)); 89 | row.add(((Document)collStats.get("indexSizes")).toJson()); 90 | table.addRow(row); 91 | } 92 | }catch (Exception e){ 93 | LOG.warn("Exception while running command", e); 94 | } 95 | 96 | return table; 97 | } 98 | 99 | 100 | 101 | private ArrayList getCollectionNames(MongoDbAccessor mongoDbAccessor, String dbName){ 102 | final ArrayList result = new ArrayList(); 103 | final MongoIterable collNames = mongoDbAccessor.getMongoDatabase(dbName).listCollectionNames(); 104 | 105 | if(collNames != null){ 106 | for(String collName : collNames){ 107 | result.add(collName); 108 | } 109 | } 110 | return result; 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/ApplicationStatusDto.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.dto; 2 | 3 | import com.google.common.collect.Lists; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.text.SimpleDateFormat; 8 | import java.util.Collections; 9 | import java.util.Date; 10 | import java.util.List; 11 | import java.util.concurrent.ConcurrentLinkedDeque; 12 | 13 | /** 14 | * Created by kay.agahd on 31.10.16. 15 | */ 16 | public class ApplicationStatusDto { 17 | 18 | private static final Logger LOG = LoggerFactory.getLogger(ApplicationStatusDto.class); 19 | 20 | public static final long DEFAULT_MAX_WEBLOG_ENTRIES = 100; 21 | 22 | private static final ConcurrentLinkedDeque WEBLOG = new ConcurrentLinkedDeque(); 23 | private static long MAX_WEBLOG_ENTRIES = DEFAULT_MAX_WEBLOG_ENTRIES; 24 | private static final String DATEFORMAT = ("yyyy/MM/dd HH:mm:ss,SSS"); 25 | 26 | private volatile List collectorStatuses = Collections.synchronizedList(Lists.newLinkedList()); 27 | private CollectorServerDto collectorServerDto; 28 | private Date collectorRunningSince; 29 | private long numberOfReads; 30 | private long numberOfWrites; 31 | private long numberOfReadsOfRemovedReaders; 32 | private long numberOfWritesOfRemovedWriters; 33 | private String config; 34 | private Date lastRefresh=null; 35 | 36 | 37 | public synchronized void addCollectorStatus(CollectorStatusDto collectorStatusDto){ 38 | collectorStatuses.add(collectorStatusDto); 39 | } 40 | 41 | 42 | public synchronized List getCollectorStatuses(){ 43 | LOG.debug("getCollectorStatuses size:" + collectorStatuses.size()); 44 | return collectorStatuses; 45 | } 46 | 47 | 48 | 49 | public CollectorServerDto getCollectorServerDto() { 50 | return collectorServerDto; 51 | } 52 | 53 | public void setCollectorServerDto(CollectorServerDto collectorServerDto) { 54 | this.collectorServerDto = collectorServerDto; 55 | } 56 | 57 | public Date getCollectorRunningSince() { 58 | return collectorRunningSince; 59 | } 60 | 61 | public void setCollectorRunningSince(Date collectorRunningSince) { this.collectorRunningSince = collectorRunningSince; } 62 | 63 | public long getNumberOfReads() { 64 | return numberOfReads; 65 | } 66 | 67 | public void setNumberOfReads(long numberOfReads) { 68 | this.numberOfReads = numberOfReads; 69 | } 70 | 71 | public long getNumberOfWrites() { 72 | return numberOfWrites; 73 | } 74 | 75 | public void setNumberOfWrites(long numberOfWrites) { 76 | this.numberOfWrites = numberOfWrites; 77 | } 78 | 79 | public long getNumberOfReadsOfRemovedReaders() { 80 | return numberOfReadsOfRemovedReaders; 81 | } 82 | 83 | public void setNumberOfReadsOfRemovedReaders(long numberOfReadsOfRemovedReaders) { 84 | this.numberOfReadsOfRemovedReaders = numberOfReadsOfRemovedReaders; 85 | } 86 | public long getNumberOfWritesOfRemovedWriters() { 87 | return numberOfWritesOfRemovedWriters; 88 | } 89 | 90 | public void setNumberOfWritesOfRemovedWriters(long numberOfWritesOfRemovedWriters) { 91 | this.numberOfWritesOfRemovedWriters = numberOfWritesOfRemovedWriters; 92 | } 93 | 94 | 95 | public String getConfig() { return config; } 96 | 97 | public void setConfig(String config) { this.config = config; } 98 | 99 | 100 | public Date getLastRefresh() {return lastRefresh; } 101 | 102 | public void setLastRefresh(Date lastRefresh) {this.lastRefresh = lastRefresh; } 103 | 104 | public synchronized CollectorStatusDto getCollectorStatus(int id) { 105 | for(CollectorStatusDto collectorStatus : collectorStatuses){ 106 | if(collectorStatus.getInstanceId() == id) return collectorStatus; 107 | } 108 | return null; 109 | } 110 | 111 | public static void setMaxWebLogEntries(long n){ 112 | MAX_WEBLOG_ENTRIES = n; 113 | while(WEBLOG.size() > MAX_WEBLOG_ENTRIES){ 114 | WEBLOG.poll(); 115 | } 116 | } 117 | 118 | public static void addWebLog(String msg){ 119 | while(WEBLOG.size() > MAX_WEBLOG_ENTRIES){ 120 | WEBLOG.poll(); 121 | } 122 | final SimpleDateFormat df = new SimpleDateFormat(DATEFORMAT); 123 | final long now = System.currentTimeMillis(); 124 | WEBLOG.offer(df.format(now) + " " + msg); 125 | } 126 | 127 | public String getWebLog(){ 128 | final StringBuilder sb = new StringBuilder(); 129 | for(String entry : WEBLOG){ 130 | sb.append(entry).append(System.lineSeparator()); 131 | } 132 | 133 | return sb.toString(); 134 | } 135 | 136 | } 137 | -------------------------------------------------------------------------------- /src/test/java/de/idealo/mongodb/slowops/util/ConfigReaderTest.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.util; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.ServerAddress; 5 | import de.idealo.mongodb.slowops.dto.CollectorServerDto; 6 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 7 | import org.bson.Document; 8 | import org.junit.Test; 9 | 10 | import java.util.List; 11 | 12 | import static org.junit.Assert.*; 13 | 14 | /** 15 | * Created by kay.agahd on 27.10.16. 16 | */ 17 | public class ConfigReaderTest { 18 | 19 | final ConfigReader configReader = new ConfigReader(); 20 | final Document doc = Document.parse("{" 21 | +"\"collector\":{" 22 | +"\"hosts\":[\"localhost:27017\"]," 23 | +"\"db\":\"profiling\"," 24 | +"\"collection\":\"slowops\"," 25 | +"\"adminUser\":\"\"," 26 | +"\"adminPw\":\"\"" 27 | +"}," 28 | +"\"profiled\":[" 29 | +"{\"label\":\"offerstore-de\"," 30 | +"\"hosts\":[\"offerstore-en-router-01.ipx:27017\"]," 31 | +"\"ns\":[\"offerStore.offer\"]," 32 | +"\"adminUser\":\"admin\"," 33 | +"\"adminPw\":\"XXX\"," 34 | +"\"slowMs\":100" 35 | +"}," 36 | +"{\"collect\":true," 37 | +"\"label\":\"offerstore-pl\"," 38 | +"\"hosts\":[\"mongo-030.ipx:27017\",\"mongo-017.ipx:27017\",\"mongo-018.ipx:27017\"]," 39 | +"\"ns\":[\"offerStore.offer\"]," 40 | +"\"adminUser\":\"admin\"," 41 | +"\"adminPw\":\"XXX\"," 42 | +"\"slowMs\":100" 43 | +"}" 44 | +"]," 45 | +"\"yAxisScale\":\"milliseconds\"" 46 | +"}"); 47 | 48 | 49 | 50 | @Test 51 | public void getString() throws Exception { 52 | assertEquals("profiling", configReader.getString(doc, "collector.db", "")); 53 | assertEquals(null, configReader.getString(doc, "collector.nonExistingField", null)); 54 | assertEquals("", configReader.getString(doc, "collector.nonExistingField", "")); 55 | assertEquals("default", configReader.getString(doc, "collector.nonExistingField", "default")); 56 | assertEquals("milliseconds", configReader.getString(doc, "yAxisScale", "")); 57 | 58 | assertNotEquals("other", configReader.getString(doc, "collector.db", "")); 59 | assertNotEquals("other", configReader.getString(doc, "collector.nonExistingField", null)); 60 | assertNotEquals("other", configReader.getString(doc, "collector.nonExistingField", "")); 61 | assertNotEquals("other", configReader.getString(doc, "collector.nonExistingField", "default")); 62 | assertNotEquals("other", configReader.getString(doc, "yAxisScale", "")); 63 | 64 | } 65 | 66 | @Test 67 | public void getList() throws Exception { 68 | assertEquals(Lists.newArrayList("localhost:27017"), configReader.getList(doc, "collector.hosts", null)); 69 | assertNotEquals(Lists.newArrayList("wronghost:27017"), configReader.getList(doc, "collector.hosts", null)); 70 | } 71 | 72 | @Test 73 | public void getCollectorServer() throws Exception { 74 | CollectorServerDto collector = configReader.getCollectorServer(doc); 75 | assertArrayEquals(new ServerAddress[]{new ServerAddress("localhost:27017")}, collector.getHosts()); 76 | assertEquals("profiling", collector.getDb()); 77 | assertEquals("slowops", collector.getCollection()); 78 | assertEquals("profiling", collector.getDb()); 79 | assertEquals("", collector.getAdminUser()); 80 | assertEquals("", collector.getAdminPw()); 81 | } 82 | 83 | @Test 84 | public void getProfiledServers() throws Exception { 85 | List dtoList = configReader.getProfiledServers(doc); 86 | assertEquals(2, dtoList.size()); 87 | 88 | ProfiledServerDto dto = dtoList.get(0); 89 | assertFalse(dto.isEnabled()); 90 | assertEquals("offerstore-de", dto.getLabel()); 91 | assertArrayEquals(new ServerAddress[]{new ServerAddress("offerstore-en-router-01.ipx:27017")}, dto.getHosts()); 92 | assertArrayEquals(new String[]{"offerStore.offer"}, dto.getNs()); 93 | assertEquals("admin", dto.getAdminUser()); 94 | assertEquals("XXX", dto.getAdminPw()); 95 | assertEquals(100l, dto.getSlowMs()); 96 | 97 | dto = dtoList.get(1); 98 | assertTrue(dto.isEnabled()); 99 | assertEquals("offerstore-pl", dto.getLabel()); 100 | assertArrayEquals(new ServerAddress[]{new ServerAddress("mongo-030.ipx:27017"),new ServerAddress("mongo-017.ipx:27017"),new ServerAddress("mongo-018.ipx:27017")}, dto.getHosts()); 101 | assertArrayEquals(new String[]{"offerStore.offer"}, dto.getNs()); 102 | assertEquals("admin", dto.getAdminUser()); 103 | assertEquals("XXX", dto.getAdminPw()); 104 | assertEquals(100l, dto.getSlowMs()); 105 | } 106 | 107 | } -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/util/Util.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.util; 5 | 6 | import com.google.common.collect.Lists; 7 | import com.mongodb.ServerAddress; 8 | import org.bson.Document; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.*; 13 | import java.util.List; 14 | import java.util.Properties; 15 | 16 | /** 17 | * 18 | * 19 | * @author kay.agahd 20 | * @since 26.03.2013 21 | * @version $Id: $ 22 | * @copyright idealo internet GmbH 23 | */ 24 | public class Util { 25 | 26 | private static final Logger LOG = LoggerFactory.getLogger(Util.class); 27 | 28 | public final static char PROPERTIES_SEPARATOR = ','; 29 | public static final int MAX_THREADS = Runtime.getRuntime().availableProcessors() * 100; 30 | 31 | public final static String Y_AXIS_SCALE = "yAxisScale"; 32 | public final static String Y_AXIS_SECONDS = "seconds"; 33 | public final static String Y_AXIS_MILLISECONDS = "milliseconds"; 34 | public final static String ADMIN_TOKEN = "adminToken"; 35 | public final static String DEFAULT_RESPONSE_TIMEOUT_IN_MS = "defaultResponseTimeoutInMs"; 36 | public final static String DEFAULT_SLOW_MS = "defaultSlowMS"; 37 | public final static String DEFAULT_EXCLUDED_DBS = "defaultExcludedDBs"; 38 | public final static String MAX_WEBLOG_ENTRIES = "maxWeblogEntries"; 39 | public final static String SYSTEM_PROFILE_MAX_SIZE_MB = "systemProfileCollectionMaxSizeInMB"; 40 | public final static String CONFIG_FILE = "config.json"; 41 | 42 | 43 | public static List getServerAddresses(String serverAddresses) { 44 | final List result = Lists.newArrayList(); 45 | final String[] servers = serverAddresses.split(""+PROPERTIES_SEPARATOR); 46 | 47 | for (String server : servers) { 48 | try { 49 | result.add(new ServerAddress(server)); 50 | } catch (Exception e) { 51 | LOG.error("Invalid server address: {}", server, e); 52 | } 53 | } 54 | 55 | return result; 56 | } 57 | 58 | 59 | 60 | private static Properties getProperties(String fileName) { 61 | Properties result = new Properties(); 62 | try { 63 | InputStream in; 64 | final File file = new File(fileName); 65 | if (file.exists() && file.isFile() && file.canRead()) { 66 | LOG.info("Try to load properties file '{}' ", file.getAbsolutePath()); 67 | in = new FileInputStream(new File(fileName)); 68 | } else { 69 | LOG.info("Try to load properties file '{}' from within jar file", fileName); 70 | in = Util.class.getClassLoader().getResourceAsStream(fileName); 71 | } 72 | if (in != null) { 73 | result.load(in); 74 | in.close(); 75 | LOG.info("Properties file '{}' successfully loaded", fileName); 76 | if (file.length() < 1024 * 1024) {//only print when less than 1 MB 77 | LOG.info("Loaded properties: {}", result); 78 | } 79 | } else { 80 | LOG.error("Properties file '{}' could not be loaded", fileName); 81 | } 82 | } catch (final FileNotFoundException e) { 83 | LOG.error("Properties file '{}' could not be found", fileName, e); 84 | } catch (final IOException e) { 85 | LOG.error("Error while reading properties file: {}", fileName, e); 86 | } 87 | return result; 88 | } 89 | 90 | 91 | public static String getString(Document doc, String propertyName, String defaultValue) { 92 | String propNameParts[] = propertyName.split("\\."); 93 | for(String part : propNameParts){ 94 | Object obj = doc.get(part); 95 | if(obj instanceof Document){ 96 | continue; 97 | }else{ 98 | return obj.toString(); 99 | } 100 | } 101 | 102 | return null; 103 | } 104 | 105 | public static List getList(Document doc, String propertyName, String defaultValue) { 106 | String propNameParts[] = propertyName.split("\\."); 107 | for(String part : propNameParts){ 108 | Object obj = doc.get(part); 109 | if(obj instanceof Document){ 110 | continue; 111 | }else if(obj instanceof List){ 112 | return (List)obj; 113 | } 114 | } 115 | 116 | return null; 117 | } 118 | 119 | public static long getNumber(Document doc, String field, long defaultValue){ 120 | long result = defaultValue; 121 | if(doc != null){ 122 | final Object obj = doc.get(field); 123 | if(obj instanceof Double){ 124 | result = Math.round((Double)obj); 125 | }else if(obj instanceof Integer) { 126 | result = (Integer) obj; 127 | }else if(obj instanceof Long) { 128 | result = (Long) obj; 129 | } 130 | } 131 | 132 | return result; 133 | } 134 | 135 | 136 | 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/HostInfoDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.dto; 5 | 6 | /** 7 | * 8 | * 9 | * @author kay.agahd 10 | * @since 18.05.2015 11 | * @version $Id: $ 12 | * @copyright idealo internet GmbH 13 | */ 14 | public class HostInfoDto { 15 | 16 | private Number memSizeMB; 17 | private Number numCores; 18 | private Number numPages; 19 | private Number maxOpenFiles; 20 | private Number pageSize; 21 | private boolean numaEnabled; 22 | private String hostName; 23 | private String osName; 24 | private String osVersion; 25 | private String versionString; 26 | private String libcVersion; 27 | private String kernelVersion; 28 | private long cpuFreqMHz; 29 | private String cpuArch; 30 | private String mongodbVersion; 31 | 32 | /** 33 | * @return the memSizeMB 34 | */ 35 | public Number getMemSizeMB() { 36 | return memSizeMB; 37 | } 38 | /** 39 | * @param memSizeMB the memSizeMB to set 40 | */ 41 | public void setMemSizeMB(Number memSizeMB) { 42 | this.memSizeMB = memSizeMB; 43 | } 44 | /** 45 | * @return the numCores 46 | */ 47 | public Number getNumCores() { 48 | return numCores; 49 | } 50 | /** 51 | * @param numCores the numCores to set 52 | */ 53 | public void setNumCores(Number numCores) { 54 | this.numCores = numCores; 55 | } 56 | /** 57 | * @return the numPages 58 | */ 59 | public Number getNumPages() { 60 | return numPages; 61 | } 62 | /** 63 | * @param numPages the numPages to set 64 | */ 65 | public void setNumPages(Number numPages) { 66 | this.numPages = numPages; 67 | } 68 | /** 69 | * @return the maxOpenFiles 70 | */ 71 | public Number getMaxOpenFiles() { 72 | return maxOpenFiles; 73 | } 74 | /** 75 | * @param maxOpenFiles the maxOpenFiles to set 76 | */ 77 | public void setMaxOpenFiles(Number maxOpenFiles) { 78 | this.maxOpenFiles = maxOpenFiles; 79 | } 80 | /** 81 | * @return the pageSize 82 | */ 83 | public Number getPageSize() { 84 | return pageSize; 85 | } 86 | /** 87 | * @param pageSize the pageSize to set 88 | */ 89 | public void setPageSize(Number pageSize) { 90 | this.pageSize = pageSize; 91 | } 92 | /** 93 | * @return the numaEnabled 94 | */ 95 | public boolean isNumaEnabled() { 96 | return numaEnabled; 97 | } 98 | /** 99 | * @param numaEnabled the numaEnabled to set 100 | */ 101 | public void setNumaEnabled(boolean numaEnabled) { 102 | this.numaEnabled = numaEnabled; 103 | } 104 | 105 | public String getHostName() { 106 | return hostName; 107 | } 108 | 109 | public void setHostName(String hostName) { 110 | this.hostName = hostName; 111 | } 112 | 113 | /** 114 | * @return the osName 115 | */ 116 | public String getOsName() { 117 | return osName; 118 | } 119 | /** 120 | * @param osName the osName to set 121 | */ 122 | public void setOsName(String osName) { 123 | this.osName = osName; 124 | } 125 | /** 126 | * @return the osVersion 127 | */ 128 | public String getOsVersion() { 129 | return osVersion; 130 | } 131 | /** 132 | * @param osVersion the osVersion to set 133 | */ 134 | public void setOsVersion(String osVersion) { 135 | this.osVersion = osVersion; 136 | } 137 | /** 138 | * @return the versionString 139 | */ 140 | public String getVersionString() { 141 | return versionString; 142 | } 143 | /** 144 | * @param versionString the versionString to set 145 | */ 146 | public void setVersionString(String versionString) { 147 | this.versionString = versionString; 148 | } 149 | /** 150 | * @return the libcVersion 151 | */ 152 | public String getLibcVersion() { 153 | return libcVersion; 154 | } 155 | /** 156 | * @param libcVersion the libcVersion to set 157 | */ 158 | public void setLibcVersion(String libcVersion) { 159 | this.libcVersion = libcVersion; 160 | } 161 | /** 162 | * @return the kernelVersion 163 | */ 164 | public String getKernelVersion() { 165 | return kernelVersion; 166 | } 167 | /** 168 | * @param kernelVersion the kernelVersion to set 169 | */ 170 | public void setKernelVersion(String kernelVersion) { 171 | this.kernelVersion = kernelVersion; 172 | } 173 | /** 174 | * @return the cpuFreqMHz 175 | */ 176 | public long getCpuFreqMHz() { 177 | return cpuFreqMHz; 178 | } 179 | /** 180 | * @param cpuFreqMHz the cpuFreqMHz to set 181 | */ 182 | public void setCpuFreqMHz(long cpuFreqMHz) { 183 | this.cpuFreqMHz = cpuFreqMHz; 184 | } 185 | 186 | public String getCpuArch() { 187 | return this.cpuArch; 188 | } 189 | public void setCpuArch(String cpuArch) { 190 | this.cpuArch = cpuArch; 191 | } 192 | 193 | public String getMongodbVersion() { 194 | return this.mongodbVersion; 195 | } 196 | 197 | public void setMongodbVersion(String mongodbVersion) { 198 | this.mongodbVersion = mongodbVersion; 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/CollectorStatusDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.dto; 5 | 6 | import com.mongodb.ServerAddress; 7 | import org.codehaus.jackson.annotate.JsonProperty; 8 | 9 | import java.text.SimpleDateFormat; 10 | import java.util.ArrayList; 11 | import java.util.Date; 12 | import java.util.List; 13 | 14 | /** 15 | * 16 | * 17 | * @author kay.agahd 18 | * @since 31.10.2016 19 | * @version $Id: $ 20 | * @copyright idealo internet GmbH 21 | */ 22 | public class CollectorStatusDto { 23 | 24 | private final int instanceId; 25 | private final String label; 26 | private final String replSetName; 27 | private final ServerAddress serverAddress; 28 | private final String database; 29 | private final List collections; 30 | private long slowMs; 31 | private long systemProfileMaxSizeInBytes; 32 | private boolean isCollecting; 33 | private boolean isProfiling; 34 | private final Date lastTs; 35 | private final ArrayList doneJobsHistory; 36 | private final String lastTsFormatted; 37 | private final String replSetStatus; 38 | private final String cpuArch; 39 | private final Number numCores; 40 | private final Number cpuFreqMHz; 41 | private final Number memSizeMB; 42 | private final String mongodbVersion; 43 | 44 | private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 45 | 46 | 47 | public CollectorStatusDto(@JsonProperty("instanceId") int instanceId, 48 | @JsonProperty("label") String label, 49 | @JsonProperty("replSetName") String replSetName, 50 | @JsonProperty("serverAddress") ServerAddress serverAddress, 51 | @JsonProperty("database") String database, 52 | @JsonProperty("collections") List collections, 53 | @JsonProperty("isCollecting") boolean isCollecting, 54 | @JsonProperty("isProfiling") boolean isProfiling, 55 | @JsonProperty("slowMs") long slowMs, 56 | @JsonProperty("systemProfileMaxSizeInBytes") long systemProfileMaxSizeInBytes, 57 | @JsonProperty("replSetStatus") String replSetStatus, 58 | @JsonProperty("lastTs") Date lastTs, 59 | @JsonProperty("doneJobsHistory") ArrayList doneJobsHistory, 60 | @JsonProperty("cpuArch") String cpuArch, 61 | @JsonProperty("numCores") Number numCores, 62 | @JsonProperty("cpuFreqMHz") Number cpuFreqMHz, 63 | @JsonProperty("memSizeMB") Number memSizeMB, 64 | @JsonProperty("mongodbVersion") String mongodbVersion) { 65 | this.instanceId = instanceId; 66 | this.label = label; 67 | this.replSetName = replSetName; 68 | this.serverAddress = serverAddress; 69 | this.database = database; 70 | this.collections = collections; 71 | this.slowMs = slowMs; 72 | this.systemProfileMaxSizeInBytes = systemProfileMaxSizeInBytes; 73 | this.replSetStatus = replSetStatus; 74 | this.isCollecting = isCollecting; 75 | this.isProfiling = isProfiling; 76 | this.lastTs = lastTs; 77 | this.doneJobsHistory = doneJobsHistory; 78 | this.lastTsFormatted = lastTs!=null&&lastTs.getTime()>0?dateFormat.format(lastTs):""; 79 | this.cpuArch = cpuArch; 80 | this.numCores = numCores; 81 | this.cpuFreqMHz = cpuFreqMHz; 82 | this.memSizeMB = memSizeMB; 83 | this.mongodbVersion = mongodbVersion; 84 | } 85 | 86 | //the following getter methods are needed for applicationStatus.jsp dataTable columns 87 | 88 | public int getInstanceId() { 89 | return instanceId; 90 | } 91 | 92 | public String getLabel() { 93 | return label; 94 | } 95 | 96 | public String getReplSetName() { 97 | return replSetName; 98 | } 99 | 100 | public String getReplSetStatus() { 101 | return replSetStatus; 102 | } 103 | 104 | public String getServerAddressAsString() { 105 | return serverAddress.getHost() + ":" + serverAddress.getPort(); 106 | } 107 | 108 | public String getDatabase() { 109 | return database; 110 | } 111 | 112 | public List getCollections() { 113 | return collections; 114 | } 115 | 116 | public String getCollectionsAsString() { 117 | StringBuffer result = new StringBuffer(); 118 | for(String c : collections) 119 | { 120 | result.append(c).append(","); 121 | } 122 | if(result.length() > 0) result.deleteCharAt(result.length()-1); 123 | 124 | return result.toString(); 125 | } 126 | 127 | public long getSlowMs() { return slowMs; } 128 | 129 | public long getSystemProfileMaxSizeInBytes() { return systemProfileMaxSizeInBytes; } 130 | 131 | public boolean isProfiling() { return isProfiling; } 132 | 133 | public boolean isCollecting() { return isCollecting; } 134 | 135 | public String getLastTsFormatted() { return lastTsFormatted;} 136 | 137 | public ArrayList getDoneJobsHistory() { 138 | return doneJobsHistory; 139 | } 140 | 141 | public String getCpuArch() {return cpuArch;} 142 | 143 | public Number getNumCores() {return numCores;} 144 | 145 | public Number getCpuFreqMHz() {return cpuFreqMHz;} 146 | 147 | public Number getMemSizeMB() {return memSizeMB;} 148 | 149 | public String getMongodbVersion() {return mongodbVersion;} 150 | } 151 | -------------------------------------------------------------------------------- /src/main/webapp/js/jquery.number.min.js: -------------------------------------------------------------------------------- 1 | (function(n){"use strict";function i(n,t){if(this.createTextRange){var i=this.createTextRange();i.collapse(!0);i.moveStart("character",n);i.moveEnd("character",t-n);i.select()}else this.setSelectionRange&&(this.focus(),this.setSelectionRange(n,t))}function r(n){var u=this.value.length,t,i,r,f;return(n=n.toLowerCase()=="start"?"Start":"End",document.selection)?(t=document.selection.createRange(),i=t.duplicate(),i.expand("textedit"),i.setEndPoint("EndToEnd",t),r=i.text.length-t.text.length,f=r+t.text.length,n=="Start"?r:f):(typeof this["selection"+n]!="undefined"&&(u=this["selection"+n]),u)}var u={codes:{46:127,188:44,109:45,190:46,191:47,192:96,220:92,222:39,221:93,219:91,173:45,187:61,186:59,189:45,110:46},shifts:{96:"~",49:"!",50:"@",51:"#",52:"$",53:"%",54:"^",55:"&",56:"*",57:"(",48:")",45:"_",61:"+",91:"{",93:"}",92:"|",59:":",39:'"',44:"<",46:">",47:"?"}},f,t;n.fn.number=function(t,f,e,o){o=typeof o=="undefined"?",":o;e=typeof e=="undefined"?".":e;f=typeof f=="undefined"?0:f;var s="\\u"+("0000"+e.charCodeAt(0).toString(16)).slice(-4),h=new RegExp("[^"+s+"0-9]","g"),c=new RegExp(s,"g");return t===!0?this.is("input:text")?this.on({"keydown.format":function(t){var w=n(this),c=w.data("numFormat"),l=t.keyCode?t.keyCode:t.which,a="",s=r.apply(this,["start"]),y=r.apply(this,["end"]),p="",v=!1,h;if(u.codes.hasOwnProperty(l)&&(l=u.codes[l]),!t.shiftKey&&l>=65&&l<=90?l+=32:!t.shiftKey&&l>=69&&l<=105?l-=48:t.shiftKey&&u.shifts.hasOwnProperty(l)&&(a=u.shifts[l]),a==""&&(a=String.fromCharCode(l)),l!=8&&l!=45&&l!=127&&a!=e&&!a.match(/[0-9]/))return(h=t.keyCode?t.keyCode:t.which,h==46||h==8||h==127||h==9||h==27||h==13||(h==65||h==82||h==80||h==83||h==70||h==72||h==66||h==74||h==84||h==90||h==61||h==173||h==48)&&(t.ctrlKey||t.metaKey)===!0||(h==86||h==67||h==88)&&(t.ctrlKey||t.metaKey)===!0||h>=35&&h<=39||h>=112&&h<=123)?void 0:(t.preventDefault(),!1);if(s==0&&y==this.value.length||w.val()==0?l==8?(s=y=1,this.value="",c.init=f>0?-1:0,c.c=f>0?-(f+1):0,i.apply(this,[0,0])):a==e?(s=y=1,this.value="0"+e+new Array(f+1).join("0"),c.init=f>0?1:0,c.c=f>0?-(f+1):0):l==45?(s=y=2,this.value="-0"+e+new Array(f+1).join("0"),c.init=f>0?1:0,c.c=f>0?-(f+1):0,i.apply(this,[2,2])):(c.init=f>0?-1:0,c.c=f>0?-f:0):c.c=y-this.value.length,c.isPartialSelection=s==y?!1:!0,f>0&&a==e&&s==this.value.length-f-1)c.c++,c.init=Math.max(0,c.init),t.preventDefault(),v=this.value.length+c.c;else if(l==45&&(s!=0||this.value.indexOf("-")==0))t.preventDefault();else if(a==e)c.init=Math.max(0,c.init),t.preventDefault();else if(f>0&&l==127&&s==this.value.length-f-1)t.preventDefault();else if(f>0&&l==8&&s==this.value.length-f)t.preventDefault(),c.c--,v=this.value.length+c.c;else if(f>0&&l==127&&s>this.value.length-f-1){if(this.value==="")return;this.value.slice(s,s+1)!="0"&&(p=this.value.slice(0,s)+"0"+this.value.slice(s+1),w.val(p));t.preventDefault();v=this.value.length+c.c}else if(f>0&&l==8&&s>this.value.length-f){if(this.value==="")return;this.value.slice(s-1,s)!="0"&&(p=this.value.slice(0,s-1)+"0"+this.value.slice(s),w.val(p));t.preventDefault();c.c--;v=this.value.length+c.c}else l==127&&this.value.slice(s,s+1)==o?t.preventDefault():l==8&&this.value.slice(s-1,s)==o?(t.preventDefault(),c.c--,v=this.value.length+c.c):f>0&&s==y&&this.value.length>f+1&&s>this.value.length-f-1&&isFinite(+a)&&!t.metaKey&&!t.ctrlKey&&!t.altKey&&a.length===1&&(p=y===this.value.length?this.value.slice(0,s-1):this.value.slice(0,s)+this.value.slice(s+1),this.value=p,v=s);v!==!1&&i.apply(this,[v,v]);w.data("numFormat",c)},"keyup.format":function(t){var o=n(this),u=o.data("numFormat"),e=t.keyCode?t.keyCode:t.which,h=r.apply(this,["start"]),c=r.apply(this,["end"]),s;(h===0&&c===0&&(e===189||e===109)&&(o.val("-"+o.val()),h=1,u.c=1-this.value.length,u.init=1,o.data("numFormat",u),s=this.value.length+u.c,i.apply(this,[s,s])),this.value===""||(e<48||e>57)&&(e<96||e>105)&&e!==8&&e!==46&&e!==110)||(o.val(o.val()),f>0&&(u.init<1?(h=this.value.length-f-(u.init<0?1:0),u.c=h-this.value.length,u.init=1,o.data("numFormat",u)):h>this.value.length-f&&e!=8&&(u.c++,o.data("numFormat",u))),e!=46||u.isPartialSelection||(u.c++,o.data("numFormat",u)),s=this.value.length+u.c,i.apply(this,[s,s]))},"paste.format":function(t){var u=n(this),i=t.originalEvent,r=null;return window.clipboardData&&window.clipboardData.getData?r=window.clipboardData.getData("Text"):i.clipboardData&&i.clipboardData.getData&&(r=i.clipboardData.getData("text/plain")),u.val(r),t.preventDefault(),!1}}).each(function(){var t=n(this).data("numFormat",{c:-(f+1),decimals:f,thousands_sep:o,dec_point:e,regex_dec_num:h,regex_dec:c,init:this.value.indexOf(".")?!0:!1});this.value!==""&&t.val(t.val())}):this.each(function(){var t=n(this),i=+t.text().replace(h,"").replace(c,".");t.number(isFinite(i)?+i:0,f,e,o)}):this.text(n.number.apply(window,arguments))};f=null;t=null;n.isPlainObject(n.valHooks.text)?(n.isFunction(n.valHooks.text.get)&&(f=n.valHooks.text.get),n.isFunction(n.valHooks.text.set)&&(t=n.valHooks.text.set)):n.valHooks.text={};n.valHooks.text.get=function(t){var u=n(t),i,r=u.data("numFormat");return r?t.value===""?"":(i=+t.value.replace(r.regex_dec_num,"").replace(r.regex_dec,"."),(t.value.indexOf("-")===0?"-":"")+(isFinite(i)?i:0)):n.isFunction(f)?f(t):undefined};n.valHooks.text.set=function(i,r){var e=n(i),u=e.data("numFormat"),f;return u?(f=n.number(r,u.decimals,u.dec_point,u.thousands_sep),n.isFunction(t)?t(i,f):i.value=f):n.isFunction(t)?t(i,r):undefined};n.number=function(n,t,i,r){var f,e;r=typeof r=="undefined"?",":r;i=typeof i=="undefined"?".":i;t=isFinite(+t)?Math.abs(t):0;f="\\u"+("0000"+i.charCodeAt(0).toString(16)).slice(-4);e="\\u"+("0000"+r.charCodeAt(0).toString(16)).slice(-4);n=(n+"").replace(".",i).replace(new RegExp(e,"g"),"").replace(new RegExp(f,"g"),".").replace(new RegExp("[^0-9+-Ee.]","g"),"");var o=isFinite(+n)?+n:0,u="",s=function(n,t){return""+(+(Math.round(n+"e+"+t)+"e-"+t))};return u=(t?s(o,t):""+Math.round(o)).split("."),u[0].length>3&&(u[0]=u[0].replace(/\B(?=(?:\d{3})+(?!\d))/g,r)),(u[1]||"").length collNames = database.listCollectionNames(); 66 | 67 | if(collNames != null){ 68 | for(String collName : collNames){ 69 | if(!"system.profile".equals(collName) && !"system.js".equals(collName)) { 70 | try { 71 | final TableDto collStats = getIndexStats(database.getCollection(collName), profilingReader.getProfiledServerDto().getLabel()); 72 | result.addRows(collStats); 73 | }catch(MongoCommandException e){ 74 | LOG.error("Error while getting index stats for namespace '{}'", database.getName() + "." + collName, e); 75 | } 76 | } 77 | } 78 | } 79 | return result; 80 | } 81 | 82 | 83 | private TableDto getIndexStats(MongoCollection collection, String dbsLabel){ 84 | final TableDto result = new TableDto(); 85 | final MongoIterable stats = collection 86 | .aggregate(Arrays.asList( 87 | new Document("$indexStats", new Document()), 88 | new Document("$sort", new Document("accesses.ops", 1)) 89 | )); 90 | final HashMap indexesProperties = getIndexesProperties(collection); 91 | 92 | for(Document doc : stats){ 93 | LOG.info("doc: {}", JSON.serialize(doc)); 94 | final ArrayList row = new ArrayList(); 95 | row.add(dbsLabel); 96 | row.add(doc.getString("host")); 97 | row.add(collection.getNamespace().getDatabaseName()); 98 | row.add(collection.getNamespace().getCollectionName()); 99 | final String indexName = doc.getString("name"); 100 | row.add(indexName); 101 | row.add(JSON.serialize(doc.get("key")));//serialization must keep the order of fields which is the case for JSON.serialize() but not for org.bson.Document.toJson() 102 | row.add(Boolean.toString(isTTL(indexesProperties, indexName))); 103 | final Object accesses = doc.get("accesses"); 104 | if(accesses instanceof Document){ 105 | final Document accDoc = (Document) accesses; 106 | row.add(Util.getNumber(accDoc,"ops", 0)); 107 | final Date date = accDoc.getDate("since"); 108 | final LocalDateTime localDateTime = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); 109 | row.add(localDateTime.format(DATE_TIME_FORMATTER)); 110 | }else{ 111 | row.add(0L); 112 | row.add(""); 113 | } 114 | 115 | result.addRow(row); 116 | 117 | } 118 | 119 | return result; 120 | } 121 | 122 | private HashMap getIndexesProperties(MongoCollection collection){ 123 | final HashMap result = new HashMap<>(); 124 | final ListIndexesIterable currentIndexes = collection.listIndexes(); 125 | for(Document doc: currentIndexes){ 126 | String indexName = doc.getString("name"); 127 | if(indexName != null){ 128 | result.put(indexName, doc); 129 | } 130 | } 131 | return result; 132 | } 133 | 134 | private boolean isTTL(HashMap indexesProperties, String indexName){ 135 | final Document indexProps = indexesProperties.get(indexName); 136 | if(indexProps != null){ 137 | return indexProps.get("expireAfterSeconds")!=null; 138 | } 139 | return false; 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /src/main/webapp/css/bootstrap-datetimepicker.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Datepicker for Bootstrap 3 | * 4 | * Copyright 2012 Stefan Petre 5 | * Licensed under the Apache License v2.0 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | */.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:"";line-height:0}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.bootstrap-datetimepicker-widget{top:0;left:0;width:250px;padding:4px;margin-top:1px;z-index:3000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.bootstrap-datetimepicker-widget:before{content:'';display:inline-block;border-left:7px solid transparent;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-bottom-color:rgba(0,0,0,0.2);position:absolute;top:-7px;left:6px}.bootstrap-datetimepicker-widget:after{content:'';display:inline-block;border-left:6px solid transparent;border-right:6px solid transparent;border-bottom:6px solid #fff;position:absolute;top:-6px;left:7px}.bootstrap-datetimepicker-widget.pull-right:before{left:auto;right:6px}.bootstrap-datetimepicker-widget.pull-right:after{left:auto;right:7px}.bootstrap-datetimepicker-widget>ul{list-style-type:none;margin:0}.bootstrap-datetimepicker-widget .timepicker-hour,.bootstrap-datetimepicker-widget .timepicker-minute,.bootstrap-datetimepicker-widget .timepicker-second{width:100%;font-weight:bold;font-size:1.2em}.bootstrap-datetimepicker-widget table[data-hour-format="12"] .separator{width:4px;padding:0;margin:0}.bootstrap-datetimepicker-widget .datepicker>div{display:none}.bootstrap-datetimepicker-widget .picker-switch{text-align:center}.bootstrap-datetimepicker-widget table{width:100%;margin:0}.bootstrap-datetimepicker-widget td,.bootstrap-datetimepicker-widget th{text-align:center;width:20px;height:20px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.bootstrap-datetimepicker-widget td.day:hover,.bootstrap-datetimepicker-widget td.hour:hover,.bootstrap-datetimepicker-widget td.minute:hover,.bootstrap-datetimepicker-widget td.second:hover{background:#eee;cursor:pointer}.bootstrap-datetimepicker-widget td.old,.bootstrap-datetimepicker-widget td.new{color:#999}.bootstrap-datetimepicker-widget td.active,.bootstrap-datetimepicker-widget td.active:hover{color:#fff;background-color:#006dcc;background-image:-moz-linear-gradient(top,#08c,#04c);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#04c));background-image:-webkit-linear-gradient(top,#08c,#04c);background-image:-o-linear-gradient(top,#08c,#04c);background-image:linear-gradient(to bottom,#08c,#04c);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0044cc',GradientType=0);border-color:#04c #04c #002a80;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);*background-color:#04c;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.bootstrap-datetimepicker-widget td.active:hover,.bootstrap-datetimepicker-widget td.active:hover:hover,.bootstrap-datetimepicker-widget td.active:active,.bootstrap-datetimepicker-widget td.active:hover:active,.bootstrap-datetimepicker-widget td.active.active,.bootstrap-datetimepicker-widget td.active:hover.active,.bootstrap-datetimepicker-widget td.active.disabled,.bootstrap-datetimepicker-widget td.active:hover.disabled,.bootstrap-datetimepicker-widget td.active[disabled],.bootstrap-datetimepicker-widget td.active:hover[disabled]{color:#fff;background-color:#04c;*background-color:#003bb3}.bootstrap-datetimepicker-widget td.active:active,.bootstrap-datetimepicker-widget td.active:hover:active,.bootstrap-datetimepicker-widget td.active.active,.bootstrap-datetimepicker-widget td.active:hover.active{background-color:#039 \9}.bootstrap-datetimepicker-widget td.disabled,.bootstrap-datetimepicker-widget td.disabled:hover{background:0;color:#999;cursor:not-allowed}.bootstrap-datetimepicker-widget td span{display:block;width:47px;height:54px;line-height:54px;float:left;margin:2px;cursor:pointer;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.bootstrap-datetimepicker-widget td span:hover{background:#eee}.bootstrap-datetimepicker-widget td span.active{color:#fff;background-color:#006dcc;background-image:-moz-linear-gradient(top,#08c,#04c);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#04c));background-image:-webkit-linear-gradient(top,#08c,#04c);background-image:-o-linear-gradient(top,#08c,#04c);background-image:linear-gradient(to bottom,#08c,#04c);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0044cc',GradientType=0);border-color:#04c #04c #002a80;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);*background-color:#04c;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.bootstrap-datetimepicker-widget td span.active:hover,.bootstrap-datetimepicker-widget td span.active:active,.bootstrap-datetimepicker-widget td span.active.active,.bootstrap-datetimepicker-widget td span.active.disabled,.bootstrap-datetimepicker-widget td span.active[disabled]{color:#fff;background-color:#04c;*background-color:#003bb3}.bootstrap-datetimepicker-widget td span.active:active,.bootstrap-datetimepicker-widget td span.active.active{background-color:#039 \9}.bootstrap-datetimepicker-widget td span.old{color:#999}.bootstrap-datetimepicker-widget td span.disabled,.bootstrap-datetimepicker-widget td span.disabled:hover{background:0;color:#999;cursor:not-allowed}.bootstrap-datetimepicker-widget th.switch{width:145px}.bootstrap-datetimepicker-widget th.next,.bootstrap-datetimepicker-widget th.prev{font-size:21px}.bootstrap-datetimepicker-widget th.disabled,.bootstrap-datetimepicker-widget th.disabled:hover{background:0;color:#999;cursor:not-allowed}.bootstrap-datetimepicker-widget thead tr:first-child th{cursor:pointer}.bootstrap-datetimepicker-widget thead tr:first-child th:hover{background:#eee}.input-append.date .add-on i,.input-prepend.date .add-on i{display:block;cursor:pointer;width:16px;height:16px}.bootstrap-datetimepicker-widget.left-oriented:before{left:auto;right:6px}.bootstrap-datetimepicker-widget.left-oriented:after{left:auto;right:7px} -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/util/MongoResolver.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.util; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.BasicDBObject; 5 | import com.mongodb.MongoCommandException; 6 | import com.mongodb.MongoException; 7 | import com.mongodb.ServerAddress; 8 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 9 | import org.bson.Document; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.concurrent.Callable; 16 | 17 | 18 | public class MongoResolver implements Callable { 19 | 20 | private static final Logger LOG = LoggerFactory.getLogger(MongoResolver.class); 21 | 22 | private final String adminUser; 23 | private final String adminPassword; 24 | private final boolean ssl; 25 | private final ServerAddress[] serverAddress; 26 | private int socketTimeout; 27 | private int responseTimeout; 28 | private List resolvedHosts; 29 | private List resolvedDatabases; 30 | 31 | 32 | public MongoResolver(String adminUser, String adminPassword, boolean ssl, ServerAddress serverAddress) { 33 | this(-1, -1, adminUser, adminPassword, ssl, serverAddress); 34 | } 35 | 36 | 37 | public MongoResolver(int socketTimeout, int responseTimeout, String adminUser, String adminPassword, boolean ssl, ServerAddress... serverAddress) { 38 | this.adminUser = adminUser; 39 | this.adminPassword = adminPassword; 40 | this.ssl = ssl; 41 | this.serverAddress = serverAddress; 42 | this.socketTimeout = socketTimeout; 43 | this.responseTimeout = responseTimeout; 44 | resolvedHosts = Lists.newLinkedList(); 45 | resolvedDatabases = Lists.newLinkedList(); 46 | } 47 | 48 | public List getResolvedHosts() { 49 | return resolvedHosts; 50 | } 51 | 52 | public List getResolvedDatabases() { 53 | return resolvedDatabases; 54 | } 55 | 56 | /** 57 | * If the given serverAddresses point to routers (mongos) of a sharded system, 58 | * it returns a list of all mongod ServerAddresses of the sharded system. 59 | * If the given serverAddresses point to mongod's, 60 | * the returned List will contain the ServerAddresses of all replSet members 61 | * or, if not a replSet, the first given ServerAddress. 62 | * 63 | * @return 64 | */ 65 | public List resolveMongodAddresses(MongoDbAccessor mongo) { 66 | 67 | final List result = Lists.newLinkedList(); 68 | 69 | try { 70 | final Document doc = mongo.runCommand("admin", new BasicDBObject("listShards", 1)); 71 | final Object shards = doc.get("shards"); 72 | if(shards != null) { 73 | final ArrayList list = (ArrayList)shards; 74 | for (Object obj : list) { 75 | final Document dbo = (Document)obj; 76 | String hosts = dbo.getString("host"); 77 | int slashIndex = hosts.indexOf("/"); 78 | if(slashIndex > -1) { 79 | hosts = hosts.substring(slashIndex+1); 80 | } 81 | //hosts = hosts.replace(',', ' '); 82 | 83 | result.addAll(Util.getServerAddresses(hosts)); 84 | } 85 | return result; 86 | } 87 | } catch (MongoCommandException e) {//replSets do not know the command listShards, thus throwing an Exception 88 | try { 89 | final Document doc = mongo.runCommand("admin", new BasicDBObject("replSetGetStatus", 1)); 90 | final Object members = doc.get("members"); 91 | if (members instanceof ArrayList) { 92 | final ArrayList list = (ArrayList) members; 93 | for (Object obj : list) { 94 | final Document dbo = (Document) obj; 95 | final String host = dbo.getString("name"); 96 | result.add(new ServerAddress(host)); 97 | } 98 | return result; 99 | } 100 | }catch (MongoCommandException e2) {//single nodes do not know the command replSets, thus throwing an Exception 101 | try { 102 | final Document doc = mongo.runCommand("admin", new BasicDBObject("serverStatus", 1)); 103 | final Object repl = doc.get("repl");//single nodes don't have serverStatus().repl 104 | if(repl == null) { 105 | if(serverAddress.length > 0) 106 | result.add(serverAddress[0]); 107 | } 108 | } catch (MongoCommandException e3) { 109 | LOG.error("Could not execute serverStatus command on server {} ", serverAddress, e3); 110 | } 111 | 112 | 113 | } 114 | } 115 | 116 | return result; 117 | 118 | } 119 | 120 | private List resolveAllDbNames(MongoDbAccessor mongoDbAccessor){ 121 | List result = Lists.newArrayList(); 122 | 123 | try{ 124 | final Document commandResultDoc = mongoDbAccessor.runCommand("admin", new BasicDBObject("listDatabases", 1)); 125 | 126 | if(commandResultDoc != null){ 127 | Object databases = commandResultDoc.get("databases"); 128 | if(databases instanceof ArrayList) { 129 | final List dbList = (ArrayList) databases; 130 | for (Object entry : dbList) { 131 | if (entry instanceof Document) { 132 | final Document entryDoc = (Document) entry; 133 | final String dbName = entryDoc.getString("name"); 134 | result.add(dbName); 135 | } 136 | } 137 | } 138 | } 139 | } 140 | catch (Exception e){ 141 | LOG.warn("Exception while running command listDatabases", e); 142 | } 143 | return result; 144 | } 145 | 146 | @Override 147 | public MongoResolver call() throws Exception { 148 | final MongoDbAccessor mongo = new MongoDbAccessor(socketTimeout, responseTimeout, adminUser, adminPassword, ssl, serverAddress); 149 | 150 | try { 151 | resolvedHosts.addAll(resolveMongodAddresses(mongo)); 152 | resolvedDatabases.addAll(resolveAllDbNames(mongo)); 153 | 154 | } catch (MongoException e) { 155 | LOG.error("Couldn't start mongo node at address {}", serverAddress, e); 156 | } 157 | return this; 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /src/main/webapp/datatables2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | slow operations 9 | 38 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 |
NamePositionOfficeAgeStart dateSalary
{"name":"Tiger Nixon", "age":50}System ArchitectEdinburgh612011/04/25$320,800
{"name":"Garrett Winters", "age":50}AccountantTokyo632011/07/25$170,750
{"name":"Ashton Cox", "age":50}Junior Technical AuthorSan Francisco662009/01/12$86,000
{"name":"Tiger Nixon", "age":50}System ArchitectEdinburgh612011/04/25$320,800
{"name":"Garrett Winters", "age":50}AccountantTokyo632011/07/25$170,750
{"name":"Ashton Cox", "age":50}Junior Technical AuthorSan Francisco662009/01/12$86,000
{"name":"Tiger Nixon", "age":50}System ArchitectEdinburgh612011/04/25$320,800
{"name":"Garrett Winters", "age":50}AccountantTokyo632011/07/25$170,750
{"name":"Ashton Cox", "age":50}Junior Technical AuthorSan Francisco662009/01/12$86,000
{"name":"Tiger Nixon", "age":50}System ArchitectEdinburgh612011/04/25$320,800
{"name":"Garrett Winters", "age":50}AccountantTokyo632011/07/25$170,750
{"name":"Ashton Cox", "age":50}Junior Technical AuthorSan Francisco662009/01/12$86,000
162 | 163 | 164 |
165 |

Actions show/hide

166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 185 | 189 | 190 | 191 |
refresh infoanalyse infocurrent ops infolist db.collections infoindex access stats info 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 |
Collecting info
startstop
184 |
186 | set slowMs  info
187 | negative values stop, positive values start profiling 188 |
192 |
193 | 194 | 195 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/dto/ProfiledServerDto.java: -------------------------------------------------------------------------------- 1 | package de.idealo.mongodb.slowops.dto; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.mongodb.ServerAddress; 5 | import de.idealo.mongodb.slowops.util.MongoResolver; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.Arrays; 10 | import java.util.HashMap; 11 | import java.util.HashSet; 12 | import java.util.List; 13 | import java.util.concurrent.locks.Lock; 14 | import java.util.concurrent.locks.ReadWriteLock; 15 | import java.util.concurrent.locks.ReentrantReadWriteLock; 16 | 17 | /** 18 | * Created by kay.agahd on 26.10.16. 19 | */ 20 | public class ProfiledServerDto { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(ProfiledServerDto.class); 23 | 24 | private boolean enabled; 25 | private String label; 26 | private ServerAddress[] hosts; 27 | private String[] ns; 28 | private String adminUser; 29 | private String adminPw; 30 | private boolean ssl; 31 | private long slowMs; 32 | private int responseTimeout; 33 | private HashSet resolvedHosts; 34 | private List resolvedDatabases; 35 | private List excludedDbs; 36 | private long systemProfileMaxSizeInMB; 37 | private final ReadWriteLock globalLock = new ReentrantReadWriteLock(); 38 | private final Lock readLock = globalLock.readLock(); 39 | private final Lock writeLock = globalLock.writeLock(); 40 | 41 | public ProfiledServerDto(boolean enabled, String label, ServerAddress[] hosts, String[] ns, String adminUser, String adminPw, boolean ssl, long slowMs, int responseTimeout, List excludedDbs, long systemProfileMaxSizeInMB) { 42 | this.enabled = enabled; 43 | this.label = label; 44 | this.hosts = hosts; 45 | this.ns = ns; 46 | this.adminUser = adminUser; 47 | this.adminPw = adminPw; 48 | this.ssl = ssl; 49 | this.slowMs = slowMs; 50 | this.responseTimeout = responseTimeout; 51 | this.resolvedHosts = new HashSet(); 52 | this.resolvedDatabases = Lists.newLinkedList(); 53 | this.excludedDbs = excludedDbs; 54 | this.systemProfileMaxSizeInMB = systemProfileMaxSizeInMB; 55 | } 56 | 57 | /** 58 | * 59 | * @return HashMap whose keys are database names and their value is a list of collection names 60 | */ 61 | public HashMap> getCollectionsPerDatabase(){ 62 | HashMap> result = new HashMap>(); 63 | for(String n : ns){ 64 | String db = null; 65 | String col = null; 66 | String[] parts = n.split("\\."); 67 | if(n.length() > 1){ 68 | db = parts[0]; 69 | col = n.substring(n.indexOf(".")+1); 70 | if("*".equals(db)){ 71 | try { 72 | readLock.lock(); 73 | for (String dbName: resolvedDatabases) { 74 | addCollection(dbName, col, result); 75 | } 76 | } finally { 77 | readLock.unlock(); 78 | } 79 | 80 | }else if(db.startsWith("!")){ 81 | String dbToRemove = db.substring(1); 82 | try { 83 | readLock.lock(); 84 | result.remove(dbToRemove); 85 | } finally { 86 | readLock.unlock(); 87 | } 88 | 89 | }else{ 90 | addCollection(db, col, result); 91 | } 92 | } 93 | } 94 | 95 | for(String exDb : excludedDbs){ 96 | result.remove(exDb); 97 | } 98 | 99 | return result; 100 | } 101 | 102 | 103 | private void addCollection(String db, String col, HashMap> collsPerDb){ 104 | List colls = collsPerDb.get(db); 105 | if(colls == null){ 106 | colls = Lists.newArrayList(); 107 | } 108 | colls.add(col); 109 | collsPerDb.put(db, colls); 110 | } 111 | 112 | public boolean isEnabled() { 113 | return enabled; 114 | } 115 | 116 | public void setEnabled(boolean enabled) { this.enabled = enabled; } 117 | 118 | public String getLabel() { 119 | return label; 120 | } 121 | 122 | public void setLabel(String label) { 123 | this.label = label; 124 | } 125 | 126 | public ServerAddress[] getHosts() { 127 | return hosts; 128 | } 129 | 130 | public void setHosts(ServerAddress[] hosts) { 131 | this.hosts = hosts; 132 | } 133 | 134 | public String[] getNs() { 135 | return ns; 136 | } 137 | 138 | public void setNs(String[] ns) { 139 | this.ns = ns; 140 | } 141 | 142 | public String getAdminUser() { 143 | return adminUser; 144 | } 145 | 146 | public void setAdminUser(String adminUser) { 147 | this.adminUser = adminUser; 148 | } 149 | 150 | public String getAdminPw() { 151 | return adminPw; 152 | } 153 | 154 | public void setAdminPw(String adminPw) { this.adminPw = adminPw; } 155 | 156 | public boolean getSsl() { 157 | return ssl; 158 | } 159 | 160 | public void setSsl(boolean ssl) { this.ssl = ssl; } 161 | 162 | public long getSlowMs() { return slowMs; } 163 | 164 | public void setSlowMs(long slowMs) { this.slowMs = slowMs; } 165 | 166 | public int getResponseTimeout() { return responseTimeout; } 167 | 168 | public void setResponseTimeout(int responseTimeout) { this.responseTimeout=responseTimeout; } 169 | 170 | public long getSystemProfileMaxSizeInMB() { return systemProfileMaxSizeInMB; } 171 | 172 | public void setSystemProfileMaxSizeInMB(long systemProfileMaxSizeInMB) { this.systemProfileMaxSizeInMB = systemProfileMaxSizeInMB; } 173 | 174 | public HashSet getResolvedHosts() { 175 | return resolvedHosts; 176 | } 177 | 178 | public void setResolvedResults(MongoResolver mongoResolver) { 179 | try { 180 | writeLock.lock(); 181 | resolvedHosts.addAll(mongoResolver.getResolvedHosts()); 182 | resolvedDatabases.addAll(mongoResolver.getResolvedDatabases()); 183 | } finally { 184 | writeLock.unlock(); 185 | } 186 | } 187 | 188 | @Override 189 | public boolean equals(Object o) { 190 | if (this == o) return true; 191 | if (o == null || getClass() != o.getClass()) return false; 192 | 193 | ProfiledServerDto that = (ProfiledServerDto) o; 194 | 195 | if (enabled != that.enabled) return false; 196 | if (slowMs != that.slowMs) return false; 197 | if (systemProfileMaxSizeInMB != that.systemProfileMaxSizeInMB) return false; 198 | if (responseTimeout != that.responseTimeout) return false; 199 | if (label != null ? !label.equals(that.label) : that.label != null) return false; 200 | if (!Arrays.equals(hosts, that.hosts)) return false; 201 | if (!Arrays.equals(ns, that.ns)) return false; 202 | if (adminUser != null ? !adminUser.equals(that.adminUser) : that.adminUser != null) return false; 203 | return adminPw != null ? adminPw.equals(that.adminPw) : that.adminPw == null; 204 | 205 | } 206 | 207 | @Override 208 | public int hashCode() { 209 | int result = (enabled ? 1 : 0); 210 | result = 31 * result + (label != null ? label.hashCode() : 0); 211 | result = 31 * result + Arrays.hashCode(hosts); 212 | result = 31 * result + Arrays.hashCode(ns); 213 | result = 31 * result + (adminUser != null ? adminUser.hashCode() : 0); 214 | result = 31 * result + (adminPw != null ? adminPw.hashCode() : 0); 215 | result = 31 * result + (Boolean.hashCode(ssl)); 216 | result = 31 * result + (int) (slowMs ^ (slowMs >>> 32)); 217 | result = 31 * result + (int) (systemProfileMaxSizeInMB ^ (systemProfileMaxSizeInMB >>> 32)); 218 | result = 31 * result + (responseTimeout ^ (responseTimeout >>> 32)); 219 | return result; 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/servlet/CommandResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.servlet; 5 | 6 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 7 | import com.mongodb.ServerAddress; 8 | import de.idealo.mongodb.slowops.collector.CollectorManagerInstance; 9 | import de.idealo.mongodb.slowops.collector.ProfilingReader; 10 | import de.idealo.mongodb.slowops.command.*; 11 | import de.idealo.mongodb.slowops.dto.CommandResultDto; 12 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 13 | import de.idealo.mongodb.slowops.dto.TableDto; 14 | import de.idealo.mongodb.slowops.util.Util; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import javax.servlet.RequestDispatcher; 19 | import javax.servlet.ServletException; 20 | import javax.servlet.annotation.WebServlet; 21 | import javax.servlet.http.HttpServlet; 22 | import javax.servlet.http.HttpServletRequest; 23 | import javax.servlet.http.HttpServletResponse; 24 | import java.io.IOException; 25 | import java.util.*; 26 | import java.util.concurrent.*; 27 | import java.util.stream.Collectors; 28 | 29 | @WebServlet("/cmd") 30 | public class CommandResult extends HttpServlet { 31 | private static final long serialVersionUID = 1L; 32 | private static final Logger LOG = LoggerFactory.getLogger(CommandResult.class); 33 | 34 | /** 35 | * @see HttpServlet#HttpServlet() 36 | */ 37 | public CommandResult() { 38 | super(); 39 | } 40 | 41 | /** 42 | * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) 43 | */ 44 | protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 45 | LOG.debug(">>> doGet"); 46 | final String cmd = request.getParameter("cmd"); 47 | final String mode = request.getParameter("mode"); 48 | CommandResultDto result = null; 49 | if (cmd != null) { 50 | LOG.info("cmd: {}", cmd); 51 | final String s_pIds = request.getParameter("pIds"); 52 | if (s_pIds != null) { 53 | LOG.info("pIds: {}", s_pIds); 54 | try { 55 | final String[] pIds = s_pIds.split(";");//convert string to string array 56 | final int[] pIdArray = Arrays.asList(pIds).stream().mapToInt(Integer::parseInt).toArray();//convert string array to int arry 57 | final Set pIdsSet = Arrays.stream(pIdArray).boxed().collect(Collectors.toSet());//convert int array to Set 58 | final List readerList = CollectorManagerInstance.getProfilingReaders(pIdsSet); 59 | 60 | 61 | ICommand command = null; 62 | if("cops".equals(cmd)){ 63 | command = new CmdCurrentOpAll(); 64 | }else if("copsns".equals(cmd)){ 65 | command = new CmdCurrentOpNs(); 66 | }else if("dbstat".equals(cmd)){ 67 | command = new CmdDatabaseStats(); 68 | }else if("collstat".equals(cmd)){ 69 | command = new CmdCollectionStats(); 70 | }else if("idxacc".equals(cmd)){ 71 | command = new CmdIdxAccessStats(); 72 | }else if("hostinfo".equals(cmd)){ 73 | command = new CmdHostInfo(); 74 | } 75 | 76 | result = executeCommand(command, readerList, mode); 77 | 78 | 79 | } catch (Exception e) { 80 | LOG.error("Exception while building command result", e); 81 | } 82 | } 83 | 84 | } 85 | 86 | request.setAttribute("commandResult", result!=null?result:new CommandResultDto()); 87 | RequestDispatcher view = request.getRequestDispatcher("/commandResult.jsp"); 88 | LOG.debug("doGet"); 89 | view.forward(request, response); 90 | LOG.debug("<<< doGet"); 91 | } 92 | 93 | /** 94 | * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) 95 | */ 96 | protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 97 | doGet(request, response); 98 | } 99 | 100 | 101 | private CommandResultDto executeCommand(ICommand command, List readerList, String mode ){ 102 | final CommandResultDto result = command.getCommandResultDto(); 103 | final int poolSize = 1 + Math.min(readerList.size(), Util.MAX_THREADS); 104 | LOG.info("TableDto poolSize:{} ", poolSize ); 105 | final ThreadFactory threadFactory = new ThreadFactoryBuilder() 106 | .setNameFormat("TableDto-%d") 107 | .setDaemon(true) 108 | .build(); 109 | final ThreadPoolExecutor threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(poolSize, threadFactory); 110 | final List> futureTableList = new ArrayList<>(); 111 | final HashSet serverAdresses = new HashSet<>(); 112 | final HashMap> dbsEntryPoints = new HashMap<>(); 113 | final boolean isMongod = "mongod".equals(mode); 114 | 115 | for (ProfilingReader reader : readerList) { 116 | 117 | if(command.isHostCommand()) { 118 | //if it's a host-command, execute command only *once* for any given server address or dbs (depending on mode) 119 | // because nodes selected by user may have same address but just different registered databases 120 | 121 | if (isMongod) { 122 | if(serverAdresses.add(reader.getServerAddress())){ 123 | submitCommand(threadPool, futureTableList, reader, command, reader.getServerAddress()); 124 | }; 125 | } else { 126 | if(dbsEntryPoints.put(reader.getProfiledServerDto(), new HashSet<>()) == null){ 127 | submitCommand(threadPool, futureTableList, reader, command, reader.getProfiledServerDto().getHosts()); 128 | }; 129 | } 130 | }else{ 131 | //if it's a database specific command, execute for every selected DB when in mongod-mode 132 | if (isMongod) { 133 | submitCommand(threadPool, futureTableList, reader, command, reader.getServerAddress()); 134 | } else { 135 | //in dbs-mode execute for all selected DB's only once per DBS 136 | //because nodes selected by user may have the same registered databases 137 | //but different addresses (i.e. Primary, Secondary or shard1, shard2) of the *same* DBS 138 | HashSet dbNames = dbsEntryPoints.get(reader.getProfiledServerDto()); 139 | if(dbNames == null) dbNames = new HashSet<>(); 140 | if(!dbNames.contains(reader.getDatabase())){ 141 | //command was not yet executed against this db of this dbs, so add this dbName to this dbs and execute command once 142 | dbNames.add(reader.getDatabase()); 143 | dbsEntryPoints.put(reader.getProfiledServerDto(), dbNames); 144 | submitCommand(threadPool, futureTableList, reader, command, reader.getProfiledServerDto().getHosts()); 145 | } 146 | } 147 | } 148 | 149 | } 150 | threadPool.shutdown(); 151 | 152 | 153 | for(Future futureTable : futureTableList){ 154 | try{ 155 | final TableDto table = futureTable.get(); 156 | result.addTableBody(table); 157 | } 158 | catch (InterruptedException | ExecutionException e){ 159 | LOG.warn("Exception while getting future command", e); 160 | } 161 | } 162 | threadPool.shutdownNow(); 163 | 164 | 165 | return result; 166 | } 167 | 168 | private void submitCommand(ThreadPoolExecutor threadPool, List> futureTableList, ProfilingReader reader, ICommand command, ServerAddress ... serverAddresses){ 169 | final CommandExecutor commandExecutor = new CommandExecutor(reader, command, serverAddresses); 170 | final Future futureTable = threadPool.submit(commandExecutor); 171 | futureTableList.add(futureTable); 172 | } 173 | 174 | 175 | } 176 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/grapher/AggregatedProfilingId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.grapher; 5 | 6 | import com.google.common.hash.Hashing; 7 | 8 | import java.nio.charset.StandardCharsets; 9 | import java.util.*; 10 | 11 | /** 12 | * 13 | * 14 | * @author kay.agahd 15 | * @since 14.03.2013 16 | * @version $Id: $ 17 | * @copyright idealo internet GmbH 18 | */ 19 | public class AggregatedProfilingId { 20 | 21 | private String lbl; 22 | private String adr; 23 | private String rs; 24 | private String db; 25 | private String col; 26 | private String op; 27 | private String user; 28 | private LinkedHashSet fields; 29 | private LinkedHashSet sort; 30 | private LinkedHashSet proj; 31 | private String year; 32 | private String month; 33 | private String week; 34 | private String dayOfMonth; 35 | private String hour; 36 | private String minute; 37 | private String second; 38 | private String milliseconds; 39 | 40 | public AggregatedProfilingId(){ 41 | } 42 | 43 | public AggregatedProfilingId(String lbl, String db, String col, String op, LinkedHashSet fields, LinkedHashSet sort, LinkedHashSet proj) { 44 | this.lbl = lbl; 45 | this.db = db; 46 | this.col = col; 47 | this.op = op; 48 | this.fields = fields; 49 | this.sort = sort; 50 | this.proj = proj; 51 | } 52 | 53 | /** 54 | * This defines a "fully qualified query shape". 55 | * 56 | * This is not foolproof, e.g. a query executed on different collections may be different 57 | * (because the same named field can be of different type and semantic) 58 | * even if all other characteristics below are identical. 59 | * But since we want to store only a minimum of example queries, this should be good enough. 60 | * 61 | * @return a fingerprint of the query shape 62 | */ 63 | public String getFingerprint(){ 64 | final StringBuffer result = new StringBuffer(); 65 | 66 | //separate each characteristics by . so even if some are null, the fingerprint will change 67 | if(op!=null) result.append(lbl).append(".").append(db).append(".").append(col).append(".").append(op); 68 | appendValues(fields, result); 69 | appendValues(sort, result); 70 | appendValues(proj, result); 71 | 72 | return Hashing.murmur3_128().hashString(result.toString(), StandardCharsets.UTF_8).toString(); 73 | } 74 | 75 | private void appendValues(LinkedHashSet fields, StringBuffer result){ 76 | result.append("."); 77 | if(fields!=null) { 78 | final Iterator fi = fields.iterator(); 79 | while(fi.hasNext()) { 80 | result.append(fi.next()); 81 | } 82 | }; 83 | } 84 | 85 | /** 86 | * Returns true if it make sense to show a slow operations document as an example. 87 | * This is the case, if at least the label, database, collection and operation are known. 88 | * 89 | * The queried, sorted and projected fields may be empty, and as such take part in defining a "fully qualified query shape". 90 | * 91 | * @return 92 | */ 93 | public boolean isFingerprintable(){ 94 | return lbl!=null && !lbl.isEmpty() && 95 | db!=null && !db.isEmpty() && 96 | col!=null && !col.isEmpty() && 97 | op!=null && !op.isEmpty(); 98 | } 99 | 100 | /** 101 | * @return the lbl 102 | */ 103 | public String getLbl() { 104 | return lbl; 105 | } 106 | 107 | 108 | 109 | /** 110 | * @return the adr 111 | */ 112 | public String getAdr() { 113 | return adr; 114 | } 115 | 116 | 117 | /** 118 | * @return the rs 119 | */ 120 | public String getRs() { 121 | return rs; 122 | } 123 | 124 | 125 | /** 126 | * @return the db 127 | */ 128 | public String getDb() { 129 | return db; 130 | } 131 | 132 | 133 | /** 134 | * @return the col 135 | */ 136 | public String getCol() { 137 | return col; 138 | } 139 | 140 | 141 | /** 142 | * @return the op 143 | */ 144 | public String getOp() { 145 | return op; 146 | } 147 | 148 | 149 | 150 | 151 | /** 152 | * @return the user 153 | */ 154 | public String getUser() { 155 | return user; 156 | } 157 | 158 | 159 | 160 | 161 | /** 162 | * @return the fields 163 | */ 164 | public LinkedHashSet getFields() { 165 | return fields; 166 | } 167 | 168 | 169 | 170 | 171 | /** 172 | * @return the sort 173 | */ 174 | public LinkedHashSet getSort() { 175 | return sort; 176 | } 177 | 178 | /** 179 | * @return the proj 180 | */ 181 | public LinkedHashSet getProj() { 182 | return proj; 183 | } 184 | 185 | 186 | 187 | /** 188 | * @return the second 189 | */ 190 | public String getSecond() { 191 | return second; 192 | } 193 | 194 | public String getLabel(boolean isHtml) { 195 | final StringBuffer result = new StringBuffer(); 196 | result.append(getField("lbl", lbl, isHtml)); 197 | result.append(getField("adr", adr, isHtml)); 198 | result.append(getField("rs", rs, isHtml)); 199 | result.append(getField("db", db, isHtml)); 200 | result.append(getField("col", col, isHtml)); 201 | result.append(getField("op", op, isHtml)); 202 | result.append(getField("user", user, isHtml)); 203 | result.append(getField("fields", getStringList(fields), isHtml)); 204 | result.append(getField("sort", getStringList(sort), isHtml)); 205 | result.append(getField("proj", getStringList(proj), isHtml)); 206 | 207 | if(result.length() > 0) { 208 | if(isHtml) { 209 | result.delete(result.length() - 4, result.length());//remove last
210 | }else { 211 | result.deleteCharAt(result.length() - 2);//remove last ;SPACE 212 | } 213 | }else { 214 | result.append("empty"); 215 | } 216 | 217 | return result.toString(); 218 | } 219 | 220 | private String getStringList(LinkedHashSet str){ 221 | if(str == null) return null; 222 | final StringBuffer result = new StringBuffer(); 223 | for (String s: str) { 224 | result.append("'").append(s).append("'").append("; "); 225 | } 226 | if(result.length()>0) result.deleteCharAt(result.length() - 2);//remove last ;SPACE 227 | 228 | return result.toString(); 229 | } 230 | 231 | private String getField(String name, String field, boolean isHtml) { 232 | if(field != null && field.length() > 0) { 233 | final StringBuffer result = new StringBuffer(); 234 | if(isHtml){ 235 | result.append(name).append("=").append(field).append("
"); 236 | }else { 237 | result.append(name).append("=").append(field).append("; "); 238 | } 239 | return result.toString(); 240 | } 241 | return ""; 242 | } 243 | 244 | public Calendar getCalendar() { 245 | final Calendar result = new GregorianCalendar(); 246 | final int offset = (result.get(Calendar.ZONE_OFFSET) + result.get(Calendar.DST_OFFSET)); 247 | 248 | result.setTimeInMillis(0);//reset 249 | 250 | if(year != null) { 251 | try { 252 | result.set(Calendar.YEAR, Integer.parseInt(year)); 253 | } catch (NumberFormatException e) { 254 | // TODO Auto-generated catch block 255 | e.printStackTrace(); 256 | } 257 | } 258 | if(month != null) { 259 | try { 260 | result.set(Calendar.MONTH, Integer.parseInt(month)-1); 261 | } catch (NumberFormatException e) { 262 | // TODO Auto-generated catch block 263 | e.printStackTrace(); 264 | } 265 | } 266 | if(week != null) { 267 | try { 268 | result.set(Calendar.WEEK_OF_YEAR, Integer.parseInt(week)); 269 | } catch (NumberFormatException e) { 270 | // TODO Auto-generated catch block 271 | e.printStackTrace(); 272 | } 273 | } 274 | if(dayOfMonth != null) { 275 | try { 276 | result.set(Calendar.DAY_OF_MONTH, Integer.parseInt(dayOfMonth)); 277 | } catch (NumberFormatException e) { 278 | // TODO Auto-generated catch block 279 | e.printStackTrace(); 280 | } 281 | } 282 | if(hour != null) { 283 | try { 284 | result.set(Calendar.HOUR_OF_DAY, Integer.parseInt(hour)); 285 | } catch (NumberFormatException e) { 286 | // TODO Auto-generated catch block 287 | e.printStackTrace(); 288 | } 289 | } 290 | if(minute != null) { 291 | try { 292 | result.set(Calendar.MINUTE, Integer.parseInt(minute)); 293 | } catch (NumberFormatException e) { 294 | // TODO Auto-generated catch block 295 | e.printStackTrace(); 296 | } 297 | } 298 | if(second != null) { 299 | try { 300 | result.set(Calendar.SECOND, Integer.parseInt(second)); 301 | } catch (NumberFormatException e) { 302 | // TODO Auto-generated catch block 303 | e.printStackTrace(); 304 | } 305 | } 306 | if(milliseconds != null) { 307 | try { 308 | result.set(Calendar.MILLISECOND, Integer.parseInt(milliseconds)); 309 | } catch (NumberFormatException e) { 310 | // TODO Auto-generated catch block 311 | e.printStackTrace(); 312 | } 313 | } 314 | 315 | result.add(Calendar.MILLISECOND, offset); 316 | 317 | return result; 318 | } 319 | 320 | } 321 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/util/ConfigReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.util; 5 | 6 | import com.google.common.collect.Lists; 7 | import com.mongodb.ServerAddress; 8 | import de.idealo.mongodb.slowops.dto.ApplicationStatusDto; 9 | import de.idealo.mongodb.slowops.dto.CollectorServerDto; 10 | import de.idealo.mongodb.slowops.dto.ProfiledServerDto; 11 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 12 | import org.bson.Document; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.io.*; 17 | import java.util.HashSet; 18 | import java.util.List; 19 | 20 | /** 21 | * 22 | * 23 | * @author kay.agahd 24 | * @since 26.10.2016 25 | * @version $Id: $ 26 | * @copyright idealo internet GmbH 27 | */ 28 | public class ConfigReader { 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(ConfigReader.class); 31 | 32 | public static Document CONFIG; 33 | 34 | 35 | static { 36 | CONFIG = getConfig(Util.CONFIG_FILE); 37 | } 38 | 39 | /** 40 | * returns true is the given cfg passed the validation test and replaced the current configuration 41 | * 42 | * @param cfg - config in json format 43 | * @return 44 | */ 45 | public static boolean reloadConfig(String cfg){ 46 | boolean result = false; 47 | final Document bak = CONFIG; 48 | try { 49 | CONFIG = Document.parse(cfg); 50 | if(isValidConfig()){ 51 | result = true; 52 | ApplicationStatusDto.setMaxWebLogEntries(getLong(CONFIG, Util.MAX_WEBLOG_ENTRIES, ApplicationStatusDto.DEFAULT_MAX_WEBLOG_ENTRIES)); 53 | }else{ 54 | CONFIG = bak; 55 | } 56 | } catch (Exception e) { 57 | LOG.error("Error while reading config: {}", cfg, e); 58 | } 59 | return result; 60 | } 61 | 62 | //rudimentary validation 63 | private static boolean isValidConfig(){ 64 | boolean result = true; 65 | final CollectorServerDto collectorServer = getCollectorServer(); 66 | 67 | result = result && !getProfiledServers(CONFIG).isEmpty(); 68 | result = result && collectorServer.getHosts().length > 0; 69 | result = result && !collectorServer.getDb().isEmpty(); 70 | result = result && !collectorServer.getCollection().isEmpty(); 71 | 72 | return result; 73 | } 74 | 75 | public static boolean getBoolean(Document doc, String propertyName, boolean defaultValue) { 76 | String propNameParts[] = propertyName.split("\\."); 77 | for(String part : propNameParts){ 78 | Object obj = doc.get(part); 79 | if(obj != null){ 80 | if(obj instanceof Boolean) { 81 | return ((Boolean)obj).booleanValue(); 82 | }else if(obj instanceof Document){ 83 | doc = (Document) obj; 84 | } 85 | } 86 | } 87 | 88 | return defaultValue; 89 | } 90 | 91 | public static String getString(Document doc, String propertyName, String defaultValue) { 92 | String propNameParts[] = propertyName.split("\\."); 93 | for(String part : propNameParts){ 94 | Object obj = doc.get(part); 95 | if(obj != null){ 96 | if(obj instanceof String) { 97 | return obj.toString(); 98 | }else if(obj instanceof Document){ 99 | doc = (Document) obj; 100 | } 101 | } 102 | } 103 | 104 | return defaultValue; 105 | } 106 | 107 | public static long getLong(Document doc, String propertyName, long defaultValue) { 108 | String propNameParts[] = propertyName.split("\\."); 109 | for(String part : propNameParts){ 110 | Object obj = doc.get(part); 111 | if(obj != null){ 112 | if(obj instanceof Number) { 113 | return Long.valueOf(obj.toString()).longValue(); 114 | }else if(obj instanceof Document){ 115 | doc = (Document) obj; 116 | } 117 | } 118 | } 119 | 120 | return defaultValue; 121 | } 122 | 123 | public static List getList(Document doc, String propertyName, List defaultValue) { 124 | String propNameParts[] = propertyName.split("\\."); 125 | for(String part : propNameParts){ 126 | Object obj = doc.get(part); 127 | if(obj != null){ 128 | if(obj instanceof List){ 129 | return (List)obj; 130 | }else if(obj instanceof Document){ 131 | doc = (Document) obj; 132 | } 133 | } 134 | } 135 | 136 | return defaultValue; 137 | } 138 | 139 | public static String getConfig() { 140 | return CONFIG.toJson(); 141 | } 142 | 143 | 144 | private static Document getConfig(String fileName) { 145 | Document result = null; 146 | try { 147 | InputStream in; 148 | final File file = new File(fileName); 149 | if (file.exists() && file.isFile() && file.canRead()) { 150 | LOG.info("Try to load config file '{}' ", file.getAbsolutePath()); 151 | in = new FileInputStream(new File(fileName)); 152 | } else { 153 | LOG.info("Try to load config file '{}' from within jar file", fileName); 154 | in = ConfigReader.class.getClassLoader().getResourceAsStream(fileName); 155 | } 156 | if (in != null) { 157 | StringBuilder builder = new StringBuilder(); 158 | int ch; 159 | while((ch = in.read()) != -1){ 160 | builder.append((char)ch); 161 | } 162 | result = Document.parse(builder.toString()); 163 | in.close(); 164 | LOG.info("Config file '{}' successfully loaded", fileName); 165 | if (file.length() < 1024 * 1024) {//only print when less than 1 MB 166 | LOG.info("Loaded config: {}", result); 167 | } 168 | } else { 169 | LOG.error("Config file '{}' could not be loaded", fileName); 170 | } 171 | } catch (final FileNotFoundException e) { 172 | LOG.error("Config file '{}' could not be found", fileName, e); 173 | } catch (final IOException e) { 174 | LOG.error("Error while reading config file: {}", fileName, e); 175 | } 176 | return result; 177 | } 178 | 179 | private static ServerAddress[] getServerAddresses(List serverAddresses) { 180 | final List result = Lists.newArrayList(); 181 | for (String serverAddress : serverAddresses) { 182 | try { 183 | result.add(new ServerAddress(serverAddress)); 184 | } catch (Exception e) { 185 | LOG.error("Invalid server address: {}", serverAddress, e); 186 | } 187 | } 188 | return result.toArray(new ServerAddress[]{}); 189 | } 190 | 191 | public static CollectorServerDto getCollectorServer(){ 192 | return getCollectorServer(CONFIG); 193 | } 194 | 195 | public static CollectorServerDto getCollectorServer(Document doc){ 196 | List hostList = getList(doc, "collector.hosts", Lists.newArrayList()); 197 | return new CollectorServerDto( 198 | getServerAddresses(hostList), 199 | getString(doc, "collector.db", "profiling"), 200 | getString(doc, "collector.collection", "slowops"), 201 | getString(doc, "collector.adminUser", null), 202 | getString(doc, "collector.adminPw", null), 203 | getBoolean(doc, "collector.ssl", false) 204 | ); 205 | } 206 | 207 | 208 | public static List getProfiledServers(Document doc){ 209 | List result = Lists.newArrayList(); 210 | List profiledServerList = getList(doc, "profiled", Lists.newArrayList()); 211 | List exDbs = getList(doc, Util.DEFAULT_EXCLUDED_DBS, Lists.newArrayList()); 212 | 213 | for(Document serverDoc : profiledServerList){ 214 | List hostList = getList(serverDoc, "hosts", Lists.newArrayList()); 215 | List nsList = getList(serverDoc, "ns", Lists.newArrayList()); 216 | 217 | ProfiledServerDto dto = new ProfiledServerDto( 218 | getBoolean(serverDoc, "collect", false), 219 | getString(serverDoc, "label", null), 220 | getServerAddresses(hostList), 221 | nsList.toArray(new String[]{}), 222 | getString(serverDoc, "adminUser", null), 223 | getString(serverDoc, "adminPw", null), 224 | getBoolean(serverDoc, "ssl", false), 225 | getLong(serverDoc, "slowMS", ConfigReader.getLong(doc, Util.DEFAULT_SLOW_MS, MongoDbAccessor.DEFAULT_SLOW_MS)), 226 | (int)getLong(serverDoc, "responseTimeoutInMs", getLong(doc, Util.DEFAULT_RESPONSE_TIMEOUT_IN_MS, MongoDbAccessor.DEFAULT_CONNECT_TIMEOUT_MS)), 227 | exDbs, 228 | getLong(serverDoc, Util.SYSTEM_PROFILE_MAX_SIZE_MB, ConfigReader.getLong(doc, Util.SYSTEM_PROFILE_MAX_SIZE_MB, 1))//if not defined, we take monogdb's default size of only 1 MB 229 | 230 | ); 231 | result.add(dto); 232 | } 233 | return result; 234 | } 235 | 236 | public static void main(String[] args) { 237 | LOG.info("{}", CONFIG.get("collector")); 238 | for(String key: CONFIG.keySet()){ 239 | LOG.info(key); 240 | } 241 | LOG.info(getString(CONFIG, "collector.db", "no db")); 242 | LOG.info(getString(CONFIG, "profiled.label", "no label")); 243 | List list = getList(CONFIG, "profiled", Lists.newArrayList()); 244 | for(Object obj : list){ 245 | if(obj instanceof Document){ 246 | LOG.info(getString((Document)obj, "label", "nix")); 247 | LOG.info(getString((Document)obj, "hosts", "nix")); 248 | List ns = getList((Document)obj, "ns", Lists.newArrayList()); 249 | for(Object nsObj : ns){ 250 | LOG.info("{}", nsObj); 251 | } 252 | } 253 | } 254 | } 255 | 256 | 257 | } 258 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/monitor/MongoDbAccessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Idealo Internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.monitor; 5 | 6 | import com.google.common.collect.Lists; 7 | import com.mongodb.*; 8 | import com.mongodb.client.MongoDatabase; 9 | import com.mongodb.client.MongoIterable; 10 | import com.mongodb.util.JSON; 11 | import org.bson.Document; 12 | import org.bson.codecs.configuration.CodecRegistry; 13 | import org.bson.conversions.Bson; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.net.UnknownHostException; 18 | import java.util.List; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | import java.util.concurrent.locks.Lock; 21 | import java.util.concurrent.locks.ReadWriteLock; 22 | import java.util.concurrent.locks.ReentrantReadWriteLock; 23 | 24 | 25 | /** 26 | * 27 | * 28 | * @author kay.agahd 29 | * @since 30.04.2013 30 | * @version $Id: $ 31 | * @copyright Idealo Internet GmbH 32 | */ 33 | public class MongoDbAccessor { 34 | 35 | private static final Logger LOG = LoggerFactory.getLogger(MongoDbAccessor.class); 36 | 37 | public static final int DEFAULT_CONNECT_TIMEOUT_MS = 2000; 38 | public static final int DEFAULT_SLOW_MS = 100; 39 | private static final int DEFAULT_SOCKET_TIMEOUT_MS = 10000; 40 | private static final ConcurrentHashMap INSTANCES = new ConcurrentHashMap(); 41 | 42 | private final ServerAddress[] serverAddresses; 43 | private final int socketTimeout; 44 | private final int connectTimeout; 45 | private final String user; 46 | private final String pw; 47 | private final boolean ssl; 48 | private final boolean isSecondaryReadPreferred; 49 | private MongoClient mongo; 50 | private final ReadWriteLock globalLock = new ReentrantReadWriteLock(); 51 | private final Lock writeLock; 52 | 53 | 54 | public static void terminate(){ 55 | if(!INSTANCES.isEmpty()) { 56 | for (MongoDbAccessor i : INSTANCES.values()) { 57 | if (i.mongo != null) { 58 | i.mongo.close(); 59 | i.mongo = null; 60 | } 61 | } 62 | INSTANCES.clear(); 63 | } 64 | } 65 | 66 | private MongoDbAccessor(){ 67 | this(-1, -1, false, null, null, false, null); 68 | }; 69 | 70 | public MongoDbAccessor(String user, String pw, boolean ssl, ServerAddress ... serverAddresses){ 71 | this(-1, -1, false, user, pw, ssl, serverAddresses); 72 | } 73 | 74 | 75 | public MongoDbAccessor(int socketTimeout, int connectTimeout, String user, String pw, boolean ssl, ServerAddress ... serverAddresses){ 76 | this(socketTimeout, connectTimeout, false, user, pw, ssl, serverAddresses); 77 | } 78 | 79 | public MongoDbAccessor(int socketTimeout, int connectTimeout, boolean isSecondaryReadPreferred, String user, String pw, boolean ssl, ServerAddress ... serverAddresses){ 80 | this.serverAddresses = serverAddresses; 81 | this.socketTimeout = socketTimeout<0? DEFAULT_SOCKET_TIMEOUT_MS :socketTimeout; 82 | this.connectTimeout = connectTimeout<0? DEFAULT_CONNECT_TIMEOUT_MS :connectTimeout; 83 | this.user = user; 84 | this.pw = pw; 85 | this.ssl = ssl; 86 | this.isSecondaryReadPreferred = isSecondaryReadPreferred; 87 | writeLock = globalLock.writeLock(); 88 | init(); 89 | 90 | } 91 | 92 | 93 | 94 | private void init() { 95 | LOG.debug(">>> init connection to servers {} ", serverAddresses); 96 | try { 97 | writeLock.lock(); 98 | final MongoDbAccessor instance = INSTANCES.get(this.hashCode()); 99 | if(instance != null && instance.mongo != null){ 100 | this.mongo = instance.mongo; 101 | LOG.debug("reuse connection to servers {} ", serverAddresses); 102 | }else { 103 | LOG.debug("create new connection to servers {} ", serverAddresses); 104 | INSTANCES.put(this.hashCode(), this); 105 | 106 | try { 107 | MongoClientOptions options = MongoClientOptions.builder(). 108 | socketTimeout(socketTimeout). 109 | connectTimeout(connectTimeout). 110 | readPreference(isSecondaryReadPreferred?ReadPreference.secondaryPreferred():ReadPreference.primaryPreferred()). 111 | writeConcern(WriteConcern.ACKNOWLEDGED). 112 | sslEnabled(ssl). 113 | sslInvalidHostNameAllowed(true). 114 | build(); 115 | 116 | if (user != null && !user.isEmpty() && pw != null && !pw.isEmpty()) { 117 | MongoCredential mc = MongoCredential.createCredential(user, "admin", pw.toCharArray()); 118 | if (serverAddresses.length == 1) { 119 | mongo = new MongoClient(serverAddresses[0], mc, options); 120 | } else { 121 | mongo = new MongoClient(Lists.newArrayList(serverAddresses), mc, options); 122 | } 123 | } else { 124 | if (serverAddresses.length == 1) { 125 | mongo = new MongoClient(serverAddresses[0], options); 126 | } else { 127 | mongo = new MongoClient(Lists.newArrayList(serverAddresses), options); 128 | } 129 | } 130 | } catch (MongoException e) { 131 | LOG.error("Error while initializing mongo at address {}", serverAddresses, e); 132 | } 133 | } 134 | }finally { 135 | writeLock.unlock(); 136 | } 137 | LOG.debug("<<< init"); 138 | } 139 | 140 | public CodecRegistry getDefaultCodecRegistry(){ 141 | return mongo.getDefaultCodecRegistry(); 142 | } 143 | 144 | public MongoDatabase getMongoDatabase(String dbName) { 145 | return mongo.getDatabase(dbName); 146 | } 147 | 148 | public DB getMongoDB(String dbName) { 149 | return mongo.getDB(dbName); 150 | } 151 | 152 | 153 | 154 | public Document runCommand(String dbName, DBObject cmd) throws IllegalStateException { 155 | 156 | if(dbName != null && !dbName.isEmpty()) { 157 | long start = System.currentTimeMillis(); 158 | Document result = new Document(); 159 | try { 160 | result = getMongoDatabase(dbName).runCommand((Bson) cmd, isSecondaryReadPreferred ? ReadPreference.secondaryPreferred() : ReadPreference.primaryPreferred()); 161 | }catch (Throwable e){ 162 | LOG.debug("runCommand failed {} on {}/{}", new Object[]{cmd.toString(), serverAddresses, dbName}); 163 | throw e; 164 | } 165 | long end = System.currentTimeMillis(); 166 | LOG.debug("runCommand {} execTime in ms: {} on {}/{}", new Object[]{cmd.toString(), (end-start), serverAddresses, dbName}); 167 | return result; 168 | } 169 | throw new IllegalStateException("Database not initialized"); 170 | } 171 | 172 | //none of both methods is able to fetch all mongod addresses of the whole cluster when router addresses are used to initialize mongo 173 | private List getAllAddresses(){ 174 | return mongo.getAllAddress(); 175 | //return mongo.getServerAddressList(); 176 | } 177 | 178 | private void find(String dbName, String collName, int limit){ 179 | 180 | final MongoIterable res = getMongoDatabase(dbName).getCollection(collName) 181 | .find() 182 | .limit(limit); 183 | 184 | if(res != null){ 185 | for(Document doc : res){ 186 | LOG.info("doc: {}", JSON.serialize(doc)); 187 | } 188 | } 189 | 190 | } 191 | 192 | @Override 193 | public boolean equals(Object o) { 194 | if (this == o) return true; 195 | if (o == null || getClass() != o.getClass()) return false; 196 | 197 | MongoDbAccessor that = (MongoDbAccessor) o; 198 | 199 | if (socketTimeout != that.socketTimeout) return false; 200 | if (connectTimeout != that.connectTimeout) return false; 201 | if (ssl != that.ssl) return false; 202 | if (isSecondaryReadPreferred != that.isSecondaryReadPreferred) return false; 203 | if (user != null ? !user.equals(that.user) : that.user != null) return false; 204 | if (pw != null ? !pw.equals(that.pw) : that.pw != null) return false; 205 | 206 | int serverAddressesHashCode = 0; 207 | for (ServerAddress sa: serverAddresses) { 208 | serverAddressesHashCode += sa.hashCode(); 209 | } 210 | int thatServerAddressesHashCode = 0; 211 | for (ServerAddress sa: that.serverAddresses) { 212 | thatServerAddressesHashCode += sa.hashCode(); 213 | } 214 | 215 | return serverAddressesHashCode == thatServerAddressesHashCode; 216 | 217 | } 218 | 219 | @Override 220 | public int hashCode() { 221 | 222 | int result = 31 * socketTimeout; 223 | result = 31 * result + connectTimeout; 224 | result = 31 * result + (user != null ? user.hashCode() : 0); 225 | result = 31 * result + (pw != null ? pw.hashCode() : 0); 226 | result = 31 * result + (ssl ? 1 : 0); 227 | result = 31 * result + (isSecondaryReadPreferred ? 1 : 0); 228 | for (ServerAddress sa: serverAddresses) { 229 | result = /*31 * */result + sa.hashCode();//order of array elements does not matter, so don't multiply by 31 230 | } 231 | 232 | return result; 233 | } 234 | 235 | public static void main(String[] args) throws UnknownHostException { 236 | /* 237 | //previously to do if SSL/TLS is to be used: keytool -importcert -file /path/to/your/ca-cert.ctr -keystore /path/to/your/truststore 238 | System.setProperty("javax.net.ssl.trustStore", "/path/to/your/truststore"); 239 | System.setProperty("javax.net.ssl.trustStorePassword", "YOUR_TRUST_STORE_PASSWORD"); 240 | 241 | ServerAddress adr = new ServerAddress("localhost:27017"); 242 | 243 | MongoDbAccessor mongoDbAccessor = new MongoDbAccessor(1000, 1000, "admin", "", true, adr); 244 | Document doc = mongoDbAccessor.runCommand("admin", new BasicDBObject("isMaster", "1")); 245 | LOG.info("doc: {}", doc); 246 | LOG.info("ismaster: {}", doc.get("ismaster")); 247 | 248 | for(ServerAddress sa : mongoDbAccessor.getAllAddresses()){ 249 | LOG.info("adr: {}", sa); 250 | } 251 | */ 252 | 253 | } 254 | 255 | 256 | 257 | } 258 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/grapher/AggregatedProfiling.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.grapher; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.Date; 10 | import java.util.HashSet; 11 | 12 | /** 13 | * 14 | * 15 | * @author kay.agahd 16 | * @since 14.03.2013 17 | * @version $Id: $ 18 | * @copyright idealo internet GmbH 19 | */ 20 | public class AggregatedProfiling { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(Grapher.class); 23 | 24 | 25 | private AggregatedProfilingId _id; 26 | private int count; 27 | private long millis; 28 | private double avgMs; 29 | private double minMs; 30 | private double maxMs; 31 | private double stdDevMs; 32 | private double nRet; 33 | private double avgRet; 34 | private double minRet; 35 | private double maxRet; 36 | private double stdDevRet; 37 | private double len; 38 | private double avgLen; 39 | private double minLen; 40 | private double maxLen; 41 | private double stdDevLen; 42 | private Date firstts; 43 | 44 | private long keys; 45 | private long docs; 46 | private long del; 47 | private long ins; 48 | private long mod; 49 | private HashSet sortstages; 50 | 51 | 52 | private AggregatedProfiling(){ 53 | 54 | } 55 | 56 | /** 57 | * @return the _id 58 | */ 59 | public AggregatedProfilingId getId() { 60 | return _id; 61 | } 62 | 63 | /** 64 | * @return the count 65 | */ 66 | public int getCount() { 67 | return count; 68 | } 69 | 70 | /** 71 | * @return the millis 72 | */ 73 | public long getMillis() { 74 | return millis; 75 | } 76 | 77 | /** 78 | * @return the avgMs 79 | */ 80 | public double getAvgMs() { 81 | return avgMs; 82 | } 83 | 84 | /** 85 | * @return the minMs 86 | */ 87 | public double getMinMs() { 88 | return minMs; 89 | } 90 | 91 | /** 92 | * @return the maxMs 93 | */ 94 | public double getMaxMs() { 95 | return maxMs; 96 | } 97 | 98 | /** 99 | * @return the stdDevMs 100 | */ 101 | public double getStdDevMs() { 102 | return stdDevMs; 103 | } 104 | 105 | /** 106 | * @return the nRet 107 | */ 108 | public double getNRet() { 109 | return nRet; 110 | } 111 | 112 | /** 113 | * @return the avgRet 114 | */ 115 | public double getAvgRet() { 116 | return avgRet; 117 | } 118 | 119 | /** 120 | * @return the minRet 121 | */ 122 | public double getMinRet() { 123 | return minRet; 124 | } 125 | 126 | /** 127 | * @return the maxRet 128 | */ 129 | public double getMaxRet() { 130 | return maxRet; 131 | } 132 | 133 | /** 134 | * @return the stdDevRet 135 | */ 136 | public double getStdDevRet() { 137 | return stdDevRet; 138 | } 139 | 140 | 141 | public double getLen() { 142 | return len; 143 | } 144 | 145 | public double getAvgLen() { 146 | return avgLen; 147 | } 148 | 149 | public double getMinLen() { 150 | return minLen; 151 | } 152 | 153 | public double getMaxLen() { 154 | return maxLen; 155 | } 156 | 157 | public double getStdDevLen() { 158 | return stdDevLen; 159 | } 160 | 161 | /** 162 | * @return the firstts 163 | */ 164 | public Date getFirstts() { 165 | return firstts; 166 | } 167 | 168 | 169 | public long getKeys() { return keys; } 170 | 171 | public long getDocs() { return docs; } 172 | 173 | public boolean hasSortStage() { return sortstages.contains(Boolean.TRUE); } 174 | 175 | public long getDel() { return del; } 176 | 177 | public long getIns() { return ins; } 178 | 179 | public long getMod() { return mod; } 180 | 181 | public long getDocsWrittenCount() { return del+ins+mod; } 182 | 183 | public HashSet getSortstages() { return sortstages; } 184 | 185 | 186 | /** 187 | * @param entry 188 | */ 189 | public void combine(AggregatedProfiling entry) { 190 | if(entry != null) { 191 | 192 | 193 | if(entry.getMinMs() < minMs) minMs = entry.getMinMs(); 194 | if(entry.getMaxMs() > maxMs) maxMs = entry.getMaxMs(); 195 | stdDevMs = combineStdDev(new Triplet(millis, stdDevMs, count), new Triplet(entry.getMillis(), entry.getStdDevMs(), entry.getCount())); 196 | if(entry.getMinRet() < minRet) minRet = entry.getMinRet(); 197 | if(entry.getMaxRet() > maxRet) maxRet = entry.getMaxRet(); 198 | stdDevRet = combineStdDev(new Triplet(nRet, stdDevRet, count), new Triplet(entry.getNRet(), entry.getStdDevRet(), entry.getCount())); 199 | if(entry.getMinLen() < minLen) minLen = entry.getMinLen(); 200 | if(entry.getMaxLen() > maxLen) maxLen = entry.getMaxLen(); 201 | stdDevLen = combineStdDev(new Triplet(len, stdDevLen, count), new Triplet(entry.getLen(), entry.getStdDevLen(), entry.getCount())); 202 | if(entry.getFirstts().before(firstts)) firstts = entry.getFirstts(); 203 | keys += entry.getKeys(); 204 | docs += entry.getDocs(); 205 | del += entry.getDel(); 206 | ins += entry.getIns(); 207 | mod += entry.getMod(); 208 | sortstages.addAll(entry.getSortstages()); 209 | nRet += entry.getNRet();//last line because it's used to calculate stdDev 210 | len += entry.getLen();//last line because it's used to calculate stdDev 211 | millis += entry.getMillis();//last line because it's used to calculate stdDev 212 | count += entry.getCount();//last line because it's used to calculate stdDev 213 | } 214 | } 215 | 216 | private double combineStdDev(Triplet t1, Triplet t2){ 217 | double bothAvg = (t1.value+t2.value)/(t1.count+t2.count); 218 | return Math.sqrt( 219 | ( 220 | (Math.pow(t1.value/t1.count + t1.stdDev - bothAvg, 2.0)*(t1.count/2.0)) + 221 | (Math.pow(t1.value/t1.count - t1.stdDev - bothAvg, 2.0)*(t1.count/2.0)) + 222 | (Math.pow(t2.value/t2.count + t2.stdDev - bothAvg, 2.0)*(t2.count/2.0)) + 223 | (Math.pow(t2.value/t2.count - t2.stdDev - bothAvg, 2.0)*(t2.count/2.0)) 224 | ) 225 | / (t1.count + t2.count) 226 | ); 227 | } 228 | 229 | @Override 230 | protected AggregatedProfiling clone() { 231 | final AggregatedProfiling result = new AggregatedProfiling(); 232 | result._id = getId(); 233 | result.count = getCount(); 234 | result.millis = getMillis(); 235 | result.avgMs = getAvgMs(); 236 | result.minMs = getMinMs(); 237 | result.maxMs = getMaxMs(); 238 | result.stdDevMs = getStdDevMs(); 239 | result.nRet = getNRet(); 240 | result.avgRet = getAvgRet(); 241 | result.minRet = getMinRet(); 242 | result.maxRet = getMaxRet(); 243 | result.stdDevRet = getStdDevRet(); 244 | result.len = getLen(); 245 | result.avgLen = getAvgLen(); 246 | result.minLen = getMinLen(); 247 | result.maxLen = getMaxLen(); 248 | result.stdDevLen = getStdDevLen(); 249 | result.firstts = (Date)getFirstts().clone(); 250 | result.keys = getKeys(); 251 | result.docs = getDocs(); 252 | result.del = getDel(); 253 | result.ins = getIns(); 254 | result.mod = getMod(); 255 | result.sortstages = getSortstages(); 256 | 257 | return result; 258 | } 259 | 260 | private class Triplet{ 261 | final double value, stdDev, count; 262 | public Triplet(double value, double stdDev, double count){ 263 | this.value = value; 264 | this.stdDev = stdDev; 265 | this.count = count; 266 | } 267 | } 268 | 269 | public static void main(String[] args) { 270 | int [][] samples = { 271 | {2,4,4,4,5,5,7,9},//first sample, avg=5, sigma=2 272 | {20,40,40,40,50,50,70,90},//second sample, , avg=50, sigma=20 273 | {2,4,4,4,5,5,7,9,20,40,40,40,50,50,70,90}//both samples merged together to one sample, , avg=27.5, sigma=26.6 274 | }; 275 | 276 | 277 | int sum[] = {0,0,0}; 278 | double avg[] = {0.0,0.0,0.0}; 279 | for (int s = 0; s < sum.length; s++) { 280 | for (int i = 0; i < samples[s].length; i++) { 281 | sum[s] += samples[s][i]; 282 | } 283 | System.out.println("sum["+s+"]=" + sum[s]); 284 | avg[s] += (double)sum[s]/(double)samples[s].length; 285 | System.out.println("avg["+s+"]=" + avg[s]); 286 | } 287 | 288 | double sumSig[] = {0.0,0.0,0.0}; 289 | double sigma[] = {0,0,0}; 290 | for (int s = 0; s < sumSig.length; s++) { 291 | for (int i = 0; i < samples[s].length; i++) { 292 | sumSig[s] += Math.pow(samples[s][i]-avg[s], 2.0); 293 | } 294 | System.out.println("sumSig["+s+"]=" + sumSig[s]); 295 | 296 | sigma[s] += Math.sqrt((double)sumSig[s]/(double)samples[s].length); 297 | System.out.println("Sigma["+s+"]=" + sigma[s]); 298 | } 299 | 300 | //double sigmaBoth = Math.sqrt( ((Math.pow(avg[0]+sigma[0]-avg[2], 2.0) )*samples[0].length + (Math.pow(avg[1]+sigma[1]-avg[2], 2.0) )*samples[1].length)/samples[2].length ); 301 | double sample1Odd = samples[0].length%2==0?0:(Math.pow(avg[0]-avg[2], 2.0) ); 302 | double sample2Odd = samples[1].length%2==0?0:(Math.pow(avg[1]-avg[2], 2.0) ); 303 | double sample1Len = samples[0].length%2==0?(samples[0].length/2.0):((samples[0].length-1)/2.0); 304 | double sample2Len = samples[1].length%2==0?(samples[1].length/2.0):((samples[1].length-1)/2.0); 305 | double sumBoth = ( 306 | ((Math.pow(avg[0]+sigma[0]-avg[2], 2.0) )*sample1Len) + 307 | ((Math.pow(avg[0]-sigma[0]-avg[2], 2.0) )*sample1Len) + 308 | sample1Odd + 309 | ((Math.pow(avg[1]+sigma[1]-avg[2], 2.0) )*sample2Len) + 310 | ((Math.pow(avg[1]-sigma[1]-avg[2], 2.0) )*sample2Len) + 311 | sample2Odd) 312 | ; 313 | System.out.println("SumBoth=" + sumBoth); 314 | 315 | double sigmaBoth = Math.sqrt( ( 316 | sumBoth 317 | ) 318 | /samples[2].length ); 319 | System.out.println("foo: " + sample1Odd); 320 | System.out.println("bar: " + sample2Odd); 321 | 322 | System.out.println("Sigma both samples=" + sigmaBoth); 323 | 324 | if(sigmaBoth - sigma[2] == 0){ 325 | System.out.println("Combined stdev of both samples are equal to stddev of merged samples."); 326 | }else{ 327 | System.out.println("Combined stddev of both samples are not equal to stddev of merged samples, but should. Diff: " + (sigmaBoth - sigma[2])); 328 | } 329 | 330 | } 331 | } 332 | -------------------------------------------------------------------------------- /src/main/java/de/idealo/mongodb/slowops/collector/ProfilingWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 idealo internet GmbH -- all rights reserved. 3 | */ 4 | package de.idealo.mongodb.slowops.collector; 5 | 6 | import com.google.common.collect.Lists; 7 | import com.mongodb.*; 8 | import com.mongodb.bulk.BulkWriteResult; 9 | import com.mongodb.client.MongoCollection; 10 | import com.mongodb.client.MongoCursor; 11 | import com.mongodb.client.MongoDatabase; 12 | import com.mongodb.client.model.IndexOptions; 13 | import com.mongodb.client.model.InsertManyOptions; 14 | import de.idealo.mongodb.slowops.dto.ApplicationStatusDto; 15 | import de.idealo.mongodb.slowops.dto.CollectorServerDto; 16 | import de.idealo.mongodb.slowops.monitor.MongoDbAccessor; 17 | import de.idealo.mongodb.slowops.util.ConfigReader; 18 | import org.bson.Document; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.util.Date; 23 | import java.util.List; 24 | import java.util.concurrent.BlockingQueue; 25 | import java.util.concurrent.atomic.AtomicLong; 26 | 27 | /** 28 | * 29 | * 30 | * @author kay.agahd 31 | * @since 25.02.2013 32 | * @version $Id: $ 33 | * @copyright idealo internet GmbH 34 | */ 35 | public class ProfilingWriter extends Thread implements Terminable{ 36 | 37 | private static final Logger LOG = LoggerFactory.getLogger(ProfilingWriter.class); 38 | private static final int RETRY_AFTER_SECONDS = 10; 39 | 40 | private final BlockingQueue jobQueue; 41 | 42 | private boolean stop = false; 43 | private AtomicLong doneJobs = new AtomicLong(0); 44 | private CollectorServerDto serverDto; 45 | private final Date runningSince; 46 | 47 | public ProfilingWriter(BlockingQueue jobQueue) { 48 | this.jobQueue = jobQueue; 49 | serverDto = ConfigReader.getCollectorServer(); 50 | runningSince = new Date(); 51 | 52 | final MongoDbAccessor mongo = getMongoDbAccessor(); 53 | try { 54 | final MongoCollection profileCollection = getProfileCollection(mongo); 55 | 56 | IndexOptions indexOptions = new IndexOptions(); 57 | indexOptions.background(true); 58 | LOG.info("Create index {lbl:1, db:1, ts:-1} in the background if it does not yet exists"); 59 | profileCollection.createIndex(new BasicDBObject("lbl",1).append("db", 1).append("ts", -1), indexOptions); 60 | LOG.info("Drop index 'ts_-1_lbl_1' (if it still exists) which has been deprecated since v3.0.0"); 61 | try { 62 | profileCollection.dropIndex("ts_-1_lbl_1"); 63 | LOG.info("Index 'ts_-1_lbl_1' successfully dropped."); 64 | }catch(MongoCommandException e){ 65 | if(e.getErrorCode() == 27) { 66 | LOG.info("Index 'ts_-1_lbl_1' not found."); 67 | }else{ 68 | LOG.error("Error while dropping index 'ts_-1_lbl_1'", e); 69 | } 70 | } 71 | LOG.info("Create index {adr:1, db:1, ts:-1} in the background if it does not yet exists"); 72 | profileCollection.createIndex(new BasicDBObject("adr",1).append("db",1).append("ts", -1), indexOptions); 73 | 74 | LOG.info("ProfilingWriter is ready at {}", serverDto.getHosts()); 75 | 76 | } catch (MongoException e) { 77 | LOG.error("Exception while connecting to: {}", serverDto.getHosts(), e); 78 | } 79 | } 80 | 81 | 82 | public MongoDbAccessor getMongoDbAccessor(){ 83 | return new MongoDbAccessor(serverDto.getAdminUser(), serverDto.getAdminPw(), serverDto.getSsl(), serverDto.getHosts()); 84 | } 85 | 86 | private MongoCollection getProfileCollection(MongoDbAccessor mongo){ 87 | final MongoDatabase db = mongo.getMongoDatabase(serverDto.getDb()); 88 | final MongoCollection result = db.getCollection(serverDto.getCollection()); 89 | 90 | if(result == null) { 91 | throw new IllegalArgumentException("Can't continue without profile collection for " + serverDto.getHosts()); 92 | } 93 | return result; 94 | } 95 | 96 | private void init(MongoDbAccessor mongo) { 97 | LOG.info(">>> init"); 98 | 99 | try { 100 | final MongoCollection profileCollection = getProfileCollection(mongo); 101 | 102 | IndexOptions indexOptions = new IndexOptions(); 103 | indexOptions.background(true); 104 | LOG.info("Create index {lbl:1, db:1, ts:-1} in the background if it does not yet exists"); 105 | profileCollection.createIndex(new BasicDBObject("lbl", 1).append("db", 1).append("ts",-1), indexOptions); 106 | LOG.info("Drop index 'ts_-1_lbl_1' (if it still exists) which has been deprecated since v3.0.0"); 107 | try { 108 | profileCollection.dropIndex("ts_-1_lbl_1"); 109 | LOG.info("Index 'ts_-1_lbl_1' successfully dropped."); 110 | }catch(MongoCommandException e){ 111 | if(e.getErrorCode() == 27) { 112 | LOG.info("Index 'ts_-1_lbl_1' not found."); 113 | }else{ 114 | LOG.error("Error while dropping index 'ts_-1_lbl_1'", e); 115 | } 116 | } 117 | LOG.info("Create index {adr:1, db:1, ts:-1} in the background if it does not yet exists"); 118 | profileCollection.createIndex(new BasicDBObject("adr",1).append("db",1).append("ts", -1), indexOptions); 119 | ApplicationStatusDto.addWebLog("ProfilingWriter is successfully connected to its collector database."); 120 | } catch (MongoException e) { 121 | LOG.error("Exception while connecting to: {}", serverDto.getHosts(), e); 122 | ApplicationStatusDto.addWebLog("ProfilingWriter could not connect to its collector database."); 123 | } 124 | 125 | LOG.info("<<< init"); 126 | } 127 | 128 | 129 | @Override 130 | public void terminate() { 131 | stop = true; 132 | interrupt();//need to interrupt when sleeping or waiting on jobQueue 133 | } 134 | 135 | 136 | @Override 137 | public long getDoneJobs() { 138 | return doneJobs.get(); 139 | } 140 | 141 | public CollectorServerDto getCollectorServerDto(){ return serverDto;} 142 | 143 | public Date getRuningSince() { return runningSince; } 144 | 145 | public Date getNewest(MongoDbAccessor mongo, ServerAddress adr, String db) { 146 | try { 147 | final MongoCollection profileCollection = getProfileCollection(mongo); 148 | 149 | if(adr != null) { 150 | final BasicDBObject query = new BasicDBObject(); 151 | final BasicDBObject fields = new BasicDBObject(); 152 | final BasicDBObject sort = new BasicDBObject(); 153 | query.put("adr", adr.getHost() + ":" + adr.getPort()); 154 | query.put("db", db); 155 | fields.put("_id", Integer.valueOf(0)); 156 | fields.put("ts", Integer.valueOf(1)); 157 | sort.put("ts", Integer.valueOf(-1)); 158 | 159 | final MongoCursor c = profileCollection.find(query).projection(fields).sort(sort).limit(1).iterator(); 160 | try { 161 | if(c.hasNext()) { 162 | final Document obj = c.next(); 163 | final Object ts = obj.get("ts"); 164 | if(ts != null) { 165 | return (Date)ts; 166 | } 167 | } 168 | }finally { 169 | c.close(); 170 | } 171 | } 172 | }catch(Exception e) { 173 | LOG.error("Couldn't get newest entry for {}/{}", new Object[]{adr, db, e}); 174 | 175 | } 176 | return null; 177 | 178 | } 179 | 180 | private void writeEntries(MongoDbAccessor mongo) { 181 | 182 | final MongoCollection profileCollection = getProfileCollection(mongo); 183 | try { 184 | final int maxBatchSize = 100; 185 | final List jobList = Lists.newArrayListWithCapacity(maxBatchSize); 186 | final List docList = Lists.newArrayListWithCapacity(maxBatchSize); 187 | final InsertManyOptions options = new InsertManyOptions(); 188 | options.ordered(false); 189 | 190 | while(!stop) { 191 | if(jobQueue.size() > 0) { 192 | jobQueue.drainTo(jobList, maxBatchSize); 193 | for (ProfilingEntry entry: jobList) { 194 | docList.add(entry.getDocument()); 195 | } 196 | try { 197 | LOG.debug("try to insertMany {} before: {}", docList.size(), doneJobs.get()); 198 | profileCollection.insertMany(docList, options); 199 | doneJobs.addAndGet(docList.size()); 200 | LOG.debug("try to insertMany after: {}", doneJobs.get()); 201 | }catch (MongoBulkWriteException e){ 202 | BulkWriteResult wr = e.getWriteResult(); 203 | doneJobs.addAndGet(wr.getInsertedCount()); 204 | LOG.error("Only {} of {} slow operations could be written to the collector.", new Object[]{wr.getInsertedCount(), jobList.size(), e}); 205 | }finally { 206 | jobList.clear(); 207 | docList.clear(); 208 | } 209 | }else{ 210 | LOG.debug("try to insertOne before: {}", doneJobs.get()); 211 | Document doc = jobQueue.take().getDocument(); 212 | profileCollection.insertOne(doc); 213 | doneJobs.incrementAndGet(); 214 | LOG.debug("try to insertOne after: {}", doneJobs.get()); 215 | } 216 | LOG.debug("doneJobs: {}", doneJobs.get()); 217 | } 218 | }catch(Exception e) { 219 | LOG.error("Exception occurred, will return and try again.", e); 220 | return; 221 | } 222 | } 223 | 224 | /* (non-Javadoc) 225 | * @see java.lang.Runnable#run() 226 | */ 227 | @Override 228 | public void run() { 229 | LOG.info("Run started."); 230 | MongoDbAccessor mongo = null; 231 | try { 232 | while(!stop) { 233 | 234 | if(jobQueue.size() > 0) { 235 | mongo = getMongoDbAccessor(); 236 | init(mongo); 237 | writeEntries(mongo); 238 | } 239 | 240 | if(!stop) { 241 | try { 242 | Thread.sleep(1000*RETRY_AFTER_SECONDS); 243 | 244 | } catch (InterruptedException e) { 245 | LOG.error("InterruptedException while sleeping: "); 246 | stop = true; 247 | } 248 | } 249 | } 250 | }finally { 251 | ApplicationStatusDto.addWebLog("ProfilingWriter terminated"); 252 | terminate(); 253 | } 254 | LOG.info("Run terminated."); 255 | } 256 | 257 | 258 | } 259 | --------------------------------------------------------------------------------