├── .cache └── v │ └── cache │ └── lastfailed ├── .coveragerc ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPTLATE.md ├── .gitignore ├── .travis.yml ├── .validateDCO ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.elk ├── Dockerfile.test ├── LICENSE ├── MAINTAINERS ├── MANIFEST.in ├── Makefile ├── README.md ├── Vagrantfile ├── crawl.sh ├── crawler ├── .gitignore ├── __init__.py ├── base_crawler.py ├── config_parser.py ├── config_spec_and_defaults.conf ├── container.py ├── containers.py ├── containers_crawler.py ├── containers_logs_linker.py ├── crawler.conf ├── crawler.py ├── crawlmodes.py ├── dockercontainer.py ├── emitters_manager.py ├── formatters.py ├── host_crawler.py ├── icrawl_plugin.py ├── iemit_plugin.py ├── plugins │ ├── __init__.py │ ├── applications │ │ ├── __init__.py │ │ ├── apache │ │ │ ├── __init__.py │ │ │ ├── apache_container_crawler.plugin │ │ │ ├── apache_container_crawler.py │ │ │ ├── apache_crawler.py │ │ │ ├── apache_host_crawler.plugin │ │ │ ├── apache_host_crawler.py │ │ │ └── feature.py │ │ ├── db2 │ │ │ ├── __init__.py │ │ │ ├── db2_container_crawler.plugin │ │ │ ├── db2_container_crawler.py │ │ │ ├── db2_crawler.py │ │ │ ├── db2_host_crawler.plugin │ │ │ ├── db2_host_crawler.py │ │ │ └── feature.py │ │ ├── liberty │ │ │ ├── __init__.py │ │ │ ├── feature.py │ │ │ ├── liberty_container_crawler.plugin │ │ │ ├── liberty_container_crawler.py │ │ │ ├── liberty_crawler.py │ │ │ ├── liberty_host_crawler.plugin │ │ │ └── liberty_host_crawler.py │ │ ├── nginx │ │ │ ├── __init__.py │ │ │ ├── feature.py │ │ │ ├── nginx_container_crawler.plugin │ │ │ ├── nginx_container_crawler.py │ │ │ ├── nginx_crawler.py │ │ │ ├── nginx_host_crawler.plugin │ │ │ └── nginx_host_crawler.py │ │ ├── redis │ │ │ ├── __init__.py │ │ │ ├── feature.py │ │ │ ├── redis_container_crawler.plugin │ │ │ ├── redis_container_crawler.py │ │ │ ├── redis_host_crawler.plugin │ │ │ └── redis_host_crawler.py │ │ └── tomcat │ │ │ ├── __init__.py │ │ │ ├── feature.py │ │ │ ├── tomcat_container_crawler.plugin │ │ │ ├── tomcat_container_crawler.py │ │ │ ├── tomcat_crawler.py │ │ │ ├── tomcat_host_crawler.plugin │ │ │ └── tomcat_host_crawler.py │ ├── emitters │ │ ├── __init__.py │ │ ├── base_emitter.py │ │ ├── base_http_emitter.py │ │ ├── cos_emitter.plugin │ │ ├── cos_emitter.py │ │ ├── file_emitter.plugin │ │ ├── file_emitter.py │ │ ├── fluentd_emitter.plugin │ │ ├── fluentd_emitter.py │ │ ├── http_emitter.plugin │ │ ├── http_emitter.py │ │ ├── https_emitter.plugin │ │ ├── https_emitter.py │ │ ├── kafka_emitter.plugin │ │ ├── kafka_emitter.py │ │ ├── mtgraphite_emitter.plugin │ │ ├── mtgraphite_emitter.py │ │ ├── sas_emitter.plugin │ │ ├── sas_emitter.py │ │ ├── stdout_emitter.plugin │ │ └── stdout_emitter.py │ ├── environments │ │ ├── cloudsight_environment.plugin │ │ ├── cloudsight_environment.py │ │ ├── icp_environment.plugin │ │ ├── icp_environment.py │ │ ├── kubernetes_environment.plugin │ │ └── kubernetes_environment.py │ └── systems │ │ ├── __init__.py │ │ ├── config_container_crawler.plugin │ │ ├── config_container_crawler.py │ │ ├── config_host_crawler.plugin │ │ ├── config_host_crawler.py │ │ ├── connection_container_crawler.plugin │ │ ├── connection_container_crawler.py │ │ ├── connection_host_crawler.plugin │ │ ├── connection_host_crawler.py │ │ ├── connection_vm_crawler.plugin │ │ ├── connection_vm_crawler.py │ │ ├── cpu_container_crawler.plugin │ │ ├── cpu_container_crawler.py │ │ ├── cpu_host_crawler.plugin │ │ ├── cpu_host_crawler.py │ │ ├── cpu_vm_crawler.plugin │ │ ├── cpu_vm_crawler.py │ │ ├── ctprobe_container_crawler.plugin │ │ ├── ctprobe_container_crawler.py │ │ ├── disk_container_crawler.plugin │ │ ├── disk_container_crawler.py │ │ ├── disk_host_crawler.plugin │ │ ├── disk_host_crawler.py │ │ ├── disk_vm_crawler.plugin │ │ ├── disk_vm_crawler.py │ │ ├── diskio_host_crawler.plugin │ │ ├── diskio_host_crawler.py │ │ ├── dockerhistory_container_crawler.plugin │ │ ├── dockerhistory_container_crawler.py │ │ ├── dockerinspect_container_crawler.plugin │ │ ├── dockerinspect_container_crawler.py │ │ ├── dockernetwork_container_crawler.plugin │ │ ├── dockernetwork_container_crawler.py │ │ ├── dockerps_host_crawler.plugin │ │ ├── dockerps_host_crawler.py │ │ ├── file_container_crawler.plugin │ │ ├── file_container_crawler.py │ │ ├── file_host_crawler.plugin │ │ ├── file_host_crawler.py │ │ ├── fprobe_container_crawler.plugin │ │ ├── fprobe_container_crawler.py │ │ ├── gpu_host_crawler.plugin │ │ ├── gpu_host_crawler.py │ │ ├── interface_container_crawler.plugin │ │ ├── interface_container_crawler.py │ │ ├── interface_host_crawler.plugin │ │ ├── interface_host_crawler.py │ │ ├── interface_vm_crawler.plugin │ │ ├── interface_vm_crawler.py │ │ ├── jar_container_crawler.plugin │ │ ├── jar_container_crawler.py │ │ ├── jar_host_crawler.plugin │ │ ├── jar_host_crawler.py │ │ ├── load_container_crawler.plugin │ │ ├── load_container_crawler.py │ │ ├── load_host_crawler.plugin │ │ ├── load_host_crawler.py │ │ ├── load_vm_crawler.plugin │ │ ├── load_vm_crawler.py │ │ ├── memory_container_crawler.plugin │ │ ├── memory_container_crawler.py │ │ ├── memory_host_crawler.plugin │ │ ├── memory_host_crawler.py │ │ ├── memory_vm_crawler.plugin │ │ ├── memory_vm_crawler.py │ │ ├── metric_container_crawler.plugin │ │ ├── metric_container_crawler.py │ │ ├── metric_host_crawler.plugin │ │ ├── metric_host_crawler.py │ │ ├── metric_vm_crawler.plugin │ │ ├── metric_vm_crawler.py │ │ ├── nodepackage_container_crawler.plugin │ │ ├── nodepackage_container_crawler.py │ │ ├── os_container_crawler.plugin │ │ ├── os_container_crawler.py │ │ ├── os_host_crawler.plugin │ │ ├── os_host_crawler.py │ │ ├── os_vm_crawler.plugin │ │ ├── os_vm_crawler.py │ │ ├── package_container_crawler.plugin │ │ ├── package_container_crawler.py │ │ ├── package_host_crawler.plugin │ │ ├── package_host_crawler.py │ │ ├── process_container_crawler.plugin │ │ ├── process_container_crawler.py │ │ ├── process_host_crawler.plugin │ │ ├── process_host_crawler.py │ │ ├── process_vm_crawler.plugin │ │ ├── process_vm_crawler.py │ │ ├── pythonpackage_container_crawler.plugin │ │ ├── pythonpackage_container_crawler.py │ │ ├── rubypackage_container_crawler.plugin │ │ └── rubypackage_container_crawler.py ├── plugins_manager.py ├── runtime_environment.py ├── utils │ ├── __init__.py │ ├── config_utils.py │ ├── connection_utils.py │ ├── crawler_exceptions.py │ ├── disk_utils.py │ ├── dockerevent.py │ ├── dockerutils.py │ ├── ethtool.py │ ├── features.py │ ├── file_utils.py │ ├── jar_utils.py │ ├── mesos.py │ ├── metric_utils.py │ ├── misc.py │ ├── mtgraphite.py │ ├── namespace.py │ ├── nvidialib │ │ ├── lib │ │ │ └── libnvidia-ml.so.375.66 │ │ └── lib64 │ │ │ └── libnvidia-ml.so.375.66 │ ├── os_utils.py │ ├── osinfo.py │ ├── package_utils.py │ ├── process_utils.py │ ├── socket_utils.py │ └── timeout_utils.py ├── virtual_machine.py ├── vms_crawler.py └── worker.py ├── dependencies ├── python-conntrackprobe_0.2.1-1_all.deb ├── python-socket-datacollector_0.1.4-1_all.deb └── softflowd_0.9.9902-1_amd64.deb ├── logstash-http-plugin.conf ├── requirements-test.txt ├── requirements.txt ├── sample-k8s-deamon-deployment.yaml ├── sample-sas-secret.yaml ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── functional │ ├── .gitignore │ ├── __init__.py │ ├── setup_logger.py │ ├── test_functional_apk_package_crawler.py │ ├── test_functional_containers_crawler.py │ ├── test_functional_cos_emitter.py │ ├── test_functional_ctprobe.py │ ├── test_functional_dockerevents.py │ ├── test_functional_dockerutils.py │ ├── test_functional_fprobe.py │ ├── test_functional_k8s_environment.py │ ├── test_functional_logs_linker.py │ ├── test_functional_namespace.py │ ├── test_functional_nodepackage_plugin.py │ ├── test_functional_plugins.py │ ├── test_functional_vm_plugins.py │ ├── test_functional_vms_crawler.py │ ├── test_logs_in_volumes1.py │ ├── test_logs_in_volumes_star.py │ └── test_logs_no_volumes.py ├── old │ ├── test_all.sh │ ├── test_crawl_all_features_container.sh │ ├── test_crawl_all_features_container_and_invm_for_10_secs.sh │ ├── test_crawl_all_features_container_crash_failure.sh │ ├── test_crawl_all_features_container_for_10_secs.sh │ ├── test_crawl_all_features_container_infinite_loop_failure.sh │ ├── test_crawl_all_features_invm.sh │ ├── test_crawl_all_features_invm_for_10_secs.sh │ ├── test_crawl_config_container.sh │ ├── test_crawl_config_invm.sh │ ├── test_crawl_config_mountpoint.sh │ ├── test_crawl_config_mountpoint_relative.sh │ ├── test_crawl_connections_container.sh │ ├── test_crawl_connections_invm.sh │ ├── test_crawl_cpu_container.sh │ ├── test_crawl_cpu_container_check_metadata.sh │ ├── test_crawl_cpu_container_dockerlog_links.sh │ ├── test_crawl_cpu_container_dockerlog_links_for_10_secs.sh │ ├── test_crawl_cpu_container_for_10_secs.sh │ ├── test_crawl_cpu_container_gzip.sh │ ├── test_crawl_cpu_container_input_logfile_links.sh │ ├── test_crawl_cpu_container_input_logfile_links_for_10_secs.sh │ ├── test_crawl_cpu_container_invalid_input_logfile_links.sh │ ├── test_crawl_cpu_container_invalid_input_logfile_links_for_10_secs.sh │ ├── test_crawl_cpu_invm.sh │ ├── test_crawl_cpu_many_containers.sh │ ├── test_crawl_cpu_many_containers_check_partitioning.sh │ ├── test_crawl_cpu_many_containers_dockerlog_links.sh │ ├── test_crawl_cpu_many_containers_dockerlog_links_for_10_secs.sh │ ├── test_crawl_cpu_many_containers_for_10_secs.sh │ ├── test_crawl_cpu_many_containers_for_10_secs_check_user_list.sh │ ├── test_crawl_cpu_many_containers_input_logfile_links.sh │ ├── test_crawl_cpu_many_containers_input_logfile_links_for_10_secs.sh │ ├── test_crawl_cpu_mem_interface_many_containers_input_logfile_links_for_10_secs.sh │ ├── test_crawl_cpu_with_varying_load_container_for_10_secs.sh │ ├── test_crawl_disk_container.sh │ ├── test_crawl_disk_invm.sh │ ├── test_crawl_dockerhistory.sh │ ├── test_crawl_dockerinspect.sh │ ├── test_crawl_dockerps.sh │ ├── test_crawl_file_container.sh │ ├── test_crawl_file_invm.sh │ ├── test_crawl_files_config_mountpoint.sh │ ├── test_crawl_interface_container.sh │ ├── test_crawl_interface_with_varying_load_container_for_10_secs.sh │ ├── test_crawl_kill_child.sh │ ├── test_crawl_kill_parent.sh │ ├── test_crawl_load_container.sh │ ├── test_crawl_load_invm.sh │ ├── test_crawl_memory_container.sh │ ├── test_crawl_metric_container.sh │ ├── test_crawl_metric_invm.sh │ ├── test_crawl_os_container.sh │ ├── test_crawl_package_container.sh │ ├── test_crawl_package_invm.sh │ ├── test_crawl_packages_disks_mountpoint.sh │ ├── test_crawl_processes_container.sh │ ├── test_crawl_processes_invm.sh │ ├── test_crawl_specific_file_container.sh │ ├── test_create_destroy_containers_w_same_name.sh │ ├── test_create_destroy_containers_w_same_name_w_input_logfiles.sh │ ├── test_create_short_lived_containers_w_same_name.sh │ ├── test_create_short_lived_containers_w_same_name_w_default_as_input_logfiles.sh │ ├── test_create_short_lived_containers_w_same_name_w_default_logfiles.sh │ ├── test_create_short_lived_containers_w_same_name_w_diff_input_logfiles.sh │ ├── test_create_short_lived_containers_w_same_name_w_input_logfiles.sh │ └── test_leaked_open_files.sh └── unit │ ├── .gitignore │ ├── __init__.py │ ├── aufs_mount_init-id │ ├── btrfs_mount_init-id │ ├── capturing.py │ ├── liberty_connection_stats │ ├── liberty_jvm_stats │ ├── liberty_mbeans │ ├── liberty_response_time_details │ ├── liberty_response_time_details_mocked │ ├── liberty_servlet_stats │ ├── liberty_session_stats │ ├── liberty_thread_pool_stats │ ├── mock_environ_file │ ├── mock_pynvml.py │ ├── proc_mounts_aufs │ ├── proc_mounts_btrfs │ ├── proc_mounts_devicemapper │ ├── proc_mounts_vfs │ ├── proc_pid_mounts_devicemapper │ ├── single_package_apk_db │ ├── test_apk_package_parser.py │ ├── test_app_apache.py │ ├── test_app_db2.py │ ├── test_app_liberty.py │ ├── test_app_nginx.py │ ├── test_app_redis.py │ ├── test_app_tomcat.py │ ├── test_container.py │ ├── test_containers.py │ ├── test_containers_crawler.py │ ├── test_diskio_host.py │ ├── test_dockercontainer.py │ ├── test_dockerutils.py │ ├── test_emitter.py │ ├── test_gpu_plugin.py │ ├── test_host_crawler.py │ ├── test_jar_plugin.py │ ├── test_jar_utils.py │ ├── test_mesos_url.py │ ├── test_misc.py │ ├── test_mtgraphite.py │ ├── test_namespace.py │ ├── test_osinfo.py │ ├── test_package_utils.py │ ├── test_plugins.py │ ├── test_vms_crawler.py │ ├── two_packages_apk_db │ └── vfs_mount_init-id └── vagrant-provision.sh /.cache/v/cache/lastfailed: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = crawler 3 | 4 | [report] 5 | exclude_lines = 6 | if self.debug: 7 | pragma: no cover 8 | raise NotImplementedError 9 | if __name__ == .__main__.: 10 | ignore_errors = True 11 | omit = 12 | tests/* 13 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | ### Python ### 2 | **/__pycache__ 3 | *.py[cod] 4 | build/ 5 | dist/ 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | 6 | ## How to Reproduce 7 | 8 | 9 | ## Log Output 10 | 11 | 12 | ## Debugging Commands Output 13 | 14 | **Output of `docker version`:** 15 | 16 | ``` 17 | (paste your output here) 18 | ``` 19 | 20 | 21 | **Output of `docker info`:** 22 | 23 | ``` 24 | (paste your output here) 25 | ``` 26 | 27 | 28 | **Output of `python --version`:** 29 | 30 | ``` 31 | (paste your output here) 32 | ``` 33 | 34 | 35 | **Output of `pip freeze`:** 36 | 37 | ``` 38 | (paste your output here) 39 | ``` 40 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPTLATE.md: -------------------------------------------------------------------------------- 1 | Please review the guidelines for contributing linked above. 2 | 3 | 4 | 5 | ## Description 6 | 7 | 8 | ## Motivation and Context 9 | 10 | 11 | Fixes # 12 | 13 | ## How Has This Been Tested? 14 | 15 | 16 | 17 | ## Checklist: 18 | 19 | 20 | 21 | - [] I have added a [Signed-off-by](https://github.com/cloudviz/agentless-system-crawler/blob/master/CONTRIBUTING.md#sign-your-prs). 22 | - [] I have either added documentation to cover my changes or this change requires no new documentation. 23 | - [] I have either added unit tests to cover my changes or this change requires no new tests. 24 | - [] I have maintained or increased the level of code coverage. 25 | - [] I have maintained or decreased the number of pylint and flake8 emissions. 26 | 27 | 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant/* 2 | ## python 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | # distribution 7 | build/ 8 | dist/ 9 | eggs/ 10 | .eggs/ 11 | .coverage* 12 | .idea 13 | .DS_Store 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | group: deprecated-2017Q4 4 | language: python 5 | 6 | services: 7 | - docker 8 | 9 | env: 10 | matrix: 11 | - DOCKER_VERSION=1.7.1-0~trusty 12 | - DOCKER_VERSION=1.9.1-0~trusty 13 | - DOCKER_VERSION=1.10.1-0~trusty 14 | - DOCKER_VERSION=1.11.1-0~trusty 15 | - DOCKER_VERSION=17.03.0~ce-0~ubuntu-trusty 16 | 17 | python: 18 | - "2.7" 19 | 20 | before_install: 21 | - ./.validateDCO 22 | - sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list' 23 | - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 24 | - sudo apt-get update 25 | - sudo apt-key update 26 | - sudo apt-get -qqy --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=${DOCKER_VERSION} 27 | - docker -v 28 | # might as well upgrade pip to support TLS and get rid of the warnings 29 | # sudo -H pip install --upgrade pip 30 | - sudo -H pip install -I pip==9.0.1 31 | - sudo -H pip install ibm_db==2.0.8a0 32 | - sudo -H pip install codecov pytest-cov==2.3.0 pytest==2.9.2 33 | # psvmi specific 34 | - sudo apt-get install qemu-kvm 35 | - git clone https://github.com/cloudviz/psvmi.git 36 | - cd psvmi/psvmi && sudo python setup.py build && sudo python setup.py install && cd - 37 | - cp -r psvmi/maps maps 38 | - cp -r psvmi/offsets offsets 39 | - cp psvmi/header.h . 40 | 41 | # command to install dependencies 42 | # XXX: Now mock complains if we don't `sudo pip install`. 43 | 44 | install: 45 | - sudo -H pip install --ignore-installed -r requirements-test.txt 46 | - echo "fix whitespace errors with 'autopep8 --select E1 --recursive --in-place .'" && flake8 --select E1 47 | 48 | # command to run tests 49 | script: 50 | - bandit -r crawler || true 51 | - flake8 --count --max-complexity=10 --statistics crawler 52 | - sudo -H python setup.py test --addopts '--cov=.' 53 | 54 | after_success: 55 | - pylint crawler 56 | - codecov 57 | -------------------------------------------------------------------------------- /.validateDCO: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script will check to make sure each commit in PR, as specified 4 | # by the TRAVIS_COMMIT_RANGE env var, is signed via a DCO 5 | 6 | set -e 7 | 8 | echo TRAVIS_COMMIT_RANGE: ${TRAVIS_COMMIT_RANGE} 9 | if [[ "${TRAVIS_COMMIT_RANGE}" != *"..."* ]]; then 10 | parent=$(git log -n 1 --format="%P" ${TRAVIS_COMMIT_RANGE}) 11 | TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE}...$parent" 12 | fi 13 | 14 | echo Commits: $(git rev-list ${TRAVIS_COMMIT_RANGE}) 15 | 16 | for commit in $(git rev-list ${TRAVIS_COMMIT_RANGE}); do 17 | echo 18 | echo Looking at commit: $commit 19 | parents=$(git log -n 1 --format="%P" $commit) 20 | if [[ "$parents" == *" "* ]]; then 21 | echo "Skipping a merge commit" 22 | continue 23 | fi 24 | 25 | set +e 26 | git log --format="%B" -n 1 $commit | \ 27 | grep -E '^Signed-off-by: ' # ([^<]+) <([^<>@]+@[^<>]+)>$' > /dev/null 2>&1 28 | if test ${PIPESTATUS[1]} -ne 0; then 29 | set -e 30 | echo "Commit msg:" 31 | echo "------------------------------------" 32 | git log --format="%B" -n 1 $commit 33 | echo "------------------------------------" 34 | echo "**** Commit $commit is missing a DCO" 35 | exit 1 36 | fi 37 | set -e 38 | 39 | done 40 | echo 41 | echo "**** All commits are signed!" 42 | exit 0 43 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | WORKDIR /crawler 4 | 5 | COPY requirements.txt /crawler/requirements.txt 6 | RUN pip install -r requirements.txt 7 | 8 | COPY \ 9 | dependencies/python-socket-datacollector_0.1.4-1_all.deb \ 10 | dependencies/softflowd_0.9.9902-1_amd64.deb \ 11 | dependencies/python-conntrackprobe_0.2.1-1_all.deb \ 12 | /tmp/ 13 | 14 | RUN dpkg -i /tmp/python-socket-datacollector_*_all.deb && \ 15 | apt-get -y update && \ 16 | apt-get -y install libpcap0.8 && \ 17 | dpkg -i /tmp/softflowd_0.9.*_amd64.deb && \ 18 | pip install pyroute2 py-radix requests-unixsocket json-rpc && \ 19 | dpkg -i /tmp/python-conntrackprobe_*_all.deb && \ 20 | rm -f /tmp/*.deb 21 | 22 | RUN apt-get upgrade -y 23 | 24 | ENV PYTHONPATH=/usr/lib/python2.7/dist-packages:/usr/local/lib/python2.7/site-packages 25 | 26 | ADD crawler /crawler 27 | 28 | ENTRYPOINT [ "python2.7", "crawler.py" ] 29 | -------------------------------------------------------------------------------- /Dockerfile.elk: -------------------------------------------------------------------------------- 1 | FROM sebp/elk:latest 2 | 3 | COPY logstash-http-plugin.conf /etc/logstash/conf.d/ 4 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | RUN apt-get update && echo "y" | apt-get install apt-transport-https 4 | RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 5 | RUN echo "deb https://apt.dockerproject.org/repo ubuntu-xenial main" > /etc/apt/sources.list.d/docker.list 6 | 7 | 8 | RUN apt-get update && \ 9 | echo "y" | apt-get install python \ 10 | python-dev \ 11 | python-pip \ 12 | build-essential \ 13 | docker-engine \ 14 | dpkg \ 15 | dpkg-dev \ 16 | qemu-kvm \ 17 | && pip install -I pip==9.0.1 \ 18 | && pip install mock 19 | 20 | COPY requirements.txt /requirements.txt 21 | COPY requirements-test.txt /requirements-test.txt 22 | RUN pip install -r requirements-test.txt 23 | 24 | WORKDIR /crawler 25 | 26 | COPY . /crawler 27 | 28 | RUN (cd psvmi/psvmi; python setup.py build && python setup.py install) 29 | 30 | COPY psvmi/maps maps 31 | COPY psvmi/offsets offsets 32 | COPY psvmi/header.h . 33 | 34 | CMD (docker daemon --storage-driver=vfs > ../docker.out 2>&1 &) && \ 35 | sleep 5 && \ 36 | flake8 --max-complexity 10 . && \ 37 | pylint crawler || true && \ 38 | python setup.py test --addopts '-s --cov=.' 39 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | ricarkol - kollerr@us.ibm.com - Ricardo Koller 2 | duglin - dug@us.ibm.com - Doug Davis 3 | sahilsuneja1 - suneja@us.ibm.com - Sahil Suneja 4 | nadgowdas - nadgowdas@us.ibm.com - Shripad Nadgowda 5 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # These assume you hav Docker installed and running 2 | SHELL = bash 3 | all: build start setupelk 4 | 5 | build: 6 | @echo -ne "Building crawler container..." 7 | @docker build -t crawler . 1>/dev/null 8 | @if [ $$? -eq 0 ];then echo "[OK]";else echo "[FAILED]"; fi 9 | 10 | start: 11 | @echo -ne "Starting crawler container..." 12 | @docker run \ 13 | --privileged \ 14 | --net=host \ 15 | --pid=host \ 16 | --name agentless-crawler\ 17 | -v /cgroup:/cgroup:ro \ 18 | -v /var/lib/docker:/var/lib/docker:ro \ 19 | -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ 20 | -v /var/run/docker.sock:/var/run/docker.sock \ 21 | -it -d crawler \ 22 | --url "http://127.0.0.1:8080/crawler/data"\ 23 | --features os,disk,process\ 24 | --frequency 5\ 25 | --format json\ 26 | --crawlmode OUTCONTAINER\ 27 | 1>/dev/null 28 | @echo "[OK]" 29 | 30 | setupelk: 31 | @echo -ne "Building elk container..." 32 | @docker build -t crawler-elk -f Dockerfile.elk . 1>/dev/null 33 | @if [ $$? -eq 0 ];then echo "[OK]";else echo "[FAILED]"; fi 34 | @echo -ne "Starting elk container..." 35 | @docker run -d -p 5601:5601 -p 9200:9200 -p 5044:5044 -p 5000:5000 -p 8080:8080 --name elk crawler-elk 1>/dev/null 36 | @echo "[OK]" 37 | @#sleep is added to allow time for elk components to bootstrap 38 | @#before crawler can send data to it 39 | @sleep 2 40 | @echo "You can view the crawl data at http://localhost:5601/app/kibana" 41 | @echo "Please create an index by @timestamp on kibana dashboard" 42 | 43 | .PHONY: test 44 | test: 45 | @if [ ! -d psvmi ]; then git clone https://github.com/cloudviz/psvmi.git; fi 46 | docker build -t agentless-system-crawler-test -f Dockerfile.test . 47 | docker run --privileged -ti --rm agentless-system-crawler-test 48 | 49 | unit: 50 | python2.7 setup.py test --addopts '--fulltrace --cov=. tests/unit' 51 | 52 | functional: 53 | python2.7 setup.py test --addopts '--fulltrace --cov=. tests/functional' 54 | 55 | clean: 56 | @echo -ne "Stopping and removing crawler container..." 57 | @docker stop agentless-crawler 1>/dev/null 58 | @docker rm agentless-crawler 1>/dev/null 59 | @echo "[OK]" 60 | @echo -ne "Stopping and removing elk container..." 61 | @docker stop elk>/dev/null 62 | @docker rm elk 1>/dev/null 63 | @echo "[OK]" 64 | 65 | covreport: unit 66 | coverage html -i 67 | $(info ************ Please open htmlcov/index.html ************) 68 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | VAGRANTFILE_API_VERSION = "2" 2 | 3 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 4 | 5 | # allow specification of base box 6 | VM_BOX=ENV["VM_BOX"] || "bento/ubuntu-14.04" 7 | config.vm.box = VM_BOX 8 | 9 | # optionally turn off vbguest auto_update 10 | # use test to stop it throwing error if vbguest gem is not installed 11 | if ENV["VBGUEST_AUTO"] == 'false' 12 | config.vbguest.auto_update = false 13 | end 14 | 15 | config.vm.provision "common" , type: "shell" do |c| 16 | c.path = "./vagrant-provision.sh" 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /crawl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start the crawler as a container. 4 | 5 | # Arguments passed to this script will be passed to the crawler. 6 | 7 | CRAWLER_ARGS=`echo $@ | awk '{for (i = 2; i <= NF; i++) print $i}'` 8 | 9 | case "$1" in 10 | help*) 11 | docker run -it crawler --help 12 | ;; 13 | host*) 14 | docker run \ 15 | --privileged \ 16 | --net=host \ 17 | --pid=host \ 18 | -v /cgroup:/cgroup:ro \ 19 | -v /var/lib/docker:/var/lib/docker:ro \ 20 | -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ 21 | -v /var/run/docker.sock:/var/run/docker.sock \ 22 | -it crawler --crawlmode INVM ${CRAWLER_ARGS} 23 | ;; 24 | containers*) 25 | docker run \ 26 | --privileged \ 27 | --net=host \ 28 | --pid=host \ 29 | -v /cgroup:/cgroup:ro \ 30 | -v /var/lib/docker:/var/lib/docker:ro \ 31 | -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ 32 | -v /var/run/docker.sock:/var/run/docker.sock \ 33 | -it crawler --crawlmode OUTCONTAINER ${CRAWLER_ARGS} 34 | ;; 35 | none*) 36 | docker run \ 37 | --privileged \ 38 | --net=host \ 39 | --pid=host \ 40 | -v /cgroup:/cgroup:ro \ 41 | -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ 42 | -v /var/run/docker.sock:/var/run/docker.sock \ 43 | --entrypoint=/bin/bash \ 44 | -it crawler 45 | ;; 46 | *) 47 | echo $"Usage: $0 {host|containers|help|none}" 48 | exit 1 49 | esac 50 | -------------------------------------------------------------------------------- /crawler/.gitignore: -------------------------------------------------------------------------------- 1 | deprecated/ 2 | *.pyc 3 | binaries/ 4 | kafka-producer.py 5 | timeout.py 6 | alchemy.py 7 | *.json 8 | *.sh 9 | -------------------------------------------------------------------------------- /crawler/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/config_parser.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from configobj import ConfigObj 4 | from validate import Validator 5 | 6 | from utils import misc 7 | 8 | CONFIG_SPEC_PATH = 'config_spec_and_defaults.conf' 9 | 10 | _config = None 11 | 12 | logger = logging.getLogger('crawlutils') 13 | 14 | 15 | def parse_crawler_config(config_path='crawler.conf'): 16 | global _config 17 | 18 | # 1. get configs 19 | _config = ConfigObj(infile=misc.execution_path(config_path), 20 | configspec=misc.execution_path(CONFIG_SPEC_PATH)) 21 | 22 | # Configspec is not being used currently 23 | # but keeping validate() and apply_user_args() for future. 24 | # Essentially NOP right now 25 | 26 | # 2. apply defaults 27 | vdt = Validator() 28 | _config.validate(vdt) 29 | 30 | 31 | def get_config(): 32 | global _config 33 | 34 | if not _config: 35 | parse_crawler_config() 36 | 37 | return _config 38 | -------------------------------------------------------------------------------- /crawler/config_spec_and_defaults.conf: -------------------------------------------------------------------------------- 1 | [ general ] 2 | features_to_crawl = string_list(default=list('os', 'cpu')) 3 | environment = string(min=1, max=30, default='cloudsight') 4 | 5 | plugin_places = string_list(default=list('plugins')) 6 | compress = boolean(default=False) 7 | 8 | link_container_log_files = boolean(default=False) 9 | default_mountpoint = string(default='/') 10 | docker_containers_list = string(default='ALL') 11 | -------------------------------------------------------------------------------- /crawler/crawlmodes.py: -------------------------------------------------------------------------------- 1 | from utils.misc import enum 2 | 3 | Modes = enum(INVM='INVM', 4 | OUTVM='OUTVM', 5 | MOUNTPOINT='MOUNTPOINT', 6 | OUTCONTAINER='OUTCONTAINER', 7 | MESOS='MESOS') 8 | -------------------------------------------------------------------------------- /crawler/host_crawler.py: -------------------------------------------------------------------------------- 1 | import plugins_manager 2 | from base_crawler import BaseCrawler, BaseFrame 3 | 4 | 5 | class HostFrame(BaseFrame): 6 | 7 | def __init__(self, feature_types, namespace): 8 | BaseFrame.__init__(self, feature_types) 9 | self.metadata['namespace'] = namespace 10 | self.metadata['system_type'] = 'host' 11 | 12 | 13 | class HostCrawler(BaseCrawler): 14 | 15 | def __init__(self, 16 | features=['os', 'cpu'], namespace='', 17 | plugin_places=['plugins'], options={}): 18 | BaseCrawler.__init__( 19 | self, 20 | features=features, 21 | plugin_places=plugin_places) 22 | plugins_manager.reload_host_crawl_plugins( 23 | features, plugin_places, options) 24 | self.plugins = plugins_manager.get_host_crawl_plugins( 25 | features=features) 26 | self.namespace = namespace 27 | 28 | def crawl(self, ignore_plugin_exception=True): 29 | """ 30 | Crawl the host with all the plugins loaded on __init__ 31 | 32 | :param ignore_plugin_exception: just ignore exceptions on a plugin 33 | :return: a list generator with a frame object 34 | """ 35 | frame = HostFrame(self.features, self.namespace) 36 | for (plugin_obj, plugin_args) in self.plugins: 37 | try: 38 | frame.add_features(plugin_obj.crawl(**plugin_args)) 39 | except Exception as exc: 40 | if not ignore_plugin_exception: 41 | raise exc 42 | yield frame 43 | -------------------------------------------------------------------------------- /crawler/icrawl_plugin.py: -------------------------------------------------------------------------------- 1 | from yapsy.IPlugin import IPlugin 2 | 3 | 4 | class IContainerCrawler(IPlugin): 5 | 6 | """ 7 | Crawler plugin interface 8 | 9 | Subclasses of this class can be used to implement crawling functions 10 | for different systems. 11 | """ 12 | 13 | def crawl(self, container_id): 14 | """ 15 | Crawling function that should return a list of features for 16 | `container_id`. This function is called once for every container 17 | at every crawling interval. 18 | """ 19 | raise NotImplementedError() 20 | 21 | def get_feature(self): 22 | """ 23 | Returns the feature type as a string. 24 | """ 25 | raise NotImplementedError() 26 | 27 | 28 | class IVMCrawler(IPlugin): 29 | 30 | """ 31 | Crawler plugin interface 32 | 33 | Subclasses of this class can be used to implement crawling functions 34 | for different systems. 35 | """ 36 | 37 | def crawl(self, vm_desc): 38 | """ 39 | Crawling function that should return a list of features for 40 | `vm_desc`. This should change to 'vm_name' after auto kernel version 41 | detection. This function is called once for every VM 42 | at every crawling interval. 43 | """ 44 | raise NotImplementedError() 45 | 46 | def get_feature(self): 47 | """ 48 | Returns the feature type as a string. 49 | """ 50 | raise NotImplementedError() 51 | 52 | 53 | class IHostCrawler(IPlugin): 54 | 55 | """ 56 | Crawler plugin interface 57 | 58 | Subclasses of this class can be used to implement crawling functions 59 | for different host features (e.g. processes running in the host). 60 | """ 61 | 62 | def crawl(self): 63 | """ 64 | Crawling function that should return a list of features for the host. 65 | This function is called once at every crawling interval. 66 | """ 67 | raise NotImplementedError() 68 | 69 | def get_feature(self): 70 | """ 71 | Returns the feature type as a string. 72 | """ 73 | raise NotImplementedError() 74 | -------------------------------------------------------------------------------- /crawler/iemit_plugin.py: -------------------------------------------------------------------------------- 1 | import cStringIO 2 | from yapsy.IPlugin import IPlugin 3 | from formatters import (write_in_csv_format, 4 | write_in_json_format, 5 | write_in_graphite_format, 6 | write_in_logstash_format) 7 | from utils.crawler_exceptions import (EmitterUnsupportedFormat) 8 | 9 | 10 | class IEmitter(IPlugin): 11 | 12 | """ 13 | Base emitter class from which emitters like FileEmitter, StdoutEmitter 14 | should inherit. The main idea is that all emitters get a url, and should 15 | implement an emit() function given an iostream (a buffer with the features 16 | to emit). 17 | """ 18 | 19 | def init(self, url, timeout=1, max_retries=5, emit_format='csv'): 20 | self.url = url 21 | self.timeout = timeout 22 | self.max_retries = max_retries 23 | self.emit_per_line = False 24 | 25 | self.supported_formats = {'csv': write_in_csv_format, 26 | 'graphite': write_in_graphite_format, 27 | 'json': write_in_json_format, 28 | 'logstash': write_in_logstash_format} 29 | 30 | if emit_format in self.supported_formats: 31 | self.formatter = self.supported_formats[emit_format] 32 | else: 33 | raise EmitterUnsupportedFormat('Not supported: %s' % emit_format) 34 | 35 | def get_emitter_protocol(self): 36 | raise NotImplementedError() 37 | 38 | def format(self, frame): 39 | # this writes the frame metadata and data into iostream 40 | # Pass iostream to the emitters so they can send its content to their 41 | # respective url 42 | iostream = cStringIO.StringIO() 43 | self.formatter(iostream, frame) 44 | return iostream 45 | 46 | def emit(self, frame, compress=False, 47 | metadata={}, snapshot_num=0, **kwargs): 48 | """ 49 | 50 | :param iostream: a CStringIO used to buffer the formatted features. 51 | :param compress: 52 | :param metadata: 53 | :param snapshot_num: 54 | :return: 55 | """ 56 | # this formats and emits an input frame 57 | raise NotImplementedError() 58 | -------------------------------------------------------------------------------- /crawler/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/apache/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/apache/apache_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_apache_container 3 | Module = apache_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Apache httpd server" 9 | 10 | -------------------------------------------------------------------------------- /crawler/plugins/applications/apache/apache_container_crawler.py: -------------------------------------------------------------------------------- 1 | from utils.namespace import run_as_another_namespace 2 | import logging 3 | import json 4 | import utils.misc 5 | import dockercontainer 6 | from icrawl_plugin import IContainerCrawler 7 | from plugins.applications.apache import apache_crawler 8 | from requests.exceptions import ConnectionError 9 | 10 | logger = logging.getLogger('crawlutils') 11 | 12 | 13 | class ApacheContainerCrawler(IContainerCrawler): 14 | feature_type = 'application' 15 | feature_key = 'apache' 16 | default_port = 80 17 | 18 | def get_feature(self): 19 | return self.feature_key 20 | 21 | def crawl(self, container_id=None, **kwargs): 22 | 23 | c = dockercontainer.DockerContainer(container_id) 24 | 25 | port = None 26 | 27 | if "annotation.io.kubernetes.container.ports" in\ 28 | c.inspect['Config']['Labels']: 29 | 30 | ports = c.inspect['Config']['Labels'][ 31 | 'annotation.io.kubernetes.container.ports'] 32 | 33 | ports = json.loads(ports) 34 | 35 | else: 36 | ports = c.get_container_ports() 37 | 38 | for each_port in ports: 39 | tmp_port = None 40 | if "containerPort" in each_port: 41 | tmp_port = int(each_port['containerPort']) 42 | else: 43 | tmp_port = int(each_port) 44 | 45 | if tmp_port == self.default_port: 46 | port = tmp_port 47 | 48 | if not port: 49 | return 50 | 51 | state = c.inspect['State'] 52 | pid = str(state['Pid']) 53 | ips = run_as_another_namespace( 54 | pid, ['net'], utils.misc.get_host_ip4_addresses) 55 | 56 | for each_ip in ips: 57 | if each_ip != "127.0.0.1": 58 | ip = each_ip 59 | break 60 | try: 61 | metrics = apache_crawler.retrieve_metrics(ip, port) 62 | return [(self.feature_key, metrics, self.feature_type)] 63 | except: 64 | logger.info("apache does not listen on port:%d", port) 65 | raise ConnectionError("apache does not listen on port:%d", port) 66 | -------------------------------------------------------------------------------- /crawler/plugins/applications/apache/apache_crawler.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | from plugins.applications.apache import feature 3 | from collections import defaultdict 4 | from utils.crawler_exceptions import CrawlError 5 | 6 | 7 | def retrieve_status_page(host, port): 8 | statusPage = "http://%s:%s/server-status?auto" % (host, port) 9 | req = urllib2.Request(statusPage) 10 | response = urllib2.urlopen(req) 11 | return response.read() 12 | 13 | 14 | def parse_score_board(line, stats): 15 | switch = { 16 | "_": 'waiting_for_connection', 17 | "S": 'starting_up', 18 | "R": 'reading_request', 19 | "W": 'sending_reply', 20 | "K": 'keepalive_read', 21 | "D": 'dns_lookup', 22 | "C": 'closing_connection', 23 | "L": 'logging', 24 | "G": 'graceful_finishing', 25 | "I": 'idle_worker_cleanup', 26 | } 27 | res = line.split(': ') 28 | 29 | workcounts = defaultdict(int) 30 | for i in res[1]: 31 | workcounts[i] += 1 32 | 33 | for x, y in workcounts.iteritems(): 34 | stats[switch.get(x)] = str(y) 35 | 36 | 37 | def retrieve_metrics(host='localhost', port=80): 38 | try: 39 | status = retrieve_status_page(host, port).splitlines() 40 | except Exception: 41 | raise CrawlError("can't access to http://%s:%s", 42 | host, port) 43 | switch = { 44 | "Total kBytes": 'Total_kBytes', 45 | "Total Accesses": 'Total_Accesses', 46 | "BusyWorkers": "BusyWorkers", 47 | "IdleWorkers": "IdleWorkers", 48 | "BytesPerSec": "BytesPerSec", 49 | "BytesPerReq": "BytesPerReq", 50 | "ReqPerSec": "ReqPerSec", 51 | "Uptime": "Uptime" 52 | } 53 | 54 | stats = {} 55 | 56 | for line in status: 57 | if "Scoreboard" in line: 58 | parse_score_board(line, stats) 59 | 60 | else: 61 | res = line.split(': ') 62 | if res[0] in switch: 63 | stats[switch.get(res[0])] = res[1] 64 | 65 | feature_attributes = feature.ApacheFeature 66 | 67 | if len(stats) == 0: 68 | raise CrawlError("failure to parse http://%s:%s", host, port) 69 | 70 | for name in feature_attributes._fields: 71 | if name not in stats: 72 | stats[name] = '0' 73 | 74 | feature_attributes = feature.get_feature(stats) 75 | return feature_attributes 76 | -------------------------------------------------------------------------------- /crawler/plugins/applications/apache/apache_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_apache_host 3 | Module = apache_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Apache httpd server" 9 | 10 | -------------------------------------------------------------------------------- /crawler/plugins/applications/apache/apache_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from plugins.applications.apache import apache_crawler 3 | import logging 4 | 5 | logger = logging.getLogger('crawlutils') 6 | 7 | 8 | class ApacheHostCrawler(IHostCrawler): 9 | feature_type = 'application' 10 | feature_key = 'apache' 11 | default_port = 80 12 | 13 | def get_feature(self): 14 | return self.feature_key 15 | 16 | def crawl(self): 17 | metrics = apache_crawler.retrieve_metrics( 18 | host='localhost', 19 | port=self.default_port 20 | ) 21 | return [(self.feature_key, metrics, self.feature_type)] 22 | -------------------------------------------------------------------------------- /crawler/plugins/applications/apache/feature.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | 4 | def get_feature(stats): 5 | feature_attributes = ApacheFeature( 6 | stats['BusyWorkers'], 7 | stats['IdleWorkers'], 8 | stats['waiting_for_connection'], 9 | stats['starting_up'], 10 | stats['reading_request'], 11 | stats['sending_reply'], 12 | stats['keepalive_read'], 13 | stats['dns_lookup'], 14 | stats['closing_connection'], 15 | stats['logging'], 16 | stats['graceful_finishing'], 17 | stats['idle_worker_cleanup'], 18 | stats['BytesPerSec'], 19 | stats['BytesPerReq'], 20 | stats['ReqPerSec'], 21 | stats['Uptime'], 22 | stats['Total_kBytes'], 23 | stats['Total_Accesses'] 24 | ) 25 | return feature_attributes 26 | 27 | ApacheFeature = namedtuple('ApacheFeature', [ 28 | 'BusyWorkers', 29 | 'IdleWorkers', 30 | 'waiting_for_connection', 31 | 'starting_up', 32 | 'reading_request', 33 | 'sending_reply', 34 | 'keepalive_read', 35 | 'dns_lookup', 36 | 'closing_connection', 37 | 'logging', 38 | 'graceful_finishing', 39 | 'idle_worker_cleanup', 40 | 'BytesPerSec', 41 | 'BytesPerReq', 42 | 'ReqPerSec', 43 | 'Uptime', 44 | 'Total_kBytes', 45 | 'Total_Accesses' 46 | ]) 47 | -------------------------------------------------------------------------------- /crawler/plugins/applications/db2/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/db2/db2_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_db2_container 3 | Module = db2_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "DB2 server" 9 | 10 | [Options] 11 | user = administrator user name, Default is db2inst1 12 | password = administrator password, Default is db2inst1-pwd 13 | db = db name to connect, Default is sample 14 | 15 | -------------------------------------------------------------------------------- /crawler/plugins/applications/db2/db2_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_db2_host 3 | Module = db2_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "DB2 server" 9 | 10 | [Options] 11 | user = administrator user name, Default is db2inst1 12 | password = administrator password, Default is db2inst1-pwd 13 | db = db name to connect, Default is sample 14 | 15 | -------------------------------------------------------------------------------- /crawler/plugins/applications/db2/db2_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from plugins.applications.db2 import db2_crawler 3 | from utils.crawler_exceptions import CrawlError 4 | import logging 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class DB2HostCrawler(IHostCrawler): 10 | feature_type = 'application' 11 | feature_key = 'db2' 12 | 13 | def get_feature(self): 14 | return self.feature_key 15 | 16 | def crawl(self, **options): 17 | password = "db2inst1-pwd" 18 | user = "db2inst1" 19 | db = "sample" 20 | 21 | if "password" in options: 22 | password = options["password"] 23 | 24 | if "user" in options: 25 | user = options["user"] 26 | 27 | if "db" in options: 28 | db = options["db"] 29 | 30 | try: 31 | metrics = db2_crawler.retrieve_metrics( 32 | host="localhost", 33 | user=user, 34 | password=password, 35 | db=db 36 | ) 37 | return [(self.feature_key, metrics, self.feature_type)] 38 | except: 39 | raise CrawlError("cannot retrice metrics db %s", db) 40 | -------------------------------------------------------------------------------- /crawler/plugins/applications/db2/feature.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | DB2Feature = namedtuple('DB2Feature', [ 4 | "dbCapacity", 5 | "dbVersion", 6 | "instanceName", 7 | "productName", 8 | "dbName", 9 | "serviceLevel", 10 | "instanceConn", 11 | "instanceUsedMem", 12 | "dbConn", 13 | "usedLog", 14 | "transcationInDoubt", 15 | "xlocksEscalation", 16 | "locksEscalation", 17 | "locksTimeOut", 18 | "deadLock", 19 | "lastBackupTime", 20 | "dbStatus", 21 | "instanceStatus", 22 | "bpIndexHitRatio", 23 | "bpDatahitRatio", 24 | "sortsInOverflow", 25 | "agetnsWait", 26 | "updateRows", 27 | "insertRows", 28 | "selectedRows", 29 | "deleteRows", 30 | "selects", 31 | "selectSQLs", 32 | "dynamicSQLs", 33 | "rollbacks", 34 | "commits", 35 | "bpTempIndexHitRatio", 36 | "bpTempDataHitRatio" 37 | ]) 38 | -------------------------------------------------------------------------------- /crawler/plugins/applications/liberty/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/liberty/feature.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | LibertyServletFeature = namedtuple('LibertyServletFeature', [ 4 | 'name', 5 | 'appName', 6 | 'reqCount', 7 | 'responseMean', 8 | 'responseMax', 9 | 'responseMin' 10 | ]) 11 | 12 | LibertyJVMFeature = namedtuple('LibertyJVMFeature', [ 13 | 'heap', 14 | 'freeMemory', 15 | 'usedMemory', 16 | 'processCPU', 17 | 'gcCount', 18 | 'gcTime', 19 | 'upTime' 20 | ]) 21 | 22 | LibertyThreadFeature = namedtuple('LibertyThreadFeature', [ 23 | 'activeThreads', 24 | 'poolSize', 25 | 'poolName' 26 | ]) 27 | 28 | LibertySessionFeature = namedtuple('LibertySessionFeature', [ 29 | 'name', 30 | 'createCount', 31 | 'liveCount', 32 | 'activeCount', 33 | 'invalidatedCount', 34 | 'invalidatedCountByTimeout', 35 | ]) 36 | 37 | LibertyMongoConnectionFeature = namedtuple('LibertyMongoConnectionFeature', [ 38 | 'checkedOutCount', 39 | 'waitQueueSize', 40 | 'maxSize', 41 | 'minSize', 42 | 'host', 43 | 'port', 44 | 'size', 45 | ]) 46 | -------------------------------------------------------------------------------- /crawler/plugins/applications/liberty/liberty_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_liberty_container 3 | Module = liberty_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Liberty crawling function for containers on the host 9 | 10 | [Options] 11 | user = administrator user name, Default is user 12 | password = administrator password, Default is password 13 | 14 | -------------------------------------------------------------------------------- /crawler/plugins/applications/liberty/liberty_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_liberty_host 3 | Module = liberty_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "WAS liberty server" 9 | 10 | [Options] 11 | user = administrator user name, Default is user 12 | password = administrator password, Default is password 13 | 14 | -------------------------------------------------------------------------------- /crawler/plugins/applications/liberty/liberty_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from plugins.applications.liberty import liberty_crawler 3 | import logging 4 | 5 | logger = logging.getLogger('crawlutils') 6 | 7 | 8 | class LibertyHostCrawler(IHostCrawler): 9 | feature_type = 'application' 10 | feature_key = 'liberty' 11 | default_port = 9443 12 | 13 | def get_feature(self): 14 | return self.feature_key 15 | 16 | def crawl(self, **options): 17 | password = "password" 18 | user = "user" 19 | 20 | if "password" in options: 21 | password = options["password"] 22 | 23 | if "user" in options: 24 | user = options["user"] 25 | 26 | return liberty_crawler.retrieve_metrics( 27 | host='localhost', 28 | port=self.default_port, 29 | user=user, 30 | password=password, 31 | feature_type=self.feature_type 32 | ) 33 | -------------------------------------------------------------------------------- /crawler/plugins/applications/nginx/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/nginx/feature.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | 4 | def get_feature(match1, match2, match3): 5 | feature_attributes = NginxFeature( 6 | int(match1.group(1)), 7 | int(match2.group(1)), 8 | int(match2.group(3)), 9 | int(match3.group(1)), 10 | int(match3.group(2)), 11 | int(match3.group(3)) 12 | ) 13 | return feature_attributes 14 | 15 | NginxFeature = namedtuple('NginxFeature', [ 16 | 'Connections', 17 | 'Accepted', 18 | 'Requests', 19 | 'Reading', 20 | 'Writing', 21 | 'Waiting' 22 | ]) 23 | -------------------------------------------------------------------------------- /crawler/plugins/applications/nginx/nginx_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_nginx_container 3 | Module = nginx_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "nginx httpd server" 9 | 10 | -------------------------------------------------------------------------------- /crawler/plugins/applications/nginx/nginx_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import dockercontainer 3 | from icrawl_plugin import IContainerCrawler 4 | from plugins.applications.nginx import nginx_crawler 5 | from utils.namespace import run_as_another_namespace 6 | from requests.exceptions import ConnectionError 7 | import json 8 | import utils.misc 9 | 10 | logger = logging.getLogger('crawlutils') 11 | 12 | 13 | class NginxContainerCrawler(IContainerCrawler): 14 | feature_type = 'application' 15 | feature_key = 'nginx' 16 | default_port = 80 17 | 18 | def get_feature(self): 19 | return self.feature_key 20 | 21 | def crawl(self, container_id=None, **kwargs): 22 | c = dockercontainer.DockerContainer(container_id) 23 | 24 | port = None 25 | 26 | if "annotation.io.kubernetes.container.ports" in\ 27 | c.inspect['Config']['Labels']: 28 | 29 | ports = c.inspect['Config']['Labels'][ 30 | 'annotation.io.kubernetes.container.ports'] 31 | 32 | ports = json.loads(ports) 33 | 34 | else: 35 | ports = c.get_container_ports() 36 | 37 | for each_port in ports: 38 | tmp_port = None 39 | if "containerPort" in each_port: 40 | tmp_port = int(each_port['containerPort']) 41 | else: 42 | tmp_port = int(each_port) 43 | 44 | if tmp_port == self.default_port: 45 | port = tmp_port 46 | 47 | if not port: 48 | return 49 | 50 | state = c.inspect['State'] 51 | pid = str(state['Pid']) 52 | ips = run_as_another_namespace( 53 | pid, ['net'], utils.misc.get_host_ip4_addresses) 54 | 55 | for each_ip in ips: 56 | if each_ip != "127.0.0.1": 57 | ip = each_ip 58 | break 59 | 60 | # crawl all candidate ports 61 | try: 62 | metrics = nginx_crawler.retrieve_metrics(ip, port) 63 | return [(self.feature_key, metrics, self.feature_type)] 64 | except: 65 | logger.error("can't find metrics endpoint at http://%s:%s", 66 | ip, 67 | port) 68 | raise ConnectionError("can't find metrics endpoint" 69 | "at http://%s:%s", ip, port) 70 | -------------------------------------------------------------------------------- /crawler/plugins/applications/nginx/nginx_crawler.py: -------------------------------------------------------------------------------- 1 | import re 2 | import urllib2 3 | 4 | from plugins.applications.nginx import feature 5 | from utils.crawler_exceptions import CrawlError 6 | 7 | 8 | def retrieve_status_page(host, port): 9 | status_page = "http://%s:%s/nginx_status" % (host, port) 10 | req = urllib2.Request(status_page) 11 | response = urllib2.urlopen(req) 12 | return response.read() 13 | 14 | 15 | def retrieve_metrics(host='localhost', port=80): 16 | try: 17 | status = retrieve_status_page(host, port) 18 | except Exception: 19 | raise CrawlError("can't access to http://%s:%s", 20 | host, port) 21 | 22 | match1 = re.search(r'Active connections:\s+(\d+)', status) 23 | match2 = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', status) 24 | match3 = re.search(r'Reading:\s*(\d+)\s*Writing:\s*(\d+)\s*' 25 | 'Waiting:\s*(\d+)', status) 26 | 27 | try: 28 | feature_attributes = feature.get_feature( 29 | match1, 30 | match2, 31 | match3) 32 | return feature_attributes 33 | except Exception: 34 | raise CrawlError("failure to parse http://%s:%s", host, port) 35 | -------------------------------------------------------------------------------- /crawler/plugins/applications/nginx/nginx_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_nginx_host 3 | Module = nginx_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "nginx httpd server" 9 | 10 | -------------------------------------------------------------------------------- /crawler/plugins/applications/nginx/nginx_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from plugins.applications.nginx import nginx_crawler 3 | import logging 4 | 5 | logger = logging.getLogger('crawlutils') 6 | 7 | 8 | class NginxHostCrawler(IHostCrawler): 9 | feature_type = 'application' 10 | feature_key = 'nginx' 11 | default_port = 80 12 | 13 | def get_feature(self): 14 | return self.feature_key 15 | 16 | def crawl(self): 17 | metrics = nginx_crawler.retrieve_metrics( 18 | host='localhost', 19 | port=self.default_port 20 | ) 21 | return [(self.feature_key, metrics, self.feature_type)] 22 | -------------------------------------------------------------------------------- /crawler/plugins/applications/redis/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/redis/redis_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_redis_container 3 | Module = redis_container_crawler 4 | 5 | [Documentation] 6 | Author = Tatsuhiro Chiba (chiba@jp.ibm.com) 7 | Version = 0.1 8 | Description = Redis crawling function for containers on the host 9 | -------------------------------------------------------------------------------- /crawler/plugins/applications/redis/redis_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_redis_host 3 | Module = redis_host_crawler 4 | 5 | [Documentation] 6 | Author = Tatsuhiro Chiba (chiba@jp.ibm.com) 7 | Version = 0.1 8 | Description = Redis crawling function for the local host 9 | -------------------------------------------------------------------------------- /crawler/plugins/applications/redis/redis_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from plugins.applications.redis import feature 3 | from requests.exceptions import ConnectionError 4 | import logging 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class RedisHostCrawler(IHostCrawler): 10 | ''' 11 | Crawling app provided metrics for redis on host. 12 | Usually redis listens on port 6379. 13 | ''' 14 | 15 | feature_type = "application" 16 | feature_key = "redis" 17 | default_port = 6379 18 | 19 | def get_feature(self): 20 | return self.feature_key 21 | 22 | # TODO: prepare an useful way to set host/port 23 | def crawl(self, root_dir='/', **kwargs): 24 | import pip 25 | pip.main(['install', 'redis']) 26 | import redis 27 | 28 | try: 29 | client = redis.Redis(host='localhost', port=self.default_port) 30 | metrics = client.info() 31 | except ConnectionError: 32 | logger.info("redis does not listen on port:%d", self.default_port) 33 | raise ConnectionError("no listen at %d", self.default_port) 34 | 35 | feature_attributes = feature.create_feature(metrics) 36 | 37 | return [(self.feature_key, feature_attributes, self.feature_type)] 38 | -------------------------------------------------------------------------------- /crawler/plugins/applications/tomcat/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /crawler/plugins/applications/tomcat/feature.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | 4 | TomcatJVMFeature = namedtuple('TomcatJVMFeature', [ 5 | 'free', 6 | 'total', 7 | 'max' 8 | ]) 9 | 10 | TomcatMemoryFeature = namedtuple('TomcatMemoryFeature', [ 11 | 'name', 12 | 'type', 13 | 'initial', 14 | 'committed', 15 | 'maximum', 16 | 'used' 17 | ]) 18 | 19 | TomcatConnectorFeature = namedtuple('TomcatConnectorFeature', [ 20 | 'connector', 21 | 'maxThread', 22 | 'currentThread', 23 | 'currentThreadBusy', 24 | 'requestMaxTime', 25 | 'processingTime', 26 | 'requestCount', 27 | 'errorCount', 28 | 'byteReceived', 29 | 'byteSent' 30 | ]) 31 | 32 | TomcatWorkerFeature = namedtuple('TomcatWorkerFeature', [ 33 | 'connector', 34 | 'stage', 35 | 'time', 36 | 'byteSent', 37 | 'byteReceived', 38 | 'client', 39 | 'vhost', 40 | 'request' 41 | ]) 42 | -------------------------------------------------------------------------------- /crawler/plugins/applications/tomcat/tomcat_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_tomcat_container 3 | Module = tomcat_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Tomcat crawling function for containers on the host 9 | 10 | [Options] 11 | user = administrator user name, Default is tomcat 12 | password = administrator password, Default is password 13 | -------------------------------------------------------------------------------- /crawler/plugins/applications/tomcat/tomcat_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = application_tomcat_host 3 | Module = tomcat_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Apache httpd server" 9 | 10 | [Options] 11 | user = administrator user name, Default is tomcat 12 | password = administrator password, Default is password 13 | -------------------------------------------------------------------------------- /crawler/plugins/applications/tomcat/tomcat_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from plugins.applications.tomcat import tomcat_crawler 3 | import logging 4 | 5 | logger = logging.getLogger('crawlutils') 6 | 7 | 8 | class TomcatHostCrawler(IHostCrawler): 9 | feature_type = 'application' 10 | feature_key = 'tomcat' 11 | default_port = 8080 12 | 13 | def get_feature(self): 14 | return self.feature_key 15 | 16 | def crawl(self, **options): 17 | password = "password" 18 | user = "tomcat" 19 | 20 | if "password" in options: 21 | password = options["password"] 22 | 23 | if "user" in options: 24 | user = options["user"] 25 | 26 | return tomcat_crawler.retrieve_metrics( 27 | host='localhost', 28 | port=self.default_port, 29 | user=user, 30 | password=password, 31 | feature_type=self.feature_type 32 | ) 33 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/crawler/plugins/emitters/__init__.py -------------------------------------------------------------------------------- /crawler/plugins/emitters/base_emitter.py: -------------------------------------------------------------------------------- 1 | 2 | class BaseEmitter: 3 | """ 4 | Base emitter class from which emitters like FileEmitter, StdoutEmitter 5 | should inherit. The main idea is that all emitters get a url, and should 6 | implement an emit() function given an iostream (a buffer with the features 7 | to emit). 8 | """ 9 | 10 | def __init__(self, url, timeout=1, max_retries=5, 11 | emit_per_line=False): 12 | self.url = url 13 | self.timeout = timeout 14 | self.max_retries = max_retries 15 | self.emit_per_line = emit_per_line 16 | 17 | def emit(self, iostream, compress=False, 18 | metadata={}, snapshot_num=0): 19 | """ 20 | 21 | :param iostream: a CStringIO used to buffer the formatted features. 22 | :param compress: 23 | :param metadata: 24 | :param snapshot_num: 25 | :return: 26 | """ 27 | pass 28 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/cos_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = COS Emitter 3 | Module = cos_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to emit frame to cloud object storage 9 | 10 | [Options] 11 | cos_accesskey_filepath = /etc/cos-secrets/access_key 12 | cos_secretkey_filepath = /etc/cos-secrets/secret_key 13 | cos_location_filepath = /etc/cos-secrets/location 14 | ssl_verifcation = True 15 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/file_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = File Emitter 3 | Module = file_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to emit frame to file 9 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/file_emitter.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import shutil 3 | 4 | from iemit_plugin import IEmitter 5 | 6 | 7 | class FileEmitter(IEmitter): 8 | 9 | """ 10 | Emitter to file. This creates one file per frame. The file names 11 | are the ones in the url. For example: for file:///tmp/a the file for 12 | the first frame would be /tmp/a.0 for a host, and /tmp/a.xyz.0 for a 13 | container with id xyz. 14 | """ 15 | 16 | def get_emitter_protocol(self): 17 | return 'file' 18 | 19 | def emit(self, frame, compress=False, 20 | metadata={}, snapshot_num=0, **kwargs): 21 | """ 22 | 23 | :param iostream: a CStringIO used to buffer the formatted features. 24 | :param compress: 25 | :param metadata: 26 | :param snapshot_num: 27 | :return: 28 | """ 29 | iostream = self.format(frame) 30 | output_path = self.url[len('file://'):] 31 | short_name = metadata.get('emit_shortname', None) 32 | if not short_name: 33 | file_suffix = str(snapshot_num) 34 | else: 35 | file_suffix = '{0}.{1}'.format(short_name, snapshot_num) 36 | output_path = '{0}.{1}'.format(output_path, file_suffix) 37 | output_path += '.gz' if compress else '' 38 | 39 | with open(output_path, 'w') as fd: 40 | if compress: 41 | gzip_file = gzip.GzipFile(fileobj=fd, mode='w') 42 | gzip_file.write(iostream.getvalue()) 43 | gzip_file.close() 44 | else: 45 | iostream.seek(0) 46 | shutil.copyfileobj(iostream, fd) 47 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/fluentd_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = Fluentd Emitter 3 | Module = fluentd_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to emit frame to Fluentd 9 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/http_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = Http Emitter 3 | Module = http_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to post frame data to http server 9 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/http_emitter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from iemit_plugin import IEmitter 4 | from plugins.emitters.base_http_emitter import BaseHttpEmitter 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class HttpEmitter(BaseHttpEmitter, IEmitter): 10 | 11 | def get_emitter_protocol(self): 12 | return 'http' 13 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/https_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = Https Emitter 3 | Module = https_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to post frame data to https server 9 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/https_emitter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from iemit_plugin import IEmitter 4 | from plugins.emitters.base_http_emitter import BaseHttpEmitter 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class HttpsEmitter(BaseHttpEmitter, IEmitter): 10 | 11 | def get_emitter_protocol(self): 12 | return 'https' 13 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/kafka_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = Kafka Emitter 3 | Module = kafka_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to emit frame over kafka 9 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/mtgraphite_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = MTGraphite Emitter 3 | Module = mtgraphite_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to emit frame to MTGraphite server 9 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/mtgraphite_emitter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from iemit_plugin import IEmitter 4 | from utils.mtgraphite import MTGraphiteClient 5 | from formatters import write_in_graphite_format 6 | from utils.crawler_exceptions import EmitterUnsupportedFormat 7 | 8 | logger = logging.getLogger('crawlutils') 9 | 10 | 11 | class MtGraphiteEmitter(IEmitter): 12 | 13 | def get_emitter_protocol(self): 14 | return 'mtgraphite' 15 | 16 | def init(self, url, timeout=1, max_retries=5, emit_format='graphite'): 17 | self.url = url 18 | self.timeout = timeout 19 | self.max_retries = max_retries 20 | self.emit_per_line = True 21 | 22 | if emit_format != 'graphite': 23 | raise EmitterUnsupportedFormat('Not supported: %s' % emit_format) 24 | 25 | self.formatter = write_in_graphite_format 26 | self.mtgraphite_client = MTGraphiteClient(self.url) 27 | 28 | def emit(self, frame, compress=False, 29 | metadata={}, snapshot_num=0, **kwargs): 30 | """ 31 | 32 | :param compress: 33 | :param metadata: 34 | :param snapshot_num: 35 | :return: 36 | """ 37 | iostream = self.format(frame) 38 | if self.emit_per_line: 39 | iostream.seek(0) 40 | num = self.mtgraphite_client.send_messages(iostream.readlines()) 41 | else: 42 | num = self.mtgraphite_client.send_messages([iostream.getvalue()]) 43 | logger.debug('Pushed %d messages to mtgraphite queue' % num) 44 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/sas_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = SAS Https Emitter 3 | Module = sas_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to post frame data to SAS (security analytics service) https server 9 | 10 | [Options] 11 | token_filepath = /etc/sas-secrets/token 12 | access_group_filepath = /etc/sas-secrets/access_group 13 | cloudoe_filepath = /etc/sas-secrets/cloudoe 14 | ssl_verifcation = False 15 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/stdout_emitter.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = Stdout Emitter 3 | Module = stdout_emitter 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Plugin to emit frame to console 9 | -------------------------------------------------------------------------------- /crawler/plugins/emitters/stdout_emitter.py: -------------------------------------------------------------------------------- 1 | import cStringIO 2 | import gzip 3 | import sys 4 | 5 | from iemit_plugin import IEmitter 6 | 7 | 8 | class StdoutEmitter(IEmitter): 9 | 10 | def get_emitter_protocol(self): 11 | return 'stdout' 12 | 13 | def emit(self, frame, compress=False, 14 | metadata={}, snapshot_num=0, **kwargs): 15 | """ 16 | 17 | :param iostream: a CStringIO used to buffer the formatted features. 18 | :param compress: 19 | :param metadata: 20 | :param snapshot_num: 21 | :return: 22 | """ 23 | iostream = self.format(frame) 24 | if self.emit_per_line: 25 | iostream.seek(0) 26 | for line in iostream.readlines(): 27 | self.emit_string(line, compress) 28 | else: 29 | self.emit_string(iostream.getvalue().strip(), compress) 30 | 31 | def emit_string(self, string, compress): 32 | if compress: 33 | tempio = cStringIO.StringIO() 34 | gzip_file = gzip.GzipFile(fileobj=tempio, mode='w') 35 | gzip_file.write(string) 36 | gzip_file.close() 37 | print tempio.getvalue() 38 | else: 39 | print "%s" % string 40 | sys.stdout.flush() 41 | -------------------------------------------------------------------------------- /crawler/plugins/environments/cloudsight_environment.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = Cloudsight Environment 3 | Module = cloudsight_environment 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Default environment 9 | -------------------------------------------------------------------------------- /crawler/plugins/environments/cloudsight_environment.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import copy 5 | import logging 6 | import os 7 | 8 | from runtime_environment import IRuntimeEnvironment 9 | 10 | try: 11 | unicode # Python 2 12 | except NameError: 13 | unicode = str # Python 3 14 | 15 | logger = logging.getLogger('crawlutils') 16 | 17 | 18 | class CloudsightEnvironment(IRuntimeEnvironment): 19 | name = 'cloudsight' 20 | 21 | def get_environment_name(self): 22 | return self.name 23 | 24 | def get_container_namespace(self, long_id, options): 25 | assert isinstance(long_id, (str, unicode)), "long_id is not a string" 26 | assert 'name' in options and 'host_namespace' in options 27 | name = (options['name'] or long_id[:12]).lstrip('/') 28 | return options['host_namespace'] + '/' + name 29 | 30 | def get_container_log_file_list(self, long_id, options): 31 | assert isinstance(long_id, (str, unicode)), "long_id is not a string" 32 | assert 'container_logs' in options 33 | container_logs = copy.deepcopy(options['container_logs']) 34 | for log in container_logs: 35 | name = log['name'] 36 | if not os.path.isabs(name) or '..' in name: 37 | container_logs.remove(log) 38 | logger.warning( 39 | 'User provided a log file path that is not absolute: %s' % 40 | name) 41 | return container_logs 42 | 43 | def get_container_log_prefix(self, long_id, options): 44 | assert isinstance(long_id, (str, unicode)), "long_id is not a string" 45 | return self.get_container_namespace(long_id, options) 46 | -------------------------------------------------------------------------------- /crawler/plugins/environments/icp_environment.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = ICp Environment 3 | Module = icp_environment 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = ICp environment 9 | -------------------------------------------------------------------------------- /crawler/plugins/environments/kubernetes_environment.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = Kubernetes Environment 3 | Module = kubernetes_environment 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Kubernetes environment 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/crawler/plugins/systems/__init__.py -------------------------------------------------------------------------------- /crawler/plugins/systems/config_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = config_container 3 | Module = config_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Config crawling function for containers 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/config_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = config_host 3 | Module = config_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Config crawling function for hosts 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/config_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IHostCrawler 4 | from utils.config_utils import crawl_config_files 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class ConfigHostCrawler(IHostCrawler): 10 | 11 | def get_feature(self): 12 | return 'config' 13 | 14 | def crawl( 15 | self, 16 | root_dir='/', 17 | exclude_dirs=[ 18 | '/dev', 19 | '/proc', 20 | '/mnt', 21 | '/tmp', 22 | '/var/cache', 23 | '/usr/share/man', 24 | '/usr/share/doc', 25 | '/usr/share/mime'], 26 | known_config_files=[ 27 | '/etc/passwd', 28 | '/etc/group', 29 | '/etc/hosts', 30 | '/etc/hostname', 31 | '/etc/mtab', 32 | '/etc/fstab', 33 | '/etc/aliases', 34 | '/etc/ssh/ssh_config', 35 | '/etc/ssh/sshd_config', 36 | '/etc/sudoers'], 37 | discover_config_files=False, 38 | **kwargs): 39 | return crawl_config_files( 40 | root_dir=root_dir, 41 | exclude_dirs=exclude_dirs, 42 | known_config_files=known_config_files, 43 | discover_config_files=discover_config_files) 44 | -------------------------------------------------------------------------------- /crawler/plugins/systems/connection_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = connection_container 3 | Module = connection_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Connection crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/connection_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import utils.dockerutils 4 | from icrawl_plugin import IContainerCrawler 5 | from utils.connection_utils import crawl_connections 6 | from utils.namespace import run_as_another_namespace, ALL_NAMESPACES 7 | 8 | logger = logging.getLogger('crawlutils') 9 | 10 | 11 | class ConnectionContainerCrawler(IContainerCrawler): 12 | 13 | def get_feature(self): 14 | return 'connection' 15 | 16 | def crawl(self, container_id, avoid_setns=False, **kwargs): 17 | inspect = utils.dockerutils.exec_dockerinspect(container_id) 18 | state = inspect['State'] 19 | pid = str(state['Pid']) 20 | logger.debug( 21 | 'Crawling %s for container %s' % 22 | (self.get_feature(), container_id)) 23 | 24 | if avoid_setns: 25 | raise NotImplementedError('avoidsetns mode not implemented') 26 | else: # in all other cases, including wrong mode set 27 | return run_as_another_namespace(pid, 28 | ALL_NAMESPACES, 29 | crawl_connections) 30 | -------------------------------------------------------------------------------- /crawler/plugins/systems/connection_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = connection_host 3 | Module = connection_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Connection crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/connection_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IHostCrawler 4 | from utils.connection_utils import crawl_connections 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class ConnectionHostCrawler(IHostCrawler): 10 | 11 | def get_feature(self): 12 | return 'connection' 13 | 14 | def crawl(self, **kwargs): 15 | logger.debug('Crawling %s' % (self.get_feature())) 16 | 17 | return crawl_connections() 18 | -------------------------------------------------------------------------------- /crawler/plugins/systems/connection_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = connection_vm 3 | Module = connection_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Connection crawling function for VMs" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/connection_vm_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import psutil 4 | 5 | from icrawl_plugin import IVMCrawler 6 | from utils.connection_utils import crawl_single_connection 7 | 8 | try: 9 | import psvmi 10 | except ImportError: 11 | psvmi = None 12 | 13 | logger = logging.getLogger('crawlutils') 14 | 15 | 16 | class ConnectionVmCrawler(IVMCrawler): 17 | 18 | def get_feature(self): 19 | return 'connection' 20 | 21 | def crawl(self, vm_desc, **kwargs): 22 | created_since = -1 23 | 24 | if psvmi is None: 25 | raise NotImplementedError() 26 | else: 27 | (domain_name, kernel_version, distro, arch) = vm_desc 28 | # XXX: this has to be read from some cache instead of 29 | # instead of once per plugin/feature 30 | vm_context = psvmi.context_init( 31 | domain_name, domain_name, kernel_version, distro, arch) 32 | proc_list = psvmi.process_iter(vm_context) 33 | 34 | for p in proc_list: 35 | pid = (p.pid() if hasattr(p.pid, '__call__') else p.pid) 36 | status = (p.status() if hasattr(p.status, '__call__' 37 | ) else p.status) 38 | if status == psutil.STATUS_ZOMBIE: 39 | continue 40 | 41 | create_time = ( 42 | p.create_time() if hasattr( 43 | p.create_time, 44 | '__call__') else p.create_time) 45 | name = (p.name() if hasattr(p.name, '__call__') else p.name) 46 | 47 | if create_time <= created_since: 48 | continue 49 | for conn in p.get_connections(): 50 | yield crawl_single_connection(conn, pid, name) 51 | -------------------------------------------------------------------------------- /crawler/plugins/systems/cpu_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = cpu_container 3 | Module = cpu_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Cpu crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/cpu_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = cpu_host 3 | Module = cpu_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Cpu crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/cpu_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import psutil 4 | 5 | from icrawl_plugin import IHostCrawler 6 | from utils.features import CpuFeature 7 | 8 | logger = logging.getLogger('crawlutils') 9 | 10 | 11 | class CpuHostCrawler(IHostCrawler): 12 | 13 | def get_feature(self): 14 | return 'cpu' 15 | 16 | def crawl(self, **kwargs): 17 | logger.debug('Crawling %s' % (self.get_feature())) 18 | 19 | for (idx, cpu) in enumerate(psutil.cpu_times_percent(percpu=True)): 20 | feature_attributes = CpuFeature( 21 | cpu.idle, 22 | cpu.nice, 23 | cpu.user, 24 | cpu.iowait, 25 | cpu.system, 26 | cpu.irq, 27 | cpu.steal, 28 | 100 - int(cpu.idle), 29 | ) 30 | feature_key = '{0}-{1}'.format('cpu', idx) 31 | yield (feature_key, feature_attributes, 'cpu') 32 | -------------------------------------------------------------------------------- /crawler/plugins/systems/cpu_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = cpu_vm 3 | Module = cpu_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Cpu crawling function for VMs" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/cpu_vm_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IVMCrawler 2 | import logging 3 | 4 | # External dependencies that must be pip install'ed separately 5 | 6 | try: 7 | import psvmi 8 | except ImportError: 9 | psvmi = None 10 | 11 | logger = logging.getLogger('crawlutils') 12 | 13 | 14 | class cpu_vm_crawler(IVMCrawler): 15 | 16 | def get_feature(self): 17 | return 'cpu' 18 | 19 | def crawl(self, vm_desc, **kwargs): 20 | raise NotImplementedError('Unsupported crawl mode') 21 | -------------------------------------------------------------------------------- /crawler/plugins/systems/ctprobe_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = ctprobe_container 3 | Module = ctprobe_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Crawling function for containers to start conntrackprobe 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/disk_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = disk_container 3 | Module = disk_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Disk crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/disk_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import utils.dockerutils 4 | from icrawl_plugin import IContainerCrawler 5 | from utils.disk_utils import crawl_disk_partitions 6 | from utils.namespace import run_as_another_namespace, ALL_NAMESPACES 7 | 8 | logger = logging.getLogger('crawlutils') 9 | 10 | 11 | class DiskContainerCrawler(IContainerCrawler): 12 | 13 | def get_feature(self): 14 | return 'disk' 15 | 16 | def crawl(self, container_id, avoid_setns=False, **kwargs): 17 | inspect = utils.dockerutils.exec_dockerinspect(container_id) 18 | state = inspect['State'] 19 | pid = str(state['Pid']) 20 | logger.debug( 21 | 'Crawling %s for container %s' % 22 | (self.get_feature(), container_id)) 23 | 24 | if avoid_setns: 25 | raise NotImplementedError('avoidsetns mode not implemented') 26 | else: # in all other cases, including wrong mode set 27 | return run_as_another_namespace(pid, 28 | ALL_NAMESPACES, 29 | crawl_disk_partitions) 30 | -------------------------------------------------------------------------------- /crawler/plugins/systems/disk_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = disk_host 3 | Module = disk_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Disk crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/disk_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IHostCrawler 4 | from utils.disk_utils import crawl_disk_partitions 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class DiskHostCrawler(IHostCrawler): 10 | 11 | def get_feature(self): 12 | return 'disk' 13 | 14 | def crawl(self, **kwargs): 15 | logger.debug('Crawling %s' % (self.get_feature())) 16 | 17 | return crawl_disk_partitions() 18 | -------------------------------------------------------------------------------- /crawler/plugins/systems/disk_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = disk_vm 3 | Module = disk_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Disk crawling function for VMs" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/disk_vm_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IVMCrawler 2 | 3 | import logging 4 | 5 | try: 6 | import psvmi 7 | except ImportError: 8 | psvmi = None 9 | 10 | logger = logging.getLogger('crawlutils') 11 | 12 | 13 | class disk_vm_crawler(IVMCrawler): 14 | 15 | def get_feature(self): 16 | return 'disk' 17 | 18 | def crawl(self, vm_desc, **kwargs): 19 | raise NotImplementedError() 20 | -------------------------------------------------------------------------------- /crawler/plugins/systems/diskio_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = diskio_host 3 | Module = diskio_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Disk I/O crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockerhistory_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = dockerhistory_container 3 | Module = dockerhistory_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Dockerhistory crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockerhistory_container_crawler.py: -------------------------------------------------------------------------------- 1 | from utils.dockerutils import exec_docker_history 2 | from icrawl_plugin import IContainerCrawler 3 | 4 | import logging 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class DockerhistoryContainerCrawler(IContainerCrawler): 10 | 11 | def get_feature(self): 12 | return 'dockerhistory' 13 | 14 | def crawl(self, container_id, avoid_setns=False, **kwargs): 15 | history = exec_docker_history(container_id) 16 | image_id = history[0]['Id'] 17 | yield (image_id, {'history': history}, 'dockerhistory') 18 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockerinspect_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = dockerinspect_container 3 | Module = dockerinspect_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Dockerinspect crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockerinspect_container_crawler.py: -------------------------------------------------------------------------------- 1 | from utils.dockerutils import exec_dockerinspect 2 | from icrawl_plugin import IContainerCrawler 3 | 4 | import logging 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class DockerinspectContainerCrawler(IContainerCrawler): 10 | 11 | def get_feature(self): 12 | return 'dockerinspect' 13 | 14 | def crawl(self, container_id, avoid_setns=False, **kwargs): 15 | inspect = exec_dockerinspect(container_id) 16 | yield (container_id, inspect, 'dockerinspect') 17 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockernetwork_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = dockernetwork_container 3 | Module = dockernetwork_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Dockernetwork crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockernetwork_container_crawler.py: -------------------------------------------------------------------------------- 1 | from utils.dockerutils import exec_dockernetwork 2 | from icrawl_plugin import IContainerCrawler 3 | 4 | import logging 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class DockernetworkContainerCrawler(IContainerCrawler): 10 | 11 | def get_feature(self): 12 | return 'dockernetwork' 13 | 14 | def crawl(self, container_id, avoid_setns=False, **kwargs): 15 | network = exec_dockernetwork(container_id) 16 | yield (container_id, network, 'dockernetwork') 17 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockerps_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = dockerps_host 3 | Module = dockerps_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Dockerps crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/dockerps_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IHostCrawler 4 | from utils.dockerutils import exec_dockerps 5 | from utils.features import DockerPSFeature 6 | 7 | logger = logging.getLogger('crawlutils') 8 | 9 | 10 | class DockerpsHostCrawler(IHostCrawler): 11 | 12 | def get_feature(self): 13 | return 'dockerps' 14 | 15 | def crawl(self, **kwargs): 16 | logger.debug('Crawling %s' % (self.get_feature())) 17 | 18 | for inspect in exec_dockerps(): 19 | yield (inspect['Id'], DockerPSFeature._make([ 20 | inspect['State']['Running'], 21 | 0, 22 | inspect['Image'], 23 | [], 24 | inspect['Config']['Cmd'], 25 | inspect['Name'], 26 | inspect['Id'], 27 | ]), 'dockerps') 28 | -------------------------------------------------------------------------------- /crawler/plugins/systems/file_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = file_container 3 | Module = file_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = File crawling function for containers 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/file_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import utils.dockerutils 4 | import utils.misc 5 | from icrawl_plugin import IContainerCrawler 6 | from utils.file_utils import crawl_files 7 | from utils.namespace import run_as_another_namespace 8 | 9 | logger = logging.getLogger('crawlutils') 10 | 11 | 12 | class FileContainerCrawler(IContainerCrawler): 13 | 14 | def get_feature(self): 15 | return 'file' 16 | 17 | def crawl( 18 | self, 19 | container_id=None, 20 | avoid_setns=False, 21 | root_dir='/', 22 | exclude_dirs=[ 23 | '/boot', 24 | '/dev', 25 | '/proc', 26 | '/sys', 27 | '/mnt', 28 | '/tmp', 29 | '/var/cache', 30 | '/usr/share/man', 31 | '/usr/share/doc', 32 | '/usr/share/mime'], 33 | **kwargs): 34 | inspect = utils.dockerutils.exec_dockerinspect(container_id) 35 | state = inspect['State'] 36 | pid = str(state['Pid']) 37 | logger.debug('Crawling file for container %s' % container_id) 38 | 39 | if avoid_setns: 40 | rootfs_dir = utils.dockerutils.get_docker_container_rootfs_path( 41 | container_id) 42 | exclude_dirs = [utils.misc.join_abs_paths(rootfs_dir, d) 43 | for d in exclude_dirs] 44 | return crawl_files( 45 | root_dir=utils.misc.join_abs_paths(rootfs_dir, root_dir), 46 | exclude_dirs=exclude_dirs, 47 | root_dir_alias=root_dir) 48 | else: # in all other cases, including wrong mode set 49 | return run_as_another_namespace(pid, 50 | ['mnt'], 51 | crawl_files, 52 | root_dir, 53 | exclude_dirs, 54 | None) 55 | -------------------------------------------------------------------------------- /crawler/plugins/systems/file_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = file_host 3 | Module = file_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = File crawling function for the local host 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/file_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from utils.file_utils import crawl_files 3 | 4 | 5 | class FileHostCrawler(IHostCrawler): 6 | 7 | def get_feature(self): 8 | return 'file' 9 | 10 | def crawl( 11 | self, 12 | root_dir='/', 13 | exclude_dirs=[ 14 | '/boot', 15 | '/dev', 16 | '/proc', 17 | '/sys', 18 | '/mnt', 19 | '/tmp', 20 | '/var/cache', 21 | '/usr/share/man', 22 | '/usr/share/doc', 23 | '/usr/share/mime'], 24 | **kwargs): 25 | return crawl_files(root_dir=root_dir, 26 | exclude_dirs=exclude_dirs) 27 | -------------------------------------------------------------------------------- /crawler/plugins/systems/fprobe_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = fprobe_container 3 | Module = fprobe_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Crawling function for containers to start fprobe 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/gpu_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = gpu_host 3 | Module = gpu_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = GPU metrics crawler for host 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/interface_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = interface_container 3 | Module = interface_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Interface crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/interface_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = interface_host 3 | Module = interface_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Interface crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/interface_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = interface_vm 3 | Module = interface_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Interface crawling function for VMs" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/jar_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = jar_container 3 | Module = jar_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Produces md5 hashes for jar files and class files in jars 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/jar_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import utils.dockerutils 4 | import utils.misc 5 | from icrawl_plugin import IContainerCrawler 6 | from utils.jar_utils import crawl_jar_files 7 | from utils.namespace import run_as_another_namespace 8 | 9 | logger = logging.getLogger('crawlutils') 10 | 11 | 12 | class JarContainerCrawler(IContainerCrawler): 13 | 14 | def get_feature(self): 15 | return 'jar' 16 | 17 | def crawl( 18 | self, 19 | container_id=None, 20 | avoid_setns=False, 21 | root_dir='/', 22 | exclude_dirs=[ 23 | '/boot', 24 | '/dev', 25 | '/proc', 26 | '/sys', 27 | '/mnt', 28 | '/tmp', 29 | '/var/cache', 30 | '/usr/share/man', 31 | '/usr/share/doc', 32 | '/usr/share/mime'], 33 | **kwargs): 34 | inspect = utils.dockerutils.exec_dockerinspect(container_id) 35 | state = inspect['State'] 36 | pid = str(state['Pid']) 37 | logger.debug('Crawling jars for container %s' % container_id) 38 | 39 | if avoid_setns: 40 | rootfs_dir = utils.dockerutils.get_docker_container_rootfs_path( 41 | container_id) 42 | exclude_dirs = [utils.misc.join_abs_paths(rootfs_dir, d) 43 | for d in exclude_dirs] 44 | return crawl_jar_files( 45 | root_dir=utils.misc.join_abs_paths(rootfs_dir, root_dir), 46 | exclude_dirs=exclude_dirs, 47 | root_dir_alias=root_dir) 48 | else: # in all other cases, including wrong mode set 49 | return run_as_another_namespace(pid, 50 | ['mnt'], 51 | crawl_jar_files, 52 | root_dir, 53 | exclude_dirs, 54 | None) 55 | -------------------------------------------------------------------------------- /crawler/plugins/systems/jar_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = jar_host 3 | Module = jar_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Produces md5 hashes for jar files and class files in jars 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/jar_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from utils.jar_utils import crawl_jar_files 3 | 4 | 5 | class JarHostCrawler(IHostCrawler): 6 | 7 | def get_feature(self): 8 | return 'jar' 9 | 10 | def crawl( 11 | self, 12 | root_dir='/', 13 | exclude_dirs=[ 14 | '/boot', 15 | '/dev', 16 | '/proc', 17 | '/sys', 18 | '/mnt', 19 | '/tmp', 20 | '/var/cache', 21 | '/usr/share/man', 22 | '/usr/share/doc', 23 | '/usr/share/mime'], 24 | **kwargs): 25 | return crawl_jar_files(root_dir=root_dir, 26 | exclude_dirs=exclude_dirs) 27 | -------------------------------------------------------------------------------- /crawler/plugins/systems/load_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = load_container 3 | Module = load_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Load crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/load_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from dockercontainer import DockerContainer 5 | from icrawl_plugin import IContainerCrawler 6 | from utils.features import LoadFeature 7 | from utils.namespace import run_as_another_namespace, ALL_NAMESPACES 8 | 9 | logger = logging.getLogger('crawlutils') 10 | 11 | 12 | class LoadContainerCrawler(IContainerCrawler): 13 | 14 | def get_feature(self): 15 | return 'load' 16 | 17 | def crawl_load(self): 18 | load = os.getloadavg() 19 | feature_key = 'load' 20 | feature_attributes = LoadFeature(load[0], load[1], load[1]) 21 | yield (feature_key, feature_attributes, 'load') 22 | 23 | def crawl(self, container_id, avoid_setns=False, **kwargs): 24 | container = DockerContainer(container_id) 25 | logger.debug( 26 | 'Crawling %s for container %s' % 27 | (self.get_feature(), container_id)) 28 | 29 | if avoid_setns: 30 | raise NotImplementedError() 31 | else: # in all other cases, including wrong mode set 32 | return run_as_another_namespace(container.pid, 33 | ALL_NAMESPACES, 34 | self.crawl_load) 35 | -------------------------------------------------------------------------------- /crawler/plugins/systems/load_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = load_host 3 | Module = load_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Load crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/load_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from icrawl_plugin import IHostCrawler 5 | from utils.features import LoadFeature 6 | 7 | logger = logging.getLogger('crawlutils') 8 | 9 | 10 | class LoadHostCrawler(IHostCrawler): 11 | 12 | def get_feature(self): 13 | return 'load' 14 | 15 | def crawl_load(self): 16 | load = os.getloadavg() 17 | feature_key = 'load' 18 | feature_attributes = LoadFeature(load[0], load[1], load[1]) 19 | yield (feature_key, feature_attributes, 'load') 20 | 21 | def crawl(self, **kwargs): 22 | logger.debug('Crawling %s' % (self.get_feature())) 23 | 24 | return self.crawl_load() 25 | -------------------------------------------------------------------------------- /crawler/plugins/systems/load_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = load_vm 3 | Module = load_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Load crawling function for VMs" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/load_vm_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IVMCrawler 2 | import logging 3 | 4 | try: 5 | import psvmi 6 | except ImportError: 7 | psvmi = None 8 | 9 | logger = logging.getLogger('crawlutils') 10 | 11 | 12 | class load_vm_crawler(IVMCrawler): 13 | 14 | def get_feature(self): 15 | return 'load' 16 | 17 | def crawl(self, vm_desc, **kwargs): 18 | raise NotImplementedError() 19 | -------------------------------------------------------------------------------- /crawler/plugins/systems/memory_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = memory_container 3 | Module = memory_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Memory crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/memory_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import psutil 4 | 5 | from dockercontainer import DockerContainer 6 | from icrawl_plugin import IContainerCrawler 7 | from utils.features import MemoryFeature 8 | 9 | logger = logging.getLogger('crawlutils') 10 | 11 | 12 | class MemoryContainerCrawler(IContainerCrawler): 13 | 14 | def get_feature(self): 15 | return 'memory' 16 | 17 | def crawl(self, container_id, avoid_setns=False, **kwargs): 18 | container = DockerContainer(container_id) 19 | 20 | used = buffered = cached = free = 'unknown' 21 | with open(container.get_memory_cgroup_path('memory.stat' 22 | ), 'r') as f: 23 | for line in f: 24 | (key, value) = line.strip().split(' ') 25 | if key == 'total_cache': 26 | cached = int(value) 27 | if key == 'total_active_file': 28 | buffered = int(value) 29 | 30 | with open(container.get_memory_cgroup_path( 31 | 'memory.limit_in_bytes'), 'r') as f: 32 | limit = int(f.readline().strip()) 33 | 34 | with open(container.get_memory_cgroup_path( 35 | 'memory.usage_in_bytes'), 'r') as f: 36 | used = int(f.readline().strip()) 37 | 38 | host_free = psutil.virtual_memory().free 39 | container_total = used + min(host_free, limit - used) 40 | free = container_total - used 41 | 42 | if 'unknown' not in [used, free] and (free + used) > 0: 43 | util_percentage = float(used) / (free + used) * 100.0 44 | else: 45 | util_percentage = 'unknown' 46 | 47 | return [('memory', MemoryFeature(used, buffered, 48 | cached, free, util_percentage), 49 | 'memory')] 50 | -------------------------------------------------------------------------------- /crawler/plugins/systems/memory_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = memory_host 3 | Module = memory_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Memory crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/memory_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import psutil 4 | 5 | from icrawl_plugin import IHostCrawler 6 | from utils.features import MemoryFeature 7 | 8 | logger = logging.getLogger('crawlutils') 9 | 10 | 11 | class MemoryHostCrawler(IHostCrawler): 12 | 13 | def get_feature(self): 14 | return 'memory' 15 | 16 | def crawl(self, **kwargs): 17 | logger.debug('Crawling %s' % (self.get_feature())) 18 | 19 | vm = psutil.virtual_memory() 20 | 21 | if (vm.free + vm.used) > 0: 22 | util_percentage = float(vm.used) / (vm.free + vm.used) * 100.0 23 | else: 24 | util_percentage = 'unknown' 25 | 26 | feature_attributes = MemoryFeature(vm.used, vm.buffers, vm.cached, 27 | vm.free, util_percentage) 28 | 29 | return [('memory', feature_attributes, 'memory')] 30 | -------------------------------------------------------------------------------- /crawler/plugins/systems/memory_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = memory_vm 3 | Module = memory_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Memory crawling function for VMs" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/memory_vm_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IVMCrawler 4 | from utils.features import MemoryFeature 5 | 6 | try: 7 | import psvmi 8 | except ImportError: 9 | psvmi = None 10 | 11 | logger = logging.getLogger('crawlutils') 12 | 13 | 14 | class MemoryVmCrawler(IVMCrawler): 15 | 16 | def get_feature(self): 17 | return 'memory' 18 | 19 | def crawl(self, vm_desc, **kwargs): 20 | if psvmi is None: 21 | raise NotImplementedError() 22 | else: 23 | (domain_name, kernel_version, distro, arch) = vm_desc 24 | # XXX: this has to be read from some cache instead of 25 | # instead of once per plugin/feature 26 | vm_context = psvmi.context_init( 27 | domain_name, domain_name, kernel_version, distro, arch) 28 | 29 | sysmem = psvmi.system_memory_info(vm_context) 30 | feature_attributes = MemoryFeature( 31 | sysmem.memory_used, 32 | sysmem.memory_buffered, 33 | sysmem.memory_cached, 34 | sysmem.memory_free, 35 | (sysmem.memory_used * 100 / (sysmem.memory_used + 36 | sysmem.memory_free))) 37 | return [('memory', feature_attributes, 'memory')] 38 | -------------------------------------------------------------------------------- /crawler/plugins/systems/metric_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = metric_container 3 | Module = metric_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Metric crawling function for containers" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/metric_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import utils.dockerutils 4 | from icrawl_plugin import IContainerCrawler 5 | from utils.metric_utils import crawl_metrics 6 | from utils.namespace import run_as_another_namespace, ALL_NAMESPACES 7 | 8 | logger = logging.getLogger('crawlutils') 9 | 10 | 11 | class MetricContainerCrawler(IContainerCrawler): 12 | 13 | def get_feature(self): 14 | return 'metric' 15 | 16 | def crawl(self, container_id, avoid_setns=False, **kwargs): 17 | inspect = utils.dockerutils.exec_dockerinspect(container_id) 18 | state = inspect['State'] 19 | pid = str(state['Pid']) 20 | logger.debug( 21 | 'Crawling %s for container %s' % 22 | (self.get_feature(), container_id)) 23 | 24 | if avoid_setns: 25 | raise NotImplementedError('avoidsetns mode not implemented') 26 | else: # in all other cases, including wrong mode set 27 | return run_as_another_namespace(pid, 28 | ALL_NAMESPACES, 29 | crawl_metrics) 30 | -------------------------------------------------------------------------------- /crawler/plugins/systems/metric_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = metric_host 3 | Module = metric_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Metric crawling function for hosts" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/metric_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IHostCrawler 4 | from utils.metric_utils import crawl_metrics 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class MetricHostCrawler(IHostCrawler): 10 | 11 | def get_feature(self): 12 | return 'metric' 13 | 14 | def crawl(self, **kwargs): 15 | logger.debug('Crawling %s' % (self.get_feature())) 16 | 17 | return crawl_metrics() 18 | -------------------------------------------------------------------------------- /crawler/plugins/systems/metric_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = metric_vm 3 | Module = metric_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "Metric crawling function for VMs" 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/nodepackage_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = node_pkg 3 | Module = nodepackage_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Crawler for Node NPM packages for containers and images 9 | Format = [(pkg_name, pkg_version)] 10 | 11 | [Options] 12 | avoid_setns = True|False. Default is True. 13 | -------------------------------------------------------------------------------- /crawler/plugins/systems/os_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = os_container 3 | Module = os_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = "OS crawling function for containers. Takes following optional arguments: 9 | crawl_mode = {MOUNPOINT, OUTCONTAINER}. Former uses docker rootfs, latter setns()" 10 | 11 | -------------------------------------------------------------------------------- /crawler/plugins/systems/os_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import utils.dockerutils 4 | from icrawl_plugin import IContainerCrawler 5 | from utils.namespace import run_as_another_namespace, ALL_NAMESPACES 6 | from utils.os_utils import crawl_os, crawl_os_mountpoint 7 | 8 | logger = logging.getLogger('crawlutils') 9 | 10 | 11 | class OSContainerCrawler(IContainerCrawler): 12 | 13 | def get_feature(self): 14 | return 'os' 15 | 16 | def crawl(self, container_id, avoid_setns=False, **kwargs): 17 | inspect = utils.dockerutils.exec_dockerinspect(container_id) 18 | state = inspect['State'] 19 | pid = str(state['Pid']) 20 | logger.debug('Crawling OS for container %s' % container_id) 21 | 22 | if avoid_setns: 23 | mp = utils.dockerutils.get_docker_container_rootfs_path( 24 | container_id) 25 | return crawl_os_mountpoint(mp) 26 | else: # in all other cases, including wrong mode set 27 | return run_as_another_namespace(pid, 28 | ALL_NAMESPACES, 29 | crawl_os) 30 | -------------------------------------------------------------------------------- /crawler/plugins/systems/os_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = os_host 3 | Module = os_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = OS crawling function for the local host 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/os_host_crawler.py: -------------------------------------------------------------------------------- 1 | from icrawl_plugin import IHostCrawler 2 | from utils.os_utils import crawl_os, crawl_os_mountpoint 3 | 4 | 5 | class OSHostCrawler(IHostCrawler): 6 | 7 | def get_feature(self): 8 | return 'os' 9 | 10 | def crawl(self, root_dir='/', **kwargs): 11 | if root_dir == '/': 12 | return crawl_os() 13 | else: 14 | return crawl_os_mountpoint(root_dir) 15 | -------------------------------------------------------------------------------- /crawler/plugins/systems/os_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = os_vm 3 | Module = os_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = OS crawling function for VMs 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/os_vm_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IVMCrawler 4 | from utils.features import OSFeature 5 | 6 | # External dependencies that must be pip install'ed separately 7 | 8 | try: 9 | import psvmi 10 | except ImportError: 11 | psvmi = None 12 | 13 | logger = logging.getLogger('crawlutils') 14 | 15 | 16 | class os_vm_crawler(IVMCrawler): 17 | 18 | def get_feature(self): 19 | return 'os' 20 | 21 | def crawl(self, vm_desc, **kwargs): 22 | if psvmi is None: 23 | raise NotImplementedError() 24 | else: 25 | (domain_name, kernel_version, distro, arch) = vm_desc 26 | # XXX: not good, context_init was being done once per VM 27 | # in previous monolithic model, now it's once per plugin/feature 28 | vm_context = psvmi.context_init( 29 | domain_name, domain_name, kernel_version, distro, arch) 30 | sys = psvmi.system_info(vm_context) 31 | feature_attributes = OSFeature( 32 | sys.boottime, 33 | 'unknown', 34 | sys.ipaddr, 35 | sys.ostype, 36 | sys.osversion, 37 | sys.osrelease, 38 | sys.osplatform 39 | ) 40 | feature_key = sys.ostype 41 | return [(feature_key, feature_attributes, 'os')] 42 | -------------------------------------------------------------------------------- /crawler/plugins/systems/package_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = package_container 3 | Module = package_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Package crawling function for containers 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/package_container_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IContainerCrawler 4 | from utils.crawler_exceptions import CrawlError 5 | from utils.dockerutils import (exec_dockerinspect, 6 | get_docker_container_rootfs_path) 7 | from utils.misc import join_abs_paths 8 | from utils.namespace import run_as_another_namespace, ALL_NAMESPACES 9 | from utils.package_utils import crawl_packages 10 | 11 | logger = logging.getLogger('crawlutils') 12 | 13 | 14 | class PackageContainerCrawler(IContainerCrawler): 15 | 16 | def get_feature(self): 17 | return 'package' 18 | 19 | def crawl(self, container_id=None, avoid_setns=False, 20 | root_dir='/', **kwargs): 21 | logger.debug('Crawling packages for container %s' % container_id) 22 | inspect = exec_dockerinspect(container_id) 23 | state = inspect['State'] 24 | pid = str(state['Pid']) 25 | 26 | if avoid_setns: 27 | rootfs_dir = get_docker_container_rootfs_path( 28 | container_id) 29 | return crawl_packages( 30 | root_dir=join_abs_paths(rootfs_dir, root_dir), 31 | reload_needed=True) 32 | else: # in all other cases, including wrong mode set 33 | try: 34 | return run_as_another_namespace(pid, 35 | ALL_NAMESPACES, 36 | crawl_packages, 37 | None, 38 | root_dir, 0, False) 39 | except CrawlError: 40 | 41 | # Retry the crawl avoiding the setns() syscall. This is 42 | # needed for PPC where we can not jump into the container and 43 | # run its apt or rpm commands. 44 | 45 | rootfs_dir = get_docker_container_rootfs_path( 46 | container_id) 47 | return crawl_packages( 48 | root_dir=join_abs_paths(rootfs_dir, root_dir), 49 | reload_needed=True) 50 | -------------------------------------------------------------------------------- /crawler/plugins/systems/package_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = package_host 3 | Module = package_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Package crawling function for hosts 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/package_host_crawler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from icrawl_plugin import IHostCrawler 4 | from utils.package_utils import crawl_packages 5 | 6 | logger = logging.getLogger('crawlutils') 7 | 8 | 9 | class PackageHostCrawler(IHostCrawler): 10 | 11 | def get_feature(self): 12 | return 'package' 13 | 14 | def crawl(self, root_dir='/', **kwargs): 15 | return crawl_packages(root_dir=root_dir, reload_needed=False) 16 | -------------------------------------------------------------------------------- /crawler/plugins/systems/process_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = process_container 3 | Module = process_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Process crawling function for containers 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/process_host_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = process_host 3 | Module = process_host_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Process crawling function for the local host 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/process_vm_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = process_vm 3 | Module = process_vm_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Process crawling function for VMs 9 | -------------------------------------------------------------------------------- /crawler/plugins/systems/pythonpackage_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = python_pkg 3 | Module = pythonpackage_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Crawler for Python PyPi packages for containers and images 9 | Format = [(pkg_name, pkg_version)] 10 | 11 | [Options] 12 | avoid_setns = True|False. Default is True. 13 | -------------------------------------------------------------------------------- /crawler/plugins/systems/rubypackage_container_crawler.plugin: -------------------------------------------------------------------------------- 1 | [Core] 2 | Name = ruby_pkg 3 | Module = rubypackage_container_crawler 4 | 5 | [Documentation] 6 | Author = IBM 7 | Version = 0.1 8 | Description = Crawler for Ruby GEM packages for containers and images 9 | Format = [(pkg_name, pkg_version)] 10 | 11 | [Options] 12 | avoid_setns = True|False. Default is True. 13 | -------------------------------------------------------------------------------- /crawler/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/crawler/utils/__init__.py -------------------------------------------------------------------------------- /crawler/utils/connection_utils.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | 3 | from utils.features import ConnectionFeature 4 | 5 | 6 | def crawl_connections(): 7 | created_since = -1 8 | 9 | proc_list = psutil.process_iter() 10 | 11 | for p in proc_list: 12 | pid = (p.pid() if hasattr(p.pid, '__call__') else p.pid) 13 | status = (p.status() if hasattr(p.status, '__call__' 14 | ) else p.status) 15 | if status == psutil.STATUS_ZOMBIE: 16 | continue 17 | 18 | create_time = ( 19 | p.create_time() if hasattr( 20 | p.create_time, 21 | '__call__') else p.create_time) 22 | name = (p.name() if hasattr(p.name, '__call__') else p.name) 23 | 24 | if create_time <= created_since: 25 | continue 26 | for conn in p.get_connections(): 27 | yield crawl_single_connection(conn, pid, name) 28 | 29 | 30 | def crawl_single_connection(c, pid, name): 31 | """Returns a ConnectionFeature""" 32 | try: 33 | (localipaddr, localport) = c.laddr[:] 34 | except: 35 | 36 | # Older version of psutil uses local_address instead of 37 | # laddr. 38 | 39 | (localipaddr, localport) = c.local_address[:] 40 | try: 41 | if c.raddr: 42 | (remoteipaddr, remoteport) = c.raddr[:] 43 | else: 44 | (remoteipaddr, remoteport) = (None, None) 45 | except: 46 | 47 | # Older version of psutil uses remote_address instead 48 | # of raddr. 49 | 50 | if c.remote_address: 51 | (remoteipaddr, remoteport) = \ 52 | c.remote_address[:] 53 | else: 54 | (remoteipaddr, remoteport) = (None, None) 55 | feature_key = '{0}/{1}/{2}'.format(pid, 56 | localipaddr, localport) 57 | return (feature_key, ConnectionFeature( 58 | localipaddr, 59 | localport, 60 | name, 61 | pid, 62 | remoteipaddr, 63 | remoteport, 64 | str(c.status), 65 | ), 'connection') 66 | -------------------------------------------------------------------------------- /crawler/utils/disk_utils.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | 3 | from utils.features import DiskFeature 4 | 5 | 6 | def crawl_disk_partitions(): 7 | partitions = [] 8 | for partition in psutil.disk_partitions(all=True): 9 | try: 10 | pdiskusage = psutil.disk_usage(partition.mountpoint) 11 | partitions.append((partition.mountpoint, DiskFeature( 12 | partition.device, 13 | 100.0 - pdiskusage.percent, 14 | partition.fstype, 15 | partition.mountpoint, 16 | partition.opts, 17 | pdiskusage.total, 18 | ), 'disk')) 19 | except OSError: 20 | continue 21 | return partitions 22 | -------------------------------------------------------------------------------- /crawler/utils/dockerevent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Docker container event object 3 | """ 4 | 5 | 6 | class DockerContainerEvent(object): 7 | def __init__(self, contId, imgId, event, etime): 8 | self.contId = contId 9 | self.imgId = imgId 10 | self.event = event 11 | self.eventTime = etime 12 | 13 | def get_containerid(self): 14 | return self.contId 15 | 16 | def get_imgageid(self): 17 | return self.imgId 18 | 19 | def get_event(self): 20 | return self.event 21 | 22 | def get_eventTime(self): 23 | return self.eventTime 24 | -------------------------------------------------------------------------------- /crawler/utils/metric_utils.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | 3 | from utils.features import MetricFeature 4 | 5 | 6 | def _crawl_metrics_cpu_percent(process): 7 | cpu_percent = ( 8 | process.get_cpu_percent( 9 | interval=0.1) if hasattr( 10 | process.get_cpu_percent, 11 | '__call__') else process.cpu_percent) 12 | return cpu_percent 13 | 14 | 15 | def crawl_metrics(): 16 | created_since = -1 17 | 18 | for p in psutil.process_iter(): 19 | create_time = ( 20 | p.create_time() if hasattr( 21 | p.create_time, 22 | '__call__') else p.create_time) 23 | if create_time <= created_since: 24 | continue 25 | 26 | name = (p.name() if hasattr(p.name, '__call__' 27 | ) else p.name) 28 | pid = (p.pid() if hasattr(p.pid, '__call__') else p.pid) 29 | status = (p.status() if hasattr(p.status, '__call__' 30 | ) else p.status) 31 | if status == psutil.STATUS_ZOMBIE: 32 | continue 33 | username = ( 34 | p.username() if hasattr( 35 | p.username, 36 | '__call__') else p.username) 37 | meminfo = ( 38 | p.get_memory_info() if hasattr( 39 | p.get_memory_info, 40 | '__call__') else p.memory_info) 41 | ioinfo = ( 42 | p.get_io_counters() if hasattr( 43 | p.get_io_counters, 44 | '__call__') else p.io_counters) 45 | 46 | cpu_percent = _crawl_metrics_cpu_percent(p) 47 | 48 | memory_percent = ( 49 | p.get_memory_percent() if hasattr( 50 | p.get_memory_percent, 51 | '__call__') else p.memory_percent) 52 | 53 | feature_key = '{0}/{1}'.format(name, pid) 54 | yield (feature_key, MetricFeature( 55 | round(cpu_percent, 2), 56 | round(memory_percent, 2), 57 | name, 58 | pid, 59 | ioinfo.read_bytes, 60 | meminfo.rss, 61 | str(status), 62 | username, 63 | meminfo.vms, 64 | ioinfo.write_bytes, 65 | ), 'metric') 66 | -------------------------------------------------------------------------------- /crawler/utils/nvidialib/lib/libnvidia-ml.so.375.66: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/crawler/utils/nvidialib/lib/libnvidia-ml.so.375.66 -------------------------------------------------------------------------------- /crawler/utils/nvidialib/lib64/libnvidia-ml.so.375.66: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/crawler/utils/nvidialib/lib64/libnvidia-ml.so.375.66 -------------------------------------------------------------------------------- /crawler/utils/os_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import platform 3 | import time 4 | 5 | import psutil 6 | 7 | import utils.misc 8 | from utils import osinfo 9 | from utils.features import OSFeature 10 | 11 | logger = logging.getLogger('crawlutils') 12 | 13 | 14 | def crawl_os(): 15 | feature_key = platform.system().lower() 16 | try: 17 | os_kernel = platform.platform() 18 | except: 19 | os_kernel = 'unknown' 20 | 21 | result = osinfo.get_osinfo(mount_point='/') 22 | os_distro = result['os'] if 'os' in result else 'unknown' 23 | os_version = result['version'] if 'version' in result else 'unknown' 24 | 25 | ips = utils.misc.get_host_ip4_addresses() 26 | 27 | boot_time = psutil.boot_time() 28 | uptime = int(time.time()) - boot_time 29 | feature_attributes = OSFeature( 30 | boot_time, 31 | uptime, 32 | ips, 33 | os_distro, 34 | os_version, 35 | os_kernel, 36 | platform.machine() 37 | ) 38 | 39 | return [(feature_key, feature_attributes, 'os')] 40 | 41 | 42 | def crawl_os_mountpoint(mountpoint='/'): 43 | result = osinfo.get_osinfo(mount_point=mountpoint) 44 | os_distro = result['os'] if 'os' in result else 'unknown' 45 | os_version = result['version'] if 'version' in result else 'unknown' 46 | 47 | feature_key = 'linux' 48 | feature_attributes = OSFeature( # boot time unknown for img 49 | # live IP unknown for img 50 | 'unsupported', 51 | 'unsupported', 52 | '0.0.0.0', 53 | os_distro, 54 | os_version, 55 | 'unknown', 56 | 'unknown' 57 | ) 58 | return [(feature_key, feature_attributes, 'os')] 59 | -------------------------------------------------------------------------------- /crawler/utils/socket_utils.py: -------------------------------------------------------------------------------- 1 | import ctypes.util 2 | import errno 3 | import os 4 | import socket 5 | 6 | libc = ctypes.CDLL(ctypes.util.find_library('c')) 7 | 8 | 9 | def if_indextoname(ifindex): 10 | libc.if_indextoname.argtypes = [ctypes.c_uint32, ctypes.c_char_p] 11 | libc.if_indextoname.restype = ctypes.c_char_p 12 | 13 | ifname = ctypes.create_string_buffer(16) 14 | ifname = libc.if_indextoname(ifindex, ifname) 15 | if not ifname: 16 | err = errno.ENXIO 17 | raise OSError(err, os.strerror(err)) 18 | return ifname 19 | 20 | 21 | def open_udp_port(bindaddr, min, max): 22 | """ 23 | Try to open a UDP listening port in the given range 24 | """ 25 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 26 | for port in range(min, max + 1): 27 | try: 28 | sock.bind((bindaddr, port)) 29 | return sock, port 30 | except: 31 | pass 32 | 33 | sock.close() 34 | return None, None 35 | -------------------------------------------------------------------------------- /crawler/utils/timeout_utils.py: -------------------------------------------------------------------------------- 1 | import signal 2 | 3 | 4 | class TimeoutError(Exception): 5 | pass 6 | 7 | 8 | class Timeout: 9 | def __init__(self, seconds=0.1, error_message='Timeout'): 10 | self.seconds = seconds 11 | self.error_message = error_message 12 | 13 | def handle_timeout(self, signum, frame): 14 | raise TimeoutError(self.error_message) 15 | 16 | def __enter__(self): 17 | signal.signal(signal.SIGALRM, self.handle_timeout) 18 | signal.setitimer(signal.ITIMER_REAL, self.seconds) 19 | 20 | def __exit__(self, type, value, traceback): 21 | signal.alarm(0) 22 | -------------------------------------------------------------------------------- /dependencies/python-conntrackprobe_0.2.1-1_all.deb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/dependencies/python-conntrackprobe_0.2.1-1_all.deb -------------------------------------------------------------------------------- /dependencies/python-socket-datacollector_0.1.4-1_all.deb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/dependencies/python-socket-datacollector_0.1.4-1_all.deb -------------------------------------------------------------------------------- /dependencies/softflowd_0.9.9902-1_amd64.deb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/dependencies/softflowd_0.9.9902-1_amd64.deb -------------------------------------------------------------------------------- /logstash-http-plugin.conf: -------------------------------------------------------------------------------- 1 | input { http { } } 2 | 3 | output { 4 | elasticsearch { 5 | hosts => ["localhost:9200"] 6 | codec => json 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | simplejson==3.8.2 3 | flake8==2.6.2 4 | pylint==1.6.2 5 | pytest==2.9.2 6 | pytest-cov==2.3.0 7 | mock==2.0.0 8 | bandit==1.4.0 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | psutil==2.2.0 2 | requests>=2.7.13 3 | netifaces==0.10.4 4 | kafka-python==1.3.1 5 | pykafka==1.1.0 6 | kafka==1.3.3 7 | docker-py==1.10.6 8 | python-dateutil==2.4.2 9 | semantic_version==2.5.0 10 | Yapsy==1.11.223 11 | configobj==4.7.0 12 | morph==0.1.2 13 | fluent-logger==0.4.6 14 | requests_unixsocket==0.1.5 15 | minio==4.0.18 16 | urllib3==1.24.2 17 | -------------------------------------------------------------------------------- /sample-k8s-deamon-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: crawler 5 | labels: 6 | tier: monitoring 7 | app: crawler 8 | version: v1 9 | spec: 10 | template: 11 | metadata: 12 | labels: 13 | name: crawler 14 | spec: 15 | hostPID: true 16 | hostIPC: true 17 | hostNetwork: true 18 | containers: 19 | - resources: 20 | requests: 21 | cpu: 0.15 22 | securityContext: 23 | privileged: true 24 | # crawler image path from image registry 25 | image: cloudviz/agentless-system-crawler 26 | name: crawler 27 | command: ["python2.7", "crawler.py", 28 | "--crawlmode", "OUTCONTAINER", 29 | "--url", "file:///tmp/test.csv", 30 | "--features", "os,disk", 31 | "--frequency", "60", 32 | "--environment", "kubernetes" ] 33 | volumeMounts: 34 | - name: crawler-cgroup 35 | mountPath: /cgroup 36 | readOnly: true 37 | - name: crawler-fs-cgroup 38 | mountPath: /sys/fs/cgroup 39 | readOnly: true 40 | - name: docker-home 41 | mountPath: /var/lib/docker 42 | readOnly: true 43 | - name: docker-sock 44 | mountPath: /var/run/docker.sock 45 | - name: crawler-output 46 | mountPath: /tmp 47 | - name: sas-secrets 48 | mountPath: /etc/sas-secrets 49 | volumes: 50 | - name: crawler-cgroup 51 | hostPath: 52 | path: /cgroup 53 | - name: crawler-fs-cgroup 54 | hostPath: 55 | path: /sys/fs/cgroup 56 | - name: docker-home 57 | hostPath: 58 | path: /var/lib/docker 59 | - name: docker-sock 60 | hostPath: 61 | path: /var/run/docker.sock 62 | - name: crawler-output 63 | hostPath: 64 | path: /tmp 65 | - name: sas-secrets 66 | secret: 67 | secretName: "bluemix-secret" 68 | -------------------------------------------------------------------------------- /sample-sas-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: bluemix-secret 5 | type: Opaque 6 | data: 7 | cloudoe: 8 | access_group: 9 | token: 10 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | from setuptools import setup, find_packages 3 | 4 | setup( 5 | name='agentless-system-crawler', 6 | version='0.0.1.dev0', 7 | # description='', 8 | # long_description=long_description, 9 | author='IBM', 10 | # author_email='', 11 | license='apache2', 12 | classifiers=[ 13 | 'Development Status :: 2 - Pre-Alpha', 14 | 'Intended Audience :: System Administrators', 15 | 'License :: OSI Approved :: Apache Software License', 16 | 'Programming Language :: Python :: 2.7', 17 | ], 18 | packages=find_packages(), 19 | install_requires=['psutil', 'netifaces', ], 20 | setup_requires=['pytest-runner>=2.0,<3dev', ], 21 | tests_require=['pytest', ], 22 | use_2to3=True, 23 | data_files=[('plugins', ['plugins/cloudsight_environment.plugin', 24 | 'plugins/cloudsight_environment.py'])] 25 | ) 26 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('crawler') 3 | -------------------------------------------------------------------------------- /tests/functional/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *alchemy* 3 | *.log* 4 | -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/tests/functional/__init__.py -------------------------------------------------------------------------------- /tests/functional/setup_logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import logging.handlers 4 | import sys 5 | 6 | 7 | def setup_logger(logger_name, logfile='crawler.log'): 8 | _logger = logging.getLogger(logger_name) 9 | _logger.setLevel(logging.INFO) 10 | logfile_name, logfile_xtnsion = os.path.splitext(logfile) 11 | fname = logfile 12 | h = logging.handlers.RotatingFileHandler( 13 | filename=fname, maxBytes=10e6, backupCount=1) 14 | f = logging.Formatter( 15 | '%(asctime)s %(processName)-10s %(levelname)-8s %(message)s') 16 | h.setFormatter(f) 17 | _logger.addHandler(h) 18 | 19 | 20 | def setup_logger_stdout(logger_name): 21 | _logger = logging.getLogger(logger_name) 22 | _logger.setLevel(logging.DEBUG) 23 | 24 | ch = logging.StreamHandler(sys.stdout) 25 | ch.setLevel(logging.DEBUG) 26 | formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') 27 | ch.setFormatter(formatter) 28 | _logger.addHandler(ch) 29 | -------------------------------------------------------------------------------- /tests/functional/test_functional_nodepackage_plugin.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import docker 3 | import requests.exceptions 4 | from plugins.systems.nodepackage_container_crawler import NodePackageCrawler 5 | 6 | 7 | # Tests conducted with a single container running. 8 | class NodePackagePluginFunctionalTests(unittest.TestCase): 9 | image_name = 'node:11.0' 10 | 11 | def setUp(self): 12 | self.docker = docker.Client( 13 | base_url='unix://var/run/docker.sock', version='auto') 14 | try: 15 | if len(self.docker.containers()) != 0: 16 | raise Exception( 17 | "Sorry, this test requires a machine with no docker" 18 | "containers running.") 19 | except requests.exceptions.ConnectionError: 20 | print ("Error connecting to docker daemon, are you in the docker" 21 | "group? You need to be in the docker group.") 22 | 23 | self.docker.pull(repository='node', tag='11.0') 24 | self.container = self.docker.create_container( 25 | image=self.image_name, command='sleep 60') 26 | self.docker.start(container=self.container['Id']) 27 | 28 | def tearDown(self): 29 | self.docker.stop(container=self.container['Id']) 30 | self.docker.remove_container(container=self.container['Id']) 31 | 32 | def test_crawl_outcontainer_node(self): 33 | fc = NodePackageCrawler() 34 | output = list(fc.crawl(self.container['Id'])) 35 | num_packages = len(output) 36 | assert num_packages > 0 37 | output = "%s" % output 38 | assert 'npm' in output 39 | 40 | if __name__ == '__main__': 41 | unittest.main() 42 | -------------------------------------------------------------------------------- /tests/old/test_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | GREEN='\033[1;32m' 4 | RED='\033[0;31m' 5 | NC='\033[0m' # No Color 6 | 7 | DEBUG=0 8 | 9 | for i in `ls test*.sh | grep -v test_all.sh | grep -v kafka | grep -v test_alchemy_crawler_services` 10 | do 11 | printf "%s " $i 12 | if [ $DEBUG == "1" ] 13 | then 14 | R=`bash -x $i` 15 | else 16 | R=`bash $i` 17 | fi 18 | if [ $R == "1" ] 19 | then 20 | printf "${GREEN}%s${NC}\n" $R 21 | else 22 | printf "${RED}%s${NC}\n" $R 23 | if [ $DEBUG == "1" ] 24 | then 25 | exit 26 | fi 27 | fi 28 | done 29 | 30 | python2.7 test_dockerutils.py 31 | python2.7 test_emitter.py 32 | python2.7 test_features_crawler.py 33 | python2.7 test_namespace.py 34 | 35 | -------------------------------------------------------------------------------- /tests/old/test_crawl_all_features_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_cpu_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_cpu_container_1 ubuntu bash -c "echo bla >> /etc/ric_config; mkdir -p /bla; echo bla >> /bla/ble; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_cpu_container_1` 14 | 15 | rm -f /tmp/test_crawl_all_features_container* 16 | 17 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 18 | --features=cpu,memory,os,config,file,package,dockerinspect,dockerhistory,metric,load --crawlContainers $ID \ 19 | --url file:///tmp/test_crawl_all_features_container --options \ 20 | '{"config": {"known_config_files":["etc/ric_config"]}, "file": {"root_dir": "/bla/"}}' 21 | 22 | COUNT_1=`grep ^cpu /tmp/test_crawl_all_features_container* | wc -l` 23 | COUNT_2=`grep ^memory /tmp/test_crawl_all_features_container* | wc -l` 24 | COUNT_3=`grep ^os /tmp/test_crawl_all_features_container* | wc -l` 25 | 26 | # This sholuld be just and exactly 1, as ther eis only /etc/ric_config 27 | COUNT_4=`grep ^config /tmp/test_crawl_all_features_container* | wc -l` 28 | 29 | # This should be just and exactly 2, as there is only /bla and /bla/ble 30 | COUNT_5=`grep ^file /tmp/test_crawl_all_features_container* | wc -l` 31 | COUNT_6=`grep ^package /tmp/test_crawl_all_features_container* | wc -l` 32 | COUNT_7=`grep ^dockerinspect /tmp/test_crawl_all_features_container* | wc -l` 33 | COUNT_8=`grep ^dockerhistory /tmp/test_crawl_all_features_container* | wc -l` 34 | COUNT_9=`grep ^metric /tmp/test_crawl_all_features_container* | wc -l` 35 | COUNT_10=`grep ^load /tmp/test_crawl_all_features_container* | wc -l` 36 | 37 | if [ $COUNT_1 -gt "0" ] && \ 38 | [ $COUNT_2 -eq "1" ] && \ 39 | [ $COUNT_3 -eq "1" ] && \ 40 | [ $COUNT_4 -eq "1" ] && \ 41 | [ $COUNT_5 -eq "2" ] && \ 42 | [ $COUNT_6 -gt "0" ] && \ 43 | [ $COUNT_7 -eq "1" ] && \ 44 | [ $COUNT_8 -gt "0" ] && \ 45 | [ $COUNT_9 -gt "0" ] && \ 46 | [ $COUNT_10 -eq "1" ] 47 | then 48 | echo 1 49 | else 50 | echo 0 51 | fi 52 | 53 | docker rm -f test_crawl_cpu_container_1 > /dev/null 54 | rm -f /tmp/test_crawl_all_features_container* 55 | -------------------------------------------------------------------------------- /tests/old/test_crawl_all_features_container_crash_failure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_cpu_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_cpu_container_1 ubuntu bash -c "echo bla >> /etc/ric_config; mkdir -p /bla; echo bla >> /bla/ble; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_cpu_container_1` 14 | 15 | rm -f /tmp/test_crawl_all_features_container* 16 | 17 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 18 | --features=cpu,memory,os,config,file,package,dockerinspect,_test_crash,dockerhistory,metric,load --crawlContainers $ID \ 19 | --url file:///tmp/test_crawl_all_features_container --options \ 20 | '{"config": {"known_config_files":["etc/ric_config"]}, "file": {"root_dir": "/bla/"}}' 21 | 22 | COUNT_1=`grep ^cpu /tmp/test_crawl_all_features_container* | wc -l` 23 | COUNT_2=`grep ^memory /tmp/test_crawl_all_features_container* | wc -l` 24 | COUNT_3=`grep ^os /tmp/test_crawl_all_features_container* | wc -l` 25 | 26 | # This sholuld be just and exactly 1, as ther eis only /etc/ric_config 27 | COUNT_4=`grep ^config /tmp/test_crawl_all_features_container* | wc -l` 28 | 29 | # This should be just and exactly 2, as there is only /bla and /bla/ble 30 | COUNT_5=`grep ^file /tmp/test_crawl_all_features_container* | wc -l` 31 | COUNT_6=`grep ^package /tmp/test_crawl_all_features_container* | wc -l` 32 | COUNT_7=`grep ^dockerinspect /tmp/test_crawl_all_features_container* | wc -l` 33 | COUNT_8=`grep ^dockerhistory /tmp/test_crawl_all_features_container* | wc -l` 34 | COUNT_9=`grep ^metric /tmp/test_crawl_all_features_container* | wc -l` 35 | COUNT_10=`grep ^load /tmp/test_crawl_all_features_container* | wc -l` 36 | 37 | if [ $COUNT_1 -gt "0" ] && \ 38 | [ $COUNT_2 -eq "1" ] && \ 39 | [ $COUNT_3 -eq "1" ] && \ 40 | [ $COUNT_4 -eq "1" ] && \ 41 | [ $COUNT_5 -eq "2" ] && \ 42 | [ $COUNT_6 -gt "0" ] && \ 43 | [ $COUNT_7 -eq "1" ] && \ 44 | [ $COUNT_8 -gt "0" ] && \ 45 | [ $COUNT_9 -gt "0" ] && \ 46 | [ $COUNT_10 -eq "1" ] 47 | then 48 | echo 1 49 | else 50 | echo 0 51 | fi 52 | 53 | docker rm -f test_crawl_cpu_container_1 > /dev/null 54 | rm -f /tmp/test_crawl_all_features_container* 55 | -------------------------------------------------------------------------------- /tests/old/test_crawl_all_features_container_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_cpu_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_cpu_container_1 ubuntu bash -c "echo bla >> /etc/ric_config; mkdir -p /bla; echo bla >> /bla/ble; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_cpu_container_1` 14 | 15 | rm -f /tmp/test_crawl_all_features_container* 16 | 17 | timeout 10 python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 18 | --features=cpu,memory,os,config,file,package,dockerinspect,dockerhistory,metric,load --crawlContainers $ID \ 19 | --url file:///tmp/test_crawl_all_features_container --frequency 1 --options \ 20 | '{"config": {"known_config_files":["etc/ric_config"]}, "file": {"root_dir": "/bla/"}}' 21 | 22 | COUNT_1=`grep ^cpu /tmp/test_crawl_all_features_container* | wc -l` 23 | COUNT_2=`grep ^memory /tmp/test_crawl_all_features_container* | wc -l` 24 | COUNT_3=`grep ^os /tmp/test_crawl_all_features_container* | wc -l` 25 | COUNT_4=`grep ^config /tmp/test_crawl_all_features_container* | wc -l` 26 | COUNT_5=`grep ^file /tmp/test_crawl_all_features_container* | wc -l` 27 | COUNT_6=`grep ^package /tmp/test_crawl_all_features_container* | wc -l` 28 | COUNT_7=`grep ^dockerinspect /tmp/test_crawl_all_features_container* | wc -l` 29 | COUNT_8=`grep ^dockerhistory /tmp/test_crawl_all_features_container* | wc -l` 30 | COUNT_9=`grep ^metric /tmp/test_crawl_all_features_container* | wc -l` 31 | COUNT_10=`grep ^load /tmp/test_crawl_all_features_container* | wc -l` 32 | 33 | if [ $COUNT_1 -gt "2" ] && \ 34 | [ $COUNT_2 -gt "2" ] && \ 35 | [ $COUNT_3 -gt "2" ] && \ 36 | [ $COUNT_4 -gt "2" ] && \ 37 | [ $COUNT_5 -gt "2" ] && \ 38 | [ $COUNT_6 -gt "2" ] && \ 39 | [ $COUNT_7 -gt "2" ] && \ 40 | [ $COUNT_8 -gt "2" ] && \ 41 | [ $COUNT_9 -gt "2" ] && \ 42 | [ $COUNT_10 -gt "2" ] 43 | then 44 | echo 1 45 | else 46 | echo 0 47 | fi 48 | 49 | docker rm -f test_crawl_cpu_container_1 > /dev/null 50 | rm -f /tmp/test_crawl_all_features_container* 51 | -------------------------------------------------------------------------------- /tests/old/test_crawl_all_features_container_infinite_loop_failure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_cpu_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_cpu_container_1 ubuntu bash -c "echo bla >> /etc/ric_config; mkdir -p /bla; echo bla >> /bla/ble; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_cpu_container_1` 14 | 15 | rm -f /tmp/test_crawl_all_features_container* 16 | 17 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 18 | --features=cpu,memory,os,config,file,package,dockerinspect,_test_infinite_loop,dockerhistory,metric,load --crawlContainers $ID \ 19 | --url file:///tmp/test_crawl_all_features_container --options \ 20 | '{"config": {"known_config_files":["etc/ric_config"]}, "file": {"root_dir": "/bla/"}}' 21 | 22 | COUNT_1=`grep ^cpu /tmp/test_crawl_all_features_container* | wc -l` 23 | COUNT_2=`grep ^memory /tmp/test_crawl_all_features_container* | wc -l` 24 | COUNT_3=`grep ^os /tmp/test_crawl_all_features_container* | wc -l` 25 | 26 | # This sholuld be just and exactly 1, as ther eis only /etc/ric_config 27 | COUNT_4=`grep ^config /tmp/test_crawl_all_features_container* | wc -l` 28 | 29 | # This should be just and exactly 2, as there is only /bla and /bla/ble 30 | COUNT_5=`grep ^file /tmp/test_crawl_all_features_container* | wc -l` 31 | COUNT_6=`grep ^package /tmp/test_crawl_all_features_container* | wc -l` 32 | COUNT_7=`grep ^dockerinspect /tmp/test_crawl_all_features_container* | wc -l` 33 | COUNT_8=`grep ^dockerhistory /tmp/test_crawl_all_features_container* | wc -l` 34 | COUNT_9=`grep ^metric /tmp/test_crawl_all_features_container* | wc -l` 35 | COUNT_10=`grep ^load /tmp/test_crawl_all_features_container* | wc -l` 36 | 37 | if [ $COUNT_1 -gt "0" ] && \ 38 | [ $COUNT_2 -eq "1" ] && \ 39 | [ $COUNT_3 -eq "1" ] && \ 40 | [ $COUNT_4 -eq "1" ] && \ 41 | [ $COUNT_5 -eq "2" ] && \ 42 | [ $COUNT_6 -gt "0" ] && \ 43 | [ $COUNT_7 -eq "1" ] && \ 44 | [ $COUNT_8 -gt "0" ] && \ 45 | [ $COUNT_9 -gt "0" ] && \ 46 | [ $COUNT_10 -eq "1" ] 47 | then 48 | echo 1 49 | else 50 | echo 0 51 | fi 52 | 53 | docker rm -f test_crawl_cpu_container_1 > /dev/null 54 | rm -f /tmp/test_crawl_all_features_container* 55 | -------------------------------------------------------------------------------- /tests/old/test_crawl_all_features_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | # start a container for dockerps to return something 12 | docker rm -f test_crawl_all_features_invm 2> /dev/null > /dev/null 13 | docker run -d --name test_crawl_all_features_invm ubuntu bash -c "echo bla >> /etc/ric_config; mkdir -p /bla; echo bla >> /bla/ble; sleep 60" 2> /dev/null > /dev/null 14 | 15 | echo ble >> /etc/ric_config 16 | mkdir -p /root/bla 17 | echo bla >> /root/bla/ble 18 | 19 | python2.7 ../../crawler/crawler.py \ 20 | --features=cpu,memory,os,config,file,package,dockerps,metric,load \ 21 | --url file:///tmp/test_crawl_all_features_invm --options \ 22 | '{"config": {"known_config_files":["etc/ric_config"]}, "file": {"root_dir": "/root/bla/"}}' 23 | 24 | COUNT_1=`grep ^cpu /tmp/test_crawl_all_features_invm.0 | wc -l` 25 | COUNT_2=`grep ^memory /tmp/test_crawl_all_features_invm.0 | wc -l` 26 | COUNT_3=`grep ^os /tmp/test_crawl_all_features_invm.0 | wc -l` 27 | 28 | # This sholuld be just and exactly 1, as ther eis only /etc/ric_config 29 | COUNT_4=`grep ^config /tmp/test_crawl_all_features_invm.0 | wc -l` 30 | 31 | # This should be just and exactly 2, as there is only /bla and /bla/ble 32 | COUNT_5=`grep ^file /tmp/test_crawl_all_features_invm.0 | wc -l` 33 | COUNT_6=`grep ^package /tmp/test_crawl_all_features_invm.0 | wc -l` 34 | COUNT_7=`grep ^dockerps /tmp/test_crawl_all_features_invm.0 | wc -l` 35 | COUNT_8=`grep ^metric /tmp/test_crawl_all_features_invm.0 | wc -l` 36 | COUNT_9=`grep ^load /tmp/test_crawl_all_features_invm.0 | wc -l` 37 | 38 | if [ $COUNT_1 -gt "0" ] && \ 39 | [ $COUNT_2 -eq "1" ] && \ 40 | [ $COUNT_3 -eq "1" ] && \ 41 | [ $COUNT_4 -eq "1" ] && \ 42 | [ $COUNT_5 -eq "2" ] && \ 43 | [ $COUNT_6 -gt "0" ] && \ 44 | [ $COUNT_7 -gt "0" ] && \ 45 | [ $COUNT_8 -gt "0" ] && \ 46 | [ $COUNT_9 -eq "1" ] 47 | then 48 | echo 1 49 | else 50 | echo 0 51 | fi 52 | 53 | docker rm -f test_crawl_all_features_invm 2> /dev/null > /dev/null 54 | rm -f /tmp/test_crawl_all_features_invm* 55 | rm -rf /root/bla 56 | rm -f /etc/ric_config 57 | -------------------------------------------------------------------------------- /tests/old/test_crawl_all_features_invm_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | # start a container for dockerps to return something 12 | docker rm -f test_crawl_all_features_invm 2> /dev/null > /dev/null 13 | docker run -d --name test_crawl_all_features_invm ubuntu bash -c "echo bla >> /etc/ric_config; mkdir -p /bla; echo bla >> /bla/ble; sleep 60" 2> /dev/null > /dev/null 14 | 15 | echo ble >> /etc/ric_config 16 | mkdir -p /root/bla 17 | echo bla >> /root/bla/ble 18 | 19 | timeout 10 python2.7 ../../crawler/crawler.py \ 20 | --features=cpu,memory,os,config,file,package,dockerps,metric,load \ 21 | --url file:///tmp/test_crawl_all_features_invm --frequency 1 --options \ 22 | '{"config": {"known_config_files":["etc/ric_config"]}, "file": {"root_dir": "/root/bla/"}}' 23 | 24 | COUNT_1=`grep ^cpu /tmp/test_crawl_all_features_invm* | wc -l` 25 | COUNT_2=`grep ^memory /tmp/test_crawl_all_features_invm* | wc -l` 26 | COUNT_3=`grep ^os /tmp/test_crawl_all_features_invm* | wc -l` 27 | COUNT_4=`grep ^config /tmp/test_crawl_all_features_invm* | wc -l` 28 | COUNT_5=`grep ^file /tmp/test_crawl_all_features_invm* | wc -l` 29 | COUNT_6=`grep ^package /tmp/test_crawl_all_features_invm* | wc -l` 30 | COUNT_7=`grep ^dockerps /tmp/test_crawl_all_features_invm* | wc -l` 31 | COUNT_8=`grep ^metric /tmp/test_crawl_all_features_invm* | wc -l` 32 | COUNT_9=`grep ^load /tmp/test_crawl_all_features_invm* | wc -l` 33 | 34 | if [ $COUNT_1 -gt "1" ] && \ 35 | [ $COUNT_2 -gt "1" ] && \ 36 | [ $COUNT_3 -gt "1" ] && \ 37 | [ $COUNT_4 -gt "1" ] && \ 38 | [ $COUNT_5 -gt "1" ] && \ 39 | [ $COUNT_6 -gt "1" ] && \ 40 | [ $COUNT_7 -gt "1" ] && \ 41 | [ $COUNT_8 -gt "1" ] && \ 42 | [ $COUNT_9 -gt "1" ] 43 | then 44 | echo 1 45 | else 46 | echo 0 47 | fi 48 | 49 | docker rm -f test_crawl_all_features_invm 2> /dev/null > /dev/null 50 | rm -f /tmp/test_crawl_all_features_invm* 51 | rm -rf /root/bla 52 | rm -f /etc/ric_config 53 | -------------------------------------------------------------------------------- /tests/old/test_crawl_config_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_config_container_1 2> /dev/null > /dev/null 12 | UUID=`uuidgen` 13 | docker run -d --name test_crawl_config_container_1 ubuntu bash -c "echo $UUID > /etc/ric_config; sleep 60" 2> /dev/null > /dev/null 14 | ID=`docker inspect -f '{{ .Id }}' test_crawl_config_container_1` 15 | 16 | rm -f /tmp/test_crawl_config_container 17 | 18 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 19 | --features=config --crawlContainers $ID --options '{"config": {"known_config_files":["etc/ric_config"]}}' \ 20 | > /tmp/test_crawl_config_container 21 | 22 | COUNT_MSG=`grep $UUID /tmp/test_crawl_config_container | wc -l` 23 | COUNT_FEATURES=`grep '^config' /tmp/test_crawl_config_container | wc -l` 24 | 25 | # 2 = METADATA + FEATURE 26 | COUNT_TOTAL=`cat /tmp/test_crawl_config_container | wc -l` 27 | 28 | if [ $COUNT_MSG == "1" ] && [ $COUNT_FEATURES == "1" ] && [ $COUNT_TOTAL == "2" ] 29 | then 30 | echo 1 31 | else 32 | echo 0 33 | fi 34 | docker rm -f test_crawl_config_container_1 > /dev/null 35 | -------------------------------------------------------------------------------- /tests/old/test_crawl_config_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | UUID=`uuidgen` 12 | echo $UUID >> /etc/ric_config 13 | 14 | python2.7 ../../crawler/crawler.py --crawlmode INVM \ 15 | --features=config --options '{"config": {"known_config_files":["etc/ric_config"]}}' \ 16 | > /tmp/test_crawl_config_invm 17 | 18 | COUNT_MSG=`grep $UUID /tmp/test_crawl_config_invm | wc -l` 19 | COUNT_FEATURES=`grep '^config' /tmp/test_crawl_config_invm | wc -l` 20 | 21 | # 2 = METADATA + FEATURE 22 | COUNT_TOTAL=`cat /tmp/test_crawl_config_invm | wc -l` 23 | 24 | if [ $COUNT_MSG == "1" ] && [ $COUNT_FEATURES == "1" ] && [ $COUNT_TOTAL == "2" ] 25 | then 26 | echo 1 27 | else 28 | echo 0 29 | fi 30 | 31 | -------------------------------------------------------------------------------- /tests/old/test_crawl_config_mountpoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | UUID=`uuidgen` 12 | echo $UUID >> /etc/ric_config 13 | 14 | python2.7 ../../crawler/crawler.py --crawlmode MOUNTPOINT \ 15 | --features=config --options '{"config": {"known_config_files":["etc/ric_config"]}}' \ 16 | --mountpoint / > /tmp/test_crawl_config_mountpoint 17 | 18 | COUNT_MSG=`grep $UUID /tmp/test_crawl_config_mountpoint | wc -l` 19 | COUNT_FEATURES=`grep '^config' /tmp/test_crawl_config_mountpoint | wc -l` 20 | 21 | # 2 = METADATA + FEATURE 22 | COUNT_TOTAL=`cat /tmp/test_crawl_config_mountpoint | wc -l` 23 | 24 | if [ $COUNT_MSG == "1" ] && [ $COUNT_FEATURES == "1" ] && [ $COUNT_TOTAL == "2" ] 25 | then 26 | echo 1 27 | else 28 | echo 0 29 | fi 30 | 31 | -------------------------------------------------------------------------------- /tests/old/test_crawl_config_mountpoint_relative.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | UUID=`uuidgen` 12 | mkdir -p /etc/etc2 13 | echo $UUID >> /etc/etc2/ric_config 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode MOUNTPOINT \ 16 | --features=config --options '{"config": {"known_config_files":["etc2/ric_config"]}}' \ 17 | --mountpoint /etc/ > /tmp/test_crawl_config_mountpoint_relative 18 | 19 | COUNT_MSG=`grep $UUID /tmp/test_crawl_config_mountpoint_relative | wc -l` 20 | COUNT_FEATURES=`grep '^config' /tmp/test_crawl_config_mountpoint_relative | wc -l` 21 | 22 | # 2 = METADATA + FEATURE 23 | COUNT_TOTAL=`cat /tmp/test_crawl_config_mountpoint_relative | wc -l` 24 | 25 | if [ $COUNT_MSG == "1" ] && [ $COUNT_FEATURES == "1" ] && [ $COUNT_TOTAL == "2" ] 26 | then 27 | echo 1 28 | else 29 | echo 0 30 | fi 31 | 32 | rm -rf /etc/etc2 33 | -------------------------------------------------------------------------------- /tests/old/test_crawl_connections_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_file_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_file_container_1 ubuntu:14.04 bash -c "nc -l -p 7777; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_file_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=connection --crawlContainers $ID | grep -c '"localport":7777' 17 | 18 | docker rm -f test_crawl_file_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_connections_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | nc -l -p 7777 2> /dev/null > /dev/null & 12 | PID=$! 13 | 14 | COUNT=`python2.7 ../../crawler/crawler.py --crawlmode INVM \ 15 | --features=connection | grep -c ^connection` 16 | 17 | # Any VM should have at least a pair of connections 18 | if [ $COUNT -gt "0" ] 19 | then 20 | echo 1 21 | else 22 | echo 0 23 | fi 24 | 25 | # Just avoid having the "Terminated ..." error showing up 26 | exec 2> /dev/null 27 | kill -9 $PID > /dev/null 2> /dev/null 28 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_cpu_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_cpu_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_cpu_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=cpu --crawlContainers $ID | grep -c cpu-0 17 | 18 | docker rm -f test_crawl_cpu_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_container_dockerlog_links.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the --linkContainerLogFiles option for the OUTCONTAINERcrawler mode . 4 | # This option maintains symlinks for some logfiles inside the container. By 5 | # default /var/log/messages and the docker (for all containers) are symlinked 6 | # to a central location: /var/log/crawl_container_logs/... 7 | # Returns 1 if success, 0 otherwise 8 | 9 | if [[ $EUID -ne 0 ]]; then 10 | echo "This script must be run as root" 11 | exit 1 12 | fi 13 | 14 | # The docker log link will be created here: 15 | # /var/log/crawler_container_logs///docker.log 16 | # I have some python code to get the local host IP. XXX replace it with bash 17 | read -d '' GET_HOST_IP_PY <<"EOF" 18 | import socket 19 | def get_host_ipaddr(): 20 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 21 | try: 22 | s.connect(('www.ibm.com', 9)) 23 | return s.getsockname()[0] 24 | except socket.error: 25 | return socket.gethostname() 26 | finally: 27 | del s 28 | print get_host_ipaddr() 29 | EOF 30 | 31 | HOST_IP=`python2.7 -c "$GET_HOST_IP_PY" 2> /dev/null` 32 | 33 | MSG=`uuidgen` 34 | NAME=test_crawl_cpu_container_log_links_1 35 | rm -rf /var/log/crawler_container_logs/$HOST_IP/$NAME/ 36 | docker rm -f $NAME 2> /dev/null > /dev/null 37 | docker run -d --name $NAME ubuntu bash -c "echo $MSG ; sleep 6000 " 2> /dev/null > /dev/null 38 | ID=`docker inspect -f '{{ .Id }}' $NAME` 39 | 40 | python2.7 ../../crawler/containers_logs_linker.py --crawlContainers $ID 41 | 42 | grep -c $MSG /var/log/crawler_container_logs/$HOST_IP/$NAME/docker.log 43 | 44 | docker rm -f $NAME > /dev/null 45 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_container_dockerlog_links_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the --linkContainerLogFiles option for the OUTCONTAINERcrawler mode . 4 | # This option maintains symlinks for some logfiles inside the container. By 5 | # default /var/log/messages and the docker (for all containers) are symlinked 6 | # to a central location: /var/log/crawl_container_logs/... 7 | # Returns 1 if success, 0 otherwise 8 | 9 | if [[ $EUID -ne 0 ]]; then 10 | echo "This script must be run as root" 11 | exit 1 12 | fi 13 | 14 | # The docker log link will be created here: 15 | # /var/log/crawler_container_logs///docker.log 16 | # I have some python code to get the local host IP. XXX replace it with bash 17 | read -d '' GET_HOST_IP_PY <<"EOF" 18 | import socket 19 | def get_host_ipaddr(): 20 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 21 | try: 22 | s.connect(('www.ibm.com', 9)) 23 | return s.getsockname()[0] 24 | except socket.error: 25 | return socket.gethostname() 26 | finally: 27 | del s 28 | print get_host_ipaddr() 29 | EOF 30 | 31 | HOST_IP=`python2.7 -c "$GET_HOST_IP_PY" 2> /dev/null` 32 | 33 | MSG=`uuidgen` 34 | NAME=test_crawl_cpu_container_log_links_1 35 | rm -rf /var/log/crawler_container_logs/$HOST_IP/$NAME/ 36 | docker rm -f $NAME 2> /dev/null > /dev/null 37 | docker run -d --name $NAME ubuntu bash -c "echo $MSG ; sleep 5; echo $MSG ; sleep 6000 " 2> /dev/null > /dev/null 38 | ID=`docker inspect -f '{{ .Id }}' $NAME` 39 | 40 | timeout 10 python2.7 ../../crawler/containers_logs_linker.py --frequency 1 2> /dev/null > /dev/null 41 | 42 | COUNT=`grep -c $MSG /var/log/crawler_container_logs/$HOST_IP/$NAME/docker.log` 43 | 44 | if [ $COUNT == "2" ] 45 | then 46 | echo 1 47 | else 48 | echo 0 49 | fi 50 | 51 | docker rm -f $NAME > /dev/null 52 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_container_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | COUNT=10 12 | docker rm -f test_crawl_cpu_container_1 2> /dev/null > /dev/null 13 | docker run -d --name test_crawl_cpu_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 14 | ID=`docker inspect -f '{{ .Id }}' test_crawl_cpu_container_1` 15 | 16 | COUNT2=`timeout $COUNT python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 17 | --features=cpu --crawlContainers $ID --frequency 1 | grep -c cpu-0` 18 | 19 | docker rm -f test_crawl_cpu_container_1 > /dev/null 20 | 21 | #if [ $COUNT == $COUNT2 ] 22 | # sometimes we can't crawl that fast 23 | if [ "6" -lt $COUNT2 ] 24 | then 25 | echo 1 26 | else 27 | echo 0 28 | fi 29 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_container_gzip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_cpu_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_cpu_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_cpu_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=cpu --crawlContainers $ID --compress | gunzip 2>/dev/null | grep -c cpu-0 17 | 18 | docker rm -f test_crawl_cpu_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the INVM crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | NUM_CORES=`grep ^processor /proc/cpuinfo | wc -l` 12 | 13 | 14 | CRAWLED_NUM_CORES=`python2.7 ../../crawler/crawler.py --crawlmode INVM \ 15 | --features=cpu | grep -c cpu-` 16 | 17 | if [ $NUM_CORES == $CRAWLED_NUM_CORES ] 18 | then 19 | echo 1 20 | else 21 | echo 0 22 | fi 23 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode for 32 containers 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | COUNT=4 12 | 13 | for i in `seq 1 $COUNT` 14 | do 15 | docker rm -f test_crawl_cpu_many_containers_$i 2> /dev/null > /dev/null 16 | docker run -d --name test_crawl_cpu_many_containers_$i ubuntu sleep 60 2> /dev/null > /dev/null 17 | done 18 | 19 | IDS=`docker ps | grep test_crawl_cpu_many_containers | awk '{printf "%s,", $1}' | sed s/,$//g` 20 | 21 | COUNT2=`python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 22 | --features=cpu --crawlContainers $IDS --numprocesses 2 | grep -c cpu-0` 23 | 24 | for i in `seq 1 $COUNT` 25 | do 26 | docker rm -f test_crawl_cpu_many_containers_$i > /dev/null 27 | done 28 | 29 | if [ $COUNT == $COUNT2 ] 30 | then 31 | echo 1 32 | else 33 | echo 0 34 | fi 35 | 36 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers_check_partitioning.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode for 32 containers 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | COUNT=8 12 | LOG_FILE=/tmp/test_crawl_cpu_many_containers_check_partitioning 13 | 14 | rm -f ${LOG_FILE}* 15 | 16 | for i in `seq 1 $COUNT` 17 | do 18 | docker rm -f test_crawl_cpu_many_containers_$i 2> /dev/null > /dev/null 19 | docker run -d --name test_crawl_cpu_many_containers_$i ubuntu sleep 60 2> /dev/null > /dev/null 20 | done 21 | 22 | IDS=`docker ps | grep test_crawl_cpu_many_containers | awk '{printf "%s,", $1}' | sed s/,$//g` 23 | 24 | rm -f *.log 25 | 26 | COUNT2=`python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 27 | --features=cpu --crawlContainers $IDS --numprocesses 2 \ 28 | --logfile ${LOG_FILE}.log | grep -c cpu-0` 29 | 30 | # Check that each crawler process got at least one container assigned to it. 31 | # It would be very VERY unlucky for one of them to get the 8 containers. 32 | COUNT_LOGS_1=`grep -c "Emitted 2 features" ${LOG_FILE}-0.log` 33 | COUNT_LOGS_2=`grep -c "Emitted 2 features" ${LOG_FILE}-1.log` 34 | 35 | COUNT_LOGS_TOTAL=$(($COUNT_LOGS_1 + $COUNT_LOGS_2)) 36 | 37 | for i in `seq 1 $COUNT` 38 | do 39 | docker rm -f test_crawl_cpu_many_containers_$i > /dev/null 40 | done 41 | 42 | if [ $COUNT == $COUNT2 ] && [ $COUNT_LOGS_1 -gt "0" ] && [ $COUNT_LOGS_2 -gt "0" ] && [ $COUNT_LOGS_TOTAL == $COUNT ] 43 | then 44 | echo 1 45 | else 46 | echo 0 47 | fi 48 | 49 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers_dockerlog_links.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode for 32 containers 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | # The docker log link will be created here: 12 | # /var/log/crawler_container_logs///docker.log 13 | # I have some python code to get the local host IP. XXX replace it with bash 14 | read -d '' GET_HOST_IP_PY <<"EOF" 15 | import socket 16 | def get_host_ipaddr(): 17 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 18 | try: 19 | s.connect(('www.ibm.com', 9)) 20 | return s.getsockname()[0] 21 | except socket.error: 22 | return socket.gethostname() 23 | finally: 24 | del s 25 | print get_host_ipaddr() 26 | EOF 27 | 28 | HOST_IP=`python2.7 -c "$GET_HOST_IP_PY"` 29 | 30 | COUNT=4 31 | MSG=`uuidgen` 32 | 33 | for i in `seq 1 $COUNT` 34 | do 35 | rm -rf /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i 36 | docker rm -f test_crawl_cpu_many_containers_$i 2> /dev/null > /dev/null 37 | docker run -d --name test_crawl_cpu_many_containers_$i ubuntu bash -c "echo $MSG ; sleep 6000 " 2> /dev/null > /dev/null 38 | done 39 | 40 | IDS=`docker ps | grep test_crawl_cpu_many_containers | awk '{printf "%s,", $1}' | sed s/,$//g` 41 | 42 | python2.7 ../../crawler/containers_logs_linker.py --crawlContainers $IDS 2> /dev/null > /dev/null 43 | 44 | COUNT2=0 45 | for i in `seq 1 $COUNT` 46 | do 47 | R=`grep -c $MSG /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i/docker.log` 48 | COUNT2=$(($COUNT2 + $R)) 49 | done 50 | 51 | for i in `seq 1 $COUNT` 52 | do 53 | docker rm -f test_crawl_cpu_many_containers_$i > /dev/null 54 | done 55 | 56 | if [ $COUNT == $COUNT2 ] 57 | then 58 | echo 1 59 | else 60 | echo 0 61 | fi 62 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers_dockerlog_links_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode for 32 containers 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | # The docker log link will be created here: 12 | # /var/log/crawler_container_logs///docker.log 13 | # I have some python code to get the local host IP. XXX replace it with bash 14 | read -d '' GET_HOST_IP_PY <<"EOF" 15 | import socket 16 | def get_host_ipaddr(): 17 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 18 | try: 19 | s.connect(('www.ibm.com', 9)) 20 | return s.getsockname()[0] 21 | except socket.error: 22 | return socket.gethostname() 23 | finally: 24 | del s 25 | print get_host_ipaddr() 26 | EOF 27 | 28 | HOST_IP=`python2.7 -c "$GET_HOST_IP_PY"` 29 | 30 | COUNT=4 31 | MSG=`uuidgen` 32 | 33 | for i in `seq 1 $COUNT` 34 | do 35 | rm -rf /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i 36 | docker rm -f test_crawl_cpu_many_containers_$i 2> /dev/null > /dev/null 37 | docker run -d --name test_crawl_cpu_many_containers_$i ubuntu bash -c "echo $MSG ; sleep 5 ; echo $MSG ; sleep 6000 " 2> /dev/null > /dev/null 38 | done 39 | 40 | IDS=`docker ps | grep test_crawl_cpu_many_containers | awk '{printf "%s,", $1}' | sed s/,$//g` 41 | 42 | timeout 10 python2.7 ../../crawler/containers_logs_linker.py --crawlContainers $IDS --frequency 1 2> /dev/null > /dev/null 43 | 44 | COUNT2=0 45 | for i in `seq 1 $COUNT` 46 | do 47 | R=`grep -c $MSG /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i/docker.log` 48 | COUNT2=$(($COUNT2 + $R)) 49 | done 50 | 51 | for i in `seq 1 $COUNT` 52 | do 53 | docker rm -f test_crawl_cpu_many_containers_$i > /dev/null 54 | done 55 | 56 | # In those 10 seconds, the containers had 2 logs 57 | COUNT=$(($COUNT * 2)) 58 | if [ $COUNT == $COUNT2 ] 59 | then 60 | echo 1 61 | else 62 | echo 0 63 | fi 64 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode for 32 containers 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | COUNT=4 12 | ITERS=10 13 | 14 | for i in `seq 1 $COUNT` 15 | do 16 | docker rm -f test_crawl_cpu_many_containers_$i 2> /dev/null > /dev/null 17 | docker run -d --name test_crawl_cpu_many_containers_$i ubuntu sleep 60 2> /dev/null > /dev/null 18 | done 19 | 20 | IDS=`docker ps | grep test_crawl_cpu_many_containers | awk '{printf "%s,", $1}' | sed s/,$//g` 21 | 22 | COUNT2=`timeout $ITERS python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 23 | --features=cpu --crawlContainers $IDS --numprocesses 2 --frequency 1 | grep -c cpu-0` 24 | 25 | for i in `seq 1 $COUNT` 26 | do 27 | docker rm -f test_crawl_cpu_many_containers_$i > /dev/null 28 | done 29 | 30 | # COUNT2 should be COUNT * 10 31 | # However, we can't crawl 4 containers with 2 32 | # processes at the expected speed of one crawl 33 | # every second. So, lets expect at least COUNT * 8 34 | if [ $(($COUNT * 6)) -lt $COUNT2 ] 35 | then 36 | echo 1 37 | else 38 | echo 0 39 | fi 40 | 41 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers_for_10_secs_check_user_list.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode by starting 4 continers, and specifying 4 | # crawl just for 2 of them. 5 | # 6 | # Returns 1 if success, 0 otherwise 7 | 8 | if [[ $EUID -ne 0 ]]; then 9 | echo "This script must be run as root" 10 | exit 1 11 | fi 12 | 13 | COUNT=4 14 | ITERS=10 15 | 16 | for i in `seq 1 $COUNT` 17 | do 18 | docker rm -f cpu_many_containers_for_10_secs_check_user_list_$i 2> /dev/null > /dev/null 19 | docker run -d --name cpu_many_containers_for_10_secs_check_user_list_$i ubuntu sleep 60 2> /dev/null > /dev/null 20 | done 21 | 22 | # get docker IDs just for the first 2 containers 23 | ID1=`docker inspect -f '{{ .Id }}' cpu_many_containers_for_10_secs_check_user_list_1` 24 | ID2=`docker inspect -f '{{ .Id }}' cpu_many_containers_for_10_secs_check_user_list_2` 25 | 26 | timeout $ITERS python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 27 | --features=cpu --crawlContainers ${ID1},${ID2} --numprocesses 2 \ 28 | --frequency 1 --url file:///tmp/cpu_many_containers_for_10_secs_check_user_list 29 | 30 | cat /tmp/cpu_many_containers_for_10_secs_check_user_list* > /tmp/cpu_many_containers_for_10_secs_check_user_list_all 31 | COUNT_METADATA_1=`grep -c cpu_many_containers_for_10_secs_check_user_list_1 /tmp/cpu_many_containers_for_10_secs_check_user_list_all` 32 | COUNT_METADATA_2=`grep -c cpu_many_containers_for_10_secs_check_user_list_2 /tmp/cpu_many_containers_for_10_secs_check_user_list_all` 33 | 34 | COUNT_CPU=`grep -c cpu-0 /tmp/cpu_many_containers_for_10_secs_check_user_list_all` 35 | 36 | for i in `seq 1 $COUNT` 37 | do 38 | docker rm -f cpu_many_containers_for_10_secs_check_user_list_$i > /dev/null 39 | done 40 | 41 | # 2 contianers for 10 seconds (but let's make it 8) 42 | # and metadata for 8 frames for container 1 43 | # and metadata for 8 frames for container 2 44 | if [ $((2 * 8)) -lt $COUNT_CPU ] && [ "8" -lt $COUNT_METADATA_1 ] && [ "8" -lt $COUNT_METADATA_2 ] 45 | then 46 | echo 1 47 | else 48 | echo 0 49 | fi 50 | 51 | rm -f /tmp/cpu_many_containers_for_10_secs_check_user_list_all* 52 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers_input_logfile_links.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo 1 4 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_many_containers_input_logfile_links_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode for 32 containers 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | # The docker log link will be created here: 12 | # /var/log/crawler_container_logs///docker.log 13 | # I have some python code to get the local host IP. XXX replace it with bash 14 | read -d '' GET_HOST_IP_PY <<"EOF" 15 | import socket 16 | def get_host_ipaddr(): 17 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 18 | try: 19 | s.connect(('www.ibm.com', 9)) 20 | return s.getsockname()[0] 21 | except socket.error: 22 | return socket.gethostname() 23 | finally: 24 | del s 25 | print get_host_ipaddr() 26 | EOF 27 | 28 | HOST_IP=`python2.7 -c "$GET_HOST_IP_PY"` 29 | 30 | COUNT=4 31 | MSG=`uuidgen` 32 | 33 | for i in `seq 1 $COUNT` 34 | do 35 | rm -rf /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i 36 | docker rm -f test_crawl_cpu_many_containers_$i 2> /dev/null > /dev/null 37 | docker run -d -e LOG_LOCATIONS=/var/log/input_file_name.log --name test_crawl_cpu_many_containers_$i \ 38 | ubuntu bash -c "echo $MSG >> /var/log/input_file_name.log ; sleep 5 ; echo $MSG >> /var/log/input_file_name.log ; sleep 6000 " 2> /dev/null > /dev/null 39 | done 40 | 41 | IDS=`docker ps | grep test_crawl_cpu_many_containers | awk '{printf "%s,", $1}' | sed s/,$//g` 42 | 43 | timeout 10 python2.7 ../../crawler/containers_logs_linker.py --crawlContainers $IDS --frequency 1 2> /dev/null > /dev/null 44 | 45 | COUNT2=0 46 | for i in `seq 1 $COUNT` 47 | do 48 | R=`grep -c $MSG /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i/var/log/input_file_name.log` 49 | COUNT2=$(($COUNT2 + $R)) 50 | done 51 | 52 | for i in `seq 1 $COUNT` 53 | do 54 | docker rm -f test_crawl_cpu_many_containers_$i > /dev/null 55 | done 56 | 57 | # In those 10 seconds, the containers had 2 logs 58 | COUNT=$(($COUNT * 2)) 59 | if [ $COUNT == $COUNT2 ] 60 | then 61 | echo 1 62 | else 63 | echo 0 64 | fi 65 | -------------------------------------------------------------------------------- /tests/old/test_crawl_cpu_mem_interface_many_containers_input_logfile_links_for_10_secs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode for 32 containers 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | # The docker log link will be created here: 12 | # /var/log/crawler_container_logs///docker.log 13 | # I have some python code to get the local host IP. XXX replace it with bash 14 | read -d '' GET_HOST_IP_PY <<"EOF" 15 | import socket 16 | def get_host_ipaddr(): 17 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 18 | try: 19 | s.connect(('www.ibm.com', 9)) 20 | return s.getsockname()[0] 21 | except socket.error: 22 | return socket.gethostname() 23 | finally: 24 | del s 25 | print get_host_ipaddr() 26 | EOF 27 | 28 | HOST_IP=`python2.7 -c "$GET_HOST_IP_PY"` 29 | 30 | COUNT=4 31 | MSG=`uuidgen` 32 | 33 | for i in `seq 1 $COUNT` 34 | do 35 | rm -rf /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i 36 | docker rm -f test_crawl_cpu_many_containers_$i 2> /dev/null > /dev/null 37 | docker run -d -e LOG_LOCATIONS=/var/log/input_file_name.log --name test_crawl_cpu_many_containers_$i \ 38 | ubuntu bash -c "echo $MSG >> /var/log/input_file_name.log ; sleep 5 ; echo $MSG >> /var/log/input_file_name.log ; sleep 6000 " 2> /dev/null > /dev/null 39 | done 40 | 41 | IDS=`docker ps | grep test_crawl_cpu_many_containers | awk '{printf "%s,", $1}' | sed s/,$//g` 42 | 43 | timeout 10 python2.7 ../../crawler/containers_logs_linker.py --crawlContainers $IDS --frequency 1 2> /dev/null > /dev/null 44 | 45 | COUNT2=0 46 | for i in `seq 1 $COUNT` 47 | do 48 | R=`grep -c $MSG /var/log/crawler_container_logs/$HOST_IP/test_crawl_cpu_many_containers_$i/var/log/input_file_name.log` 49 | COUNT2=$(($COUNT2 + $R)) 50 | done 51 | 52 | for i in `seq 1 $COUNT` 53 | do 54 | docker rm -f test_crawl_cpu_many_containers_$i > /dev/null 55 | done 56 | 57 | # In those 10 seconds, the containers had 2 logs 58 | COUNT=$(($COUNT * 2)) 59 | if [ $COUNT == $COUNT2 ] 60 | then 61 | echo 1 62 | else 63 | echo 0 64 | fi 65 | -------------------------------------------------------------------------------- /tests/old/test_crawl_disk_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_file_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_file_container_1 ubuntu bash -c "echo test > /etc/test.test; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_file_container_1` 14 | 15 | # pretty much only docker contianers mount a /etc/resolv.conf partition 16 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 17 | --features=disk --crawlContainers $ID | grep -c "/etc/resolv.conf" 18 | 19 | docker rm -f test_crawl_file_container_1 > /dev/null 20 | -------------------------------------------------------------------------------- /tests/old/test_crawl_disk_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | python2.7 ../../crawler/crawler.py --crawlmode INVM \ 12 | --features=disk --crawlContainers $ID | grep -c '"/"' 13 | -------------------------------------------------------------------------------- /tests/old/test_crawl_dockerhistory.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_os_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_os_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_os_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=dockerhistory --crawlContainers $ID | grep -c '/bin/bash' 17 | 18 | docker rm -f test_crawl_os_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_dockerinspect.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_os_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_os_container_1 ubuntu sleep 600 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_os_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=dockerinspect --crawlContainers $ID | grep -c 'sleep' 17 | 18 | docker rm -f test_crawl_os_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_dockerps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_os_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_os_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_os_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode INVM \ 16 | --features=dockerps | grep -c $ID 17 | 18 | docker rm -f test_crawl_os_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_file_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_file_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_file_container_1 ubuntu bash -c "echo test > /etc/test.test; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_file_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=file --crawlContainers $ID | grep -c "/etc/test.test" 17 | 18 | docker rm -f test_crawl_file_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_file_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | UUID=`uuidgen` 12 | mkdir -p /bla 13 | touch /bla/$UUID 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode INVM \ 16 | --features=file --options '{"file": {"root_dir": "/bla/"}}' | grep -c "/bla/$UUID" 17 | 18 | rm -rf /bla 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_files_config_mountpoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the MOUNTPOINT crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | touch /etc/test.test 12 | 13 | python2.7 ../../crawler/crawler.py --crawlmode MOUNTPOINT --features file,config \ 14 | --mountpoint '/etc/' --option '{"config": {"known_config_files":["test.test"]}}' > /tmp/test_crawl_config_packages_mountpoint 15 | 16 | COUNT_CONFIG=`grep '^config' /tmp/test_crawl_config_packages_mountpoint | wc -l` 17 | COUNT_FILES=`grep '^file' /tmp/test_crawl_config_packages_mountpoint | wc -l` 18 | 19 | rm /etc/test.test 20 | 21 | if [ $COUNT_CONFIG -gt "0" ] && [ $COUNT_FILES -gt "10" ] 22 | then 23 | echo 1 24 | else 25 | echo 0 26 | fi 27 | -------------------------------------------------------------------------------- /tests/old/test_crawl_interface_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_interface_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_interface_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_interface_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=interface --crawlContainers $ID | grep -c eth0 17 | 18 | docker rm -f test_crawl_interface_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_load_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_os_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_os_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_os_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=load --crawlContainers $ID | grep -c shortterm 17 | 18 | docker rm -f test_crawl_os_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_load_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | python2.7 ../../crawler/crawler.py --crawlmode INVM \ 12 | --features=load | grep -c shortterm 13 | -------------------------------------------------------------------------------- /tests/old/test_crawl_memory_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_memory_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_memory_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_memory_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=memory --crawlContainers $ID | grep -c ^memory 17 | 18 | docker rm -f test_crawl_memory_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_metric_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_process_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_process_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_process_container_1` 14 | 15 | # sleep should be process with pid 1 (/1) 16 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 17 | --features=metric --crawlContainers $ID | grep -c "sleep/1" 18 | 19 | docker rm -f test_crawl_process_container_1 > /dev/null 20 | -------------------------------------------------------------------------------- /tests/old/test_crawl_metric_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | python2.7 ../../crawler/crawler.py --crawlmode INVM \ 12 | --features=metric | grep '/1\"' | grep '"pid":1,' | grep -c cpupct 13 | -------------------------------------------------------------------------------- /tests/old/test_crawl_os_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_os_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_os_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_os_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=os --crawlContainers $ID | grep -c linux 17 | 18 | docker rm -f test_crawl_os_container_1 > /dev/null 19 | -------------------------------------------------------------------------------- /tests/old/test_crawl_package_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_package_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_package_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_package_container_1` 14 | 15 | 16 | COUNT=`python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 17 | --features=package --crawlContainers $ID | grep ^package | grep -c ubuntu` 18 | 19 | docker rm -f test_crawl_package_container_1 > /dev/null 20 | 21 | # Should have at least 10 packages 22 | if [ $COUNT -gt "10" ] 23 | then 24 | echo 1 25 | else 26 | echo 0 27 | fi 28 | -------------------------------------------------------------------------------- /tests/old/test_crawl_package_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | COUNT=`python2.7 ../../crawler/crawler.py --crawlmode INVM \ 12 | --features=package | grep -c ^package` 13 | 14 | if [ $COUNT -gt "100" ] 15 | then 16 | echo 1 17 | else 18 | echo 0 19 | fi 20 | -------------------------------------------------------------------------------- /tests/old/test_crawl_packages_disks_mountpoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the MOUNTPOINT crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | python2.7 ../../crawler/crawler.py --crawlmode MOUNTPOINT --features package,disk \ 12 | --mountpoint '/' > /tmp/test_crawl_packages_disks_mountpoint 13 | 14 | COUNT_PACKAGE=`grep '^package' /tmp/test_crawl_packages_disks_mountpoint | wc -l` 15 | COUNT_DISK=`grep '^disk' /tmp/test_crawl_packages_disks_mountpoint | wc -l` 16 | 17 | if [ $COUNT_PACKAGE -gt "10" ] && [ $COUNT_DISK -gt "0" ] 18 | then 19 | echo 1 20 | else 21 | echo 0 22 | fi 23 | -------------------------------------------------------------------------------- /tests/old/test_crawl_processes_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_process_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_process_container_1 ubuntu sleep 60 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_process_container_1` 14 | 15 | # sleep should be process with pid 1 (/1) 16 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 17 | --features=process --crawlContainers $ID | grep -c "sleep/1" 18 | 19 | docker rm -f test_crawl_process_container_1 > /dev/null 20 | -------------------------------------------------------------------------------- /tests/old/test_crawl_processes_invm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the INVM crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | # In a VM process with pid 1 should be init 12 | python2.7 ../../crawler/crawler.py --crawlmode INVM \ 13 | --features=process | grep -c '/1"' 14 | -------------------------------------------------------------------------------- /tests/old/test_crawl_specific_file_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Tests the OUTCONTAINER crawler mode 4 | # Returns 1 if success, 0 otherwise 5 | 6 | if [[ $EUID -ne 0 ]]; then 7 | echo "This script must be run as root" 8 | exit 1 9 | fi 10 | 11 | docker rm -f test_crawl_file_container_1 2> /dev/null > /dev/null 12 | docker run -d --name test_crawl_file_container_1 ubuntu bash -c "mkdir /bla; echo test > /bla/ble; sleep 60" 2> /dev/null > /dev/null 13 | ID=`docker inspect -f '{{ .Id }}' test_crawl_file_container_1` 14 | 15 | python2.7 ../../crawler/crawler.py --crawlmode OUTCONTAINER \ 16 | --features=file --crawlContainers $ID --options \ 17 | '{"config": {"known_config_files":["etc/test.test"]}, "file": {"root_dir": "/bla/"}}' > \ 18 | /tmp/test_crawl_config_container 19 | 20 | # 2 as there is one feature for /bla and one for /bla/ble 21 | COUNT_MSG=`grep "/bla" /tmp/test_crawl_config_container | wc -l` 22 | 23 | # 2 as there is one feature for /bla and one for /bla/ble 24 | COUNT_FEATURES=`grep '^file' /tmp/test_crawl_config_container | wc -l` 25 | 26 | # 3 = metadata and 2 features 27 | COUNT_TOTAL=`cat /tmp/test_crawl_config_container | wc -l` 28 | 29 | if [ $COUNT_MSG == "2" ] && [ $COUNT_FEATURES == "2" ] && [ $COUNT_TOTAL == "3" ] 30 | then 31 | echo 1 32 | else 33 | echo 0 34 | fi 35 | 36 | docker rm -f test_crawl_file_container_1 > /dev/null 37 | -------------------------------------------------------------------------------- /tests/old/test_leaked_open_files.sh: -------------------------------------------------------------------------------- 1 | echo 1 2 | -------------------------------------------------------------------------------- /tests/unit/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *alchemy* 3 | *.log* 4 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudviz/agentless-system-crawler/376cc211b0037f4e2a9cc7df39127325dae4e39e/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/aufs_mount_init-id: -------------------------------------------------------------------------------- 1 | vol1/id/rootfs-a-b-c 2 | -------------------------------------------------------------------------------- /tests/unit/btrfs_mount_init-id: -------------------------------------------------------------------------------- 1 | vol1/id/rootfs-a-b-c 2 | -------------------------------------------------------------------------------- /tests/unit/capturing.py: -------------------------------------------------------------------------------- 1 | from cStringIO import StringIO 2 | import sys 3 | 4 | # Class used to capture the stdout of a function 5 | 6 | 7 | class Capturing(list): 8 | 9 | def __enter__(self): 10 | self._stdout = sys.stdout 11 | sys.stdout = self._stringio = StringIO() 12 | return self 13 | 14 | def __exit__(self, *args): 15 | self.extend(self._stringio.getvalue().splitlines()) 16 | sys.stdout = self._stdout 17 | -------------------------------------------------------------------------------- /tests/unit/liberty_response_time_details: -------------------------------------------------------------------------------- 1 | {"value":{"count":"292","description":"Average Response Time for servlet","maximumValue":"129746827","mean":"1646404.6780821919","minimumValue":"257689","reading":{"count":"292","maximumValue":"129746827","mean":"1646404.6780821919","minimumValue":"257689","standardDeviation":"7747033.106769906","timestamp":"1479283670331","total":"4.80750166E8","unit":"ns","variance":"6.001652195738899E13"},"standardDeviation":"7746816.577149615","total":"4.80750166E8","unit":"ns","variance":"5.980967601894751E13"},"type":{"className":"javax.management.openmbean.CompositeDataSupport","openType":"0"},"openTypes":[{"openTypeClass":"javax.management.openmbean.CompositeType","className":"javax.management.openmbean.CompositeData","typeName":"com.ibm.websphere.monitor.meters.StatisticsMeter","description":"com.ibm.websphere.monitor.meters.StatisticsMeter","items":[{"key":"count","description":"count","type":"1"},{"key":"description","description":"description","type":"2"},{"key":"maximumValue","description":"maximumValue","type":"1"},{"key":"mean","description":"mean","type":"3"},{"key":"minimumValue","description":"minimumValue","type":"1"},{"key":"reading","description":"reading","type":"4"},{"key":"standardDeviation","description":"standardDeviation","type":"3"},{"key":"total","description":"total","type":"3"},{"key":"unit","description":"unit","type":"2"},{"key":"variance","description":"variance","type":"3"}]},"java.lang.Long","java.lang.String","java.lang.Double",{"openTypeClass":"javax.management.openmbean.CompositeType","className":"javax.management.openmbean.CompositeData","typeName":"com.ibm.websphere.monitor.meters.StatisticsReading","description":"com.ibm.websphere.monitor.meters.StatisticsReading","items":[{"key":"count","description":"count","type":"1"},{"key":"maximumValue","description":"maximumValue","type":"1"},{"key":"mean","description":"mean","type":"3"},{"key":"minimumValue","description":"minimumValue","type":"1"},{"key":"standardDeviation","description":"standardDeviation","type":"3"},{"key":"timestamp","description":"timestamp","type":"1"},{"key":"total","description":"total","type":"3"},{"key":"unit","description":"unit","type":"2"},{"key":"variance","description":"variance","type":"3"}]}]} -------------------------------------------------------------------------------- /tests/unit/liberty_response_time_details_mocked: -------------------------------------------------------------------------------- 1 | {"className":"com.ibm.ws.webcontainer.monitor.ServletStats","URL":"/IBMJMXConnectorREST/mbeans/WebSphere%3Aname%3Dcom.ibm.ws.jmx.connector.server.rest.JMXRESTProxyServlet%2Ctype%3DServletStats","value":{"count":"292","description":"Average Response Time for servlet","maximumValue":"129746827","mean":"1646404.6780821919","minimumValue":"257689","reading":{"count":"292","maximumValue":"129746827","mean":"1646404.6780821919","minimumValue":"257689","standardDeviation":"7747033.106769906","timestamp":"1479283670331","total":"4.80750166E8","unit":"ns","variance":"6.001652195738899E13"},"standardDeviation":"7746816.577149615","total":"4.80750166E8","unit":"ns","variance":"5.980967601894751E13"},"type":{"className":"javax.management.openmbean.CompositeDataSupport","openType":"0"},"openTypes":[{"openTypeClass":"javax.management.openmbean.CompositeType","className":"javax.management.openmbean.CompositeData","typeName":"com.ibm.websphere.monitor.meters.StatisticsMeter","description":"com.ibm.websphere.monitor.meters.StatisticsMeter","items":[{"key":"count","description":"count","type":"1"},{"key":"description","description":"description","type":"2"},{"key":"maximumValue","description":"maximumValue","type":"1"},{"key":"mean","description":"mean","type":"3"},{"key":"minimumValue","description":"minimumValue","type":"1"},{"key":"reading","description":"reading","type":"4"},{"key":"standardDeviation","description":"standardDeviation","type":"3"},{"key":"total","description":"total","type":"3"},{"key":"unit","description":"unit","type":"2"},{"key":"variance","description":"variance","type":"3"}]},"java.lang.Long","java.lang.String","java.lang.Double",{"openTypeClass":"javax.management.openmbean.CompositeType","className":"javax.management.openmbean.CompositeData","typeName":"com.ibm.websphere.monitor.meters.StatisticsReading","description":"com.ibm.websphere.monitor.meters.StatisticsReading","items":[{"key":"count","description":"count","type":"1"},{"key":"maximumValue","description":"maximumValue","type":"1"},{"key":"mean","description":"mean","type":"3"},{"key":"minimumValue","description":"minimumValue","type":"1"},{"key":"standardDeviation","description":"standardDeviation","type":"3"},{"key":"timestamp","description":"timestamp","type":"1"},{"key":"total","description":"total","type":"3"},{"key":"unit","description":"unit","type":"2"},{"key":"variance","description":"variance","type":"3"}]}]} 2 | -------------------------------------------------------------------------------- /tests/unit/mock_environ_file: -------------------------------------------------------------------------------- 1 | HOME=/TERM=linuxPATH=/sbin:/bin 2 | -------------------------------------------------------------------------------- /tests/unit/mock_pynvml.py: -------------------------------------------------------------------------------- 1 | #class pynvml() 2 | import collections 3 | 4 | Memory = collections.namedtuple('Memory', 'total used free') 5 | Utilization = collections.namedtuple('Utilization', 'gpu memory') 6 | 7 | NVML_TEMPERATURE_GPU = 0 8 | 9 | class DummyProcess(): 10 | pid = 1234 11 | usedGpuMemory = 273285120 12 | 13 | def nvmlInit(): 14 | pass 15 | 16 | def nvmlShutdown(): 17 | pass 18 | 19 | def nvmlDeviceGetCount(): 20 | return 1 21 | 22 | def nvmlDeviceGetHandleByIndex(arg): 23 | return 0 24 | 25 | def nvmlDeviceGetTemperature(arg1, arg2): 26 | return 31 27 | 28 | def nvmlDeviceGetMemoryInfo(arg): 29 | retVal = 12205 * 1024 * 1024 30 | return Memory(total=retVal, used=0, free=retVal) 31 | 32 | def nvmlDeviceGetPowerUsage(arg): 33 | return 27000 34 | 35 | def nvmlDeviceGetEnforcedPowerLimit(arg): 36 | return 149000 37 | 38 | def nvmlDeviceGetUtilizationRates(arg): 39 | return Utilization(gpu=0, memory=0) 40 | 41 | def nvmlDeviceGetComputeRunningProcesses(arg): 42 | p = DummyProcess() 43 | return [p] 44 | #return [{'pid': 1234, 'usedGpuMemory': 273285120}] 45 | -------------------------------------------------------------------------------- /tests/unit/proc_mounts_aufs: -------------------------------------------------------------------------------- 1 | rootfs / rootfs rw 0 0 2 | sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 3 | proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 4 | udev /dev devtmpfs rw,relatime,size=4008360k,nr_inodes=1002090,mode=755 0 0 5 | devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 6 | tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=804824k,mode=755 0 0 7 | /dev/dm-1 / ext4 rw,relatime,errors=remount-ro,data=ordered 0 0 8 | none /sys/fs/cgroup tmpfs rw,relatime,size=4k,mode=755 0 0 9 | none /sys/fs/fuse/connections fusectl rw,relatime 0 0 10 | none /sys/kernel/debug debugfs rw,relatime 0 0 11 | none /sys/kernel/security securityfs rw,relatime 0 0 12 | cgroup /sys/fs/cgroup/cpuset cgroup rw,relatime,cpuset 0 0 13 | cgroup /sys/fs/cgroup/cpu cgroup rw,relatime,cpu 0 0 14 | cgroup /sys/fs/cgroup/cpuacct cgroup rw,relatime,cpuacct 0 0 15 | none /sys/firmware/efi/efivars efivarfs rw,relatime 0 0 16 | none /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0 17 | none /run/shm tmpfs rw,nosuid,nodev,relatime 0 0 18 | cgroup /sys/fs/cgroup/memory cgroup rw,relatime,memory 0 0 19 | none /run/user tmpfs rw,nosuid,nodev,noexec,relatime,size=102400k,mode=755 0 0 20 | none /sys/fs/pstore pstore rw,relatime 0 0 21 | cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 22 | cgroup /sys/fs/cgroup/freezer cgroup rw,relatime,freezer 0 0 23 | cgroup /sys/fs/cgroup/blkio cgroup rw,relatime,blkio 0 0 24 | cgroup /sys/fs/cgroup/perf_event cgroup rw,relatime,perf_event 0 0 25 | cgroup /sys/fs/cgroup/hugetlb cgroup rw,relatime,hugetlb 0 0 26 | /dev/sda2 /boot ext2 rw,relatime 0 0 27 | /dev/sda1 /boot/efi vfat rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro 0 0 28 | systemd /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,name=systemd 0 0 29 | /dev/dm-1 /var/lib/docker/aufs ext4 rw,relatime,errors=remount-ro,data=ordered 0 0 30 | gaufsd-fuse /run/user/1000/gaufs fuse.gaufsd-fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=1001 0 0 31 | none /var/lib/docker/aufs/mnt/e29e2a3403c1825008a1c53c61d4bd41774df3833cc5d8a12926fc6de39a466b aufs rw,relatime,si=90ef4e398f8a3d10,dio 0 0 32 | shm /var/lib/docker/containers/c751d4e5a334df29466b0fff65ea721317372d2c5b56012e371923ddaaa4f95a/shm tmpfs rw,nosuid,nodev,noexec,relatime,size=65536k 0 0 33 | proc /run/docker/netns/41815b5eedd6 proc rw,nosuid,nodev,noexec,relatime 0 0 34 | -------------------------------------------------------------------------------- /tests/unit/proc_mounts_btrfs: -------------------------------------------------------------------------------- 1 | rootfs / rootfs rw 0 0 2 | sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 3 | proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 4 | udev /dev devtmpfs rw,relatime,size=4008360k,nr_inodes=1002090,mode=755 0 0 5 | devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 6 | tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=804824k,mode=755 0 0 7 | /dev/dm-1 / ext4 rw,relatime,errors=remount-ro,data=ordered 0 0 8 | none /sys/fs/cgroup tmpfs rw,relatime,size=4k,mode=755 0 0 9 | none /sys/fs/fuse/connections fusectl rw,relatime 0 0 10 | none /sys/kernel/debug debugfs rw,relatime 0 0 11 | none /sys/kernel/security securityfs rw,relatime 0 0 12 | cgroup /sys/fs/cgroup/cpuset cgroup rw,relatime,cpuset 0 0 13 | cgroup /sys/fs/cgroup/cpu cgroup rw,relatime,cpu 0 0 14 | cgroup /sys/fs/cgroup/cpuacct cgroup rw,relatime,cpuacct 0 0 15 | none /sys/firmware/efi/efivars efivarfs rw,relatime 0 0 16 | none /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0 17 | none /run/shm tmpfs rw,nosuid,nodev,relatime 0 0 18 | cgroup /sys/fs/cgroup/memory cgroup rw,relatime,memory 0 0 19 | none /run/user tmpfs rw,nosuid,nodev,noexec,relatime,size=102400k,mode=755 0 0 20 | none /sys/fs/pstore pstore rw,relatime 0 0 21 | cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 22 | cgroup /sys/fs/cgroup/freezer cgroup rw,relatime,freezer 0 0 23 | cgroup /sys/fs/cgroup/blkio cgroup rw,relatime,blkio 0 0 24 | cgroup /sys/fs/cgroup/perf_event cgroup rw,relatime,perf_event 0 0 25 | cgroup /sys/fs/cgroup/hugetlb cgroup rw,relatime,hugetlb 0 0 26 | /dev/sda2 /boot ext2 rw,relatime 0 0 27 | /dev/sda1 /boot/efi vfat rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro 0 0 28 | systemd /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,name=systemd 0 0 29 | /dev/dm-1 /var/lib/docker/btrfs ext4 rw,relatime,errors=remount-ro,data=ordered 0 0 30 | gbtrfsd-fuse /run/user/1000/gbtrfs fuse.gbtrfsd-fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=1001 0 0 31 | none /var/lib/docker/btrfs/mnt/e29e2a3403c1825008a1c53c61d4bd41774df3833cc5d8a12926fc6de39a466b btrfs rw,relatime,si=90ef4e398f8a3d10,dio 0 0 32 | shm /var/lib/docker/containers/c751d4e5a334df29466b0fff65ea721317372d2c5b56012e371923ddaaa4f95a/shm tmpfs rw,nosuid,nodev,noexec,relatime,size=65536k 0 0 33 | proc /run/docker/netns/41815b5eedd6 proc rw,nosuid,nodev,noexec,relatime 0 0 34 | -------------------------------------------------------------------------------- /tests/unit/proc_mounts_devicemapper: -------------------------------------------------------------------------------- 1 | rootfs / rootfs rw 0 0 2 | proc /proc proc rw,relatime 0 0 3 | sysfs /sys sysfs rw,seclabel,relatime 0 0 4 | devtmpfs /dev devtmpfs rw,seclabel,relatime,size=3916200k,nr_inodes=979050,mode=755 0 0 5 | devpts /dev/pts devpts rw,seclabel,relatime,gid=5,mode=620,ptmxmode=000 0 0 6 | tmpfs /dev/shm tmpfs rw,seclabel,relatime 0 0 7 | /dev/mapper/vg_oc3262877066-lv_root / ext4 rw,seclabel,relatime,data=ordered 0 0 8 | none /selinux selinuxfs rw,relatime 0 0 9 | devtmpfs /dev devtmpfs rw,seclabel,relatime,size=3916200k,nr_inodes=979050,mode=755 0 0 10 | /dev/sda1 /boot ext4 rw,seclabel,relatime,data=ordered 0 0 11 | none /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0 12 | cgroup /cgroup/cpuset cgroup rw,relatime,cpuset 0 0 13 | cgroup /cgroup/cpu cgroup rw,relatime,cpu 0 0 14 | cgroup /cgroup/cpuacct cgroup rw,relatime,cpuacct 0 0 15 | cgroup /cgroup/memory cgroup rw,relatime,memory 0 0 16 | cgroup /cgroup/devices cgroup rw,relatime,devices 0 0 17 | cgroup /cgroup/freezer cgroup rw,relatime,freezer 0 0 18 | cgroup /cgroup/net_cls cgroup rw,relatime,net_cls 0 0 19 | cgroup /cgroup/blkio cgroup rw,relatime,blkio 0 0 20 | /etc/auto.misc /misc autofs rw,relatime,fd=7,pgrp=2897,timeout=300,minproto=5,maxproto=5,indirect 0 0 21 | -hosts /net autofs rw,relatime,fd=13,pgrp=2897,timeout=300,minproto=5,maxproto=5,indirect 0 0 22 | /etc/auto.gsa /gsa autofs rw,relatime,fd=19,pgrp=2897,timeout=300,minproto=5,maxproto=5,indirect 0 0 23 | /etc/auto.gsaro /gsaro autofs rw,relatime,fd=25,pgrp=2897,timeout=300,minproto=5,maxproto=5,indirect 0 0 24 | /dev/mapper/vg_oc3262877066-lv_root /var/lib/docker/devicemapper ext4 rw,seclabel,relatime,data=ordered 0 0 25 | gvfs-fuse-daemon /root/.gvfs fuse.gvfs-fuse-daemon rw,nosuid,nodev,relatime,user_id=0,group_id=0 0 0 26 | sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 27 | /dev/mapper/docker-253:2-29492713-65fe676c24fe1faea1f06e222cc3811cc9b651c381702ca4f787ffe562a5e39b /var/lib/docker/devicemapper/mnt/65fe676c24fe1faea1f06e222cc3811cc9b651c381702ca4f787ffe562a5e39b ext4 rw,seclabel,relatime,stripe=16,data=ordered 0 0 28 | proc /var/run/docker/netns/65fe676c24fe proc rw,relatime 0 0 29 | -------------------------------------------------------------------------------- /tests/unit/proc_mounts_vfs: -------------------------------------------------------------------------------- 1 | rootfs / rootfs rw 0 0 2 | sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 3 | proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 4 | udev /dev devtmpfs rw,relatime,size=4008360k,nr_inodes=1002090,mode=755 0 0 5 | devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 6 | tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=804824k,mode=755 0 0 7 | /dev/dm-1 / ext4 rw,relatime,errors=remount-ro,data=ordered 0 0 8 | none /sys/fs/cgroup tmpfs rw,relatime,size=4k,mode=755 0 0 9 | none /sys/fs/fuse/connections fusectl rw,relatime 0 0 10 | none /sys/kernel/debug debugfs rw,relatime 0 0 11 | none /sys/kernel/security securityfs rw,relatime 0 0 12 | cgroup /sys/fs/cgroup/cpuset cgroup rw,relatime,cpuset 0 0 13 | cgroup /sys/fs/cgroup/cpu cgroup rw,relatime,cpu 0 0 14 | cgroup /sys/fs/cgroup/cpuacct cgroup rw,relatime,cpuacct 0 0 15 | none /sys/firmware/efi/efivars efivarfs rw,relatime 0 0 16 | none /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0 17 | none /run/shm tmpfs rw,nosuid,nodev,relatime 0 0 18 | cgroup /sys/fs/cgroup/memory cgroup rw,relatime,memory 0 0 19 | none /run/user tmpfs rw,nosuid,nodev,noexec,relatime,size=102400k,mode=755 0 0 20 | none /sys/fs/pstore pstore rw,relatime 0 0 21 | cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 22 | cgroup /sys/fs/cgroup/freezer cgroup rw,relatime,freezer 0 0 23 | cgroup /sys/fs/cgroup/blkio cgroup rw,relatime,blkio 0 0 24 | cgroup /sys/fs/cgroup/perf_event cgroup rw,relatime,perf_event 0 0 25 | cgroup /sys/fs/cgroup/hugetlb cgroup rw,relatime,hugetlb 0 0 26 | /dev/sda2 /boot ext2 rw,relatime 0 0 27 | /dev/sda1 /boot/efi vfat rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro 0 0 28 | systemd /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,name=systemd 0 0 29 | /dev/dm-1 /var/lib/docker/vfs ext4 rw,relatime,errors=remount-ro,data=ordered 0 0 30 | gvfsd-fuse /run/user/1000/gvfs fuse.gvfsd-fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=1001 0 0 31 | none /var/lib/docker/vfs/mnt/e29e2a3403c1825008a1c53c61d4bd41774df3833cc5d8a12926fc6de39a466b vfs rw,relatime,si=90ef4e398f8a3d10,dio 0 0 32 | shm /var/lib/docker/containers/c751d4e5a334df29466b0fff65ea721317372d2c5b56012e371923ddaaa4f95a/shm tmpfs rw,nosuid,nodev,noexec,relatime,size=65536k 0 0 33 | proc /run/docker/netns/41815b5eedd6 proc rw,nosuid,nodev,noexec,relatime 0 0 34 | -------------------------------------------------------------------------------- /tests/unit/proc_pid_mounts_devicemapper: -------------------------------------------------------------------------------- 1 | rootfs / rootfs rw 0 0 2 | /dev/mapper/docker-253:2-29492713-65fe676c24fe1faea1f06e222cc3811cc9b651c381702ca4f787ffe562a5e39b / ext4 rw,seclabel,relatime,stripe=16,data=ordered 0 0 3 | proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 4 | tmpfs /dev tmpfs rw,seclabel,nosuid,mode=755 0 0 5 | devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666 0 0 6 | shm /dev/shm tmpfs rw,seclabel,nosuid,nodev,noexec,relatime,size=65536k 0 0 7 | mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0 8 | sysfs /sys sysfs ro,seclabel,nosuid,nodev,noexec,relatime 0 0 9 | /dev/mapper/vg_oc3262877066-lv_root /etc/resolv.conf ext4 rw,seclabel,relatime,data=ordered 0 0 10 | /dev/mapper/vg_oc3262877066-lv_root /etc/hostname ext4 rw,seclabel,relatime,data=ordered 0 0 11 | /dev/mapper/vg_oc3262877066-lv_root /etc/hosts ext4 rw,seclabel,relatime,data=ordered 0 0 12 | devpts /dev/console devpts rw,seclabel,relatime,gid=5,mode=620,ptmxmode=000 0 0 13 | proc /proc/asound proc ro,nosuid,nodev,noexec,relatime 0 0 14 | proc /proc/bus proc ro,nosuid,nodev,noexec,relatime 0 0 15 | proc /proc/fs proc ro,nosuid,nodev,noexec,relatime 0 0 16 | proc /proc/irq proc ro,nosuid,nodev,noexec,relatime 0 0 17 | proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0 18 | proc /proc/sysrq-trigger proc ro,nosuid,nodev,noexec,relatime 0 0 19 | tmpfs /proc/kcore tmpfs rw,seclabel,nosuid,mode=755 0 0 20 | tmpfs /proc/timer_stats tmpfs rw,seclabel,nosuid,mode=755 0 0 21 | -------------------------------------------------------------------------------- /tests/unit/single_package_apk_db: -------------------------------------------------------------------------------- 1 | C:Q128iBSO2PpkDnOWZqp3Watn4H20U= 2 | P:test-package 3 | V:999.9.9 4 | A:x86_64 5 | S:999 6 | I:581632 7 | T:the musl c library (libc) implementation 8 | U:http://www.musl-libc.org/ 9 | L:MIT 10 | o:musl 11 | m:Timo Teräs 12 | t:1485463794 13 | c:2e4493888fff74afc6a6ef6257aeea469df32af5 14 | p:so:libc.musl-x86_64.so.1=1 15 | F:lib 16 | R:libc.musl-x86_64.so.1 17 | a:0:0:777 18 | Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI= 19 | R:ld-musl-x86_64.so.1 20 | a:0:0:755 21 | Z:Q1OSxgRxSfAKG2m68kTiuRMQclyD8= 22 | F:usr 23 | F:usr/lib 24 | 25 | -------------------------------------------------------------------------------- /tests/unit/test_container.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import unittest 3 | 4 | from container import Container 5 | 6 | 7 | def mocked_exists(pid): 8 | return True 9 | 10 | 11 | class ContainerTests(unittest.TestCase): 12 | 13 | def setUp(self): 14 | pass 15 | 16 | def tearDown(self): 17 | pass 18 | 19 | def test_non_implemented_methods(self): 20 | c = Container(1) 21 | with self.assertRaises(NotImplementedError): 22 | c.get_memory_cgroup_path() 23 | with self.assertRaises(NotImplementedError): 24 | c.get_cpu_cgroup_path() 25 | 26 | @mock.patch('crawler.container.os.path.exists', side_effect=mocked_exists) 27 | def test_is_running(self, mock_exists): 28 | c = Container(1) 29 | assert c.is_running() 30 | 31 | def test_eq_ne(self): 32 | c1 = Container(1) 33 | c2 = Container(2) 34 | c3 = Container(2) 35 | assert c1 != c2 36 | assert c2 == c3 37 | 38 | def test_is_docker(self): 39 | c = Container(1) 40 | assert not c.is_docker_container() 41 | 42 | def test_to_str(self): 43 | c = Container(1) 44 | print(c) 45 | -------------------------------------------------------------------------------- /tests/unit/test_gpu_plugin.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import mock 4 | sys.path.append('tests/unit/') 5 | sys.modules['pynvml'] = __import__('mock_pynvml') 6 | from plugins.systems.gpu_host_crawler import GPUHostCrawler 7 | 8 | class GPUPluginTests(unittest.TestCase): 9 | 10 | def setUp(self): 11 | pass 12 | 13 | def tearDown(self): 14 | pass 15 | 16 | @mock.patch( 17 | 'plugins.systems.gpu_host_crawler.get_host_ipaddr', 18 | side_effect=lambda: "127.0.0.1") 19 | @mock.patch( 20 | 'plugins.systems.gpu_host_crawler.GPUHostCrawler._load_nvidia_lib', 21 | side_effect=lambda: 1) 22 | def test_os_gpu_host_crawler_plugin(self, *args): 23 | fc = GPUHostCrawler() 24 | for gpu_metrics in fc.crawl(): 25 | print gpu_metrics 26 | assert gpu_metrics == ( 27 | '127/0/0/1.gpu0.NA.NA', 28 | { 29 | "memory": {"total": 12205, "used": 0, "free": 12205}, 30 | "temperature": 31, 31 | "power": {"draw": 27, "limit": 149}, 32 | "utilization": {"gpu": 0, "memory": 0} 33 | }, 34 | 'gpu') 35 | -------------------------------------------------------------------------------- /tests/unit/test_jar_plugin.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import os 4 | import sys 5 | import tempfile 6 | from zipfile import ZipFile, ZipInfo 7 | 8 | from utils import jar_utils 9 | from utils.features import JarFeature 10 | 11 | # 12 | # https://security.openstack.org/guidelines/dg_using-temporary-files-securely.html 13 | # 14 | 15 | sys.path.append('tests/unit/') 16 | from plugins.systems.jar_host_crawler import JarHostCrawler 17 | 18 | 19 | class JarHashesPluginTests(unittest.TestCase): 20 | 21 | def setUp(self): 22 | pass 23 | 24 | def tearDown(self): 25 | pass 26 | 27 | def test_jar_host_crawler_plugin(self, *args): 28 | tmpdir = tempfile.mkdtemp() 29 | jar_file_name = 'myfile.jar' 30 | 31 | # Ensure the file is read/write by the creator only 32 | saved_umask = os.umask(0077) 33 | 34 | path = os.path.join(tmpdir, jar_file_name) 35 | try: 36 | with ZipFile(path, "w") as myjar: 37 | myjar.writestr(ZipInfo('first.class',(1980,1,1,1,1,1)), "first secrets!") 38 | myjar.writestr(ZipInfo('second.class',(1980,1,1,1,1,1)), "second secrets!") 39 | myjar.writestr(ZipInfo('second.txt',(1980,1,1,1,1,1)), "second secrets!") 40 | 41 | fc = JarHostCrawler() 42 | jars = list(fc.crawl(root_dir=tmpdir)) 43 | #jars = list(jar_utils.crawl_jar_files(root_dir=tmpdir)) 44 | print jars 45 | jar_feature = jars[0][1] 46 | assert 'myfile.jar' == jar_feature.name 47 | assert '48ac85a26ffa7ff5cefdd5c73a9fb888' == jar_feature.jarhash 48 | assert ['ddc6eff37020aa858e26b1ba8a49ee0e', 49 | 'cbe2a13eb99c1c8ac5f30d0a04f8c492'] == jar_feature.hashes 50 | assert 'jar' == jars[0][2] 51 | 52 | except IOError as e: 53 | print 'IOError' 54 | finally: 55 | os.remove(path) 56 | os.umask(saved_umask) 57 | -------------------------------------------------------------------------------- /tests/unit/test_jar_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import os 4 | import tempfile 5 | from zipfile import ZipFile, ZipInfo 6 | 7 | from utils import jar_utils 8 | from utils.features import JarFeature 9 | 10 | # 11 | # https://security.openstack.org/guidelines/dg_using-temporary-files-securely.html 12 | # 13 | 14 | class JarUtilsTests(unittest.TestCase): 15 | 16 | def setUp(self): 17 | pass 18 | 19 | def tearDown(self): 20 | pass 21 | 22 | def test_get_jar_features(self): 23 | tmpdir = tempfile.mkdtemp() 24 | jar_file_name = 'myfile.jar' 25 | 26 | # Ensure the file is read/write by the creator only 27 | saved_umask = os.umask(0077) 28 | 29 | path = os.path.join(tmpdir, jar_file_name) 30 | try: 31 | with ZipFile(path, "w") as myjar: 32 | myjar.writestr(ZipInfo('first.class',(1980,1,1,1,1,1)), "first secrets!") 33 | myjar.writestr(ZipInfo('second.class',(1980,1,1,1,1,1)), "second secrets!") 34 | myjar.writestr(ZipInfo('second.txt',(1980,1,1,1,1,1)), "second secrets!") 35 | 36 | jars = list(jar_utils.crawl_jar_files(root_dir=tmpdir)) 37 | print jars 38 | jar_feature = jars[0][1] 39 | assert 'myfile.jar' == jar_feature.name 40 | assert '48ac85a26ffa7ff5cefdd5c73a9fb888' == jar_feature.jarhash 41 | assert ['ddc6eff37020aa858e26b1ba8a49ee0e', 42 | 'cbe2a13eb99c1c8ac5f30d0a04f8c492'] == jar_feature.hashes 43 | assert 'jar' == jars[0][2] 44 | 45 | except IOError as e: 46 | print 'IOError' 47 | finally: 48 | os.remove(path) 49 | os.umask(saved_umask) 50 | os.rmdir(tmpdir) 51 | -------------------------------------------------------------------------------- /tests/unit/test_mesos_url.py: -------------------------------------------------------------------------------- 1 | # test.py 2 | from mock import patch, Mock 3 | 4 | from utils.mesos import fetch_stats 5 | 6 | 7 | @patch('utils.mesos.urllib2.urlopen') 8 | def mytest(mock_urlopen): 9 | a = Mock() 10 | a.read.side_effect = ['{}', None] 11 | mock_urlopen.return_value = a 12 | res = fetch_stats("0.22.0") 13 | print res 14 | if res is None: 15 | assert res 16 | 17 | 18 | mytest() 19 | -------------------------------------------------------------------------------- /tests/unit/two_packages_apk_db: -------------------------------------------------------------------------------- 1 | P:first-package 2 | V:111.1.1 3 | A:x86_64 4 | S:111 5 | 6 | P:second-package 7 | V:222.2.2 8 | A:x86_64 9 | S:222 10 | -------------------------------------------------------------------------------- /tests/unit/vfs_mount_init-id: -------------------------------------------------------------------------------- 1 | vol1/id/rootfs-a-b-c 2 | -------------------------------------------------------------------------------- /vagrant-provision.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # docker as per https://docs.docker.com/installation/ubuntulinux/ 4 | apt-key adv --keyserver hkp://pgp.mit.edu:80 \ 5 | --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 6 | 7 | echo deb https://apt.dockerproject.org/repo ubuntu-trusty main \ 8 | >> /etc/apt/sources.list.d/docker.list 9 | apt-get update 10 | 11 | apt-get -y install docker-engine 12 | 13 | # Prereqs for agentless-crawler 14 | apt-get -y install python-dev 15 | apt-get -y install python-pip 16 | pip install simplejson 17 | pip install psutil 18 | pip install netifaces 19 | pip install bottle 20 | --------------------------------------------------------------------------------