├── devbin ├── vsbug │ ├── .gitignore │ ├── CMakeLists.txt │ └── Makefile ├── vfprog │ ├── .gitignore │ ├── CMakeLists.txt │ ├── CIncrementer.cc │ └── Makefile ├── analyze_test │ ├── .gitignore │ ├── CMakeLists.txt │ ├── Makefile │ └── Main.cc ├── model_extractor │ ├── .gitignore │ ├── CMakeLists.txt │ └── Makefile ├── move_copy_swap │ ├── .gitignore │ ├── CMakeLists.txt │ └── Makefile ├── unixtime_to_string │ ├── .gitignore │ ├── CMakeLists.txt │ └── Makefile ├── state_search_splitter │ ├── .gitignore │ ├── CMakeLists.txt │ └── Makefile ├── CMakeLists.txt └── Makefile ├── .buildkite ├── ml_pipeline │ ├── __init__.py │ └── step.py ├── scripts │ ├── common │ │ └── base.sh │ └── steps │ │ ├── check-style.sh │ │ └── upload_dra_to_s3.sh ├── pipelines │ ├── send_email_notification.sh │ ├── format_and_validation.yml.sh │ ├── send_slack_notification.sh │ ├── upload_dra_to_gcs.yml.sh │ ├── upload_dra_to_s3.yml.sh │ ├── run_pytorch_tests.yml.sh │ └── create_dra.yml.sh └── pull-requests.json ├── 3rd_party ├── licenses │ ├── boost-NOTICE.txt │ ├── libxml2-NOTICE.txt │ ├── moby-NOTICE.txt │ ├── scowl-NOTICE.txt │ ├── strptime-NOTICE.txt │ ├── zlib-NOTICE.txt │ ├── gcc-runtime-NOTICE.txt │ ├── visualstudio2019-NOTICE.txt │ ├── boost-INFO.csv │ ├── pytorch-INFO.csv │ ├── moby-INFO.csv │ ├── libxml2-INFO.csv │ ├── zlib-INFO.csv │ ├── eigen-INFO.csv │ ├── scowl-INFO.csv │ ├── strptime-INFO.csv │ ├── visualstudio2019-INFO.csv │ ├── gcc-runtime-INFO.csv │ ├── intel-mkl-INFO.csv │ ├── eigen-NOTICE.txt │ ├── moby-LICENSE.txt │ └── zlib-LICENSE.txt ├── .gitignore ├── pull-valijson.sh └── pull-valijson.cmake ├── lib ├── ver │ ├── .gitignore │ └── unittest │ │ ├── CMakeLists.txt │ │ └── Main.cc ├── api │ ├── dump_state │ │ ├── .gitignore │ │ └── CMakeLists.txt │ ├── unittest │ │ ├── .gitignore │ │ ├── testfiles │ │ │ ├── new_mlfields_timechart.conf │ │ │ ├── eventConfig.json │ │ │ ├── new_mlfields_summarycount.conf │ │ │ ├── new_mlfields_sos_message_cat.conf │ │ │ ├── new_mlfields_with_utf8_bom.conf │ │ │ ├── new_persist_count_influence.conf │ │ │ ├── new_persist_dc_influence.conf │ │ │ ├── new_mlfields_bracket_percent.conf │ │ │ ├── new_populationmlfields.conf │ │ │ ├── incomplete_initial_bucket.txt │ │ │ ├── filterConfig.json │ │ │ ├── new_mlfields_categorization_filters.conf │ │ │ ├── new_persist_count.json │ │ │ ├── new_persist_dc.json │ │ │ ├── new_persist_categorization.json │ │ │ ├── new_mlfields_influencers.conf │ │ │ ├── resource_limits_3_2over_3partition_first8.csv │ │ │ ├── testLogErrors.boost.log.ini │ │ │ ├── new_persist_per_partition_categorization.json │ │ │ ├── time_messages.csv │ │ │ ├── new_mlfields_excludefrequent.conf │ │ │ ├── pop_sum_bytes_by_status_over_clientip.json │ │ │ ├── scheduled_events.conf │ │ │ ├── new_mlfields.json │ │ │ ├── new_mlfields_partition.json │ │ │ ├── s.csv │ │ │ ├── new_mlfields_over.json │ │ │ ├── count_over_ip_config.json │ │ │ ├── normalizerInput.csv │ │ │ └── instrumentation │ │ │ │ └── memory_usage.schema.json │ │ ├── CMockSearcher.cc │ │ └── Main.cc │ ├── ElasticsearchStateIndex.cc │ ├── CNoopCategoryIdMapper.cc │ ├── CSimpleOutputWriter.cc │ └── CCategoryIdMapper.cc ├── core │ ├── unittest │ │ ├── testfiles │ │ │ ├── mdhmon.xml │ │ │ ├── boost.log.ini │ │ │ ├── CXmlParser3.xml │ │ │ ├── CXmlParser4.xml │ │ │ ├── p2ps.xml │ │ │ ├── testLogWarnings.boost.log.ini │ │ │ ├── CXmlParser3_2.xml │ │ │ ├── CXmlParser3_1.xml │ │ │ ├── withNs.xml │ │ │ └── CXmlParser2.xml │ │ ├── .gitignore │ │ ├── CProcessStatsTest_Linux.cc │ │ ├── CProcessStatsTest_MacOSX.cc │ │ ├── CProcessStatsTest_Windows.cc │ │ ├── CProcessPriorityTest.cc │ │ ├── CProcessTest.cc │ │ ├── CMutexTest.cc │ │ ├── Main.cc │ │ └── CWindowsErrorTest.cc │ ├── boost.log.ini │ ├── CCompressedDictionary.cc │ ├── CContainerPrinter.cc │ ├── CTimeGm.cc │ ├── CTimeGm_Windows.cc │ ├── CCTimeR.cc │ ├── CDataSearcher.cc │ ├── CPersistUtils.cc │ ├── README.scowl │ ├── CCrashHandler.cc │ ├── CUnSetEnv.cc │ ├── CGmTimeR.cc │ ├── CStrPTime.cc │ ├── CGmTimeR_Windows.cc │ ├── CStrFTime.cc │ ├── CStrTokR.cc │ ├── CStrTokR_Windows.cc │ ├── CSetEnv.cc │ ├── CSetMode.cc │ ├── CBase64Filter.cc │ ├── CScopedLock.cc │ ├── CScopedFastLock.cc │ ├── CSetMode_Windows.cc │ ├── CSetEnv_Windows.cc │ ├── CStrCaseCmp.cc │ ├── CStrCaseCmp_Windows.cc │ ├── CUnSetEnv_Windows.cc │ ├── CFastMutex_MacOSX.cc │ ├── CProcessPriority_Windows.cc │ ├── CScopedReadLock.cc │ ├── CScopedWriteLock.cc │ ├── CFastMutex_Windows.cc │ ├── CBoostJsonUnbufferedIStreamWrapper.cc │ ├── CCTimeR_Windows.cc │ ├── CProcessPriority.cc │ ├── CWindowsError.cc │ ├── CMemoryCircuitBreaker.cc │ ├── CMutex_Windows.cc │ ├── CReadWriteLock_Windows.cc │ └── CProcessStats_MacOSX.cc ├── maths │ ├── common │ │ ├── unittest │ │ │ └── testfiles │ │ │ │ ├── sparse_guass_quadrature_test_d2_l1 │ │ │ │ ├── sparse_guass_quadrature_test_d2_l2 │ │ │ │ └── testLogErrors.boost.log.ini │ │ ├── CLinearAlgebra.cc │ │ ├── CLeastSquaresOnlineRegression.cc │ │ ├── COrderings.cc │ │ └── CFuzzyLogic.cc │ └── analytics │ │ ├── unittest │ │ └── CMakeLists.txt │ │ └── CMakeLists.txt ├── model │ ├── unittest │ │ ├── testfiles │ │ │ ├── status503.txt │ │ │ ├── mysqlabort.txt │ │ │ ├── invalidmllimits.conf │ │ │ └── mllimits.conf │ │ └── Main.cc │ ├── CProcessMemoryUsage_Linux.cc │ ├── CProcessMemoryUsage.cc │ ├── CMonitoredResource.cc │ ├── FrequencyPredicates.cc │ └── CHierarchicalResultsProbabilityFinalizer.cc ├── seccomp │ ├── CMakeLists.txt │ └── unittest │ │ ├── CMakeLists.txt │ │ └── Main.cc ├── CMakeLists.txt └── test │ ├── CProgramCounterClearingFixture.cc │ ├── CShellCmdEscape_Windows.cc │ ├── CMakeLists.txt │ └── CBoostTestXmlOutput.cc ├── bin ├── normalize │ ├── .gitignore │ └── CMakeLists.txt ├── autodetect │ ├── .gitignore │ └── CMakeLists.txt ├── controller │ ├── unittest │ │ ├── testfiles │ │ │ ├── slogan1.txt │ │ │ └── slogan2.txt │ │ └── CMakeLists.txt │ └── CMakeLists.txt ├── pytorch_inference │ ├── unittest │ │ ├── .gitignore │ │ └── CMakeLists.txt │ ├── examples │ │ └── control_message │ │ │ └── thread_control_message.json │ └── CMakeLists.txt ├── categorize │ └── CMakeLists.txt ├── data_frame_analyzer │ └── CMakeLists.txt └── CMakeLists.txt ├── settings.gradle ├── dev-tools ├── build_macos_third_party_deps.sh ├── minimal.zip ├── docker │ ├── ml_cpp_jdk │ │ └── Dockerfile │ ├── run_docker_clang_format.sh │ ├── style_checker │ │ └── Dockerfile │ ├── check_style_image │ │ └── Dockerfile │ ├── linux_dependency_builder │ │ └── Dockerfile │ ├── linux_builder │ │ └── Dockerfile │ └── linux_tester │ │ └── Dockerfile └── autodetect_profiling_macos │ ├── record_memgraph.sh │ └── record_xctrace.sh ├── cmake ├── ml.ico ├── architecture │ ├── aarch64.cmake │ └── x86_64.cmake ├── os │ ├── linux.cmake │ ├── windows.cmake │ └── darwin.cmake ├── test-clean-results.cmake ├── Info.plist.in ├── test-check-success.cmake ├── linux-x86_64.cmake └── linux-aarch64.cmake ├── NOTICE.txt ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle.properties ├── .github └── ISSUE_TEMPLATE.md ├── .entitlements.xml ├── .backportrc.json ├── .git-blame-ignore-revs ├── copyright_code_header.txt ├── devlib ├── CMakeLists.txt ├── vflib │ ├── CMakeLists.txt │ ├── Makefile │ └── CIncrementer.cc └── Makefile ├── .clang-format ├── .ci └── orka │ ├── gobld-bootstrap.sh │ └── gobld-bootstrap.plist ├── include ├── api │ ├── ApiTypes.h │ └── ElasticsearchStateIndex.h └── core │ ├── CSmallVectorFwd.h │ ├── CStreamUtils.h │ ├── CLoggerTrace.h │ ├── CStreamWriter.h │ ├── CoreTypes.h │ ├── CTimeGm.h │ ├── CStrTokR.h │ ├── CSetEnv.h │ ├── CScopedLock.h │ └── CCTimeR.h ├── .dockerignore └── .gitignore /devbin/vsbug/.gitignore: -------------------------------------------------------------------------------- 1 | vsbug 2 | -------------------------------------------------------------------------------- /.buildkite/ml_pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3rd_party/licenses/boost-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3rd_party/licenses/libxml2-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3rd_party/licenses/moby-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3rd_party/licenses/scowl-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3rd_party/licenses/strptime-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3rd_party/licenses/zlib-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /devbin/vfprog/.gitignore: -------------------------------------------------------------------------------- 1 | vfprog 2 | -------------------------------------------------------------------------------- /lib/ver/.gitignore: -------------------------------------------------------------------------------- 1 | CBuildInfo.cc 2 | -------------------------------------------------------------------------------- /3rd_party/licenses/gcc-runtime-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /3rd_party/.gitignore: -------------------------------------------------------------------------------- 1 | eigen/ 2 | valijson/ 3 | -------------------------------------------------------------------------------- /3rd_party/licenses/visualstudio2019-NOTICE.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/normalize/.gitignore: -------------------------------------------------------------------------------- 1 | *.csv 2 | *.json 3 | -------------------------------------------------------------------------------- /devbin/analyze_test/.gitignore: -------------------------------------------------------------------------------- 1 | analyze_test 2 | -------------------------------------------------------------------------------- /lib/api/dump_state/.gitignore: -------------------------------------------------------------------------------- 1 | dump_state 2 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'ml-cpp' 2 | -------------------------------------------------------------------------------- /devbin/model_extractor/.gitignore: -------------------------------------------------------------------------------- 1 | model_extractor 2 | -------------------------------------------------------------------------------- /devbin/move_copy_swap/.gitignore: -------------------------------------------------------------------------------- 1 | move_copy_swap 2 | -------------------------------------------------------------------------------- /devbin/unixtime_to_string/.gitignore: -------------------------------------------------------------------------------- 1 | unixtime_to_string 2 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/mdhmon.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /bin/autodetect/.gitignore: -------------------------------------------------------------------------------- 1 | *.csv 2 | *.conf 3 | *.tsv 4 | *.json 5 | -------------------------------------------------------------------------------- /bin/controller/unittest/testfiles/slogan1.txt: -------------------------------------------------------------------------------- 1 | Elastic is great! 2 | -------------------------------------------------------------------------------- /devbin/state_search_splitter/.gitignore: -------------------------------------------------------------------------------- 1 | state_search_splitter 2 | -------------------------------------------------------------------------------- /bin/controller/unittest/testfiles/slogan2.txt: -------------------------------------------------------------------------------- 1 | You know, for search! 2 | -------------------------------------------------------------------------------- /dev-tools/build_macos_third_party_deps.sh: -------------------------------------------------------------------------------- 1 | ../.ci/orka/third_party_deps.sh -------------------------------------------------------------------------------- /lib/api/unittest/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore temporary test files 2 | test.log -------------------------------------------------------------------------------- /cmake/ml.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ml-cpp/HEAD/cmake/ml.ico -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_timechart.conf: -------------------------------------------------------------------------------- 1 | timechart=1 2 | 3 | -------------------------------------------------------------------------------- /lib/core/unittest/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore temporary test files 2 | test.log 3 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/eventConfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "events": [ 3 | ] 4 | } 5 | -------------------------------------------------------------------------------- /lib/maths/common/unittest/testfiles/sparse_guass_quadrature_test_d2_l1: -------------------------------------------------------------------------------- 1 | .5, .5, 1 -------------------------------------------------------------------------------- /bin/pytorch_inference/unittest/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore temporary test files 2 | test.log 3 | -------------------------------------------------------------------------------- /dev-tools/minimal.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ml-cpp/HEAD/dev-tools/minimal.zip -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | ml-cpp (implements core analytics for machine learning) 2 | Copyright 2012-2021 Elasticsearch B.V. 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/ml-cpp/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_summarycount.conf: -------------------------------------------------------------------------------- 1 | detector.1.clause = count by airline summarycountfield=count 2 | 3 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_sos_message_cat.conf: -------------------------------------------------------------------------------- 1 | detector.1.clause = count by mlcategory categorizationfield=message 2 | 3 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_with_utf8_bom.conf: -------------------------------------------------------------------------------- 1 | 2 | detector.1.clause = responsetime by airline 3 | influencer.1 = airline 4 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_persist_count_influence.conf: -------------------------------------------------------------------------------- 1 | [default] 2 | detector.1.clause = count by program 3 | 4 | influencer.1 = user 5 | 6 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_persist_dc_influence.conf: -------------------------------------------------------------------------------- 1 | detector.1.clause = distinct_count(file) by program 2 | 3 | influencer.1 = user 4 | 5 | -------------------------------------------------------------------------------- /3rd_party/licenses/boost-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | Boost C++ Libraries,1.86.0,,http://www.boost.org,BSL-1.0,, 3 | -------------------------------------------------------------------------------- /3rd_party/licenses/pytorch-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | PyTorch,2.7.1,e2d141dbde55c2a4370fac5165b0561b6af4798b,https://pytorch.org,BSD-3-Clause,, 3 | -------------------------------------------------------------------------------- /3rd_party/licenses/moby-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | Moby lexicon project,1,,https://web.archive.org/web/20170930060409/http://icon.shef.ac.uk/Moby/,Public Domain,, 3 | -------------------------------------------------------------------------------- /bin/pytorch_inference/examples/control_message/thread_control_message.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "request_id": "1", 5 | "control": 0, 6 | "num_allocations": 4 7 | } 8 | } 9 | ] -------------------------------------------------------------------------------- /lib/maths/common/unittest/testfiles/sparse_guass_quadrature_test_d2_l2: -------------------------------------------------------------------------------- 1 | .21132486540518713, .5, .5 2 | .5, .21132486540518713, .5 3 | .5, .5, -1 4 | .5, .78867513459481287, .5 5 | .78867513459481287, .5, .5 -------------------------------------------------------------------------------- /3rd_party/licenses/libxml2-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | Libxml2,2.10.4,7846b0a677f8d3ce72486125fa281e92ac9970e8,http://xmlsoft.org/,MIT,Copyright (C) 1998-2012 Daniel Veillard, 3 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | org.gradle.daemon=false 2 | 3 | elasticsearchVersion=9.4.0 4 | 5 | artifactName=ml-cpp 6 | 7 | # Enforce the build to fail on deprecated gradle api usage 8 | systemProp.org.gradle.warning.mode=fail 9 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_bracket_percent.conf: -------------------------------------------------------------------------------- 1 | detector.1.clause = max("Level 1 (Urgent)") by "10%" over "%10" partitionfield="Percentage (%)" categorizationfield="This string should have quotes removed" 2 | -------------------------------------------------------------------------------- /3rd_party/licenses/zlib-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | zlib,1.2.13,04f42ceca40f73e2978b50e93806c2a18c1281fc,http://zlib.net,Zlib,Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler, 3 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_populationmlfields.conf: -------------------------------------------------------------------------------- 1 | detector.1.sourcetype = netflow 2 | detector.1.clause = count by SRC 3 | detector.2.sourcetype = netflow 4 | detector.2.clause = count by DPT over SRC usenull=1 5 | 6 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/incomplete_initial_bucket.txt: -------------------------------------------------------------------------------- 1 | {"time":"1585698939", "clientip":"172.245.214.190", "status":"200", "bytes":"25108"} 2 | {"time":"1585698945", "clientip":"172.245.214.190", "status":"404", "bytes":"16201"} -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/filterConfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "filters": [ 3 | { 4 | "filter_id": "safe_ips", 5 | "items": [ 6 | "111.111.111.111", 7 | "222.222.222.222" 8 | ] 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /3rd_party/licenses/eigen-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | Eigen,3.4.0,3147391d946bb4b6c68edd901f2add6ac1f31f8c,http://eigen.tuxfamily.org,MPL-2.0,,https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.zip 3 | -------------------------------------------------------------------------------- /3rd_party/licenses/scowl-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | SCOWL,7.1,,http://wordlist.aspell.net,Custom;https://raw.githubusercontent.com/en-wl/wordlist/scowl-7.1/scowl/Copyright,Copyright 2000-2011 by Kevin Atkinson, 3 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/boost.log.ini: -------------------------------------------------------------------------------- 1 | 2 | [Core] 3 | Filter="%Severity% >= TRACE" 4 | 5 | [Sinks.Stderr] 6 | Destination=Console 7 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 8 | 9 | -------------------------------------------------------------------------------- /3rd_party/licenses/strptime-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | strptime,1.35,,http://cvsweb.netbsd.org/bsdweb.cgi/src/lib/libc/time/strptime.c,BSD-2-Clause-NetBSD,"Copyright (c) 1997, 1998, 2005, 2008 The NetBSD Foundation, Inc.", 3 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_categorization_filters.conf: -------------------------------------------------------------------------------- 1 | detector.1.clause = count by mlcategory categorizationfield=message 2 | categorizationfilter.0 = foo 3 | categorizationfilter.1 = " " 4 | categorizationfilter.2 = 5 | categorizationfilter.3 = foo,bar 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 7 | -------------------------------------------------------------------------------- /3rd_party/licenses/visualstudio2019-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | Visual Studio 2019 Runtime Libraries,14.2,,https://www.visualstudio.com,Custom;https://www.visualstudio.com/license-terms/mlt031619/,Copyright (C) Microsoft Corporation, 3 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_persist_count.json: -------------------------------------------------------------------------------- 1 | { 2 | "job_id": "new_persist_count", 3 | "analysis_config": { 4 | "detectors": [ 5 | { 6 | "function": "count", 7 | "by_field_name": "program" 8 | } 9 | ] 10 | } 11 | } 12 | 13 | -------------------------------------------------------------------------------- /lib/model/unittest/testfiles/status503.txt: -------------------------------------------------------------------------------- 1 | 1346763641 2 | 1346787300 3 | 1346832480 4 | 1346863305 5 | 1346870562 6 | 1346870570 7 | 1346870700 8 | 1347035640 9 | 1347050369 10 | 1347071990 11 | 1347134940 12 | 1347177855 13 | 1347178041 14 | 1347201170 15 | 1347201240 16 | -------------------------------------------------------------------------------- /.entitlements.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | com.apple.security.get-task-allow 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_persist_dc.json: -------------------------------------------------------------------------------- 1 | { 2 | "job_id": "new_persist_dc", 3 | "analysis_config": { 4 | "detectors": [ 5 | { 6 | "function": "distinct_count", 7 | "field_name": "file", 8 | "by_field_name": "program" 9 | } 10 | ] 11 | } 12 | } 13 | 14 | -------------------------------------------------------------------------------- /.backportrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "upstream" : "elastic/ml-cpp", 3 | "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "7.17" ], 4 | "targetPRLabels" : [ "backport" ], 5 | "branchLabelMapping" : { 6 | "^v8.17.0$" : "main", 7 | "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2" 8 | } 9 | } -------------------------------------------------------------------------------- /3rd_party/licenses/gcc-runtime-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | GCC Runtime Library,13.3.0,f00b5710a30f22efc3171c393e56aeb335c3cd39,https://gcc.gnu.org,GPL-3.0 WITH GCC-exception-3.1,"Copyright (C) 2021 Free Software Foundation, Inc.",http://ftpmirror.gnu.org/gcc/gcc-10.3.0/gcc-10.3.0.tar.gz 3 | -------------------------------------------------------------------------------- /dev-tools/docker/ml_cpp_jdk/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | RUN apt-get update && apt-get upgrade -y 4 | RUN DEBIAN_FRONTEND=noninteractive apt-get install aptitude ssh git openjdk-17-jre openjdk-17-jdk jq zip -y 5 | RUN useradd -ms /bin/bash -u 1000 elasticsearch 6 | 7 | USER elasticsearch 8 | WORKDIR /home/elasticsearch 9 | -------------------------------------------------------------------------------- /3rd_party/licenses/intel-mkl-INFO.csv: -------------------------------------------------------------------------------- 1 | name,version,revision,url,license,copyright,sourceURL 2 | Intel oneAPI Math Kernel Library (oneMKL),2024.0,,https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html,ISSL;https://cdrdv2-public.intel.com/749362/intel-simplified-license-software-october-2022.pdf,(c) Intel Corporation, 3 | -------------------------------------------------------------------------------- /lib/core/boost.log.ini: -------------------------------------------------------------------------------- 1 | 2 | [Core] 3 | Filter="%Severity% >= DEBUG" 4 | 5 | [Sinks.FileExample] 6 | Destination=TextFile 7 | FileName=ml_%3N.log 8 | Append=true 9 | AutoFlush=true 10 | RotationSize=104857600 11 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 12 | 13 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/CXmlParser3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/CXmlParser4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/p2ps.xml: -------------------------------------------------------------------------------- 1 | 426391232558807Open Failure192.168.9.242CUBE_LIFFEInfoml00-4201.1.p2psERROR : Invalid Product Code426_dave 2 | -------------------------------------------------------------------------------- /3rd_party/licenses/eigen-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Eigen is licensed under Mozilla Public License Version 2.0 (MPL 2.0). A MPL 2.0 2 | and FAQ's for MPL 2.0 can be found at: https://www.mozilla.org/en-US/MPL/2.0/ 3 | and https://www.mozilla.org/en-US/MPL/2.0/FAQ/. 4 | 5 | A copy of the Eigen source code may be obtained from 6 | https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.zip 7 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_persist_categorization.json: -------------------------------------------------------------------------------- 1 | { 2 | "job_id":"new_persist_categorization", 3 | "analysis_config": 4 | { 5 | "detectors":[ 6 | { 7 | "function":"count", 8 | "by_field_name":"mlcategory" 9 | } 10 | ], 11 | "categorization_field_name":"message" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_influencers.conf: -------------------------------------------------------------------------------- 1 | detector.1.sourcetype = event_count 2 | detector.1.clause = count by mlcategory 3 | detector.2.sourcetype = event_count 4 | detector.2.clause = count by processx 5 | detector.3.sourcetype = event_count 6 | detector.3.clause = count by vlog 7 | 8 | influencer.1 = host 9 | influencer.2 = socket 10 | influencer.3 = warehouse 11 | 12 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionSha256Sum=47a5bfed9ef814f90f8debcbbb315e8e7c654109acd224595ea39fca95c5d4da 4 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-all.zip 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | distributionSha256Sum=47a5bfed9ef814f90f8debcbbb315e8e7c654109acd224595ea39fca95c5d4da 8 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/resource_limits_3_2over_3partition_first8.csv: -------------------------------------------------------------------------------- 1 | time,value,colour,species,greenhouse 2 | 1407438000,200,black,begonia,N7 3 | 1407438000,200,black,petunia,N7 4 | 1407438000,200,black,begonia,N7 5 | 1407438000,200,black,petunia,N7 6 | 1407438000,200,black,begonia,N7 7 | 1407438000,200,black,petunia,N7 8 | 1407438000,200,black,begonia,N7 9 | 1407438000,200,black,petunia,N7 10 | 1407438000,200,black,begonia,N7 11 | 1407438000,200,black,petunia,N7 12 | 1407454843,200,black,begonia,SE1 13 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/testLogErrors.boost.log.ini: -------------------------------------------------------------------------------- 1 | 2 | [Core] 3 | Filter="%Severity% >= DEBUG" 4 | 5 | [Sinks.Stderr] 6 | Destination=Console 7 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 8 | 9 | [Sinks.ErrorLog] 10 | Destination=TextFile 11 | FileName=test.log 12 | AutoFlush=true 13 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 14 | Filter="%Severity% >= ERROR" 15 | 16 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/testLogWarnings.boost.log.ini: -------------------------------------------------------------------------------- 1 | 2 | [Core] 3 | Filter="%Severity% >= DEBUG" 4 | 5 | [Sinks.Stderr] 6 | Destination=Console 7 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 8 | 9 | [Sinks.WarnLog] 10 | Destination=TextFile 11 | FileName=test.log 12 | AutoFlush=true 13 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 14 | Filter="%Severity% >= WARN" 15 | 16 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/CXmlParser3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Transport read error 4 | 5 | 6 | 7 | ^[[:digit:]]* 8 | 9 | 10 | source 11 | [[:space:]]*)?]]> 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /lib/maths/common/unittest/testfiles/testLogErrors.boost.log.ini: -------------------------------------------------------------------------------- 1 | 2 | [Core] 3 | Filter="%Severity% >= DEBUG" 4 | 5 | [Sinks.Stderr] 6 | Destination=Console 7 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 8 | 9 | [Sinks.ErrorLog] 10 | Destination=TextFile 11 | FileName=test.log 12 | AutoFlush=true 13 | Format="%TimeStamp(format=\"%Y-%m-%d %H:%M:%S,%f\")% UTC [%ProcessID%] %Severity% %File%@%Line% %Message%" 14 | Filter="%Severity% >= ERROR" 15 | 16 | -------------------------------------------------------------------------------- /lib/model/unittest/testfiles/mysqlabort.txt: -------------------------------------------------------------------------------- 1 | 1346763647 2 | 1346763647 3 | 1346787304 4 | 1346787304 5 | 1346832485 6 | 1346832485 7 | 1346863312 8 | 1346863312 9 | 1346870568 10 | 1346870568 11 | 1346870576 12 | 1346870576 13 | 1346870706 14 | 1346870706 15 | 1347035646 16 | 1347035646 17 | 1347050376 18 | 1347050376 19 | 1347071996 20 | 1347071996 21 | 1347134945 22 | 1347134945 23 | 1347177860 24 | 1347177860 25 | 1347178046 26 | 1347178046 27 | 1347201176 28 | 1347201176 29 | 1347201247 30 | 1347201247 31 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_persist_per_partition_categorization.json: -------------------------------------------------------------------------------- 1 | { 2 | "job_id":"new_persist_per_partition_categorization", 3 | "analysis_config": { 4 | "detectors":[ 5 | { 6 | "function":"count", 7 | "by_field_name":"mlcategory", 8 | "partition_field_name":"event.dataset" 9 | } 10 | ], 11 | "per_partition_categorization":{ 12 | "enabled":true, 13 | "stop_on_warn":false 14 | }, 15 | "categorization_field_name":"message" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/CXmlParser3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Transport node error 4 | 5 | 6 | 7 | ^[[:space:]]* 8 | 9 | 10 | source 11 | [[:space:]]*)?]]> 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /3rd_party/licenses/moby-LICENSE.txt: -------------------------------------------------------------------------------- 1 | The Moby lexicon project is complete and has 2 | been place into the public domain. Use, sell, 3 | rework, excerpt and use in any way on any platform. 4 | 5 | Placing this material on internal or public servers is 6 | also encouraged. The compiler is not aware of any 7 | export restrictions so freely distribute world-wide. 8 | 9 | You can verify the public domain status by contacting 10 | 11 | Grady Ward 12 | 3449 Martha Ct. 13 | Arcata, CA 95521-4884 14 | 15 | daedal@myrealbox.com >> 16 | -------------------------------------------------------------------------------- /lib/model/unittest/testfiles/invalidmllimits.conf: -------------------------------------------------------------------------------- 1 | 2 | # This file configures various limits to Ml's search commands. 3 | # Improperly configured limits may result in Ml search command crashes 4 | # and/or memory overuse. 5 | 6 | [anomaly] 7 | # Maximum number of distinct values of a single field before analysis of that 8 | # field will be halted 9 | maxfieldvalues = 1000 10 | # Maximum number of time buckets to process during anomaly detection before 11 | # ceasing to output results 12 | maxtimebuckets = ten thousand 13 | 14 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/time_messages.csv: -------------------------------------------------------------------------------- 1 | time,message 2 | 1505292200,Node 1 started 3 | 1505293202,Failed to shutdown [error org.aaaa.bbbb.Cccc line 54 caused by foo exception] 4 | 1505294203,Node 2 started 5 | 1505295204,Failed to shutdown [error but this time completely different] 6 | 1505296205,Node 3 started 7 | 1505297207,Node 2 started 8 | 1505298209,Failed to shutdown [error org.aaaa.bbbb.Cccc line 66 caused by bar exception] 9 | 1505299212,Node 1 started 10 | 1505300213,Failed to shutdown [error org.dd.ee.ff line 88] 11 | 1505311214,Node 3 started 12 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # This file contains SHAs of changes that should be ignored when e.g. 2 | # running `git blame` on a file. Do not edit any of the existing commits. 3 | # 4 | # Use this file by running: 5 | # 6 | # git blame --ignore-revs-file=.git-blame-ignore-rev 7 | # 8 | # or by configuring `blame.ignoreRevsFile`. The latter ought to also work 9 | # with IDEs such as CLion. 10 | 11 | 12 | # Soft column limit of 80 for clang-format 13 | 78e15c472a71edb5a324957228e3ee929f626feb 14 | 15 | # Reformat native code using clang-format 16 | 2945e2b7e2628daf07a9c117ec550ccfd73c7548 17 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_excludefrequent.conf: -------------------------------------------------------------------------------- 1 | detector.1.clause = sum(bytes) by dest_ip over src_ip excludefrequent=all 2 | detector.2.clause = metric(responsetime) by airline 3 | detector.3.clause = sum(bytes) by dest_ip excludefrequent=by 4 | detector.4.clause = sum(bytes) by src_ip over dest_ip excludefrequent=by 5 | detector.5.clause = sum(bytes) by src_ip partitionfield=host excludefrequent=by 6 | detector.6.clause = sum(bytes) over dest_ip excludefrequent=over 7 | detector.7.clause = rare by process excludefrequent=by 8 | detector.8.clause = rare by client excludefrequent=none 9 | 10 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/pop_sum_bytes_by_status_over_clientip.json: -------------------------------------------------------------------------------- 1 | { 2 | "job_id": "pop_sum_bytes_by_status_over_clientip", 3 | "analysis_config": { 4 | "bucket_span": "15m", 5 | "detectors": [ 6 | { 7 | "detector_description": "sum(bytes) by status partitionfield=clientip", 8 | "function": "sum", 9 | "field_name": "bytes", 10 | "by_field_name": "status", 11 | "partition_field_name": "clientip" 12 | } 13 | ], 14 | "influencers": [ 15 | "clientip", 16 | "status" 17 | ], 18 | "model_prune_window": "30d" 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /copyright_code_header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/scheduled_events.conf: -------------------------------------------------------------------------------- 1 | detector.1.clause = count by mlcategory categorizationfield=_raw 2 | scheduledevent.1.description = May Bank Holiday 3 | scheduledevent.1.rules = [{"actions":["skip_result","skip_model_update"],"conditions":[{"applies_to":"time","operator":"gte","value":1525132800.0},{"applies_to":"time","operator":"lt","value":1525219200.0}]}] 4 | scheduledevent.2.description = New Years Day 5 | scheduledevent.2.rules = [{"actions":["skip_result","skip_model_update"],"conditions":[{"applies_to":"time","operator":"gte","value":1514764800.0},{"applies_to":"time","operator":"lt","value":1514851200.0}]}] 6 | -------------------------------------------------------------------------------- /devlib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | add_subdirectory(vflib) 13 | -------------------------------------------------------------------------------- /lib/model/unittest/testfiles/mllimits.conf: -------------------------------------------------------------------------------- 1 | 2 | # This file configures various limits to Ml's search commands. 3 | # Improperly configured limits may result in Ml search command crashes 4 | # and/or memory overuse. 5 | 6 | [memory] 7 | # Maximum allowed memory used by the internal models before further analysis 8 | # that would increase memory usage is halted. In MiB. 9 | modelmemorylimit = 4567 10 | 11 | [results] 12 | # Probability threshold (as a percentage) for results to be output 13 | unusualprobabilitythreshold = 0.5 14 | 15 | # Max number of examples per category that are stored in case of categorization. 16 | maxexamples = 8 17 | 18 | -------------------------------------------------------------------------------- /lib/core/unittest/testfiles/withNs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | foo_top 4 | 5 | foo_sub 6 | 7 | 8 | Ml 9 | 0 10 | 11 | 12 | 13 | 14 | ml 15 | 16 | 17 | Ml 18 | 1 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /devbin/vsbug/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML VSBug") 13 | 14 | ml_add_non_distributed_executable(vsbug 15 | Main.cc 16 | ) 17 | -------------------------------------------------------------------------------- /dev-tools/autodetect_profiling_macos/record_memgraph.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | while true; do 3 | autodetect_pid=$(pgrep autodetect) 4 | 5 | if [[ -n $autodetect_pid ]]; then 6 | echo "Process 'autodetect' found with PID $autodetect_pid." 7 | 8 | # Start 'leaks' utility from Xcode to generate memgraph 9 | leaks_command="leaks $autodetect_pid --outputGraph=autodetect_${autodetect_pid}" 10 | eval $leaks_command 11 | 12 | if [[ $? -eq 0 ]]; then 13 | echo "Memgraph generated successfully." 14 | break 15 | else 16 | echo "Failed to generate memgraph." 17 | break 18 | fi 19 | fi 20 | 21 | # Sleep for 0.1 seconds before checking again 22 | sleep 0.1 23 | done -------------------------------------------------------------------------------- /lib/maths/common/CLinearAlgebra.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | // Template implementations in header. 15 | -------------------------------------------------------------------------------- /devbin/analyze_test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Analyze Test") 13 | 14 | ml_add_non_distributed_executable(analyze_test 15 | Main.cc 16 | ) 17 | -------------------------------------------------------------------------------- /lib/core/CCompressedDictionary.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace core {} 16 | } 17 | -------------------------------------------------------------------------------- /cmake/architecture/aarch64.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | message(STATUS "aarch64 detected for target") 13 | set(ARCHCFLAGS "-march=armv8-a+crc+crypto" "-ffp-contract=on") 14 | -------------------------------------------------------------------------------- /cmake/os/linux.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | message(STATUS "Linux detected") 13 | 14 | set(EXE_DIR bin) 15 | set(PLATFORM_NAME "Linux") 16 | add_compile_definitions(Linux) 17 | 18 | -------------------------------------------------------------------------------- /cmake/os/windows.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | message(STATUS "Windows detected") 13 | set(EXE_DIR bin) 14 | set(PLATFORM_NAME "Windows") 15 | add_compile_definitions(Windows) 16 | 17 | -------------------------------------------------------------------------------- /cmake/test-clean-results.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | file(GLOB_RECURSE TEST_RESULTS "${TEST_DIR}/*.failed" ) 13 | if (TEST_RESULTS) 14 | file(REMOVE ${TEST_RESULTS}) 15 | endif() 16 | -------------------------------------------------------------------------------- /dev-tools/autodetect_profiling_macos/record_xctrace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while true; do 4 | autodetect_pid=$(pgrep autodetect) 5 | 6 | if [[ -n $autodetect_pid ]]; then 7 | echo "Process 'autodetect' found with PID $autodetect_pid." 8 | 9 | # Start 'xctrace' to record memgraph 10 | xctrace_command="xctrace record --output autodetect_${autodetect_pid}.trace --template 'Allocations' --attach 'autodetect'" 11 | eval $xctrace_command 12 | 13 | if [[ $? -eq 0 ]]; then 14 | echo "Trace file generated successfully." 15 | break 16 | else 17 | echo "Failed to generate the trace file." 18 | break 19 | fi 20 | fi 21 | 22 | # Sleep for 0.1 seconds before checking again 23 | sleep 0.1 24 | done 25 | -------------------------------------------------------------------------------- /lib/seccomp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Seccomp") 13 | 14 | set(SRCS 15 | CSystemCallFilter.cc 16 | ) 17 | 18 | ml_add_library(MlSeccomp STATIC ${SRCS}) 19 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: LLVM 2 | AccessModifierOffset: -4 3 | AllowAllParametersOfDeclarationOnNextLine: false 4 | AllowShortBlocksOnASingleLine: true 5 | AllowShortFunctionsOnASingleLine: InlineOnly 6 | AlwaysBreakTemplateDeclarations: true 7 | FixNamespaceComments: false 8 | IndentCaseLabels: false 9 | IndentWidth: 4 10 | TabWidth: 4 11 | BinPackParameters: false 12 | PenaltyBreakAssignment: 20 13 | PenaltyBreakBeforeFirstCallParameter: 15 14 | PenaltyBreakComment: 300 15 | PenaltyBreakFirstLessLess: 1 16 | PenaltyBreakString: 30 17 | PenaltyExcessCharacter: 1 18 | PenaltyReturnTypeOnItsOwnLine: 30 19 | PointerAlignment: Left 20 | SpaceAfterTemplateKeyword: false 21 | ReflowComments: false 22 | SortIncludes: true 23 | BreakStringLiterals: false 24 | -------------------------------------------------------------------------------- /cmake/Info.plist.in: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | English 7 | CFBundlePackageType 8 | APPL 9 | CFBundleIdentifier 10 | co.elastic.ml-cpp.@ML_APP_NAME@ 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | @ML_APP_NAME@ 15 | CFBundleDisplayName 16 | @ML_APP_NAME@ 17 | CFBundleVersion 18 | @ML_VERSION_NUM@ 19 | 20 | 21 | -------------------------------------------------------------------------------- /.buildkite/scripts/common/base.sh: -------------------------------------------------------------------------------- 1 | # This file contains useful bash functions or variables. 2 | # It should be sourced in each bash script. 3 | 4 | # ---------------------------------- 5 | # Colors 6 | # 7 | # example: 8 | # echo -e "${RED}ALERT${NOCOLOR} occurred." 9 | # # -e is required to enable interpretation of backslash escapes 10 | # ---------------------------------- 11 | NOCOLOR='\033[0m' 12 | RED='\033[0;31m' 13 | GREEN='\033[0;32m' 14 | ORANGE='\033[0;33m' 15 | BLUE='\033[0;34m' 16 | PURPLE='\033[0;35m' 17 | CYAN='\033[0;36m' 18 | LIGHTGRAY='\033[0;37m' 19 | DARKGRAY='\033[1;30m' 20 | LIGHTRED='\033[1;31m' 21 | LIGHTGREEN='\033[1;32m' 22 | YELLOW='\033[1;33m' 23 | LIGHTBLUE='\033[1;34m' 24 | LIGHTPURPLE='\033[1;35m' 25 | LIGHTCYAN='\033[1;36m' 26 | WHITE='\033[1;37m' 27 | -------------------------------------------------------------------------------- /dev-tools/docker/run_docker_clang_format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 4 | # or more contributor license agreements. Licensed under the Elastic License 5 | # 2.0 and the following additional limitation. Functionality enabled by the 6 | # files subject to the Elastic License 2.0 may only be used in production when 7 | # invoked by an Elasticsearch process with a license key installed that permits 8 | # use of machine learning features. You may not use this file except in 9 | # compliance with the Elastic License 2.0 and the foregoing additional 10 | # limitation. 11 | # 12 | 13 | docker run --rm -v $CPP_SRC_HOME:/ml-cpp -u $(id -u):$(id -g) docker.elastic.co/ml-dev/ml-check-style:2 /ml-cpp/dev-tools/clang-format.sh 14 | -------------------------------------------------------------------------------- /lib/core/CContainerPrinter.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace core { 16 | const std::string CContainerPrinter::NULL_STR{"\"null\""}; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /lib/core/CTimeGm.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | time_t CTimeGm::timeGm(struct tm* ts) { 17 | return ::timegm(ts); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /devbin/move_copy_swap/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Move/Copy/Swap") 13 | 14 | set(ML_LINK_LIBRARIES 15 | MlCore 16 | ) 17 | 18 | ml_add_non_distributed_executable(move_copy_swap 19 | Main.cc 20 | ) 21 | -------------------------------------------------------------------------------- /cmake/architecture/x86_64.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | message(STATUS "x86_64 detected for target") 13 | set (ARCHCFLAGS "-msse4.2") 14 | if(CMAKE_SYSTEM_NAME STREQUAL "Linux") 15 | list(APPEND ARCHCFLAGS "-mfpmath=sse") 16 | endif() 17 | -------------------------------------------------------------------------------- /devlib/vflib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML vflib") 13 | 14 | include_directories(${CMAKE_SOURCE_DIR}/devinclude) 15 | 16 | ml_add_non_distributed_library(Vf SHARED 17 | CIncrementer.cc 18 | CLooper.cc 19 | ) 20 | -------------------------------------------------------------------------------- /lib/core/CTimeGm_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | time_t CTimeGm::timeGm(struct tm* ts) { 17 | return ::_mkgmtime(ts); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /devlib/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | include $(CPP_SRC_HOME)/mk/defines.mk 13 | 14 | .PHONY: test 15 | .PHONY: build 16 | .PHONY: install 17 | 18 | COMPONENTS= \ 19 | vflib \ 20 | 21 | include $(CPP_SRC_HOME)/mk/toplevel.mk 22 | 23 | -------------------------------------------------------------------------------- /devbin/vfprog/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML VfProg") 13 | 14 | set(ML_LINK_LIBRARIES 15 | MlCore 16 | Vf 17 | ) 18 | 19 | ml_add_non_distributed_executable(vfprog 20 | Main.cc 21 | CIncrementer.cc 22 | CLooper.cc 23 | ) 24 | -------------------------------------------------------------------------------- /devbin/vsbug/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=vsbug$(EXE_EXT) 14 | 15 | all: build 16 | 17 | SRCS= \ 18 | Main.cc \ 19 | 20 | NO_TEST_CASES=1 21 | 22 | include $(CPP_SRC_HOME)/mk/stddevapp.mk 23 | 24 | -------------------------------------------------------------------------------- /lib/core/CCTimeR.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | char* CCTimeR::cTimeR(const time_t* clock, char* result) { 17 | return ::ctime_r(clock, result); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/core/CDataSearcher.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | const std::string CDataSearcher::EMPTY_STRING; 17 | 18 | CDataSearcher::~CDataSearcher() { 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /devbin/state_search_splitter/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML State Splitter") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | ) 18 | 19 | ml_add_non_distributed_executable(state_splitter 20 | Main.cc 21 | ) 22 | -------------------------------------------------------------------------------- /lib/core/CPersistUtils.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace core { 16 | 17 | const char CPersistUtils::DELIMITER(':'); 18 | const char CPersistUtils::PAIR_DELIMITER(';'); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/core/README.scowl: -------------------------------------------------------------------------------- 1 | The file scowl.dict was generated using the open source word list SCOWL version 7.1 2 | 3 | The command used to generate the file (run from the scowl-7.1 directory) was: 4 | 5 | ./mk-list -f -v1 american british canadian 55 | grep -v roman-numerals | grep -v proper-names | sed 's~^~final/~' | xargs cat | tr 'A-Z' 'a-z' | grep -v '[^a-z]' | grep -v '^[b-hj-z]$' | sort -u > scowl.dict 6 | 7 | The intention is to generate a word list that: 8 | 1) Contains US, UK and Canadian spelling variants 9 | 2) Does not contain roman numerals 10 | 3) Contains very few proper names 11 | 4) Is all lower case 12 | 5) Contains only ASCII letters (no apostrophes, hyphens or accented letters) 13 | 6) Does not contain single letter "words" except a and i 14 | 7) Is sorted into ASCII order 15 | 8) Contains no duplicates 16 | 17 | -------------------------------------------------------------------------------- /lib/core/CCrashHandler.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | void CCrashHandler::installCrashHandler() { 17 | // do nothing, see platform specific actions 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/core/CUnSetEnv.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | int CUnSetEnv::unSetEnv(const char* name) { 19 | return ::unsetenv(name); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /.ci/orka/gobld-bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | retry() { 4 | local retries=$1; shift 5 | local delay=$1; shift 6 | local attempts=1 7 | 8 | until "$@"; do 9 | retry_exit_status=$? 10 | echo "Exited with $retry_exit_status" >&2 11 | if (( retries == "0" )); then 12 | return $retry_exit_status 13 | elif (( attempts == retries )); then 14 | echo "Failed $attempts retries" >&2 15 | return $retry_exit_status 16 | else 17 | echo "Retrying $((retries - attempts)) more times..." >&2 18 | attempts=$((attempts + 1)) 19 | sleep "$delay" 20 | fi 21 | done 22 | } 23 | 24 | # Retry up to ~80 seconds as network is not guaranteed to be available when the script starts. 25 | retry 16 5 curl -s https://storage.googleapis.com/ci-systems-orka-provisioner-prod/gobld-bootstrap.sh | bash 26 | -------------------------------------------------------------------------------- /lib/core/CGmTimeR.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | struct tm* CGmTimeR::gmTimeR(const time_t* clock, struct tm* result) { 17 | return ::gmtime_r(clock, result); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/api/dump_state/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Dump State") 13 | 14 | set(SRCS 15 | ) 16 | 17 | set(ML_LINK_LIBRARIES ${Boost_LIBRARIES} 18 | MlCore 19 | MlApi 20 | MlSeccomp 21 | MlVer 22 | ) 23 | 24 | ml_add_executable(dump_state ${SRCS}) 25 | -------------------------------------------------------------------------------- /.buildkite/scripts/steps/check-style.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | set -euo pipefail 13 | 14 | source ${REPO_ROOT}/.buildkite/scripts/common/base.sh 15 | 16 | # override the setting in the check_style docker image 17 | unset CPP_SRC_HOME 18 | 19 | ./dev-tools/check-style.sh ${@} 20 | -------------------------------------------------------------------------------- /bin/normalize/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Normalize") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | MlModel 18 | MlApi 19 | MlSeccomp 20 | MlVer 21 | ) 22 | 23 | ml_add_executable(normalize 24 | CCmdLineParser.cc 25 | ) 26 | -------------------------------------------------------------------------------- /lib/core/CStrPTime.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | char* CStrPTime::strPTime(const char* buf, const char* format, struct tm* tm) 17 | 18 | { 19 | return ::strptime(buf, format, tm); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /devbin/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | add_subdirectory(unixtime_to_string) 13 | add_subdirectory(model_extractor) 14 | add_subdirectory(state_search_splitter) 15 | add_subdirectory(analyze_test) 16 | add_subdirectory(move_copy_swap) 17 | add_subdirectory(vfprog) 18 | add_subdirectory(vsbug) 19 | -------------------------------------------------------------------------------- /lib/core/CGmTimeR_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | struct tm* CGmTimeR::gmTimeR(const time_t* clock, struct tm* result) { 17 | ::gmtime_s(result, clock); 18 | 19 | return result; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/core/CStrFTime.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | size_t CStrFTime::strFTime(char* buf, size_t maxSize, const char* format, struct tm* tm) { 17 | return ::strftime(buf, maxSize, format, tm); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/core/CStrTokR.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | char* CStrTokR::strTokR(char* str, const char* sep, char** lasts) { 19 | return ::strtok_r(str, sep, lasts); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /devbin/unixtime_to_string/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Unix-Time to String") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | MlVer 18 | ) 19 | 20 | ml_add_non_distributed_executable(unixtime_to_string 21 | Main.cc 22 | CCmdLineParser.cc 23 | ) 24 | -------------------------------------------------------------------------------- /lib/core/CStrTokR_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | char* CStrTokR::strTokR(char* str, const char* sep, char** lasts) { 19 | return ::strtok_s(str, sep, lasts); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/core/CSetEnv.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | int CSetEnv::setEnv(const char* name, const char* value, int overwrite) { 19 | return ::setenv(name, value, overwrite); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/core/CSetMode.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | int CSetMode::setMode(int /* fd */, int /* mode */) { 17 | return 0; 18 | } 19 | 20 | int CSetMode::setBinaryMode(int /* fd */) { 21 | return 0; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /bin/categorize/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Categorize") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | MlMathsCommon 18 | MlModel 19 | MlApi 20 | MlSeccomp 21 | MlVer 22 | ) 23 | 24 | ml_add_executable(categorize 25 | CCmdLineParser.cc 26 | ) 27 | -------------------------------------------------------------------------------- /devbin/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | include $(CPP_SRC_HOME)/mk/defines.mk 13 | 14 | .PHONY: test 15 | .PHONY: build 16 | 17 | COMPONENTS= \ 18 | unixtime_to_string \ 19 | model_extractor \ 20 | state_search_splitter \ 21 | 22 | include $(CPP_SRC_HOME)/mk/toplevel.mk 23 | 24 | -------------------------------------------------------------------------------- /devbin/analyze_test/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=analyze_test$(EXE_EXT) 14 | 15 | INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR) 16 | 17 | all: build 18 | 19 | SRCS= \ 20 | Main.cc \ 21 | 22 | NO_TEST_CASES=1 23 | 24 | include $(CPP_SRC_HOME)/mk/stddevapp.mk 25 | 26 | -------------------------------------------------------------------------------- /devbin/model_extractor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Model Extractor") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | MlApi 18 | MlModel 19 | MlVer 20 | ) 21 | 22 | ml_add_non_distributed_executable(model_extractor 23 | Main.cc 24 | CCmdLineParser.cc 25 | ) 26 | -------------------------------------------------------------------------------- /dev-tools/docker/style_checker/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | # Increment the version here when a new check style image is built 13 | FROM docker.elastic.co/ml-dev/ml-check-style:2 14 | 15 | MAINTAINER David Roberts 16 | 17 | # Copy the current Git repository into the container 18 | COPY . /ml-cpp/ 19 | 20 | -------------------------------------------------------------------------------- /devbin/move_copy_swap/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=move_copy_swap$(EXE_EXT) 14 | 15 | ML_LIBS=$(LIB_ML_CORE) 16 | 17 | LIBS=$(ML_LIBS) 18 | 19 | all: build 20 | 21 | SRCS= \ 22 | Main.cc \ 23 | 24 | NO_TEST_CASES=1 25 | 26 | include $(CPP_SRC_HOME)/mk/stddevapp.mk 27 | 28 | -------------------------------------------------------------------------------- /devlib/vflib/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=$(OBJS_DIR)/libVf$(DYNAMIC_LIB_EXT) 14 | 15 | INCLUDE_PATH=-I$(CPP_SRC_HOME)/devinclude 16 | 17 | all: build 18 | 19 | SRCS= \ 20 | CIncrementer.cc \ 21 | CLooper.cc \ 22 | 23 | NO_TEST_CASES=1 24 | 25 | include $(CPP_SRC_HOME)/mk/dynamiclib.mk 26 | -------------------------------------------------------------------------------- /lib/ver/unittest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Ver unit tests") 13 | 14 | set (SRCS 15 | Main.cc 16 | CBuildInfoTest.cc 17 | ) 18 | 19 | set(ML_LINK_LIBRARIES 20 | ${Boost_LIBRARIES_WITH_UNIT_TEST} 21 | ${LIBXML2_LIBRARIES} 22 | MlCore 23 | MlVer 24 | MlTest 25 | ) 26 | 27 | ml_add_test_executable(ver ${SRCS}) 28 | -------------------------------------------------------------------------------- /lib/core/CBase64Filter.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace core { 16 | 17 | CBase64Encoder::CBase64Encoder() : m_Buffer(4096) { 18 | } 19 | 20 | CBase64Decoder::CBase64Decoder() 21 | : m_BufferIn(4096), m_BufferOut(4096), m_Eos(false) { 22 | } 23 | 24 | } // core 25 | } // ml 26 | -------------------------------------------------------------------------------- /lib/core/CScopedLock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | CScopedLock::CScopedLock(CMutex& mutex) : m_Mutex(mutex) { 19 | m_Mutex.lock(); 20 | } 21 | 22 | CScopedLock::~CScopedLock() { 23 | m_Mutex.unlock(); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lib/maths/common/CLeastSquaresOnlineRegression.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace maths { 16 | namespace common { 17 | namespace least_squares_online_regression_detail { 18 | const double CMaxCondition::VALUE = 1e7; 19 | } 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /bin/data_frame_analyzer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML DataFrame Analyzer") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | MlMathsCommon 18 | MlMathsAnalytics 19 | MlModel 20 | MlApi 21 | MlSeccomp 22 | MlVer 23 | ) 24 | 25 | ml_add_executable(data_frame_analyzer 26 | CCmdLineParser.cc 27 | ) 28 | -------------------------------------------------------------------------------- /bin/controller/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Controller") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | MlSeccomp 18 | MlVer 19 | ) 20 | 21 | ml_add_executable(controller 22 | CBlockingCallCancellingStreamMonitor.cc 23 | CCmdLineParser.cc 24 | CCommandProcessor.cc 25 | CResponseJsonWriter.cc 26 | ) 27 | -------------------------------------------------------------------------------- /devbin/state_search_splitter/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=state_search_splitter$(EXE_EXT) 14 | 15 | ML_LIBS=$(LIB_ML_CORE) 16 | 17 | USE_BOOST=1 18 | 19 | LIBS=$(ML_LIBS) 20 | 21 | all: build 22 | 23 | SRCS= \ 24 | Main.cc \ 25 | 26 | NO_TEST_CASES=1 27 | 28 | include $(CPP_SRC_HOME)/mk/stddevapp.mk 29 | 30 | -------------------------------------------------------------------------------- /include/api/ApiTypes.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #ifndef INCLUDED_ml_api_ApiTypes_h 13 | #define INCLUDED_ml_api_ApiTypes_h 14 | 15 | namespace ml { 16 | namespace api_t { 17 | enum EDataFrameTrainBoostedTreeTask { 18 | E_Encode = 0, 19 | E_Train, 20 | E_Update, 21 | E_Predict 22 | }; 23 | } 24 | } 25 | 26 | #endif // INCLUDED_ml_api_ApiTypes_h 27 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields.json: -------------------------------------------------------------------------------- 1 | { 2 | "job_id": "new_ml_fields", 3 | "analysis_config": { 4 | "detectors": [ 5 | { 6 | "function": "count", 7 | "by_field_name": "mlcategory", 8 | "partition_field_name": "agent" 9 | }, 10 | { 11 | "function": "count", 12 | "by_field_name": "remote_ip" 13 | }, 14 | { 15 | "function": "count", 16 | "by_field_name": "remote_user" 17 | }, 18 | { 19 | "function": "count", 20 | "by_field_name": "request" 21 | }, 22 | { 23 | "function": "count", 24 | "by_field_name": "response" 25 | }, 26 | { 27 | "function": "metric", 28 | "field_name": "bytes", 29 | "by_field_name": "referrer" 30 | }, 31 | { 32 | "function": "count", 33 | "by_field_name": "agent" 34 | } 35 | ] 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /lib/model/CProcessMemoryUsage_Linux.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace model { 16 | 17 | // On Linux the process memory usage is determined by the OS. 18 | const CProcessMemoryUsage::EMemoryStrategy CProcessMemoryUsage::MEMORY_STRATEGY{ 19 | EMemoryStrategy::E_System}; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dev-tools/docker/check_style_image/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | FROM alpine:3.8 13 | 14 | # Docker image containing the correct clang-format for check-style.sh 15 | 16 | MAINTAINER Valeriy Khakhutskyy 17 | 18 | RUN apk update && \ 19 | apk add --no-cache clang bash git 20 | 21 | WORKDIR /ml-cpp 22 | ENV CPP_SRC_HOME=/ml-cpp 23 | 24 | -------------------------------------------------------------------------------- /lib/seccomp/unittest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Seccomp unit tests") 13 | 14 | set (SRCS 15 | Main.cc 16 | CSystemCallFilterTest.cc 17 | ) 18 | 19 | set(ML_LINK_LIBRARIES 20 | ${Boost_LIBRARIES_WITH_UNIT_TEST} 21 | ${LIBXML2_LIBRARIES} 22 | MlCore 23 | MlSeccomp 24 | MlTest 25 | ) 26 | 27 | ml_add_test_executable(seccomp ${SRCS}) 28 | -------------------------------------------------------------------------------- /devbin/unixtime_to_string/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=unixtime_to_string$(EXE_EXT) 14 | 15 | ML_LIBS=$(LIB_ML_CORE) 16 | 17 | USE_XML=1 18 | USE_BOOST=1 19 | 20 | LIBS=$(ML_LIBS) 21 | 22 | all: build 23 | 24 | SRCS= \ 25 | Main.cc \ 26 | 27 | NO_TEST_CASES=1 28 | 29 | include $(CPP_SRC_HOME)/mk/stddevapp.mk 30 | 31 | -------------------------------------------------------------------------------- /devbin/vfprog/CIncrementer.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include "CIncrementer.h" 12 | 13 | namespace ml { 14 | namespace vfprog { 15 | 16 | CIncrementer::~CIncrementer() { 17 | } 18 | 19 | size_t CIncrementer::nonVirtualIncrement(size_t val) { 20 | return val + 1; 21 | } 22 | 23 | size_t CIncrementer::virtualIncrement(size_t val) { 24 | return val + 1; 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /lib/core/CScopedFastLock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | CScopedFastLock::CScopedFastLock(CFastMutex& mutex) : m_Mutex(mutex) { 19 | m_Mutex.lock(); 20 | } 21 | 22 | CScopedFastLock::~CScopedFastLock() { 23 | m_Mutex.unlock(); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lib/core/CSetMode_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | #include 15 | 16 | namespace ml { 17 | namespace core { 18 | 19 | int CSetMode::setMode(int fd, int mode) { 20 | return _setmode(fd, mode); 21 | } 22 | 23 | int CSetMode::setBinaryMode(int fd) { 24 | return _setmode(fd, _O_BINARY); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /devlib/vflib/CIncrementer.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace vflib { 15 | 16 | CIncrementer::~CIncrementer() { 17 | } 18 | 19 | size_t CIncrementer::nonVirtualIncrement(size_t val) { 20 | return val + 1; 21 | } 22 | 23 | size_t CIncrementer::virtualIncrement(size_t val) { 24 | return val + 1; 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /lib/model/CProcessMemoryUsage.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace model { 16 | // On platforms other than Linux the process memory usage is the estimated size of the models. 17 | const CProcessMemoryUsage::EMemoryStrategy CProcessMemoryUsage::MEMORY_STRATEGY{ 18 | EMemoryStrategy::E_Estimated}; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.buildkite/scripts/steps/upload_dra_to_s3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | # Upload the artifacts to S3, where day-to-day Elasticsearch 12 | # builds will download them from. 13 | # 14 | . .buildkite/scripts/common/aws_creds_from_vault.sh 15 | echo 'Uploading daily releasable artifacts to S3' 16 | ./gradlew --info -Dbuild.version_qualifier=$VERSION_QUALIFIER -Dbuild.snapshot=$BUILD_SNAPSHOT uploadAll 17 | -------------------------------------------------------------------------------- /lib/core/CSetEnv_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | int CSetEnv::setEnv(const char* name, const char* value, int overwrite) { 19 | if (overwrite == 0 && ::getenv(name) != 0) { 20 | return 0; 21 | } 22 | 23 | return ::_putenv_s(name, value); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.buildkite/pipelines/send_email_notification.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | # Create a unique job that sends email notification only. 12 | cat < 13 | 14 | namespace ml { 15 | namespace api { 16 | 17 | namespace { 18 | const std::string STATE_ID_SUFFIX{"_state"}; 19 | } 20 | 21 | std::string getStateId(const std::string& jobId, const std::string& analysisName) { 22 | return jobId + '_' + analysisName + STATE_ID_SUFFIX; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /lib/core/CStrCaseCmp.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | int CStrCaseCmp::strCaseCmp(const char* s1, const char* s2) { 19 | return ::strcasecmp(s1, s2); 20 | } 21 | 22 | int CStrCaseCmp::strNCaseCmp(const char* s1, const char* s2, size_t n) { 23 | return ::strncasecmp(s1, s2, n); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lib/core/CStrCaseCmp_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | int CStrCaseCmp::strCaseCmp(const char* s1, const char* s2) { 19 | return ::_stricmp(s1, s2); 20 | } 21 | 22 | int CStrCaseCmp::strNCaseCmp(const char* s1, const char* s2, size_t n) { 23 | return ::_strnicmp(s1, s2, n); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lib/core/CUnSetEnv_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | int CUnSetEnv::unSetEnv(const char* name) { 19 | // The Microsoft C runtime library treats a request to set an environment 20 | // variable to an empty string as a request to delete it 21 | return ::_putenv_s(name, ""); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /lib/core/CFastMutex_MacOSX.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | CFastMutex::CFastMutex() : m_Mutex(OS_UNFAIR_LOCK_INIT) { 17 | } 18 | 19 | CFastMutex::~CFastMutex() { 20 | } 21 | 22 | void CFastMutex::lock() { 23 | os_unfair_lock_lock(&m_Mutex); 24 | } 25 | 26 | void CFastMutex::unlock() { 27 | os_unfair_lock_unlock(&m_Mutex); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /lib/maths/common/COrderings.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace maths { 16 | namespace common { 17 | const std::less<> COrderings::SDerefLess::s_Less; 18 | const std::greater<> COrderings::SDerefGreater::s_Greater; 19 | const std::less<> COrderings::SReferenceLess::s_Less; 20 | const std::greater<> COrderings::SReferenceGreater::s_Greater; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.buildkite/pipelines/format_and_validation.yml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | 11 | cat < 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | void CProcessPriority::reduceMemoryPriority() { 17 | // Default is to do nothing - see platform-specific implementation files for 18 | // platforms where we do more 19 | } 20 | 21 | void CProcessPriority::reduceCpuPriority() { 22 | // Nothing at present on Windows 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /lib/core/CScopedReadLock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | CScopedReadLock::CScopedReadLock(CReadWriteLock& readWriteLock) 19 | : m_ReadWriteLock(readWriteLock) { 20 | m_ReadWriteLock.readLock(); 21 | } 22 | 23 | CScopedReadLock::~CScopedReadLock() { 24 | m_ReadWriteLock.readUnlock(); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /3rd_party/pull-valijson.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 4 | # or more contributor license agreements. Licensed under the Elastic License 5 | # 2.0 and the following additional limitation. Functionality enabled by the 6 | # files subject to the Elastic License 2.0 may only be used in production when 7 | # invoked by an Elasticsearch process with a license key installed that permits 8 | # use of machine learning features. You may not use this file except in 9 | # compliance with the Elastic License 2.0 and the foregoing additional 10 | # limitation. 11 | # 12 | 13 | # Script to get the appropriate version of Valijson, if not already present. 14 | # 15 | # Valijson must only be used in test code, _not_ any form of redistributable code. 16 | 17 | 18 | cd `dirname "$BASH_SOURCE"` 19 | 20 | if [ ! -d "valijson" ] ; then 21 | git -c advice.detachedHead=false clone --depth=1 --branch=v1.0.2 https://github.com/tristanpenman/valijson.git 22 | fi 23 | 24 | -------------------------------------------------------------------------------- /lib/core/CScopedWriteLock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | CScopedWriteLock::CScopedWriteLock(CReadWriteLock& readWriteLock) 19 | : m_ReadWriteLock(readWriteLock) { 20 | m_ReadWriteLock.writeLock(); 21 | } 22 | 23 | CScopedWriteLock::~CScopedWriteLock() { 24 | m_ReadWriteLock.writeUnlock(); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /bin/controller/unittest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML controller unit tests") 13 | 14 | set (SRCS 15 | Main.cc 16 | CBlockingCallCancellingStreamMonitorTest.cc 17 | CCommandProcessorTest.cc 18 | CResponseJsonWriterTest.cc 19 | ) 20 | 21 | set(ML_LINK_LIBRARIES 22 | ${Boost_LIBRARIES_WITH_UNIT_TEST} 23 | ${LIBXML2_LIBRARIES} 24 | MlCore 25 | MlTest 26 | MlVer 27 | ) 28 | 29 | ml_add_test_executable(controller ${SRCS}) 30 | -------------------------------------------------------------------------------- /include/api/ElasticsearchStateIndex.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #ifndef INCLUDED_ml_api_ElasticsearchStateIndex_h 12 | #define INCLUDED_ml_api_ElasticsearchStateIndex_h 13 | 14 | #include 15 | 16 | #include 17 | 18 | namespace ml { 19 | namespace api { 20 | API_EXPORT std::string getStateId(const std::string& jobId, const std::string& analysisName); 21 | } 22 | } 23 | 24 | #endif // INCLUDED_ml_api_ElasticsearchStateIndex_h 25 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/new_mlfields_partition.json: -------------------------------------------------------------------------------- 1 | { 2 | "job_id": "new_ml_fields_partition", 3 | "analysis_config": { 4 | "detectors": [ 5 | { 6 | "function": "count", 7 | "partition_field_name": "mlcategory" 8 | }, 9 | { 10 | "function": "count", 11 | "partition_field_name": "remote_ip" 12 | }, 13 | { 14 | "function": "count", 15 | "partition_field_name": "remote_user" 16 | }, 17 | { 18 | "function": "count", 19 | "partition_field_name": "request" 20 | }, 21 | { 22 | "function": "count", 23 | "partition_field_name": "response" 24 | }, 25 | { 26 | "function": "avg", 27 | "field_name": "bytes", 28 | "partition_field_name": "request" 29 | }, 30 | { 31 | "function": "high_sum", 32 | "field_name": "bytes", 33 | "partition_field_name": "response" 34 | } 35 | ], 36 | "categorization_field_name": "agent" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bin/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | if(CMAKE_SYSTEM_NAME STREQUAL "Linux") 12 | set(CMAKE_INSTALL_RPATH $ORIGIN/../lib) 13 | endif() 14 | 15 | if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") 16 | set(CMAKE_INSTALL_RPATH "@loader_path/../lib") 17 | endif() 18 | 19 | add_subdirectory(autodetect) 20 | add_subdirectory(categorize) 21 | add_subdirectory(controller) 22 | add_subdirectory(data_frame_analyzer) 23 | add_subdirectory(normalize) 24 | add_subdirectory(pytorch_inference) 25 | -------------------------------------------------------------------------------- /lib/maths/common/CFuzzyLogic.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace maths { 16 | namespace common { 17 | const CFuzzyTruthValue CFuzzyTruthValue::TRUE_VALUE{1.0}; 18 | const CFuzzyTruthValue CFuzzyTruthValue::FALSE_VALUE{0.0}; 19 | const CFuzzyTruthValue CFuzzyTruthValue::OR_UNDETERMINED_VALUE{0.0, 0.0}; 20 | const CFuzzyTruthValue CFuzzyTruthValue::AND_UNDETERMINED_VALUE{1.0, 1.0}; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /devbin/model_extractor/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=model_extractor$(EXE_EXT) 14 | 15 | ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_API) $(LIB_ML_MODEL) 16 | 17 | USE_XML=1 18 | USE_BOOST=1 19 | USE_BOOST_PROGRAMOPTIONS_LIBS=1 20 | USE_EIGEN=1 21 | USE_RAPIDJSON=1 22 | 23 | LIBS=$(ML_LIBS) 24 | 25 | all: build 26 | 27 | SRCS= \ 28 | Main.cc \ 29 | CCmdLineParser.cc \ 30 | 31 | NO_TEST_CASES=1 32 | 33 | include $(CPP_SRC_HOME)/mk/stddevapp.mk 34 | 35 | -------------------------------------------------------------------------------- /devbin/vfprog/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | include $(CPP_SRC_HOME)/mk/defines.mk 12 | 13 | TARGET=vfprog$(EXE_EXT) 14 | 15 | INCLUDE_PATH=-I$(CPP_SRC_HOME)/devinclude 16 | 17 | ML_LIBS=$(LIB_ML_CORE) 18 | 19 | ifeq ($(OS),Windows) 20 | ML_LIBS+= libVf.lib 21 | else 22 | ML_LIBS+= -lVf 23 | endif 24 | 25 | LIBS=$(ML_LIBS) 26 | 27 | all: build 28 | 29 | SRCS= \ 30 | Main.cc \ 31 | CIncrementer.cc \ 32 | CLooper.cc \ 33 | 34 | NO_TEST_CASES=1 35 | 36 | include $(CPP_SRC_HOME)/mk/stddevapp.mk 37 | 38 | -------------------------------------------------------------------------------- /cmake/os/darwin.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | message(STATUS "Darwin detected") 13 | 14 | set(EXE_DIR MacOS) 15 | set(CMAKE_MACOSX_RPATH 1) 16 | add_compile_definitions(MacOSX) 17 | set(PLATFORM_NAME "MacOSX") 18 | 19 | # Xcode code signs the binary artifacts by default, 20 | # which we don't want as it is invalidated when 21 | # the RPATH is updated upon install. Hence we disable 22 | # code signing here. 23 | set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "") 24 | set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO") 25 | -------------------------------------------------------------------------------- /lib/core/unittest/CProcessStatsTest_Linux.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | #include 15 | 16 | BOOST_AUTO_TEST_SUITE(CProcessStatsTest) 17 | 18 | BOOST_AUTO_TEST_CASE(testResidentSetSize) { 19 | BOOST_REQUIRE_GT(ml::core::CProcessStats::residentSetSize(), 0); 20 | } 21 | 22 | BOOST_AUTO_TEST_CASE(testMaxResidentSetSize) { 23 | BOOST_REQUIRE_GT(ml::core::CProcessStats::maxResidentSetSize(), 0); 24 | } 25 | 26 | BOOST_AUTO_TEST_SUITE_END() 27 | -------------------------------------------------------------------------------- /lib/core/CFastMutex_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | CFastMutex::CFastMutex() { 17 | InitializeSRWLock(&m_Mutex); 18 | } 19 | 20 | CFastMutex::~CFastMutex() { 21 | // There is no function to destroy the read/write lock on Windows 22 | } 23 | 24 | void CFastMutex::lock() { 25 | AcquireSRWLockExclusive(&m_Mutex); 26 | } 27 | 28 | void CFastMutex::unlock() { 29 | ReleaseSRWLockExclusive(&m_Mutex); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /lib/core/unittest/CProcessStatsTest_MacOSX.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | #include 15 | 16 | BOOST_AUTO_TEST_SUITE(CProcessStatsTest) 17 | 18 | BOOST_AUTO_TEST_CASE(testResidentSetSize) { 19 | BOOST_REQUIRE_EQUAL(ml::core::CProcessStats::residentSetSize(), 0); 20 | } 21 | 22 | BOOST_AUTO_TEST_CASE(testMaxResidentSetSize) { 23 | BOOST_REQUIRE_GT(ml::core::CProcessStats::maxResidentSetSize(), 0); 24 | } 25 | 26 | BOOST_AUTO_TEST_SUITE_END() 27 | -------------------------------------------------------------------------------- /lib/core/unittest/CProcessStatsTest_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | #include 15 | 16 | BOOST_AUTO_TEST_SUITE(CProcessStatsTest) 17 | 18 | BOOST_AUTO_TEST_CASE(testResidentSetSize) { 19 | BOOST_REQUIRE_GT(ml::core::CProcessStats::residentSetSize(), 0); 20 | } 21 | 22 | BOOST_AUTO_TEST_CASE(testMaxResidentSetSize) { 23 | BOOST_REQUIRE_GT(ml::core::CProcessStats::maxResidentSetSize(), 0); 24 | } 25 | 26 | BOOST_AUTO_TEST_SUITE_END() 27 | -------------------------------------------------------------------------------- /3rd_party/licenses/zlib-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler 2 | 3 | This software is provided 'as-is', without any express or implied 4 | warranty. In no event will the authors be held liable for any damages 5 | arising from the use of this software. 6 | 7 | Permission is granted to anyone to use this software for any purpose, 8 | including commercial applications, and to alter it and redistribute it 9 | freely, subject to the following restrictions: 10 | 11 | 1. The origin of this software must not be misrepresented; you must not 12 | claim that you wrote the original software. If you use this software 13 | in a product, an acknowledgment in the product documentation would be 14 | appreciated but is not required. 15 | 2. Altered source versions must be plainly marked as such, and must not be 16 | misrepresented as being the original software. 17 | 3. This notice may not be removed or altered from any source distribution. 18 | 19 | Jean-loup Gailly Mark Adler 20 | jloup@gzip.org madler@alumni.caltech.edu 21 | -------------------------------------------------------------------------------- /lib/core/unittest/CProcessPriorityTest.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | #include 15 | 16 | BOOST_AUTO_TEST_SUITE(CProcessPriorityTest) 17 | 18 | BOOST_AUTO_TEST_CASE(testReduceMemoryPriority) { 19 | BOOST_REQUIRE_NO_THROW(ml::core::CProcessPriority::reduceMemoryPriority()); 20 | } 21 | 22 | BOOST_AUTO_TEST_CASE(testReduceCpuPriority) { 23 | BOOST_REQUIRE_NO_THROW(ml::core::CProcessPriority::reduceCpuPriority()); 24 | } 25 | 26 | BOOST_AUTO_TEST_SUITE_END() 27 | -------------------------------------------------------------------------------- /bin/pytorch_inference/unittest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML pytorch_inference unit tests") 13 | 14 | set (SRCS 15 | Main.cc 16 | CCommandParserTest.cc 17 | CResultWriterTest.cc 18 | CThreadSettingsTest.cc 19 | ) 20 | 21 | set(ML_LINK_LIBRARIES 22 | ${Boost_LIBRARIES_WITH_UNIT_TEST} 23 | ${LIBXML2_LIBRARIES} 24 | MlCore 25 | MlMathsCommon 26 | MlMathsAnalytics 27 | MlModel 28 | MlApi 29 | MlTest 30 | MlVer 31 | ${TORCH_LIB} 32 | ${C10_LIB} 33 | ) 34 | 35 | ml_add_test_executable(pytorch_inference ${SRCS}) 36 | -------------------------------------------------------------------------------- /.ci/orka/gobld-bootstrap.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | EnvironmentVariables 6 | 7 | PATH 8 | /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin: 9 | 10 | Label 11 | co.elastic.gobld-bootstrap 12 | Program 13 | /usr/local/bin/gobld-bootstrap.sh 14 | RunAtLoad 15 | 16 | KeepAlive 17 | 18 | SessionCreate 19 | 20 | LaunchOnlyOnce 21 | 22 | StandardOutPath 23 | /tmp/gobld-bootstrap.stdout 24 | StandardErrorPath 25 | /tmp/gobld-bootstrap.stderr 26 | UserName 27 | admin 28 | GroupName 29 | staff 30 | InitGroups 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /lib/core/CBoostJsonUnbufferedIStreamWrapper.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace core { 16 | 17 | CBoostJsonUnbufferedIStreamWrapper::CBoostJsonUnbufferedIStreamWrapper(std::istream& strm) 18 | : m_Stream{strm} {}; 19 | 20 | char CBoostJsonUnbufferedIStreamWrapper::take() { 21 | int c{m_Stream.get()}; 22 | if (c == std::istream::traits_type::eof()) { 23 | return '\0'; 24 | } 25 | ++m_Count; 26 | return static_cast(c); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.buildkite/pull-requests.json: -------------------------------------------------------------------------------- 1 | { 2 | "jobs": [ 3 | { 4 | "enabled": true, 5 | "pipeline_slug": "ml-cpp-pr-builds", 6 | "allow_org_users": true, 7 | "allowed_repo_permissions": ["admin", "write"], 8 | "set_commit_status": false, 9 | "commit_status_context": "ml-cpp-ci", 10 | "build_on_commit": true, 11 | "build_on_comment": true, 12 | "trigger_comment_regex": "^(?:(?:buildkite +)(?build|debug|run_qa_tests|run_pytorch_tests)(=(?(?:[^ ]+)))? *(?: for ES_BRANCH=(?([.0-9a-zA-Z]+)))? *(?:with STACK_VERSION=(?([.0-9]+)))? *(?: *on *(?(?:[ ,]*(?:windows|linux|mac(os)?))+))?) *(?(?:[, ]*aarch64|x86_64)+)?$", 13 | "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", 14 | "skip_ci_labels": ["skip-ci", "jenkins-ci", ">test-mute", ">docs"], 15 | "skip_target_branches": ["6.8", "7.11", "7.12"], 16 | "skip_ci_on_only_changed": ["^docs/", "^3rd_party/licenses/", "README.*", "\\.md$", "\\.mdx$", "^\\.buildkite/pull-requests\\.json$"], 17 | "buildkite_branch_name_separator": "+" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /3rd_party/pull-valijson.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | # Script to get the latest version of Valijson, if not already present. 13 | # 14 | # Valijson must only be used in test code, _not_ any form of redistributable code. 15 | 16 | # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location 17 | 18 | if ( NOT EXISTS valijson ) 19 | execute_process( COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=v1.0.2 https://github.com/tristanpenman/valijson.git WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) 20 | endif() 21 | -------------------------------------------------------------------------------- /include/core/CSmallVectorFwd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #ifndef INCLUDED_ml_core_CSmallVectorFwd_h 13 | #define INCLUDED_ml_core_CSmallVectorFwd_h 14 | 15 | #include 16 | 17 | #include 18 | 19 | namespace ml { 20 | namespace core { 21 | //! Map boost::container::small_vector_base for consistent naming. 22 | template 23 | using CSmallVectorBase = boost::container::small_vector_base; 24 | template 25 | class CSmallVector; 26 | } 27 | } 28 | 29 | #endif // INCLUDED_ml_core_CSmallVectorFwd_h 30 | -------------------------------------------------------------------------------- /cmake/linux-x86_64.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | set(CPP_PLATFORM_HOME $ENV{CPP_SRC_HOME}/build/distribution/platform/linux-x86_64) 13 | 14 | message(STATUS "CMAKE_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}") 15 | 16 | # this must be first 17 | include("${CMAKE_CURRENT_LIST_DIR}/functions.cmake") 18 | 19 | # include linux specific settings 20 | include("${CMAKE_CURRENT_LIST_DIR}/os/linux.cmake") 21 | 22 | # set the architecture bits 23 | include("${CMAKE_CURRENT_LIST_DIR}/architecture/x86_64.cmake") 24 | 25 | # include gcc specific settings 26 | include("${CMAKE_CURRENT_LIST_DIR}/compiler/gcc.cmake") 27 | -------------------------------------------------------------------------------- /.buildkite/pipelines/send_slack_notification.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | # Create a unique job that sends slack notification only. 12 | cat < 13 | 14 | #include 15 | 16 | namespace ml { 17 | namespace test { 18 | 19 | CProgramCounterClearingFixture::CProgramCounterClearingFixture() { 20 | core::CProgramCounters& counters{core::CProgramCounters::instance()}; 21 | 22 | // Set all counters to 0 23 | for (std::size_t i = 0; i < counter_t::NUM_COUNTERS; ++i) { 24 | counters.counter(i) = 0; 25 | } 26 | 27 | // Clear the cache 28 | counters.m_Cache.clear(); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lib/test/CShellCmdEscape_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace test { 17 | 18 | void CShellCmdEscape::escapeCmd(std::string& cmd) { 19 | // Special characters are ^ < > & | 20 | // Escape character is ^ on Windows 21 | 22 | core::CStringUtils::replace("^", "^^", cmd); 23 | core::CStringUtils::replace("<", "^<", cmd); 24 | core::CStringUtils::replace(">", "^>", cmd); 25 | core::CStringUtils::replace("&", "^&", cmd); 26 | core::CStringUtils::replace("|", "^|", cmd); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/s.csv: -------------------------------------------------------------------------------- 1 | time,airline,responsetime,sourcetype 2 | 1359331200,EGF,197.4412,farequote 3 | 1359331200,JAL,503.7342,farequote 4 | 1359331207,AWE,20.4649,farequote 5 | 1359331220,JAL,525.3308,farequote 6 | 1359331259,AAL,136.2361,farequote 7 | 1359331262,VRD,282.9847,farequote 8 | 1359331269,JAL,493.0338,farequote 9 | 1359331270,UAL,8.4275,farequote 10 | 1359331272,FFT,221.8693,farequote 11 | 1359331296,UAL,9.946,farequote 12 | 1359331301,AWE,20.0674,farequote 13 | 1359331311,ACA,23.1251,farequote 14 | 1359331314,JAL,491.3751,farequote 15 | 1359331315,UAL,10.7749,farequote 16 | 1359331316,SWR,2668.3092,farequote 17 | 1359331321,AWE,19.2948,farequote 18 | 1359331328,EGF,205.1143,farequote 19 | 1359331333,ASA,66.3888,farequote 20 | 1359331349,AMX,13.3429,farequote 21 | 1359331352,AAL,104.9828,farequote 22 | 1359331370,JAL,507.8157,farequote 23 | 1359331382,BAW,188.0154,farequote 24 | 1359331385,JBU,1037.7588,farequote 25 | 1359331386,FFT,221.1585,farequote 26 | 1359331395,JZA,995.4091,farequote 27 | 1359331404,AMX,12.3134,farequote 28 | 1359331416,UAL,10.1147,farequote 29 | 1359331416,ASA,85.1724,farequote 30 | 1359331424,SWR,1965.1186,farequote 31 | 1359331429,SWA,237.5185,farequote 32 | -------------------------------------------------------------------------------- /lib/core/CCTimeR_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | char* CCTimeR::cTimeR(const time_t* clock, char* result) { 17 | // This is effectively bypassing the security feature of the Windows 18 | // ctime_s() call, but the wrapper function has the arguments of the 19 | // vulnerable Unix ctime_r() function, so we don't know the real buffer 20 | // size, and must assume it's big enough 21 | static const size_t MIN_BUF_SIZE(26); 22 | 23 | ::ctime_s(result, MIN_BUF_SIZE, clock); 24 | 25 | return result; 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /lib/model/CMonitoredResource.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace model { 15 | 16 | bool CMonitoredResource::supportsPruning() const { 17 | return false; 18 | } 19 | 20 | bool CMonitoredResource::initPruneWindow(std::size_t& /*defaultPruneWindow*/, 21 | std::size_t& /*minimumPruneWindow*/) const { 22 | return false; 23 | } 24 | 25 | core_t::TTime CMonitoredResource::bucketLength() const { 26 | return 0; 27 | } 28 | 29 | void CMonitoredResource::prune(std::size_t /*maximumAge*/) { 30 | // NO-OP 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /lib/api/CNoopCategoryIdMapper.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace { 15 | std::string EMPTY_STRING; 16 | } 17 | 18 | namespace ml { 19 | namespace api { 20 | 21 | CGlobalCategoryId CNoopCategoryIdMapper::map(model::CLocalCategoryId localCategoryId) { 22 | return CGlobalCategoryId{localCategoryId.id()}; 23 | } 24 | 25 | const std::string& CNoopCategoryIdMapper::categorizerKey() const { 26 | return EMPTY_STRING; 27 | } 28 | 29 | CCategoryIdMapper::TCategoryIdMapperPtr CNoopCategoryIdMapper::clone() const { 30 | return std::make_shared(*this); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /lib/model/FrequencyPredicates.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace model { 16 | 17 | CPersonFrequencyGreaterThan::CPersonFrequencyGreaterThan(const CAnomalyDetectorModel& model, 18 | double threshold) 19 | : m_Model(&model), m_Threshold(threshold) { 20 | } 21 | 22 | CAttributeFrequencyGreaterThan::CAttributeFrequencyGreaterThan(const CAnomalyDetectorModel& model, 23 | double threshold) 24 | : m_Model(&model), m_Threshold(threshold) { 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.buildkite/pipelines/upload_dra_to_gcs.yml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | # Upload all artifacts, both platform-specific and all-platforms, to 12 | # GCS, where release manager builds will download them from. 13 | # 14 | 15 | . .buildkite/scripts/common/base.sh 16 | 17 | cat < 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | namespace ml { 20 | namespace core { 21 | 22 | void CProcessPriority::reduceMemoryPriority() { 23 | // Default is to do nothing - see platform-specific implementation files for 24 | // platforms where we do more 25 | } 26 | 27 | void CProcessPriority::reduceCpuPriority() { 28 | errno = 0; 29 | if (::nice(5) == -1 && errno != 0) { 30 | LOG_ERROR(<< "Failed to reduce process priority: " << ::strerror(errno)); 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /.buildkite/pipelines/upload_dra_to_s3.yml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | 11 | . .buildkite/scripts/common/base.sh 12 | 13 | cat < 13 | #include 14 | 15 | template 16 | struct STemplated { 17 | void printFirst() { std::cout << s_First << std::endl; } 18 | 19 | T s_First; 20 | std::vector s_Second; 21 | }; 22 | 23 | struct SSimple { 24 | void printFirst() { std::cout << s_First << std::endl; } 25 | 26 | int s_First; 27 | std::vector s_Second; 28 | }; 29 | 30 | int main(int, char**) { 31 | { 32 | SSimple obj; 33 | obj.printFirst(); 34 | } 35 | 36 | { 37 | STemplated obj; 38 | obj.printFirst(); 39 | } 40 | 41 | return 0; 42 | } 43 | -------------------------------------------------------------------------------- /lib/core/unittest/CProcessTest.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | BOOST_AUTO_TEST_SUITE(CProcessTest) 18 | 19 | BOOST_AUTO_TEST_CASE(testPids) { 20 | ml::core::CProcess& process = ml::core::CProcess::instance(); 21 | ml::core::CProcess::TPid pid = process.id(); 22 | ml::core::CProcess::TPid ppid = process.parentId(); 23 | 24 | LOG_DEBUG(<< "PID = " << pid << " and parent PID = " << ppid); 25 | 26 | BOOST_TEST_REQUIRE(pid != 0); 27 | BOOST_TEST_REQUIRE(ppid != 0); 28 | BOOST_TEST_REQUIRE(pid != ppid); 29 | } 30 | 31 | BOOST_AUTO_TEST_SUITE_END() 32 | -------------------------------------------------------------------------------- /lib/core/unittest/CMutexTest.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | #include 15 | 16 | BOOST_AUTO_TEST_SUITE(CMutexTest) 17 | 18 | BOOST_AUTO_TEST_CASE(testRecursive) { 19 | ml::core::CMutex mutex; 20 | 21 | mutex.lock(); 22 | mutex.unlock(); 23 | 24 | mutex.lock(); 25 | // If the mutex isn't recursive as expected then this test 26 | // will deadlock rather than fail :-( 27 | // The assertion here is just to stop Boost.Test complaining 28 | // that there are no assertions 29 | BOOST_REQUIRE_NO_THROW(mutex.lock()); 30 | mutex.unlock(); 31 | mutex.unlock(); 32 | } 33 | 34 | BOOST_AUTO_TEST_SUITE_END() 35 | -------------------------------------------------------------------------------- /lib/model/CHierarchicalResultsProbabilityFinalizer.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | #include 15 | 16 | namespace ml { 17 | namespace model { 18 | 19 | void CHierarchicalResultsProbabilityFinalizer::visit(const CHierarchicalResults& /*results*/, 20 | const TNode& node, 21 | bool /*pivot*/) { 22 | if (node.s_RawAnomalyScore > 0.0) { 23 | node.s_AnnotatedProbability.s_Probability = 24 | maths::common::CTools::inverseAnomalyScore(node.s_RawAnomalyScore); 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /.buildkite/pipelines/run_pytorch_tests.yml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | 11 | cat < 16 | 17 | #include 18 | 19 | namespace ml { 20 | namespace core { 21 | 22 | //! \brief Stream utility functions. 23 | class CORE_EXPORT CStreamUtils { 24 | public: 25 | //! boost::ini_parser doesn't like UTF-8 ini files that begin 26 | //! with byte order markers. This function advances the seek 27 | //! pointer of the stream over a UTF-8 BOM, but only if one 28 | //! exists. 29 | static void skipUtf8Bom(std::ifstream& strm); 30 | }; 31 | } 32 | } 33 | 34 | #endif // INCLUDED_ml_core_CStreamUtils_h 35 | -------------------------------------------------------------------------------- /.buildkite/pipelines/create_dra.yml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | 11 | cat < 2 | 3 | 4 | 5 | 6 | Transport node error 7 | 8 | 9 | 10 | ^[[:space:]]* 11 | 12 | 13 | source 14 | [[:space:]]*)?]]> 15 | 16 | 17 | 18 | 19 | Transport read error 20 | 21 | 22 | 23 | ^[[:digit:]]* 24 | 25 | 26 | source 27 | [[:space:]]*)?]]> 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /lib/maths/analytics/unittest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Maths Analytics unit tests") 13 | 14 | set (SRCS 15 | Main.cc 16 | BoostedTreeTestData.cc 17 | CBoostedTreeLeafNodeStatisticsTest.cc 18 | CBoostedTreeLossTest.cc 19 | CBoostedTreeTest.cc 20 | CBoostedTreeUtilsTest.cc 21 | CDataFrameCategoryEncoderTest.cc 22 | CDataFrameUtilsTest.cc 23 | CMicTest.cc 24 | COutliersTest.cc 25 | CTreeShapFeatureImportanceTest.cc 26 | TestUtils.cc 27 | ) 28 | 29 | set(ML_LINK_LIBRARIES 30 | ${Boost_LIBRARIES_WITH_UNIT_TEST} 31 | ${LIBXML2_LIBRARIES} 32 | MlCore 33 | MlMathsCommon 34 | MlMathsAnalytics 35 | MlTest 36 | ) 37 | 38 | ml_add_test_executable(maths_analytics ${SRCS}) 39 | -------------------------------------------------------------------------------- /include/core/CLoggerTrace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #ifndef INCLUDED_ml_core_CLoggerTrace_h 13 | #define INCLUDED_ml_core_CLoggerTrace_h 14 | 15 | #ifdef EXCLUDE_TRACE_LOGGING 16 | // Compiling trace logging is expensive so if we don't want it just discard 17 | // the code in the preprocessor. 18 | #ifdef LOG_TRACE 19 | #undef LOG_TRACE 20 | #endif 21 | #define LOG_TRACE(message) 22 | // Avoids compiler warning in the case a variable is only used in LOG_TRACE. 23 | #ifdef SUPPRESS_USAGE_WARNING 24 | #undef SUPPRESS_USAGE_WARNING 25 | #endif 26 | #define SUPPRESS_USAGE_WARNING(variable) static_cast(&(variable)) 27 | #else 28 | #include 29 | #endif 30 | 31 | #endif // INCLUDED_ml_core_CLoggerTrace_h 32 | -------------------------------------------------------------------------------- /lib/core/CWindowsError.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace core { 17 | 18 | CWindowsError::CWindowsError() : m_ErrorCode(0) { 19 | } 20 | 21 | CWindowsError::CWindowsError(std::uint32_t /* errorCode */) : m_ErrorCode(0) { 22 | } 23 | 24 | std::uint32_t CWindowsError::errorCode() const { 25 | return m_ErrorCode; 26 | } 27 | 28 | std::string CWindowsError::errorString() const { 29 | return "Asking for Windows error message on Unix!"; 30 | } 31 | 32 | std::ostream& operator<<(std::ostream& os, const CWindowsError& /* windowsError */) { 33 | os << "Asking for Windows error message on Unix!"; 34 | return os; 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dev-tools/docker/linux_dependency_builder/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | FROM docker.elastic.co/ml-dev/ml-linux-dependency-build:pytorch_viable_strict 13 | 14 | MAINTAINER Ed Savage 15 | 16 | # Copy the current Git repository into the container 17 | COPY . /ml-cpp/ 18 | 19 | # Pass through any version qualifier (default none) 20 | ARG VERSION_QUALIFIER= 21 | 22 | # Pass through whether this is a snapshot build (default yes if not specified) 23 | ARG SNAPSHOT=yes 24 | 25 | # Pass through ML debug option (default blank) 26 | ARG ML_DEBUG= 27 | 28 | ENV CMAKE_FLAGS -DCMAKE_TOOLCHAIN_FILE=/ml-cpp/cmake/linux-x86_64.cmake 29 | 30 | # Run the build 31 | RUN \ 32 | /ml-cpp/dev-tools/docker/docker_entrypoint.sh 33 | 34 | -------------------------------------------------------------------------------- /lib/core/CMemoryCircuitBreaker.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace core { 16 | 17 | // To ensure the singleton is constructed before multiple threads may require it 18 | // call instance() during the static initialisation phase of the program. Of 19 | // course, the instance may already be constructed before this if another static 20 | // object has used it. 21 | // const ml::core::CMemoryCircuitBreakerStub& DO_NOT_USE_THIS_VARIABLE = 22 | // ml::core::CMemoryCircuitBreakerStub::instance(); 23 | 24 | CMemoryCircuitBreakerStub& CMemoryCircuitBreakerStub::instance() { 25 | static CMemoryCircuitBreakerStub instance; 26 | return instance; 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /lib/maths/analytics/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML Maths Analytics") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlMathsCommon 17 | MlCore) 18 | 19 | ml_add_library(MlMathsAnalytics SHARED 20 | CBoostedTree.cc 21 | CBoostedTreeFactory.cc 22 | CBoostedTreeHyperparameters.cc 23 | CBoostedTreeImpl.cc 24 | CBoostedTreeLeafNodeStatistics.cc 25 | CBoostedTreeLeafNodeStatisticsIncremental.cc 26 | CBoostedTreeLeafNodeStatisticsScratch.cc 27 | CBoostedTreeLeafNodeStatisticsThreading.cc 28 | CBoostedTreeLoss.cc 29 | CBoostedTreeUtils.cc 30 | CDataFrameCategoryEncoder.cc 31 | CDataFramePredictiveModel.cc 32 | CDataFrameUtils.cc 33 | CMic.cc 34 | COutliers.cc 35 | CTreeShapFeatureImportance.cc 36 | ) 37 | -------------------------------------------------------------------------------- /lib/core/CMutex_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace { 14 | // 4000 is a value that Microsoft uses in some of their code, so it's 15 | // hopefully a reasonably sensible setting 16 | const DWORD SPIN_COUNT(4000); 17 | } 18 | 19 | namespace ml { 20 | namespace core { 21 | 22 | CMutex::CMutex() { 23 | // On Windows Vista and above this function cannot fail, hence no need to 24 | // check the return code 25 | InitializeCriticalSectionAndSpinCount(&m_Mutex, SPIN_COUNT); 26 | } 27 | 28 | CMutex::~CMutex() { 29 | DeleteCriticalSection(&m_Mutex); 30 | } 31 | 32 | void CMutex::lock() { 33 | EnterCriticalSection(&m_Mutex); 34 | } 35 | 36 | void CMutex::unlock() { 37 | LeaveCriticalSection(&m_Mutex); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /lib/core/CReadWriteLock_Windows.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace core { 15 | 16 | CReadWriteLock::CReadWriteLock() { 17 | InitializeSRWLock(&m_ReadWriteLock); 18 | } 19 | 20 | CReadWriteLock::~CReadWriteLock() { 21 | // There is no function to destroy the read/write lock on Windows 22 | } 23 | 24 | void CReadWriteLock::readLock() { 25 | AcquireSRWLockShared(&m_ReadWriteLock); 26 | } 27 | 28 | void CReadWriteLock::readUnlock() { 29 | ReleaseSRWLockShared(&m_ReadWriteLock); 30 | } 31 | 32 | void CReadWriteLock::writeLock() { 33 | AcquireSRWLockExclusive(&m_ReadWriteLock); 34 | } 35 | 36 | void CReadWriteLock::writeUnlock() { 37 | ReleaseSRWLockExclusive(&m_ReadWriteLock); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /include/core/CStreamWriter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #ifndef INCLUDED_ml_core_CStreamWriter_h 13 | #define INCLUDED_ml_core_CStreamWriter_h 14 | 15 | #include 16 | 17 | #include 18 | 19 | namespace ml { 20 | namespace core { 21 | using TBoostJsonLineWriterStream = core::CBoostJsonLineWriter; 22 | 23 | class CStreamWriter : public TBoostJsonLineWriterStream { 24 | public: 25 | CStreamWriter(std::ostream& os) : TBoostJsonLineWriterStream(os) {} 26 | CStreamWriter() : TBoostJsonLineWriterStream() {} 27 | 28 | void append(const std::string_view& str) override { (*m_Os) << str; } 29 | 30 | void put(char c) override { m_Os->put(c); } 31 | }; 32 | } 33 | } 34 | 35 | #endif //INCLUDED_ml_core_CStreamWriter_h 36 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/normalizerInput.csv: -------------------------------------------------------------------------------- 1 | level,partition_field_name,partition_field_value,person_field_name,person_field_value,function_name,value_field_name,probability 2 | root,,,bucketTime,,,,2.56098e-205 3 | inflb,,,status,,,,2.93761e-203 4 | infl,,,status,,,,5.56572e-204 5 | leaf,,,status,,count,,0 6 | leaf,,,status,,count,,1e-300 7 | root,,,bucketTime,,,,1e-200 8 | leaf,,,status,,count,,1e-200 9 | root,,,bucketTime,,,,1e-100 10 | leaf,,,status,,count,,1e-100 11 | root,,,bucketTime,,,,1e-75 12 | leaf,,,status,,count,,1e-75 13 | root,,,bucketTime,,,,1e-50 14 | leaf,,,status,,count,,1e-50 15 | root,,,bucketTime,,,,1e-25 16 | leaf,,,status,,count,,1e-25 17 | root,,,bucketTime,,,,1e-10 18 | leaf,,,status,,count,,1e-10 19 | root,,,bucketTime,,,,1e-9 20 | leaf,,,status,,count,,1e-9 21 | root,,,bucketTime,,,,1e-8 22 | leaf,,,status,,count,,1e-8 23 | root,,,bucketTime,,,,1e-7 24 | leaf,,,status,,count,,1e-7 25 | root,,,bucketTime,,,,1e-6 26 | leaf,,,status,,count,,1e-6 27 | root,,,bucketTime,,,,1e-5 28 | leaf,,,status,,count,,1e-5 29 | root,,,bucketTime,,,,1e-4 30 | leaf,,,status,,count,,1e-4 31 | root,,,bucketTime,,,,1e-3 32 | leaf,,,status,,count,,1e-3 33 | root,,,bucketTime,,,,1e-2 34 | leaf,,,status,,count,,1e-2 35 | root,,,bucketTime,,,,1e-1 36 | leaf,,,status,,count,,1e-1 37 | root,,,bucketTime,,,,1 38 | infl,,,status,,,,1 39 | leaf,,,status,,count,,1 40 | -------------------------------------------------------------------------------- /lib/core/CProcessStats_MacOSX.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | namespace ml { 21 | namespace core { 22 | 23 | std::size_t CProcessStats::residentSetSize() { 24 | // not supported on osx 25 | return 0; 26 | } 27 | 28 | std::size_t CProcessStats::maxResidentSetSize() { 29 | struct rusage rusage; 30 | 31 | if (getrusage(RUSAGE_SELF, &rusage) != 0) { 32 | LOG_DEBUG(<< "failed to get resource usage(getrusage): " << ::strerror(errno)); 33 | return 0; 34 | } 35 | 36 | // ru_maxrss is in bytes 37 | return static_cast(rusage.ru_maxrss); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /include/core/CoreTypes.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #ifndef INCLUDED_ml_core_t_CoreTypes_h 12 | #define INCLUDED_ml_core_t_CoreTypes_h 13 | 14 | #include 15 | 16 | namespace ml { 17 | namespace core_t { 18 | 19 | //! For now just use seconds as the ml time granularity 20 | //! This is a UTC value 21 | using TTime = time_t; 22 | 23 | //! The standard line ending for the platform - DON'T make this std::string as 24 | //! that would cause many strings to be constructed (since the variable is 25 | //! const at the namespace level, so is internal to each file this header is 26 | //! included in) 27 | #ifdef Windows 28 | const char* const LINE_ENDING = "\r\n"; 29 | #else 30 | const char* const LINE_ENDING = "\n"; 31 | #endif 32 | } 33 | } 34 | 35 | #endif // INCLUDED_ml_core_t_CoreTypes_h 36 | -------------------------------------------------------------------------------- /include/core/CTimeGm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #ifndef INCLUDED_ml_core_CTimeGm_h 12 | #define INCLUDED_ml_core_CTimeGm_h 13 | 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | namespace ml { 20 | namespace core { 21 | 22 | //! \brief 23 | //! Convert tm into a time_t without making a timezone adjustment. 24 | //! See timegm man page for details. 25 | //! 26 | //! DESCRIPTION:\n 27 | //! Convert tm into a time_t without making a timezone adjustment. 28 | //! See timegm man page for details. 29 | //! 30 | //! IMPLEMENTATION DECISIONS:\n 31 | //! 32 | class CORE_EXPORT CTimeGm : private CNonInstantiatable { 33 | public: 34 | static time_t timeGm(struct tm* ts); 35 | }; 36 | } 37 | } 38 | 39 | #endif // INCLUDED_ml_core_CTimeGm_h 40 | -------------------------------------------------------------------------------- /lib/api/CSimpleOutputWriter.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | namespace ml { 14 | namespace api { 15 | 16 | // Initialise statics 17 | const CSimpleOutputWriter::TStrVec CSimpleOutputWriter::EMPTY_FIELD_NAMES; 18 | const CSimpleOutputWriter::TStrStrUMap CSimpleOutputWriter::EMPTY_FIELD_OVERRIDES; 19 | 20 | bool CSimpleOutputWriter::fieldNames(const TStrVec& fieldNames) { 21 | return this->fieldNames(fieldNames, EMPTY_FIELD_NAMES); 22 | } 23 | 24 | bool CSimpleOutputWriter::writeRow(const TStrStrUMap& dataRowFields) { 25 | // Since the overrides are checked first, but we know there aren't any, it's 26 | // most efficient to pretend everything's an override 27 | return this->writeRow(EMPTY_FIELD_OVERRIDES, dataRowFields); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dev-tools/docker/linux_builder/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | # Increment the version here when a new tools/3rd party components image is built 13 | FROM docker.elastic.co/ml-dev/ml-linux-build:34 14 | 15 | LABEL maintainer="Valeriy Khakhutskyy " 16 | 17 | # Copy the current Git repository into the container 18 | COPY . /ml-cpp/ 19 | 20 | # Pass through any version qualifier (default none) 21 | ARG VERSION_QUALIFIER= 22 | 23 | # Pass through whether this is a snapshot build (default yes if not specified) 24 | ARG SNAPSHOT=yes 25 | 26 | # Pass through ML debug option (default blank) 27 | ARG ML_DEBUG= 28 | 29 | ENV CMAKE_FLAGS=-DCMAKE_TOOLCHAIN_FILE=/ml-cpp/cmake/linux-x86_64.cmake 30 | 31 | # Run the build 32 | RUN \ 33 | /ml-cpp/dev-tools/docker/docker_entrypoint.sh 34 | 35 | -------------------------------------------------------------------------------- /include/core/CStrTokR.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #ifndef INCLUDED_ml_core_CStrTokR_h 12 | #define INCLUDED_ml_core_CStrTokR_h 13 | 14 | #include 15 | #include 16 | 17 | namespace ml { 18 | namespace core { 19 | 20 | //! \brief 21 | //! Portable wrapper for the strtok_r() function. 22 | //! 23 | //! DESCRIPTION:\n 24 | //! Portable wrapper for the strtok_r() function. 25 | //! 26 | //! IMPLEMENTATION DECISIONS:\n 27 | //! This has been broken into a class of its own because Windows has a 28 | //! strtok_s() function rather than Unix's strtok_r(). 29 | //! 30 | class CORE_EXPORT CStrTokR : private CNonInstantiatable { 31 | public: 32 | static char* strTokR(char* str, const char* sep, char** lasts); 33 | }; 34 | } 35 | } 36 | 37 | #endif // INCLUDED_ml_core_CStrTokR_h 38 | -------------------------------------------------------------------------------- /lib/api/CCategoryIdMapper.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | 14 | namespace ml { 15 | namespace api { 16 | 17 | CCategoryIdMapper::TGlobalCategoryIdVec 18 | CCategoryIdMapper::mapVec(const TLocalCategoryIdVec& localCategoryIds) { 19 | TGlobalCategoryIdVec mapped; 20 | mapped.reserve(localCategoryIds.size()); 21 | 22 | for (const auto& localCategoryId : localCategoryIds) { 23 | mapped.emplace_back(this->map(localCategoryId)); 24 | } 25 | 26 | return mapped; 27 | } 28 | 29 | void CCategoryIdMapper::acceptPersistInserter(core::CStatePersistInserter& /*inserter*/) const { 30 | // No-op 31 | } 32 | 33 | bool CCategoryIdMapper::acceptRestoreTraverser(core::CStateRestoreTraverser& /*traverser*/) { 34 | // No-op 35 | return true; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /lib/api/unittest/testfiles/instrumentation/memory_usage.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "$id": "https://raw.githubusercontent.com/elastic/ml-json-schemas-private/master/schemas/instrumentation/memory_usage.schema.json", 4 | "description": "Data frame analytics peak memory usage", 5 | "title": "analytics_memory_usage", 6 | "type": "object", 7 | "properties": { 8 | "job_id": { 9 | "description": "Data Frame Analytics Job ID. Populated by Java.", 10 | "type": "string" 11 | }, 12 | "timestamp": { 13 | "description": "Milliseconds since Unix Epoch", 14 | "type": "integer" 15 | }, 16 | "peak_usage_bytes": { 17 | "description": "Peak memory usage for the data frame analytics job in bytes", 18 | "type": "integer" 19 | }, 20 | "status": { 21 | "type":"string", 22 | "enum": ["ok", "hard-limit"], 23 | "description": "Describes how current memory usage relates to the configured memory limit. If the limit is exceeded, 'status' is 'hard-limit'." 24 | }, 25 | "memory_reestimate_bytes": { 26 | "type": "integer", 27 | "description": "If 'status' is set to 'hard-limit', this optional field provides the new advised memory limit." 28 | } 29 | }, 30 | "required": [ 31 | "peak_usage_bytes", 32 | "status" 33 | ], 34 | "additionalProperties": false 35 | } -------------------------------------------------------------------------------- /lib/test/CBoostTestXmlOutput.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include 12 | 13 | #include 14 | 15 | namespace ml { 16 | namespace test { 17 | 18 | std::ofstream CBoostTestXmlOutput::ms_XmlOutputFile("boost_test_results.xml"); 19 | 20 | bool CBoostTestXmlOutput::init() { 21 | // Add XML output to boost_test_results.xml as an extra output over and above 22 | // the default or whatever was specified on the command line 23 | boost::unit_test::unit_test_log.add_format(boost::unit_test::OF_XML); 24 | boost::unit_test::unit_test_log.set_stream(boost::unit_test::OF_XML, ms_XmlOutputFile); 25 | boost::unit_test::unit_test_log.set_threshold_level( 26 | boost::unit_test::OF_XML, boost::unit_test::log_test_units); 27 | return true; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /bin/pytorch_inference/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | project("ML PyTorch Inference") 13 | 14 | set(ML_LINK_LIBRARIES 15 | ${Boost_LIBRARIES} 16 | MlCore 17 | MlApi 18 | MlSeccomp 19 | MlVer 20 | ${TORCH_LIB} 21 | ${C10_LIB} 22 | ) 23 | 24 | if (LINK_TCMALLOC) 25 | message(AUTHOR_WARNING "Linking libtcmalloc. Build is not for production release.") 26 | list(APPEND ML_LINK_LIBRARIES tcmalloc) 27 | endif () 28 | 29 | if (LINK_PROFILER) 30 | message(AUTHOR_WARNING "Linking libprofiler. Build is not for production release.") 31 | list(APPEND ML_LINK_LIBRARIES profiler) 32 | endif () 33 | 34 | ml_add_executable(pytorch_inference 35 | CBufferedIStreamAdapter.cc 36 | CCmdLineParser.cc 37 | CCommandParser.cc 38 | CResultWriter.cc 39 | CThreadSettings.cc 40 | ) 41 | 42 | ml_codesign(pytorch_inference) 43 | -------------------------------------------------------------------------------- /dev-tools/docker/linux_tester/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | # 11 | 12 | # Increment the version here when a new tools/3rd party components image is built 13 | FROM docker.elastic.co/ml-dev/ml-linux-build:34 14 | 15 | LABEL maintainer="Valeriy Khakhutskyy " 16 | 17 | # Copy the current Git repository into the container 18 | COPY . /ml-cpp/ 19 | 20 | # Pass through any version qualifier (default none) 21 | ARG VERSION_QUALIFIER= 22 | 23 | # Pass through whether this is a snapshot build (default yes if not specified) 24 | ARG SNAPSHOT=yes 25 | 26 | # Pass through ML debug option (default blank) 27 | ARG ML_DEBUG= 28 | 29 | ENV CMAKE_FLAGS=-DCMAKE_TOOLCHAIN_FILE=/ml-cpp/cmake/linux-x86_64.cmake 30 | 31 | # Run the build and unit tests 32 | RUN \ 33 | /ml-cpp/dev-tools/docker/docker_entrypoint.sh --test 34 | 35 | -------------------------------------------------------------------------------- /.buildkite/ml_pipeline/step.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | # or more contributor license agreements. Licensed under the Elastic License 4 | # 2.0 and the following additional limitation. Functionality enabled by the 5 | # files subject to the Elastic License 2.0 may only be used in production when 6 | # invoked by an Elasticsearch process with a license key installed that permits 7 | # use of machine learning features. You may not use this file except in 8 | # compliance with the Elastic License 2.0 and the foregoing additional 9 | # limitation. 10 | 11 | class PipelineStep(list): 12 | def generate_step(self, label, command): 13 | command = command + " | buildkite-agent pipeline upload" 14 | step = { 15 | "label": label, 16 | "depends_on": None, 17 | "command": command, 18 | "agents": { 19 | "image": "python", 20 | } 21 | } 22 | return step 23 | 24 | def generate_step_template(self, platform, action, build_aarch64, build_x86_64): 25 | platform_lower = platform.lower() 26 | platform_emoji = ":"+platform_lower+":" 27 | label = f"Upload {action} pipeline for {platform} {platform_emoji}" 28 | command = f"python3 .buildkite/pipelines/build_{platform_lower}.json.py --action={action} {build_aarch64} {build_x86_64}" 29 | return self.generate_step(label, command) 30 | -------------------------------------------------------------------------------- /include/core/CSetEnv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #ifndef INCLUDED_ml_core_CSetEnv_h 12 | #define INCLUDED_ml_core_CSetEnv_h 13 | 14 | #include 15 | #include 16 | 17 | namespace ml { 18 | namespace core { 19 | 20 | //! \brief 21 | //! Portable wrapper for the setenv() function. 22 | //! 23 | //! DESCRIPTION:\n 24 | //! Portable wrapper for the setenv() function. 25 | //! 26 | //! IMPLEMENTATION DECISIONS:\n 27 | //! This has been broken into a class of its own because Windows has a 28 | //! _putenv_s() function with slightly different semantics to Unix's 29 | //! setenv(). 30 | //! 31 | class CORE_EXPORT CSetEnv : private CNonInstantiatable { 32 | public: 33 | static int setEnv(const char* name, const char* value, int overwrite); 34 | }; 35 | } 36 | } 37 | 38 | #endif // INCLUDED_ml_core_CSetEnv_h 39 | -------------------------------------------------------------------------------- /lib/api/unittest/CMockSearcher.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #include "CMockSearcher.h" 12 | 13 | #include 14 | 15 | #include "CMockDataAdder.h" 16 | 17 | CMockSearcher::CMockSearcher(const CMockDataAdder& mockDataAdder) 18 | : m_MockDataAdder{mockDataAdder} { 19 | } 20 | 21 | CMockSearcher::TIStreamP CMockSearcher::search(std::size_t currentDocNum, std::size_t /*limit*/) { 22 | if (currentDocNum == 0) { 23 | LOG_ERROR(<< "Current doc number cannot be 0 - data store requires 1-based numbers"); 24 | return TIStreamP{}; 25 | } 26 | 27 | const CMockDataAdder::TStrVec& events = m_MockDataAdder.events(); 28 | if (currentDocNum > events.size()) { 29 | return TIStreamP{new std::istringstream("[ ]")}; 30 | } 31 | return TIStreamP{new std::istringstream(events[currentDocNum - 1])}; 32 | } 33 | -------------------------------------------------------------------------------- /lib/api/unittest/Main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #define BOOST_TEST_MODULE lib.api 13 | // Defining BOOST_TEST_MODULE usually auto-generates main(), but we don't want 14 | // this as we need custom initialisation to allow for output in both console and 15 | // Boost.Test XML formats 16 | #define BOOST_TEST_NO_MAIN 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | int main(int argc, char** argv) { 24 | ml::test::CTestObserver observer; 25 | boost::unit_test::framework::register_observer(observer); 26 | int result{boost::unit_test::unit_test_main(&ml::test::CBoostTestXmlOutput::init, 27 | argc, argv)}; 28 | boost::unit_test::framework::deregister_observer(observer); 29 | return result; 30 | } 31 | -------------------------------------------------------------------------------- /lib/ver/unittest/Main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #define BOOST_TEST_MODULE lib.ver 13 | // Defining BOOST_TEST_MODULE usually auto-generates main(), but we don't want 14 | // this as we need custom initialisation to allow for output in both console and 15 | // Boost.Test XML formats 16 | #define BOOST_TEST_NO_MAIN 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | int main(int argc, char** argv) { 24 | ml::test::CTestObserver observer; 25 | boost::unit_test::framework::register_observer(observer); 26 | int result{boost::unit_test::unit_test_main(&ml::test::CBoostTestXmlOutput::init, 27 | argc, argv)}; 28 | boost::unit_test::framework::deregister_observer(observer); 29 | return result; 30 | } 31 | -------------------------------------------------------------------------------- /include/core/CScopedLock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #ifndef INCLUDED_ml_core_CScopedLock_h 12 | #define INCLUDED_ml_core_CScopedLock_h 13 | 14 | #include 15 | #include 16 | 17 | namespace ml { 18 | namespace core { 19 | class CMutex; 20 | 21 | //! \brief 22 | //! Implementation of Scoped Locking idiom/pattern. 23 | //! 24 | //! DESCRIPTION:\n 25 | //! Implementation of Scoped Locking idiom/pattern. 26 | //! 27 | //! IMPLEMENTATION DECISIONS:\n 28 | //! See Schmidt etc. for details. 29 | //! 30 | class CORE_EXPORT CScopedLock : private CNonCopyable { 31 | public: 32 | //! Lock specified mutex 33 | CScopedLock(CMutex& mutex); 34 | 35 | //! Unlock specified mutex 36 | ~CScopedLock(); 37 | 38 | private: 39 | CMutex& m_Mutex; 40 | }; 41 | } 42 | } 43 | 44 | #endif // INCLUDED_ml_core_CScopedLock_h 45 | -------------------------------------------------------------------------------- /lib/core/unittest/Main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #define BOOST_TEST_MODULE lib.core 13 | // Defining BOOST_TEST_MODULE usually auto-generates main(), but we don't want 14 | // this as we need custom initialisation to allow for output in both console and 15 | // Boost.Test XML formats 16 | #define BOOST_TEST_NO_MAIN 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | int main(int argc, char** argv) { 24 | ml::test::CTestObserver observer; 25 | boost::unit_test::framework::register_observer(observer); 26 | int result{boost::unit_test::unit_test_main(&ml::test::CBoostTestXmlOutput::init, 27 | argc, argv)}; 28 | boost::unit_test::framework::deregister_observer(observer); 29 | return result; 30 | } 31 | -------------------------------------------------------------------------------- /lib/model/unittest/Main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #define BOOST_TEST_MODULE lib.model 13 | // Defining BOOST_TEST_MODULE usually auto-generates main(), but we don't want 14 | // this as we need custom initialisation to allow for output in both console and 15 | // Boost.Test XML formats 16 | #define BOOST_TEST_NO_MAIN 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | int main(int argc, char** argv) { 24 | ml::test::CTestObserver observer; 25 | boost::unit_test::framework::register_observer(observer); 26 | int result{boost::unit_test::unit_test_main(&ml::test::CBoostTestXmlOutput::init, 27 | argc, argv)}; 28 | boost::unit_test::framework::deregister_observer(observer); 29 | return result; 30 | } 31 | -------------------------------------------------------------------------------- /include/core/CCTimeR.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | #ifndef INCLUDED_ml_core_CCTimeR_h 12 | #define INCLUDED_ml_core_CCTimeR_h 13 | 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | namespace ml { 20 | namespace core { 21 | 22 | //! \brief 23 | //! Portable wrapper for the ctime_r() function. 24 | //! 25 | //! DESCRIPTION:\n 26 | //! Portable wrapper for the ctime_r() function. 27 | //! 28 | //! IMPLEMENTATION DECISIONS:\n 29 | //! This has been broken into a class of its own because Windows has a 30 | //! ctime_s() function with slightly different semantics to Unix's 31 | //! ctime_r(). 32 | //! 33 | class CORE_EXPORT CCTimeR : private CNonInstantiatable { 34 | public: 35 | static char* cTimeR(const time_t* clock, char* result); 36 | }; 37 | } 38 | } 39 | 40 | #endif // INCLUDED_ml_core_CCTimeR_h 41 | -------------------------------------------------------------------------------- /lib/core/unittest/CWindowsErrorTest.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | BOOST_AUTO_TEST_SUITE(CWindowsErrorTest) 18 | 19 | BOOST_AUTO_TEST_CASE(testErrors) { 20 | LOG_INFO(<< "Windows error 1 is : " << ml::core::CWindowsError(1)); 21 | LOG_INFO(<< "Windows error 2 is : " << ml::core::CWindowsError(2)); 22 | LOG_INFO(<< "Windows error 3 is : " << ml::core::CWindowsError(3)); 23 | LOG_INFO(<< "Windows error 4 is : " << ml::core::CWindowsError(4)); 24 | LOG_INFO(<< "Windows error 5 is : " << ml::core::CWindowsError(5)); 25 | LOG_INFO(<< "Windows error 6 is : " << ml::core::CWindowsError(6)); 26 | 27 | BOOST_TEST_REQUIRE(ml::core::CWindowsError(7).errorString().length() > 0); 28 | } 29 | 30 | BOOST_AUTO_TEST_SUITE_END() 31 | -------------------------------------------------------------------------------- /lib/seccomp/unittest/Main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0 and the following additional limitation. Functionality enabled by the 5 | * files subject to the Elastic License 2.0 may only be used in production when 6 | * invoked by an Elasticsearch process with a license key installed that permits 7 | * use of machine learning features. You may not use this file except in 8 | * compliance with the Elastic License 2.0 and the foregoing additional 9 | * limitation. 10 | */ 11 | 12 | #define BOOST_TEST_MODULE lib.seccomp 13 | // Defining BOOST_TEST_MODULE usually auto-generates main(), but we don't want 14 | // this as we need custom initialisation to allow for output in both console and 15 | // Boost.Test XML formats 16 | #define BOOST_TEST_NO_MAIN 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | int main(int argc, char** argv) { 24 | ml::test::CTestObserver observer; 25 | boost::unit_test::framework::register_observer(observer); 26 | int result{boost::unit_test::unit_test_main(&ml::test::CBoostTestXmlOutput::init, 27 | argc, argv)}; 28 | boost::unit_test::framework::deregister_observer(observer); 29 | return result; 30 | } 31 | --------------------------------------------------------------------------------