├── .circleci └── config.yml ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE.txt ├── README.md ├── amqp ├── CMakeLists.txt ├── amqp.c ├── amqp.def ├── index.md └── sandboxes │ └── heka │ └── input │ └── amqp.lua ├── artifact_push.sh ├── aws ├── CMakeLists.txt ├── aws.def ├── index.md ├── kinesis.cpp ├── sandboxes │ └── heka │ │ ├── input │ │ └── kinesis.lua │ │ └── output │ │ └── kinesis.lua └── tests │ └── hindsight │ ├── hindsight.cfg │ ├── run.sh │ └── run │ ├── analysis │ └── placeholder.txt │ ├── input │ ├── generate_data.cfg │ └── generate_data.lua │ └── output │ └── placeholder.txt ├── bit ├── CMakeLists.txt ├── CMakeLists.txt.bit └── index.md ├── bloom_filter ├── CMakeLists.txt ├── bloom_filter.c ├── bloom_filter.def ├── index.md ├── test_sandbox.c └── tests │ ├── benchmark.lua │ ├── test.lua │ └── test_sandbox.lua ├── circular_buffer ├── CMakeLists.txt ├── circular_buffer.c ├── circular_buffer.def ├── index.md ├── modules │ └── lpeg │ │ └── cbufd.lua ├── sandboxes │ └── heka │ │ ├── analysis │ │ └── throughput.lua │ │ └── output │ │ └── cbuf2tsv.lua ├── test_sandbox.c └── tests │ ├── benchmark.lua │ ├── benchmark_output.lua │ ├── cbufd.lua │ ├── test.lua │ ├── test_sandbox.lua │ ├── test_sandbox_annotation.lua │ └── test_sandbox_delta.lua ├── cjson ├── CMakeLists.txt ├── CMakeLists.txt.cjson ├── index.md ├── test_sandbox.c └── tests │ ├── empty_array.lua │ ├── limited.lua │ ├── test.lua │ └── unlimited.lua ├── cmake ├── CTestCustom.in.cmake ├── FindMYSQL.cmake ├── sandbox_ep_test.cmake ├── sandbox_module.cmake └── test_module.in.h ├── common ├── test_verify_message.lua ├── xxhash.c └── xxhash.h ├── compat ├── CMakeLists.txt ├── CMakeLists.txt.compat └── index.md ├── cuckoo_filter ├── CMakeLists.txt ├── common.c ├── common.h ├── cuckoo_filter.c ├── cuckoo_filter.def ├── cuckoo_filter_expire.c ├── cuckoo_filter_expire.def ├── index.md ├── test_sandbox.c └── tests │ ├── benchmark.lua │ ├── test.lua │ ├── test_sandbox.lua │ └── test_sandbox_expire.lua ├── docker.arrow.keys.txt ├── docker_push.sh ├── elasticsearch ├── CMakeLists.txt ├── io_modules │ └── encoders │ │ └── elasticsearch │ │ ├── common.lua │ │ ├── mozlog.lua │ │ └── payload.lua ├── sandboxes │ └── heka │ │ └── output │ │ └── elasticsearch_bulk_api.lua ├── test_sandbox.c └── tests │ └── test.lua ├── gcp ├── CMakeLists.txt ├── common.cpp ├── common.h ├── gcp.def ├── index.md ├── logging.cpp ├── logging.md ├── pubsub.cpp ├── pubsub.md ├── sandboxes │ └── heka │ │ ├── input │ │ └── gcp_pubsub.lua │ │ └── output │ │ ├── gcp_bigquery.lua │ │ ├── gcp_logging.lua │ │ └── gcp_pubsub.lua └── tests │ └── hindsight │ ├── hindsight.cfg │ ├── run.sh │ └── run │ ├── analysis │ ├── throughput.cfg │ └── throughput.lua │ ├── input │ ├── load.cfg │ ├── load1.cfg │ ├── load2.cfg │ ├── load3.cfg │ ├── logdata.cfg │ ├── logdata.lua │ ├── prune.cfg │ ├── sub.cfg │ ├── testdata.cfg │ └── testdata.lua │ └── output │ ├── full_log.lua │ ├── logging.cfg │ ├── pub.cfg │ └── pub1.cfg ├── gen_gh_pages.lua ├── geoip ├── CMakeLists.txt ├── CMakeLists.txt.geoip ├── index.md ├── io_modules │ └── geoip │ │ └── heka.lua ├── test_sandbox.c └── tests │ └── test.lua ├── gzfile ├── CMakeLists.txt ├── gzfile.c ├── gzfile.def ├── index.md ├── test_sandbox.c └── tests │ ├── compressed.log │ ├── test.lua │ └── uncompressed.log ├── heka ├── CMakeLists.txt ├── io_modules │ ├── decoders │ │ ├── heka │ │ │ ├── framed_protobuf.lua │ │ │ ├── json.lua │ │ │ ├── json_mutate.lua │ │ │ ├── protobuf.lua │ │ │ ├── table_to_fields.lua │ │ │ └── table_to_message.lua │ │ └── payload.lua │ ├── encoders │ │ ├── heka │ │ │ ├── framed_protobuf.lua │ │ │ └── protobuf.lua │ │ └── payload.lua │ └── heka │ │ ├── derived_stream.lua │ │ └── derived_stream │ │ ├── heka_protobuf.lua │ │ ├── redshift.lua │ │ ├── redshift │ │ ├── psv.lua │ │ └── sql.lua │ │ └── tsv.lua ├── modules │ └── heka │ │ ├── alert.lua │ │ ├── msg_interpolate.lua │ │ └── util.lua ├── sandboxes │ └── heka │ │ ├── analysis │ │ └── heka_message_schema.lua │ │ ├── input │ │ ├── file.lua │ │ ├── heka_file.lua │ │ └── heka_s3.lua │ │ └── output │ │ ├── alert.lua │ │ ├── heka_debug.lua │ │ ├── heka_inject_payload.lua │ │ ├── logfile_roll_by_size.lua │ │ ├── logfile_rotate.lua │ │ └── near_real_time_report.lua ├── test_sandbox.c └── tests │ ├── integration │ ├── decoder_table_to_fields │ │ ├── input.json │ │ └── run │ │ │ ├── analysis │ │ │ ├── verify_function.cfg │ │ │ └── verify_function.lua │ │ │ ├── input │ │ │ └── function.cfg │ │ │ └── output │ │ │ └── placeholder.txt │ ├── decoder_table_to_message │ │ ├── input.json │ │ ├── input.logfmt │ │ └── run │ │ │ ├── analysis │ │ │ ├── verify_function.cfg │ │ │ ├── verify_function.lua │ │ │ ├── verify_grammar.cfg │ │ │ └── verify_grammar.lua │ │ │ ├── input │ │ │ ├── function.cfg │ │ │ └── grammar.cfg │ │ │ └── output │ │ │ └── placeholder.txt │ ├── hindsight.cfg │ ├── input_file │ │ ├── multiline.log │ │ ├── run │ │ │ ├── analysis │ │ │ │ ├── verify_multiline.cfg │ │ │ │ ├── verify_multiline.lua │ │ │ │ ├── verify_singleline.cfg │ │ │ │ └── verify_singleline.lua │ │ │ ├── input │ │ │ │ ├── multiline.cfg │ │ │ │ └── singleline.cfg │ │ │ └── output │ │ │ │ └── placeholder.txt │ │ └── singleline.log │ └── run.sh │ └── test.lua ├── hindsight ├── CMakeLists.txt ├── sandboxes │ └── heka │ │ ├── analysis │ │ └── plugin_monitor.lua │ │ └── input │ │ ├── plugins_tsv.lua │ │ └── prune_input.lua └── tests │ └── integration │ ├── hindsight.cfg │ ├── plugin_monitor │ └── run │ │ ├── analysis │ │ ├── plugin_monitor.cfg │ │ ├── terminate.cfg │ │ └── terminate.lua │ │ ├── input │ │ ├── stats_generate.cfg │ │ └── stats_generate.lua │ │ └── output │ │ ├── plugin_monitor_test.lua │ │ ├── test_alert.cfg │ │ ├── test_alert.lua │ │ ├── test_minimum.cfg │ │ ├── test_percent.cfg │ │ └── test_success.cfg │ └── run.sh ├── hyperloglog ├── CMakeLists.txt ├── hyperloglog.c ├── hyperloglog.def ├── index.md ├── redis_hyperloglog.c ├── redis_hyperloglog.h ├── test_sandbox.c └── tests │ ├── test.lua │ └── test_sandbox.lua ├── irc ├── CMakeLists.txt ├── index.md ├── io_modules │ └── alert │ │ └── irc.lua ├── irc.c └── modules │ └── heka │ └── alert │ └── irc.lua ├── jose ├── CMakeLists.txt ├── index.md ├── jose.c ├── jose.def ├── test_sandbox.c └── tests │ └── test.lua ├── kafka ├── CMakeLists.txt ├── index.md ├── kafka.c ├── kafka.def ├── sandboxes │ └── heka │ │ ├── input │ │ └── kafka.lua │ │ └── output │ │ └── kafka.lua ├── test_sandbox.c └── tests │ ├── test.lua │ ├── test_errors.lua │ ├── test_sandbox_consumer.lua │ └── test_sandbox_producer.lua ├── lfs ├── CMakeLists.txt ├── CMakeLists.txt.lfs ├── index.md ├── sandboxes │ └── heka │ │ └── input │ │ └── tail.lua └── tests │ └── integration │ ├── hindsight.cfg │ ├── input_tail │ ├── compressed.log.gz │ ├── multiline.log │ ├── run │ │ ├── analysis │ │ │ ├── verify_messages.lua │ │ │ ├── verify_multiline.cfg │ │ │ ├── verify_rotate.cfg │ │ │ └── verify_singleline.cfg │ │ ├── input │ │ │ ├── generate.cfg │ │ │ └── generate.lua │ │ └── output │ │ │ ├── shutdown.cfg │ │ │ └── shutdown.lua │ └── singleline.log │ └── run.sh ├── libinjection ├── CMakeLists.txt ├── CMakeLists.txt.libinjection ├── index.md ├── libinjection_wrap.c ├── test_sandbox.c └── tests │ └── test.lua ├── lpeg ├── CMakeLists.txt ├── CMakeLists.txt.lpeg ├── extract_printf.lua ├── files.txt ├── index.md ├── io_modules │ ├── decoders │ │ └── nginx │ │ │ ├── access.lua │ │ │ └── error.lua │ └── lpeg │ │ ├── heka.lua │ │ └── sub_decoder_util.lua ├── modules │ └── lpeg │ │ ├── common_log_format.lua │ │ ├── date_time.lua │ │ ├── escape_sequences.lua │ │ ├── ip_address.lua │ │ ├── logfmt.lua │ │ ├── mysql.lua │ │ ├── phabricator.lua │ │ ├── postfix.lua │ │ ├── printf.lua │ │ └── uri.lua ├── printf.c ├── test_sandbox.c └── tests │ ├── common_log_format.lua │ ├── date_time.lua │ ├── escape_sequences.lua │ ├── ip_address.lua │ ├── logfmt.lua │ ├── lpeg_heka.lua │ ├── mysql.lua │ ├── phabricator.lua │ ├── postfix.lua │ ├── printf.lua │ ├── printf_m1.lua │ ├── printf_m2.lua │ ├── printf_m3.lua │ ├── printf_m4.lua │ ├── test.lua │ └── uri.lua ├── lsb ├── CMakeLists.txt ├── modules │ └── lsb │ │ ├── stats.lua │ │ └── util.lua ├── test_sandbox.c └── tests │ ├── stats.lua │ ├── test.lua │ └── util.lua ├── lyaml ├── CMakeLists.txt ├── CMakeLists.txt.lyaml └── index.md ├── maxminddb ├── CMakeLists.txt ├── CMakeLists.txt.maxminddb ├── index.md ├── io_modules │ └── maxminddb │ │ └── heka.lua ├── test_sandbox.c └── tests │ ├── GeoIP2-City-Test.mmdb │ ├── GeoIP2-ISP-Test.mmdb │ └── test.lua ├── moz_ingest ├── CMakeLists.txt ├── io_modules │ └── decoders │ │ └── moz_ingest │ │ ├── common.lua │ │ ├── gcp.lua │ │ ├── gcp_ingestion.lua │ │ └── json.lua ├── modules │ └── moz_ingest │ │ └── util.lua ├── sandboxes │ └── heka │ │ ├── analysis │ │ └── moz_ingest_doctype_monitor.lua │ │ └── output │ │ └── moz_ingest_doctype_bootstrap.lua ├── test_sandbox.c └── tests │ ├── cities1000-test.txt │ ├── integration │ ├── decoder_common │ │ ├── input.hpb │ │ └── run │ │ │ ├── analysis │ │ │ ├── verify_decoder.cfg │ │ │ └── verify_decoder.lua │ │ │ ├── input │ │ │ ├── generate_data.lua │ │ │ ├── generate_data.off │ │ │ ├── test_decoder.cfg │ │ │ └── test_decoder.lua │ │ │ └── output │ │ │ └── placeholder.txt │ ├── decoder_json │ │ ├── input.hpb │ │ ├── namespaces │ │ │ └── foo │ │ │ │ └── bar │ │ │ │ └── bar.1.schema.json │ │ └── run │ │ │ ├── analysis │ │ │ ├── verify_decoder.cfg │ │ │ └── verify_decoder.lua │ │ │ ├── input │ │ │ ├── generate_data.lua │ │ │ ├── generate_data.off │ │ │ ├── test_decoder.cfg │ │ │ └── test_decoder.lua │ │ │ └── output │ │ │ └── placeholder.txt │ ├── doctype_monitor │ │ └── run │ │ │ ├── analysis │ │ │ └── doctype_monitor.cfg │ │ │ ├── input │ │ │ ├── doctype_monitor.cfg │ │ │ └── doctype_monitor.lua │ │ │ └── output │ │ │ ├── dashboard.cfg │ │ │ ├── doctype_monitor.cfg │ │ │ └── doctype_monitor.lua │ ├── hindsight.cfg │ └── run.sh │ └── test.lua ├── moz_logging ├── CMakeLists.txt ├── io_modules │ └── decoders │ │ └── moz_logging │ │ ├── json_heka.lua │ │ ├── json_heka_fields.lua │ │ └── line_splitter.lua ├── sandboxes │ └── heka │ │ └── analysis │ │ ├── fxa_abuse.lua │ │ ├── fxa_account.lua │ │ ├── fxa_active_daily_users.lua │ │ ├── fxa_auth_memory.lua │ │ ├── fxa_auth_mysql.lua │ │ ├── fxa_auth_rate_limit.lua │ │ ├── fxa_auth_request_time.lua │ │ ├── fxa_auth_resend.lua │ │ ├── fxa_content_oauth_errors.lua │ │ ├── fxa_content_signup_metrics.lua │ │ ├── fxa_creates.lua │ │ ├── fxa_sync_11_migrations.lua │ │ ├── moz_logging_active_counts.lua │ │ ├── moz_logging_http_status.lua │ │ ├── moz_logging_ingestion_error_monitor.lua │ │ └── moz_logging_unique_items.lua └── tests │ └── integration │ ├── decoder_json_heka │ └── run │ │ ├── analysis │ │ ├── verify_decoder.cfg │ │ └── verify_decoder.lua │ │ ├── input │ │ ├── test_decoder.cfg │ │ └── test_decoder.lua │ │ └── output │ │ └── placeholder.txt │ ├── decoder_json_heka_fields │ └── run │ │ ├── analysis │ │ ├── verify_decoder.cfg │ │ └── verify_decoder.lua │ │ ├── input │ │ ├── test_decoder.cfg │ │ └── test_decoder.lua │ │ └── output │ │ └── placeholder.txt │ ├── decoder_line_splitter │ └── run │ │ ├── analysis │ │ ├── verify_decoder.cfg │ │ └── verify_decoder.lua │ │ ├── input │ │ ├── test_decoder.cfg │ │ └── test_decoder.lua │ │ └── output │ │ └── placeholder.txt │ ├── hindsight.cfg │ └── run.sh ├── moz_pioneer ├── CMakeLists.txt ├── io_modules │ └── decoders │ │ └── moz_ingest │ │ └── pioneer.lua ├── sandboxes │ └── heka │ │ └── analysis │ │ └── moz_pioneer_opt_in_estimates.lua └── tests │ └── integration │ ├── decoder_moz_ingest_pioneer │ ├── input.hpb │ └── run │ │ ├── analysis │ │ ├── verify_decoder.cfg │ │ └── verify_decoder.lua │ │ ├── input │ │ ├── generate_data.lua │ │ ├── generate_data.off │ │ ├── test_decoder.cfg │ │ └── test_decoder.lua │ │ └── output │ │ └── placeholder.txt │ ├── hindsight.cfg │ └── run.sh ├── moz_security ├── CMakeLists.txt ├── index.md ├── io_modules │ ├── hawk.lua │ └── taskcluster.lua ├── modules │ ├── heka │ │ ├── alert │ │ │ └── idrouter.lua │ │ ├── iprepd.lua │ │ ├── secmetrics.lua │ │ └── selprinc.lua │ └── iputils.lua ├── sandboxes │ └── heka │ │ ├── analysis │ │ ├── moz_security_auth_any.lua │ │ ├── moz_security_auth_ip_geo.lua │ │ ├── moz_security_auth_lastx.lua │ │ ├── moz_security_cloudtrail.lua │ │ ├── moz_security_duo.lua │ │ ├── moz_security_hh.lua │ │ ├── moz_security_http_error_rate.lua │ │ ├── moz_security_iprepd_alerts.lua │ │ ├── moz_security_pathsetfreq.lua │ │ ├── moz_security_simuse.lua │ │ ├── moz_security_sshd_login_monitor.lua │ │ └── moz_security_webinj.lua │ │ └── output │ │ ├── moz_security_iprepd.lua │ │ └── moz_security_metrics.lua ├── test_sandbox.c └── tests │ ├── hawk.lua │ ├── hindsight │ ├── hindsight.cfg │ ├── iprepd_submit │ │ └── run │ │ │ ├── analysis │ │ │ └── placeholder.txt │ │ │ ├── input │ │ │ ├── generate_data.cfg │ │ │ └── generate_data.lua │ │ │ └── output │ │ │ └── iprepd.cfg │ ├── run.sh │ └── secmetrics │ │ └── run │ │ ├── analysis │ │ └── placeholder.txt │ │ ├── input │ │ ├── generate_data.cfg │ │ └── generate_data.lua │ │ └── output │ │ └── secmetrics.cfg │ ├── integration │ ├── analysis_auth_any │ │ └── run │ │ │ ├── analysis │ │ │ └── auth_any_1.cfg │ │ │ ├── input │ │ │ ├── generate_auth.cfg │ │ │ └── generate_auth.lua │ │ │ └── output │ │ │ ├── auth_any_verification.cfg │ │ │ └── auth_any_verification.lua │ ├── analysis_auth_ip_geo │ │ └── run │ │ │ ├── analysis │ │ │ ├── auth_ip_geo_1.cfg │ │ │ ├── auth_ip_geo_2.cfg │ │ │ └── auth_ip_geo_3.cfg │ │ │ ├── input │ │ │ ├── generate_auth.cfg │ │ │ └── generate_auth.lua │ │ │ └── output │ │ │ ├── auth_ip_geo_verification.cfg │ │ │ └── auth_ip_geo_verification.lua │ ├── analysis_auth_lastx │ │ └── run │ │ │ ├── analysis │ │ │ ├── auth_lastx_1.cfg │ │ │ ├── auth_lastx_2.cfg │ │ │ ├── auth_lastx_3.cfg │ │ │ ├── auth_lastx_4.cfg │ │ │ ├── auth_lastx_5.cfg │ │ │ ├── auth_lastx_6.cfg │ │ │ ├── auth_lastx_7.cfg │ │ │ └── auth_lastx_8.cfg │ │ │ ├── input │ │ │ ├── generate_auth.cfg │ │ │ ├── generate_auth.lua │ │ │ ├── generate_auth_duo.cfg │ │ │ ├── generate_auth_duo.lua │ │ │ ├── generate_auth_multi.cfg │ │ │ └── generate_auth_multi.lua │ │ │ └── output │ │ │ ├── auth_lastx_verification.cfg │ │ │ └── auth_lastx_verification.lua │ ├── analysis_cloudtrail │ │ └── run │ │ │ ├── analysis │ │ │ └── cloudtrail.cfg │ │ │ ├── input │ │ │ ├── cloudtrail.log │ │ │ └── generate_data.cfg │ │ │ └── output │ │ │ ├── verification.cfg │ │ │ └── verification.lua │ ├── analysis_duo │ │ └── run │ │ │ ├── analysis │ │ │ └── duo.cfg │ │ │ ├── input │ │ │ ├── duo.log │ │ │ └── generate_data.cfg │ │ │ └── output │ │ │ ├── verification.cfg │ │ │ └── verification.lua │ ├── analysis_error_rate │ │ └── run │ │ │ ├── analysis │ │ │ ├── err.cfg │ │ │ └── err_hawk.cfg │ │ │ ├── input │ │ │ ├── generate_err.cfg │ │ │ ├── generate_err.lua │ │ │ └── generate_err_hawk.cfg │ │ │ └── output │ │ │ ├── err_hawk_verification.cfg │ │ │ ├── err_hawk_verification.lua │ │ │ ├── err_verification.cfg │ │ │ └── err_verification.lua │ ├── analysis_hh │ │ └── run │ │ │ ├── analysis │ │ │ ├── hh.cfg │ │ │ ├── hh_hawk.cfg │ │ │ └── hh_min.cfg │ │ │ ├── input │ │ │ ├── generate_hh.cfg │ │ │ ├── generate_hh.lua │ │ │ ├── generate_hh_hawk.cfg │ │ │ └── generate_hh_min.cfg │ │ │ └── output │ │ │ ├── hh_hawk_verification.cfg │ │ │ ├── hh_hawk_verification.lua │ │ │ ├── hh_min_verification.cfg │ │ │ ├── hh_min_verification.lua │ │ │ ├── hh_verification.cfg │ │ │ └── hh_verification.lua │ ├── analysis_iprepd_alerts │ │ └── run │ │ │ ├── analysis │ │ │ └── iprepd_alerts.cfg │ │ │ ├── input │ │ │ ├── generate_data.cfg │ │ │ └── generate_data.lua │ │ │ └── output │ │ │ ├── verification.cfg │ │ │ └── verification.lua │ ├── analysis_pathsetfreq │ │ └── run │ │ │ ├── analysis │ │ │ ├── pathsetfreq.cfg │ │ │ ├── pathsetfreq_hawk.cfg │ │ │ └── pathsetfreq_nosingle.cfg │ │ │ ├── input │ │ │ ├── access.log │ │ │ └── generate_pathsetfreq.cfg │ │ │ └── output │ │ │ ├── pathsetfreq_hawk_verification.cfg │ │ │ ├── pathsetfreq_hawk_verification.lua │ │ │ ├── pathsetfreq_nosingle_verification.cfg │ │ │ ├── pathsetfreq_nosingle_verification.lua │ │ │ ├── pathsetfreq_verification.cfg │ │ │ └── pathsetfreq_verification.lua │ ├── analysis_simuse │ │ └── run │ │ │ ├── analysis │ │ │ ├── simuse_1.cfg │ │ │ ├── simuse_2.cfg │ │ │ └── simuse_3.cfg │ │ │ ├── input │ │ │ ├── generate_data.cfg │ │ │ └── generate_data.lua │ │ │ └── output │ │ │ ├── verification.cfg │ │ │ └── verification.lua │ ├── analysis_sshd_login_monitor │ │ └── run │ │ │ ├── analysis │ │ │ ├── sshd_login_monitor_1.cfg │ │ │ ├── sshd_login_monitor_2.cfg │ │ │ ├── sshd_login_monitor_3.cfg │ │ │ └── sshd_login_monitor_4.cfg │ │ │ ├── input │ │ │ ├── generate_sshd.cfg │ │ │ └── generate_sshd.lua │ │ │ └── output │ │ │ ├── sshd_login_verification.cfg │ │ │ └── sshd_login_verification.lua │ ├── analysis_webinj │ │ └── run │ │ │ ├── analysis │ │ │ ├── webinj.cfg │ │ │ └── webinj_hawk.cfg │ │ │ ├── input │ │ │ ├── access.log │ │ │ └── generate_webinj.cfg │ │ │ └── output │ │ │ ├── webinj_hawk_verification.cfg │ │ │ ├── webinj_hawk_verification.lua │ │ │ ├── webinj_verification.cfg │ │ │ └── webinj_verification.lua │ ├── hindsight.cfg │ └── run.sh │ ├── iputils.lua │ └── taskcluster.lua ├── moz_telemetry ├── CMakeLists.txt ├── io_modules │ ├── decoders │ │ ├── moz_ingest │ │ │ ├── gcp_telemetry.lua │ │ │ ├── gcp_telemetry_error.lua │ │ │ └── telemetry.lua │ │ └── moz_telemetry │ │ │ └── tls.lua │ ├── encoders │ │ ├── elasticsearch │ │ │ └── moz_telemetry │ │ │ │ └── telemetry.lua │ │ └── heka │ │ │ └── moz_telemetry │ │ │ └── webrtc.lua │ └── moz_telemetry │ │ └── s3.lua ├── modules │ └── moz_telemetry │ │ ├── histogram.lua │ │ ├── normalize.lua │ │ └── ping.lua ├── sandboxes │ └── heka │ │ ├── analysis │ │ ├── moz_shield_study_enrollment.lua │ │ ├── moz_telemetry_build_monitor.lua │ │ ├── moz_telemetry_doctype_monitor.lua │ │ ├── moz_telemetry_histogram_monitor.lua │ │ ├── moz_telemetry_landfill_error.lua │ │ ├── moz_telemetry_new_doctype_monitor.lua │ │ ├── moz_telemetry_new_experiment_monitor.lua │ │ ├── moz_telemetry_parquet_schema.lua │ │ ├── moz_telemetry_payload_size.lua │ │ └── moz_testpilot_installation.lua │ │ ├── input │ │ ├── moz_serverlog_s3_bootstrap.lua │ │ ├── moz_telemetry_s3_bootstrap.lua │ │ ├── moz_telemetry_s3_landfill.lua │ │ └── moz_telemetry_s3_snappy.lua │ │ └── output │ │ ├── moz_telemetry_crash_summary.lua │ │ ├── moz_telemetry_executive_summary.lua │ │ ├── moz_telemetry_http.lua │ │ ├── moz_telemetry_main_summary.lua │ │ └── moz_telemetry_s3.lua ├── test_sandbox.c └── tests │ ├── integration │ ├── decoder_telemetry │ │ ├── input.hpb │ │ └── run │ │ │ ├── analysis │ │ │ ├── verify_decoder.cfg │ │ │ └── verify_decoder.lua │ │ │ ├── input │ │ │ ├── generate_data.lua │ │ │ ├── generate_data.off │ │ │ ├── test_decoder.cfg │ │ │ └── test_decoder.lua │ │ │ └── output │ │ │ ├── s3_parquet_all.cfg │ │ │ └── s3_parquet_select.cfg │ ├── doctype_monitor │ │ └── run │ │ │ ├── analysis │ │ │ └── doctype_monitor.cfg │ │ │ ├── input │ │ │ ├── doctype_monitor.cfg │ │ │ └── doctype_monitor.lua │ │ │ └── output │ │ │ ├── dashboard.cfg │ │ │ ├── doctype_monitor.cfg │ │ │ └── doctype_monitor.lua │ ├── hindsight.cfg │ ├── moz_telemetry_s3 │ │ ├── moz_telemetry_s3.json │ │ ├── moz_telemetry_s3_experiments.json │ │ └── run │ │ │ ├── analysis │ │ │ └── placeholder.txt │ │ │ ├── input │ │ │ ├── moz_telemetry_s3.cfg │ │ │ └── moz_telemetry_s3.lua │ │ │ └── output │ │ │ └── moz_telemetry_s3.cfg │ ├── new_doctype_monitor │ │ └── run │ │ │ ├── analysis │ │ │ └── new_doctype_monitor.cfg │ │ │ ├── input │ │ │ ├── new_doctype_monitor.cfg │ │ │ └── new_doctype_monitor.lua │ │ │ └── output │ │ │ ├── new_doctype_monitor.cfg │ │ │ └── new_doctype_monitor.lua │ ├── new_experiment_monitor │ │ └── run │ │ │ ├── analysis │ │ │ └── new_experiment_monitor.cfg │ │ │ ├── input │ │ │ ├── generator.cfg │ │ │ └── generator.lua │ │ │ └── output │ │ │ ├── results.cfg │ │ │ └── results.lua │ └── run.sh │ └── test.lua ├── openssl ├── CMakeLists.txt ├── CMakeLists.txt.openssl ├── config.h.in ├── index.md └── patch.diff ├── papertrail ├── CMakeLists.txt ├── sandboxes │ └── heka │ │ └── input │ │ ├── papertrail.lua │ │ ├── papertrail_delay.lua │ │ ├── papertrail_generic.lua │ │ └── papertrail_monitor.lua └── tests │ └── hindsight │ ├── hindsight.cfg │ ├── papertrail │ └── run │ │ ├── analysis │ │ └── placeholder.txt │ │ ├── input │ │ └── papertrail.cfg │ │ └── output │ │ └── placeholder.txt │ └── run.sh ├── parquet ├── CMakeLists.txt ├── index.md ├── io_modules │ └── lpeg │ │ └── parquet.lua ├── parquet.cpp ├── parquet.def ├── sandboxes │ └── heka │ │ └── output │ │ └── s3_parquet.lua ├── test_sandbox.c └── tests │ ├── test.lua │ ├── test_sandbox_full.lua │ └── test_sandbox_min.lua ├── postgres ├── CMakeLists.txt ├── CMakeLists.txt.postgres ├── index.md └── patch.txt ├── rjson ├── CMakeLists.txt ├── index.md ├── rjson.cpp ├── rjson.def ├── test_sandbox.c └── tests │ ├── test.lua │ └── test_sandbox.lua ├── sax ├── CMakeLists.txt ├── CMakeLists.txt.sax └── index.md ├── snappy ├── CMakeLists.txt ├── CMakeLists.txt.snappy └── index.md ├── socket ├── CMakeLists.txt ├── CMakeLists.txt.socket ├── index.md ├── io_modules │ └── alert │ │ └── email.lua ├── modules │ └── heka │ │ └── alert │ │ └── email.lua └── sandboxes │ └── heka │ ├── input │ ├── heka_tcp.lua │ ├── tcp.lua │ └── udp.lua │ └── output │ ├── heka_tcp.lua │ └── heka_tcp_matcher.lua ├── ssl ├── CMakeLists.txt ├── CMakeLists.txt.ssl └── index.md ├── struct ├── CMakeLists.txt ├── CMakeLists.txt.struct └── index.md ├── syslog ├── CMakeLists.txt ├── io_modules │ └── decoders │ │ └── syslog.lua ├── modules │ └── lpeg │ │ ├── bsd │ │ └── filterlog.lua │ │ ├── linux │ │ ├── cron.lua │ │ ├── dhclient.lua │ │ ├── dhcpd.lua │ │ ├── groupadd.lua │ │ ├── groupdel.lua │ │ ├── kernel.lua │ │ ├── login.lua │ │ ├── named.lua │ │ ├── pam.lua │ │ ├── puppet_agent.lua │ │ ├── sshd.lua │ │ ├── su.lua │ │ ├── sudo.lua │ │ ├── systemd_logind.lua │ │ └── useradd.lua │ │ ├── openssh_portable.lua │ │ ├── sendmail.lua │ │ └── syslog.lua ├── sandboxes │ └── heka │ │ └── input │ │ └── klog.lua ├── test_sandbox.c └── tests │ ├── bsd │ └── filterlog.lua │ ├── integration │ ├── hindsight.cfg │ ├── run.sh │ └── tutorial │ │ ├── adhoc.log │ │ ├── decoder_module.log │ │ ├── grammar_module.log │ │ ├── modules │ │ └── syslog_docs.lua │ │ ├── run │ │ ├── analysis │ │ │ ├── verify_adhoc.cfg │ │ │ ├── verify_adhoc.lua │ │ │ ├── verify_decoder_module.cfg │ │ │ ├── verify_decoder_module.lua │ │ │ ├── verify_function.cfg │ │ │ ├── verify_function.lua │ │ │ ├── verify_grammar_args.cfg │ │ │ ├── verify_grammar_args.lua │ │ │ ├── verify_grammar_args_nil.cfg │ │ │ ├── verify_grammar_args_nil.lua │ │ │ ├── verify_grammar_module.cfg │ │ │ ├── verify_grammar_module.lua │ │ │ ├── verify_grammar_module_keep.cfg │ │ │ ├── verify_grammar_module_keep.lua │ │ │ ├── verify_grammar_transform.cfg │ │ │ ├── verify_grammar_transform.lua │ │ │ ├── verify_printf.cfg │ │ │ ├── verify_printf.lua │ │ │ ├── verify_printf_transform.cfg │ │ │ ├── verify_printf_transform.lua │ │ │ ├── verify_syslog.cfg │ │ │ └── verify_syslog.lua │ │ ├── input │ │ │ ├── adhoc.cfg │ │ │ ├── decoder_module.cfg │ │ │ ├── function.cfg │ │ │ ├── grammar_args.cfg │ │ │ ├── grammar_args_nil.cfg │ │ │ ├── grammar_module.cfg │ │ │ ├── grammar_module_keep.cfg │ │ │ ├── grammar_transform.cfg │ │ │ ├── printf.cfg │ │ │ ├── printf_transform.cfg │ │ │ └── syslog.cfg │ │ └── output │ │ │ └── placeholder.txt │ │ └── syslog.log │ ├── linux │ ├── cron.lua │ ├── dhclient.lua │ ├── dhcpd.lua │ ├── groupadd.lua │ ├── groupdel.lua │ ├── kernel.lua │ ├── login.lua │ ├── named.lua │ ├── pam.lua │ ├── puppet_agent.lua │ ├── sshd.lua │ ├── su.lua │ ├── sudo.lua │ ├── systemd_logind.lua │ └── useradd.lua │ ├── sendmail.lua │ ├── syslog.lua │ └── test.lua ├── systemd ├── CMakeLists.txt ├── CMakeLists.txt.systemd ├── index.md └── sandboxes │ └── heka │ └── input │ └── journald.lua ├── taskcluster ├── CMakeLists.txt ├── bin │ ├── realorg.lua │ └── releng_cost.lua ├── cloud_functions │ └── mach_try_table_exports │ │ ├── mach_try_table_exports.py │ │ └── requirements.txt ├── io_modules │ └── decoders │ │ └── taskcluster │ │ ├── live_backing_log.lua │ │ └── worker_metrics.lua ├── modules │ └── taskcluster │ │ └── util.lua ├── sandboxes │ └── heka │ │ ├── analysis │ │ ├── taskcluster_influx_workertype_lag.lua │ │ └── taskcluster_influx_workertype_tasks.lua │ │ ├── input │ │ ├── taskcluster_backfill.lua │ │ ├── taskcluster_capacity.lua │ │ ├── taskcluster_treeherder.lua │ │ └── taskcluster_usage_anomaly.lua │ │ └── output │ │ └── influx_output.lua ├── schemas │ └── taskcluster │ │ ├── artifact_list.1.schema.json │ │ ├── drain.1.bq.json │ │ ├── perfherder.1.schema.json │ │ ├── pulse_task.1.schema.json │ │ ├── resource_monitor.49.schema.json │ │ ├── task_definition.1.schema.json │ │ └── worker_metrics.1.schema.json ├── sql │ ├── datastudio │ │ └── derived_missing_workertype_costs.sql │ ├── derived_daily_cost_per_workertype.sql │ ├── derived_kind_costs.sql │ ├── derived_task_summary.sql │ ├── derived_task_summary_community.sql │ ├── derived_task_summary_stage.sql │ ├── derived_taskgroup_costs.sql │ ├── derived_test_suite_costs.sql │ ├── derived_worker_concurrency.sql │ ├── derived_worker_metrics_utilization.sql │ ├── derived_workertype_costs.sql │ ├── derived_workertype_costs_community.sql │ ├── derived_workertype_costs_stage.sql │ ├── machtry_quantiles.sql │ └── task_duration_estimates.sql ├── test.lua ├── test_sandbox.c └── tests │ ├── integration │ ├── hindsight.cfg │ ├── run.sh │ ├── taskcluster │ │ ├── C0PGASalTF6MZNoTg0eLGQ_pulse.json │ │ ├── ZYFKUNR5RhmrBAq4Z4KB9g_pulse.json │ │ ├── android_sp-cm_Rap_pulse.json │ │ ├── android_tp6m-c-2-vismet_Btime-fenix_pulse.json │ │ ├── linux_B_Bpgo_pulse.json │ │ ├── linux_GTest_nil_pulse.json │ │ ├── linux_R1_R-1proc_pulse.json │ │ ├── linux_ach_p_pulse.json │ │ ├── linux_ach_ps_pulse.json │ │ ├── osx_1_M_pulse.json │ │ ├── osx_tp6-c_Rap_pulse.json │ │ ├── run │ │ │ ├── analysis │ │ │ │ ├── verify_artifact_list.cfg │ │ │ │ ├── verify_artifact_list.lua │ │ │ │ ├── verify_error.cfg │ │ │ │ ├── verify_error.lua │ │ │ │ ├── verify_perfherder.cfg │ │ │ │ ├── verify_perfherder.lua │ │ │ │ ├── verify_project.cfg │ │ │ │ ├── verify_project.lua │ │ │ │ ├── verify_resolution_definition_cnts.cfg │ │ │ │ ├── verify_resolution_definition_cnts.lua │ │ │ │ ├── verify_resource_monitor.cfg │ │ │ │ ├── verify_resource_monitor.lua │ │ │ │ ├── verify_timing.cfg │ │ │ │ └── verify_timing.lua │ │ │ ├── input │ │ │ │ ├── tc.cfg │ │ │ │ └── tc.lua │ │ │ └── output │ │ │ │ └── placeholder.txt │ │ ├── task_AuIAPvWhSiyp0eUh5vAdCw │ │ ├── task_AuIAPvWhSiyp0eUh5vAdCw_runs_0_artifacts │ │ ├── task_AuIAPvWhSiyp0eUh5vAdCw_runs_0_artifacts_continuationToken_more │ │ ├── task_AuIAPvWhSiyp0eUh5vAdCw_runs_0_artifacts_continuationToken_more1 │ │ ├── task_AuIAPvWhSiyp0eUh5vAdCw_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_B02j0uS8SOGRfB4TrE2q4w │ │ ├── task_B02j0uS8SOGRfB4TrE2q4w_runs_0_artifacts │ │ ├── task_B02j0uS8SOGRfB4TrE2q4w_runs_1_artifacts │ │ ├── task_B02j0uS8SOGRfB4TrE2q4w_runs_1_artifacts_public_logs_live_backing.log │ │ ├── task_C0PGASalTF6MZNoTg0eLGQ │ │ ├── task_C0PGASalTF6MZNoTg0eLGQ_runs_0_artifacts │ │ ├── task_C0PGASalTF6MZNoTg0eLGQ_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_CX96USXgR3CvJWA2ZsBYrA │ │ ├── task_CX96USXgR3CvJWA2ZsBYrA_runs_0_artifacts │ │ ├── task_CX96USXgR3CvJWA2ZsBYrA_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_CZ9BVge4QdKrJ3ZWdVbsTQ │ │ ├── task_CZ9BVge4QdKrJ3ZWdVbsTQ_runs_0_artifacts │ │ ├── task_CZ9BVge4QdKrJ3ZWdVbsTQ_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_GCqbg2tVRxiT2JaYrqev3g │ │ ├── task_GCqbg2tVRxiT2JaYrqev3g_runs_0_artifacts │ │ ├── task_GCqbg2tVRxiT2JaYrqev3g_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_HbxirNOCRYiM1yUmLBMeYQ │ │ ├── task_HbxirNOCRYiM1yUmLBMeYQ_runs_0_artifacts │ │ ├── task_HbxirNOCRYiM1yUmLBMeYQ_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_HbxirNOCRYiM1yUmLBMeYQ_runs_0_artifacts_public_test_info_perfherder-data.json │ │ ├── task_IViCwC-nTMi_gXPE3ckjPg │ │ ├── task_IViCwC-nTMi_gXPE3ckjPg_runs_0_artifacts │ │ ├── task_IViCwC-nTMi_gXPE3ckjPg_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_KcwJfqv_Sqi6e_2KKFS1Ow │ │ ├── task_KcwJfqv_Sqi6e_2KKFS1Ow_runs_0_artifacts │ │ ├── task_KcwJfqv_Sqi6e_2KKFS1Ow_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_SSXGXyVIRQGQqVKGym90mQ │ │ ├── task_SSXGXyVIRQGQqVKGym90mQ_runs_0_artifacts │ │ ├── task_SSXGXyVIRQGQqVKGym90mQ_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_USQ8K5YcQJKQgybb28cyXg │ │ ├── task_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts │ │ ├── task_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts_public_monitoring_resource-monitor.json │ │ ├── task_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts_public_test_info_perfherder-data.json │ │ ├── task_ZYFKUNR5RhmrBAq4Z4KB9g │ │ ├── task_ZYFKUNR5RhmrBAq4Z4KB9g_runs_0_artifacts │ │ ├── task_ZYFKUNR5RhmrBAq4Z4KB9g_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_a46tmlKvRFuopDLt8E7IEQ │ │ ├── task_a46tmlKvRFuopDLt8E7IEQ_runs_0_artifacts │ │ ├── task_a46tmlKvRFuopDLt8E7IEQ_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_aZ19zQ0SS8OLeFnQoHSwNA │ │ ├── task_aZ19zQ0SS8OLeFnQoHSwNA_runs_0_artifacts │ │ ├── task_aZ19zQ0SS8OLeFnQoHSwNA_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_bZPP4WKDR5Whwge70OcW9g │ │ ├── task_bZPP4WKDR5Whwge70OcW9g_runs_0_artifacts │ │ ├── task_bZPP4WKDR5Whwge70OcW9g_runs_0_artifacts_public_logs_live_backing.log │ │ ├── tests.txt │ │ ├── win_X1_X_pulse.json │ │ ├── win_sy-tp6_SY_pulse.json │ │ ├── win_tp6-c_Btime-Prof_pulse.json │ │ └── win_x_T_pulse.json │ ├── taskcluster_backfill │ │ ├── C0PGASalTF6MZNoTg0eLGQ_pulse.json │ │ ├── USQ8K5YcQJKQgybb28cyXg_pulse.json │ │ ├── ZYFKUNR5RhmrBAq4Z4KB9g_pulse.json │ │ ├── run │ │ │ ├── analysis │ │ │ │ ├── verify_backfill_cnts.cfg │ │ │ │ └── verify_backfill_cnts.lua │ │ │ ├── input │ │ │ │ ├── backfill.cfg │ │ │ │ ├── tc.cfg │ │ │ │ └── tc.lua │ │ │ └── output │ │ │ │ ├── verify_error.cfg │ │ │ │ └── verify_error.lua │ │ ├── task_BF_C0PGASalTF6MZNoTg0eLGQ │ │ ├── task_BF_C0PGASalTF6MZNoTg0eLGQ_runs_0_artifacts │ │ ├── task_BF_C0PGASalTF6MZNoTg0eLGQ_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_BF_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts_public_monitoring_resource-monitor.json │ │ ├── task_BF_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts_public_test_info_perfherder-data.json │ │ ├── task_BF_ZYFKUNR5RhmrBAq4Z4KB9g_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_USQ8K5YcQJKQgybb28cyXg │ │ ├── task_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts │ │ ├── task_USQ8K5YcQJKQgybb28cyXg_runs_0_artifacts_public_logs_live_backing.log │ │ ├── task_ZYFKUNR5RhmrBAq4Z4KB9g │ │ ├── task_ZYFKUNR5RhmrBAq4Z4KB9g_runs_0_artifacts │ │ └── tests.txt │ └── worker_metrics │ │ └── run │ │ ├── analysis │ │ ├── verify.cfg │ │ ├── verify.lua │ │ ├── verify_error.cfg │ │ └── verify_error.lua │ │ ├── input │ │ ├── worker_metrics.cfg │ │ └── worker_metrics.lua │ │ └── output │ │ └── placeholder.txt │ └── test.lua ├── xxhash ├── CMakeLists.txt ├── index.md ├── test_sandbox.c ├── tests │ └── test.lua ├── xxhash.c └── xxhash.def └── zlib ├── CMakeLists.txt ├── CMakeLists.txt.zlib └── index.md /.gitignore: -------------------------------------------------------------------------------- 1 | release/ 2 | gh-pages/ 3 | *.swp 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: generic 2 | sudo: required 3 | services: 4 | - docker 5 | 6 | script: 7 | - docker build -t mozilla/lua_sandbox_extensions . 8 | - | 9 | docker run --rm mozilla/lua_sandbox_extensions \ 10 | bash -c 'cd /root/lua_sandbox_extensions/release && ctest -V -C integration' 11 | - mkdir dist 12 | - | 13 | docker run -v `pwd`/dist:/dist --rm mozilla/lua_sandbox_extensions \ 14 | bash -c 'cp /root/lua_sandbox_extensions/release/*.rpm /dist' 15 | - sha256sum dist/* 16 | 17 | deploy: 18 | - provider: script 19 | script: bash artifact_push.sh 20 | skip_cleanup: true 21 | on: 22 | all_branches: true 23 | condition: $TRAVIS_BRANCH =~ ^dev|master$ 24 | - provider: script 25 | script: bash docker_push.sh $TRAVIS_BRANCH 26 | on: 27 | all_branches: true 28 | condition: $TRAVIS_BRANCH =~ ^dev|master|test$ 29 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | See: https://github.com/mozilla-services/hindsight/wiki/Change-Log 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | 10 | 16 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /amqp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.5) 6 | project(amqp VERSION 0.0.3 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "AMQP RabbitMQ Lua module") 8 | 9 | find_package(PkgConfig REQUIRED) 10 | pkg_search_module(RABBITMQ REQUIRED librabbitmq) 11 | 12 | set(MODULE_SRCS amqp.c amqp.def) 13 | set(INSTALL_MODULE_PATH ${INSTALL_IOMODULE_PATH}) 14 | include(sandbox_module) 15 | target_link_libraries(${MODULE_NAME} ${RABBITMQ_LIBRARIES}) 16 | -------------------------------------------------------------------------------- /amqp/amqp.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_amqp 3 | -------------------------------------------------------------------------------- /artifact_push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/main/install | bash 3 | tar -C dist -zcvf all.tgz . 4 | 5 | export ARTIFACTS_PERMISSIONS="public-read" 6 | export ARTIFACTS_TARGET_PATHS="/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_JOB_NUMBER}/centos7/" 7 | export ARTIFACTS_PATHS="all.tgz:dist/" 8 | artifacts upload 9 | 10 | echo "$ARTIFACTS_TARGET_PATHS" > latest.txt 11 | export ARTIFACTS_TARGET_PATHS="/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/centos7/" 12 | export ARTIFACTS_PATHS="all.tgz:latest.txt" 13 | artifacts upload 14 | -------------------------------------------------------------------------------- /aws/aws.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_aws_kinesis 3 | -------------------------------------------------------------------------------- /aws/tests/hindsight/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /aws/tests/hindsight/run/analysis/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/aws/tests/hindsight/run/analysis/placeholder.txt -------------------------------------------------------------------------------- /aws/tests/hindsight/run/input/generate_data.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /aws/tests/hindsight/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/aws/tests/hindsight/run/output/placeholder.txt -------------------------------------------------------------------------------- /bit/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.6 FATAL_ERROR) 6 | project(bit LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_bit 10 | GIT_REPOSITORY https://github.com/LuaDist/luabitop.git 11 | GIT_TAG 81bb23b0e737805442033535de8e6d204d0e5381 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_bit copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /bit/index.md: -------------------------------------------------------------------------------- 1 | # Lua Bit Operations Module 2 | 3 | http://bitop.luajit.org/index.html 4 | -------------------------------------------------------------------------------- /bloom_filter/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(bloom-filter VERSION 1.0.0 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Lua bloom filter module (membership test)") 8 | set(MODULE_SRCS bloom_filter.c ../common/xxhash.c bloom_filter.def) 9 | include(sandbox_module) 10 | -------------------------------------------------------------------------------- /bloom_filter/bloom_filter.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_bloom_filter 3 | -------------------------------------------------------------------------------- /bloom_filter/tests/benchmark.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "bloom_filter" 6 | 7 | bf = bloom_filter.new(2e6, 0.01) 8 | 9 | function process(ts) 10 | bf:add(ts) 11 | return 0 12 | end 13 | 14 | function report(tc) 15 | write_output(bf:count()) 16 | end 17 | 18 | -------------------------------------------------------------------------------- /bloom_filter/tests/test_sandbox.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "bloom_filter" 6 | 7 | bf = bloom_filter.new(20, 0.01) 8 | 9 | local ok, err = pcall(bf.fromstring, bf, {}) 10 | assert(not ok) --incorrect argument type 11 | 12 | local ok, err = pcall(bf.fromstring, bf, " ") 13 | assert(not ok) --incorrect argument length 14 | 15 | function process(ts) 16 | if not bf:query(ts) then 17 | if not bf:add(ts) then 18 | error("key existed") 19 | end 20 | end 21 | 22 | return 0 23 | end 24 | 25 | function report(tc) 26 | if tc == 99 then 27 | bf:clear() 28 | else 29 | write_output(bf:count()) 30 | end 31 | end 32 | 33 | -------------------------------------------------------------------------------- /circular_buffer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(circular-buffer VERSION 1.0.2 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Lua circular buffer module (in-memory time series data store)") 8 | set(MODULE_SRCS circular_buffer.c circular_buffer.def) 9 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-lpeg (>= 1.0), ${PACKAGE_PREFIX}-cjson (>= 2.1)") 10 | string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS}) 11 | include(sandbox_module) 12 | -------------------------------------------------------------------------------- /circular_buffer/circular_buffer.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_circular_buffer 3 | -------------------------------------------------------------------------------- /circular_buffer/tests/benchmark.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "circular_buffer" 6 | 7 | data = circular_buffer.new(1440, 10, 1) 8 | 9 | function process(ts) 10 | data:add(ts, 4, 1) 11 | return 0 12 | end 13 | -------------------------------------------------------------------------------- /circular_buffer/tests/benchmark_output.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "circular_buffer" 6 | 7 | local cbuf = circular_buffer.new(1440, 3, 60) 8 | 9 | function process(tc) 10 | write_output(cbuf) 11 | return 0 12 | end 13 | -------------------------------------------------------------------------------- /circular_buffer/tests/cbufd.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | -------------------------------------------------------------------------------- /circular_buffer/tests/test_sandbox.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "circular_buffer" 6 | 7 | data = circular_buffer.new(3, 3, 1) 8 | local ADD_COL = data:set_header(1, "Add column") 9 | local SET_COL = data:set_header(2, "Set column", "count") 10 | local GET_COL = data:set_header(3, "Get column", "count", "sum") 11 | 12 | function process(ts) 13 | if data:add(ts, ADD_COL, 1) then 14 | data:set(ts, GET_COL, data:get(ts, ADD_COL)) 15 | end 16 | data:set(ts, SET_COL, 1) 17 | return 0 18 | end 19 | 20 | function report(tc) 21 | write_output(data) 22 | end 23 | -------------------------------------------------------------------------------- /circular_buffer/tests/test_sandbox_annotation.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "circular_buffer" 6 | 7 | local cbuf = circular_buffer.new(1440, 3, 60) 8 | local test = circular_buffer.new(2, 2, 60) 9 | 10 | function process(tc) 11 | if tc == 0 then 12 | test:annotate(0, 1, "info", "annotation\"\t\b\r\n\240 end") 13 | test:annotate(60e9, 2, "alert", "alert") 14 | test:annotate(120e9, 2, "info", "out of range, should be ignored") 15 | write_output(test) 16 | elseif tc == 1 then 17 | test:set(120e9, 1, 0/0) -- advance the buffer (pruning the annotation) 18 | write_output(test) 19 | end 20 | return 0 21 | end 22 | -------------------------------------------------------------------------------- /cjson/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(cjson LANGUAGES C) 7 | 8 | externalproject_add( 9 | ep_cjson 10 | GIT_REPOSITORY https://github.com/trink/lua-cjson.git 11 | GIT_TAG lua_sandbox 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_cjson copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | include(sandbox_ep_test) 19 | -------------------------------------------------------------------------------- /cjson/index.md: -------------------------------------------------------------------------------- 1 | # Lua CJSON Module 2 | 3 | [CJSON](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) parser with the following modifications: 4 | - Loads the cjson module in a global cjson table 5 | - The encode buffer is limited to the sandbox output_limit. 6 | - The decode buffer will be roughly limited to one half of the sandbox memory_limit. 7 | - The NULL value is not decoded to cjson.null it is simply discarded. 8 | If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. 9 | - The new() function has been disabled so only a single cjson parser can be created. 10 | - The encode_keep_buffer() function has been disabled (the buffer is always reused). 11 | -------------------------------------------------------------------------------- /cjson/tests/empty_array.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "cjson" 6 | 7 | local ea = '[]' 8 | local lea = {nil} 9 | 10 | local eh = '{}' 11 | local leh = {} 12 | 13 | local sparse = "[1,2,null,4]" 14 | 15 | function process() 16 | local t = cjson.decode(ea) 17 | local es = cjson.encode(t) 18 | assert(ea == es, es) 19 | 20 | es = cjson.encode(lea) 21 | assert(ea == es, es) 22 | 23 | t = cjson.decode(eh) 24 | es = cjson.encode(t) 25 | assert(eh == es, es) 26 | 27 | es = cjson.encode(leh) 28 | assert(eh == es, es) 29 | 30 | t = cjson.decode(sparse) 31 | es = cjson.encode(t) 32 | assert(sparse == es, es) 33 | return 0 34 | end 35 | -------------------------------------------------------------------------------- /cjson/tests/limited.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | require "table" 7 | local cj = require "cjson" 8 | local js = '["this is a test","this is a test","this is a test","this is a test","this is a test"]' 9 | 10 | function process() 11 | local t = cjson.decode(js) 12 | assert(#t == 5, "can decode a JSON string bigger than the output buffer") 13 | 14 | local ok, json = pcall(cjson.encode, t) 15 | assert(not ok, "cannot encode an array bigger than the the output buffer") 16 | return 0 17 | end 18 | 19 | -------------------------------------------------------------------------------- /cjson/tests/test.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | require "table" 7 | local cj = require "cjson" 8 | local js = '["this is a test","this is a test","this is a test","this is a test","this is a test"]' 9 | 10 | assert(cj == cjson, "cjson not creating a global table") 11 | 12 | local value = cjson.decode("[ true, { \"foo\": \"bar\" } ]") 13 | assert("bar" == value[2].foo, string.format("bar: %s", tostring(value[2].foo))) 14 | 15 | local null_json = '{"test" : 1, "null" : null}' 16 | local value = cjson.decode(null_json) 17 | assert(value.null == nil, "null not discarded") 18 | 19 | cjson.decode_null(true) 20 | value = cjson.decode(null_json) 21 | assert(type(value.null) == "userdata", "null discarded") 22 | 23 | assert(not cjson.new) 24 | 25 | assert(not cjson.encode_keep_buffer) 26 | -------------------------------------------------------------------------------- /cjson/tests/unlimited.lua: -------------------------------------------------------------------------------- 1 | require "cjson" 2 | 3 | function process() 4 | local ls = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" 5 | local t = {} 6 | 7 | for i=1, 1000 do 8 | t[#t+1] = ls 9 | end 10 | write_output(cjson.encode(t)) 11 | return 0 12 | end 13 | -------------------------------------------------------------------------------- /cmake/CTestCustom.in.cmake: -------------------------------------------------------------------------------- 1 | set(CTEST_CUSTOM_PRE_TEST "${CMAKE_MAKE_PROGRAM} install DESTDIR=install") 2 | set(CTEST_CUSTOM_PRE_TEST "${CMAKE_COMMAND} -DCMAKE_INSTALL_PREFIX=install -P cmake_install.cmake") 3 | -------------------------------------------------------------------------------- /cmake/test_module.in.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 | /* vim: set ts=2 et sw=2 tw=80: */ 3 | /* This Source Code Form is subject to the terms of the Mozilla Public 4 | * License, v. 2.0. If a copy of the MPL was not distributed with this 5 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 | 7 | /** @brief Test module configuration @file */ 8 | 9 | #define TEST_MODULE_PATH \ 10 | "path = [[${TEST_MODULE_PATH};${TEST_IOMODULE_PATH}]]\n" \ 11 | "cpath = [[${TEST_MODULE_CPATH};${TEST_IOMODULE_CPATH}]]\n" 12 | -------------------------------------------------------------------------------- /compat/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(compat LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_compat 10 | GIT_REPOSITORY https://github.com/keplerproject/lua-compat-5.2.git 11 | GIT_TAG ad22f7e8a6ee2445f3085b572ba925d2561c7359 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_compat copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /compat/index.md: -------------------------------------------------------------------------------- 1 | # Compatibility module providing Lua-5.2-style APIs for Lua 5.1 2 | 3 | https://github.com/keplerproject/lua-compat-5.2 4 | -------------------------------------------------------------------------------- /cuckoo_filter/cuckoo_filter.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_cuckoo_filter 3 | -------------------------------------------------------------------------------- /cuckoo_filter/cuckoo_filter_expire.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_cuckoo_filter_expire 3 | -------------------------------------------------------------------------------- /cuckoo_filter/tests/benchmark.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "cuckoo_filter" 6 | 7 | cf = cuckoo_filter.new(2e6) 8 | 9 | function process(ts) 10 | cf:add(ts) 11 | return 0 12 | end 13 | 14 | function report(tc) 15 | write_output(cf:count()) 16 | end 17 | 18 | -------------------------------------------------------------------------------- /cuckoo_filter/tests/test_sandbox.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "cuckoo_filter" 6 | 7 | cf = cuckoo_filter.new(16) 8 | 9 | function process(ts) 10 | if not cf:query(ts) then 11 | if not cf:add(ts) then 12 | error("key existed") 13 | end 14 | end 15 | 16 | return 0 17 | end 18 | 19 | function report(tc) 20 | if tc == 98 then 21 | cf:delete(1); 22 | write_output(cf:count()) 23 | elseif tc == 99 then 24 | cf:clear() 25 | write_output(cf:count()) 26 | else 27 | write_output(cf:count()) 28 | end 29 | end 30 | 31 | -------------------------------------------------------------------------------- /cuckoo_filter/tests/test_sandbox_expire.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "cuckoo_filter_expire" 6 | 7 | cf = cuckoo_filter_expire.new(512, 1) 8 | 9 | function process(ts) 10 | if not cf:query(ts) then 11 | if not cf:add(ts, 1) then 12 | error("key existed") 13 | end 14 | end 15 | 16 | return 0 17 | end 18 | 19 | function report(tc) 20 | if tc == 98 then 21 | cf:delete(1); 22 | write_output(cf:count()) 23 | elseif tc == 99 then 24 | cf:clear() 25 | write_output(cf:count()) 26 | else 27 | write_output(cf:count()) 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /docker_push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | tag="" 4 | if [[ $1 = "main" ]]; then 5 | tag="mozilla/lua_sandbox_extensions:main" 6 | elif [[ $1 = "dev" ]]; then 7 | tag="mozilla/lua_sandbox_extensions:dev" 8 | elif [[ $1 = "test" ]]; then 9 | tag="mozilla/lua_sandbox_extensions:test" 10 | else 11 | exit 1 12 | fi 13 | 14 | docker tag local/lua_sandbox_extensions $tag 15 | docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" 16 | docker push $tag 17 | -------------------------------------------------------------------------------- /elasticsearch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(elasticsearch VERSION 1.0.8 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Elasticsearch encoders and outputs") 8 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-heka (>= 1.0), ${PACKAGE_PREFIX}-rjson (>= 1.0), ${PACKAGE_PREFIX}-cjson (>= 2.1), ${PACKAGE_PREFIX}-socket (>= 3.0)") 9 | string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS}) 10 | include(sandbox_module) 11 | -------------------------------------------------------------------------------- /elasticsearch/tests/test.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | return -- no-op on Lua 5.1 this module will only work with the lua_sandbox 6 | -------------------------------------------------------------------------------- /gcp/gcp.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_gcp_logging 3 | luaopen_gcp_pubsub 4 | -------------------------------------------------------------------------------- /gcp/index.md: -------------------------------------------------------------------------------- 1 | # GCP Lua Module 2 | 3 | ## Overview 4 | Lua wrapper for the GCP gRPC APIs. 5 | 6 | ## Modules 7 | 8 | * [Logging](logging.md) 9 | * [Pub/Sub](pubsub.md) 10 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_run_path = "run" 3 | --sandbox_load_path = "load" 4 | analysis_lua_path = "/usr/lib/luasandbox/modules/?.lua" 5 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 6 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 7 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 8 | output_size = 1024 * 1024 * 1024 9 | 10 | analysis_defaults = {} 11 | input_defaults = {} 12 | output_defaults = {} 13 | 14 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/analysis/throughput.cfg: -------------------------------------------------------------------------------- 1 | filename = "throughput.lua" 2 | message_matcher = "TRUE" 3 | ticker_interval = 10 4 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/analysis/throughput.lua: -------------------------------------------------------------------------------- 1 | require "circular_buffer" 2 | 3 | inputs = {} 4 | 5 | local function get_input(logger) 6 | local i = inputs[logger] 7 | if not i then 8 | i = circular_buffer.new(60, 1, 1) 9 | i:set_header(1, "messages") 10 | inputs[logger] = i 11 | end 12 | return i 13 | end 14 | 15 | 16 | function process_message() 17 | local cb = get_input(read_message("Logger")) 18 | cb:add(read_message("Timestamp"), 1, 1) 19 | return 0 20 | end 21 | 22 | 23 | function timer_event(ns, shutdown) 24 | for k,v in pairs(inputs) do 25 | v:add(ns, 1, 0) 26 | inject_payload("cbuf", k, v) 27 | end 28 | end 29 | 30 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/input/logdata.cfg: -------------------------------------------------------------------------------- 1 | filename = "logdata.lua" 2 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/input/logdata.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | 7 | local msg = { 8 | Payload = '127.0.0.1 - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"', 9 | Fields = { 10 | body_bytes_sent = 222, 11 | request = "GET / HTTP/1.1", 12 | remote_addr = "127.0.0.1", 13 | remote_user = "-", 14 | status = 304, 15 | http_user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0", 16 | referer = "-", 17 | other = "bogus" 18 | } 19 | } 20 | 21 | function process_message() 22 | inject_message(msg) 23 | return 0 24 | end 25 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/input/prune.cfg: -------------------------------------------------------------------------------- 1 | filename = "prune_input.lua" 2 | ticker_interval = 60 3 | input = true 4 | analysis = true 5 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/input/testdata.cfg: -------------------------------------------------------------------------------- 1 | filename = "testdata.lua" 2 | instruction_limit = 0 3 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/input/testdata.lua: -------------------------------------------------------------------------------- 1 | require "string" 2 | 3 | local msg = { 4 | Payload = '127.0.0.1 - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0', 5 | Fields = { 6 | id = {value = 0, value_type = 2}, 7 | strs = {value = {"one", "two", "thr\tee"}}, 8 | ints = {value = {1, 2, 3}, value_type = 2}, 9 | dbls = {value = {1.1, 1.2, 1.3}, value_type = 3}, 10 | bools = {value = {true, false, true}, value_type = 4}, 11 | bin = {value = "", value_type = 1} 12 | } 13 | } 14 | 15 | function process_message() 16 | for i=1, 10000000 do 17 | --for i=1, 1000000 do 18 | --for i=1, 1 do 19 | msg.Fields.id.value = i 20 | inject_message(msg) 21 | end 22 | return 0 23 | end 24 | 25 | 26 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/output/pub.cfg: -------------------------------------------------------------------------------- 1 | filename = "gcp_pubsub.lua" 2 | message_matcher = "Logger == 'input.testdata' && Uuid < '\128' && Fields[bin] != NIL" 3 | ticker_interval = 1 4 | 5 | channel = "pubsub.googleapis.com" 6 | project = "projects/mozilla-data-poc-198117" 7 | topic = "pub_grpc" 8 | batch_size = 1000 -- default/maximum 9 | max_async_requests = 20 -- default (0 for synchronous) 10 | async_buffer_size = max_async_requests * batch_size 11 | 12 | -- Specify a module that will encode/convert the Heka message into its output representation. 13 | encoder_module = "encoders.payload" 14 | 15 | -------------------------------------------------------------------------------- /gcp/tests/hindsight/run/output/pub1.cfg: -------------------------------------------------------------------------------- 1 | filename = "gcp_pubsub.lua" 2 | message_matcher = "Logger == 'input.testdata' && Uuid > '\128' && Fields[bin] != NIL" 3 | ticker_interval = 1 4 | 5 | channel = "pubsub.googleapis.com" 6 | project = "projects/mozilla-data-poc-198117" 7 | topic = "pub_grpc" 8 | batch_size = 1000 -- default/maximum 9 | max_async_requests = 20 -- default (0 for synchronous) 10 | async_buffer_size = max_async_requests * batch_size 11 | 12 | -- Specify a module that will encode/convert the Heka message into its output representation. 13 | encoder_module = "encoders.payload" 14 | 15 | -------------------------------------------------------------------------------- /geoip/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(geoip LANGUAGES C) 7 | 8 | externalproject_add( 9 | ep_geoip 10 | GIT_REPOSITORY https://github.com/agladysh/lua-geoip.git 11 | GIT_TAG 06548fd6dac30d4ba0ea5489c3cffe8c20b526c0 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} -DPARENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_geoip copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | include(sandbox_ep_test) 19 | -------------------------------------------------------------------------------- /geoip/index.md: -------------------------------------------------------------------------------- 1 | # Lua GeoIP Module (DEPRECATED) 2 | 3 | https://github.com/agladysh/lua-geoip 4 | 5 | See: https://mozilla-services.github.io/lua_sandbox_extensions/maxminddb/ 6 | -------------------------------------------------------------------------------- /gzfile/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(gzfile VERSION 0.0.3 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "gzip File Access Functions") 8 | set(CPACK_DEBIAN_PACKAGE_DEPENDS " zlib1g (>= 1:1.1.4)") 9 | find_package(ZLIB) 10 | set(MODULE_SRCS gzfile.c gzfile.def) 11 | set(INSTALL_MODULE_PATH ${INSTALL_IOMODULE_PATH}) 12 | include(sandbox_module) 13 | 14 | target_link_libraries(gzfile ${ZLIB_LIBRARIES}) 15 | if(MSVC) 16 | target_link_libraries(gzfile ${LUA_LIBRARIES}) 17 | endif() 18 | -------------------------------------------------------------------------------- /gzfile/gzfile.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_gzfile 3 | -------------------------------------------------------------------------------- /gzfile/tests/compressed.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/gzfile/tests/compressed.log -------------------------------------------------------------------------------- /gzfile/tests/uncompressed.log: -------------------------------------------------------------------------------- 1 | line one 2 | line two 3 | This is a long line three that should be truncated at eighty characters 12345678 truncated 4 | line four no terminating new line -------------------------------------------------------------------------------- /heka/io_modules/decoders/heka/protobuf.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Heka Protobuf Message Decoder Module 7 | 8 | ## Decoder Configuration Table 9 | * none 10 | 11 | ## Functions 12 | 13 | ### decode 14 | 15 | Decode and inject the resulting message 16 | 17 | *Arguments* 18 | - data (string) - Protobuf message with a Heka schema 19 | 20 | *Return* 21 | - nil - throws an error on an invalid data type, inject_message failure etc. 22 | 23 | --]] 24 | 25 | -- Imports 26 | local inject_message = inject_message 27 | 28 | local M = {} 29 | setfenv(1, M) -- Remove external access to contain everything in the module 30 | 31 | function decode(data) 32 | inject_message(data) 33 | end 34 | 35 | return M 36 | -------------------------------------------------------------------------------- /heka/io_modules/encoders/heka/framed_protobuf.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Heka Framed Protobuf Message Encoder Module 7 | 8 | ## Encoder Configuration Table 9 | * none 10 | 11 | ## Functions 12 | 13 | ### encode 14 | 15 | Returns the read_message userdata reference to retrieve the framed Heka protobuf 16 | message. 17 | 18 | *Arguments* 19 | - none 20 | 21 | *Return* 22 | - data (userdata) - reference to the framed message 23 | 24 | --]] 25 | 26 | -- Imports 27 | local framed = read_message("framed", nil, nil, true) 28 | 29 | local M = {} 30 | setfenv(1, M) -- Remove external access to contain everything in the module 31 | 32 | function encode() 33 | return framed 34 | end 35 | 36 | return M 37 | -------------------------------------------------------------------------------- /heka/io_modules/encoders/heka/protobuf.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Heka Protobuf Message Encoder Module 7 | 8 | ## Encoder Configuration Table 9 | * none 10 | 11 | ## Functions 12 | 13 | ### encode 14 | 15 | Returns the read_message userdata reference to retrieve the raw Heka protobuf 16 | message. 17 | 18 | *Arguments* 19 | - none 20 | 21 | *Return* 22 | - data (userdata) - reference to the raw message 23 | 24 | --]] 25 | 26 | -- Imports 27 | 28 | local raw = read_message("raw", nil, nil, true) 29 | 30 | local M = {} 31 | setfenv(1, M) -- Remove external access to contain everything in the module 32 | 33 | function encode() 34 | return raw 35 | end 36 | 37 | return M 38 | -------------------------------------------------------------------------------- /heka/io_modules/encoders/payload.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Heka Payload Encoder Module 7 | Extracts the payload from the provided Heka message and converts it into a byte 8 | stream for delivery to an external resource. 9 | 10 | ## Encoder Configuration Table 11 | * none 12 | 13 | ## Functions 14 | 15 | ### encode 16 | 17 | Returns the read_message reference to retrieve the payload from the Heka 18 | message. 19 | 20 | *Arguments* 21 | - none 22 | 23 | *Return* 24 | - data (userdata) - reference to the message Payload 25 | 26 | --]] 27 | 28 | -- Imports 29 | 30 | local payload = read_message("Payload", nil, nil, true) 31 | 32 | local M = {} 33 | setfenv(1, M) -- Remove external access to contain everything in the module 34 | 35 | function encode() 36 | return payload 37 | end 38 | 39 | return M 40 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_fields/input.json: -------------------------------------------------------------------------------- 1 | {"foo":"bar", "len":10, "time":123456789000000000, "Pid":3432, "nested": { "a":"string", "b":1 }, "captain": { "jean-luc": { "picard": { "v":"uss enterprise" }}}} 2 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_fields/run/analysis/verify_function.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_function.lua" 2 | message_matcher = "Logger == 'input.function'" 3 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_fields/run/input/function.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "input.json" 3 | default_headers = {Type = "default_header"} 4 | send_decode_failures = true 5 | decoder_module = "decoders.heka.table_to_fields" 6 | decoders_heka_table_to_fields = { 7 | module_name = "cjson", 8 | module_function = "decode", 9 | max_depth = 2 10 | } 11 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_fields/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/heka/tests/integration/decoder_table_to_fields/run/output/placeholder.txt -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_message/input.json: -------------------------------------------------------------------------------- 1 | {"foo":"bar", "len":10, "time":123456789000000000, "Pid":3432} 2 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_message/input.logfmt: -------------------------------------------------------------------------------- 1 | Type=logfmt foo=bar a=14 baz="hello kitty" cool%story=bro f %^asdf 2 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_message/run/analysis/verify_function.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_function.lua" 2 | message_matcher = "Logger == 'input.function'" 3 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_message/run/analysis/verify_grammar.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_grammar.lua" 2 | message_matcher = "Logger == 'input.grammar'" 3 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_message/run/input/function.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "input.json" 3 | default_headers = {Type = "default_header"} 4 | send_decode_failures = true 5 | decoder_module = "decoders.heka.table_to_message" 6 | decoders_heka_table_to_message = { 7 | module_name = "cjson", 8 | module_function = "decode", 9 | map = { -- optional if not provided a default mapping will be used 10 | time = {header = "Timestamp"}, 11 | len = {field = "length", type = "int", representation = "inches"} 12 | -- default mapping for anything else 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_message/run/input/grammar.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "input.logfmt" 3 | default_headers = {Type = "default_header"} 4 | send_decode_failures = true 5 | decoder_module = "decoders.heka.table_to_message" 6 | decoders_heka_table_to_message = { 7 | module_name = "lpeg.logfmt", 8 | module_grammar = "grammar", 9 | map = { -- optional if not provided a default mapping will be used 10 | a = {field = "a", type = "int"} 11 | -- default mapping for anything else 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /heka/tests/integration/decoder_table_to_message/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/heka/tests/integration/decoder_table_to_message/run/output/placeholder.txt -------------------------------------------------------------------------------- /heka/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_run_path = "run" 3 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 4 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 5 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 6 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 7 | hostname = "integration_test" 8 | 9 | analysis_defaults = {} 10 | input_defaults = {} 11 | output_defaults = {} 12 | -------------------------------------------------------------------------------- /heka/tests/integration/input_file/multiline.log: -------------------------------------------------------------------------------- 1 | This is 2 | a test 3 | item 4 | long 5 | long 6 | line 7 | 8 | This is log line 2 9 | 10 | This is log 11 | line 12 | three 13 | 14 | 15 | -------------------------------------------------------------------------------- /heka/tests/integration/input_file/run/analysis/verify_multiline.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_multiline.lua" 2 | message_matcher = "Logger == 'input.multiline'" 3 | -------------------------------------------------------------------------------- /heka/tests/integration/input_file/run/analysis/verify_singleline.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_singleline.lua" 2 | message_matcher = "Logger == 'input.singleline'" 3 | -------------------------------------------------------------------------------- /heka/tests/integration/input_file/run/input/multiline.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "multiline.log" 3 | delimiter = "^$" 4 | decoder_module = "decoders.payload" 5 | -------------------------------------------------------------------------------- /heka/tests/integration/input_file/run/input/singleline.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "singleline.log" 3 | decoder_module = "decoders.payload" 4 | -------------------------------------------------------------------------------- /heka/tests/integration/input_file/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/heka/tests/integration/input_file/run/output/placeholder.txt -------------------------------------------------------------------------------- /heka/tests/integration/input_file/singleline.log: -------------------------------------------------------------------------------- 1 | line 1 2 | line 2 3 | line 3 4 | -------------------------------------------------------------------------------- /heka/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /hindsight/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(hindsight VERSION 0.0.1 LANGUAGES NONE) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Hindsight Utility Sandboxes and Modules") 8 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-lpeg (>= 1.0.10)") 9 | string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS}) 10 | include(sandbox_module) 11 | 12 | add_test(NAME ${PROJECT_NAME}_plugin_monitor 13 | COMMAND ../run.sh 14 | CONFIGURATIONS integration 15 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/integration/plugin_monitor) 16 | -------------------------------------------------------------------------------- /hindsight/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_load_path = "load" 3 | sandbox_run_path = "run" 4 | analysis_lua_path = "/usr/lib/luasandbox/modules/?.lua" 5 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 6 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 7 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 8 | hostname = "integration_test" 9 | 10 | analysis_defaults = {} 11 | input_defaults = {} 12 | output_defaults = {} 13 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/analysis/plugin_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "plugin_monitor.lua" 2 | preserve_data = false 3 | process_message_inject_limit = 1 4 | message_matcher = "Type =~ '^hindsight%.plugin%.'" 5 | alert = { 6 | disabled = false, 7 | prefix = false, 8 | throttle = 1440, 9 | modules = { 10 | email = {recipients = {"trink@mozilla.com"}}, 11 | }, 12 | 13 | thresholds = { 14 | ["output.test_minimum"] = { 15 | process_message_failures = { 16 | percent = 5, 17 | minimum_sample = 200 18 | } 19 | }, 20 | ["output.test_success"] = { 21 | process_message_failures = { 22 | percent = 5, 23 | minimum_sample = 50 24 | } 25 | }, 26 | ["output.test_percent"] = { 27 | process_message_failures = { 28 | percent = 5, 29 | minimum_sample = 50 30 | } 31 | }, 32 | _default_ = { 33 | terminated = true 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/analysis/terminate.cfg: -------------------------------------------------------------------------------- 1 | filename = "terminate.lua" 2 | message_matcher = "FALSE" 3 | ticker_interval = 1 4 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/analysis/terminate.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | function process_message() 6 | return 0 7 | end 8 | 9 | local cnt = 0 10 | function timer_event() 11 | cnt = cnt + 1 12 | if cnt == 6 then error"boom" end 13 | end 14 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/input/stats_generate.cfg: -------------------------------------------------------------------------------- 1 | filename = "stats_generate.lua" 2 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/output/plugin_monitor_test.lua: -------------------------------------------------------------------------------- 1 | function process_message() 2 | local n = tonumber(read_message("Payload")) 3 | if n < 0 then 4 | return -1 5 | end 6 | return 0 7 | end 8 | 9 | function timer_event () 10 | end 11 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/output/test_alert.cfg: -------------------------------------------------------------------------------- 1 | filename = "test_alert.lua" 2 | message_matcher = "Type == 'alert'" 3 | ticker_interval = 1 4 | shutdown_on_terminate = true 5 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/output/test_minimum.cfg: -------------------------------------------------------------------------------- 1 | filename = "plugin_monitor_test.lua" 2 | message_matcher = "Type == 'plugin_monitor_test_minimum'" 3 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/output/test_percent.cfg: -------------------------------------------------------------------------------- 1 | filename = "plugin_monitor_test.lua" 2 | message_matcher = "Type == 'plugin_monitor_test_percent'" 3 | -------------------------------------------------------------------------------- /hindsight/tests/integration/plugin_monitor/run/output/test_success.cfg: -------------------------------------------------------------------------------- 1 | filename = "plugin_monitor_test.lua" 2 | message_matcher = "Type == 'plugin_monitor_test_success'" 3 | -------------------------------------------------------------------------------- /hindsight/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | rm -rf output 3 | mkdir -p load/input 4 | mkdir -p load/analysis 5 | mkdir -p load/output 6 | hindsight_cli ../hindsight.cfg 7 7 | rc=$?; if [[ $rc != 4 ]]; then exit $rc; fi 8 | -------------------------------------------------------------------------------- /hyperloglog/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(hyperloglog VERSION 1.0.1 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Lua hyperloglog module (distinct count)") 8 | set(MODULE_SRCS hyperloglog.c redis_hyperloglog.c hyperloglog.def) 9 | include(sandbox_module) 10 | -------------------------------------------------------------------------------- /hyperloglog/hyperloglog.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_hyperloglog 3 | -------------------------------------------------------------------------------- /hyperloglog/tests/test_sandbox.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "hyperloglog" 6 | 7 | hll = hyperloglog.new() 8 | 9 | function process(ts) 10 | hll:add(ts) 11 | return 0 12 | end 13 | 14 | function report(tc) 15 | if tc == 99 then 16 | hll:clear() 17 | else 18 | write_output(hll:count()) 19 | end 20 | end 21 | 22 | -------------------------------------------------------------------------------- /irc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(irc VERSION 0.0.1 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "IRC module") 8 | find_package(OpenSSL REQUIRED) 9 | set(MODULE_SRCS irc.c) 10 | include(sandbox_module) 11 | string(REPLACE "-std=c99" "-std=gnu99" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) 12 | target_link_libraries(irc ${OPENSSL_LIBRARIES}) 13 | -------------------------------------------------------------------------------- /jose/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.6) 6 | project(jose VERSION 0.0.1 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Lua JOSE module") 8 | 9 | find_package(PkgConfig REQUIRED) 10 | pkg_search_module(CJOSE REQUIRED cjose) 11 | 12 | set(MODULE_SRCS jose.c jose.def) 13 | set(CPACK_DEBIAN_PACKAGE_RELEASE 2) 14 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "libjansson4, libssl1.0.0 (>= 1.0.1)|libssl1.0.2 (>= 1.0.2)") 15 | find_package(ZLIB) 16 | if(ZLIB_FOUND) 17 | add_definitions(-DHAVE_ZLIB) 18 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, zlib1g (>= 1:1.1.4)") 19 | endif() 20 | include(sandbox_module) 21 | 22 | target_link_libraries(jose ${CJOSE_STATIC_LIBRARIES}) 23 | if(ZLIB_FOUND) 24 | target_link_libraries(jose ${ZLIB_LIBRARIES}) 25 | endif() 26 | -------------------------------------------------------------------------------- /jose/jose.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_jose 3 | -------------------------------------------------------------------------------- /kafka/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(kafka VERSION 1.0.7 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Lua Kafka producer/consumer module") 8 | 9 | # todo add a more robust kafka check 10 | find_library(LIBRDKAFKA_LIBRARY rdkafka REQUIRED) 11 | 12 | set(MODULE_SRCS kafka.c kafka.def) 13 | set(INSTALL_MODULE_PATH ${INSTALL_IOMODULE_PATH}) 14 | set(TEST_CONFIGURATION "kafka") 15 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "librdkafka-dev (>= 0.11), luasandbox-lpeg (>= 1.0.9)") 16 | set(CPACK_RPM_PACKAGE_REQUIRES "luasandbox-lpeg >= 1.0.9") 17 | include(sandbox_module) 18 | target_link_libraries(kafka ${LIBRDKAFKA_LIBRARY}) 19 | -------------------------------------------------------------------------------- /kafka/kafka.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_kafka 3 | -------------------------------------------------------------------------------- /lfs/index.md: -------------------------------------------------------------------------------- 1 | # Lua FileSystem Module 2 | 3 | https://github.com/keplerproject/luafilesystem 4 | -------------------------------------------------------------------------------- /lfs/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_load_path = "load" 3 | sandbox_run_path = "run" 4 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 5 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 6 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 7 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 8 | hostname = "integration_test" 9 | 10 | analysis_defaults = {} 11 | input_defaults = {} 12 | output_defaults = {} 13 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/compressed.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/lfs/tests/integration/input_tail/compressed.log.gz -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/multiline.log: -------------------------------------------------------------------------------- 1 | This is 2 | a test 3 | item 4 | long 5 | long 6 | line 7 | 8 | This is log line 2 9 | 10 | This is log 11 | line 12 | three 13 | 14 | 15 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/run/analysis/verify_messages.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | local test = require "test_verify_message" 7 | 8 | local messages = read_config("messages") 9 | local len = #messages 10 | local cnt = 0 11 | function process_message() 12 | cnt = cnt + 1 13 | local received = decode_message(read_message("raw")) 14 | local ok, err = pcall(test.verify_msg, messages[cnt], received, cnt) 15 | if not ok or cnt == len then 16 | inject_message({Type = "shutdown", Payload = received.Logger}) 17 | if not ok then error(err) end 18 | end 19 | return 0 20 | end 21 | 22 | function timer_event(ns) 23 | assert(cnt == len, string.format("%d of %d tests ran", cnt, len)) 24 | end 25 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/run/analysis/verify_multiline.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_messages.lua" 2 | message_matcher = "Logger == 'input.multiline'" 3 | process_message_inject_limit = 1 4 | 5 | messages = { 6 | { 7 | Logger = "input.multiline", 8 | Hostname = "integration_test", 9 | Payload = "This is\na test\nitem\nlong\nlong\nline\n" 10 | }, 11 | { 12 | Logger = "input.multiline", 13 | Hostname = "integration_test", 14 | Payload = "This is log line 2\n" 15 | }, 16 | { 17 | Logger = "input.multiline", 18 | Hostname = "integration_test", 19 | Payload = "This is log\nline\nthree\n\n" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/run/analysis/verify_rotate.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_messages.lua" 2 | message_matcher = "Logger == 'input.rotate'" 3 | process_message_inject_limit = 1 4 | 5 | messages = { 6 | { 7 | Logger = "input.rotate", 8 | Hostname = "integration_test", 9 | Payload = "log 1 line one" 10 | }, 11 | { 12 | Logger = "input.rotate", 13 | Hostname = "integration_test", 14 | Payload = "log 1 start of line two end of line two" 15 | }, 16 | { 17 | Logger = "input.rotate", 18 | Hostname = "integration_test", 19 | Payload = "log 2 line one beyond the checkpoint of the initial file" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/run/analysis/verify_singleline.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_messages.lua" 2 | message_matcher = "Logger == 'input.singleline'" 3 | process_message_inject_limit = 1 4 | 5 | messages = { 6 | { 7 | Logger = "input.singleline", 8 | Hostname = "integration_test", 9 | Payload = "line 1" 10 | }, 11 | { 12 | Logger = "input.singleline", 13 | Hostname = "integration_test", 14 | Payload = "line 2" 15 | }, 16 | { 17 | Logger = "input.singleline", 18 | Hostname = "integration_test", 19 | Payload = "line 3" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/run/input/generate.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate.lua" 2 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/run/output/shutdown.cfg: -------------------------------------------------------------------------------- 1 | filename = "shutdown.lua" 2 | message_matcher = "Type == 'shutdown'" 3 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/run/output/shutdown.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "io" 6 | require "string" 7 | local load_path = read_config("sandbox_load_path") 8 | 9 | function process_message() 10 | local fn = string.format("%s/%s/%s.off", load_path, 11 | string.match(read_message("Payload"), "([^.]+)%.(.*)")) 12 | local fh = assert(io.open(fn, "w+")) 13 | fh:close() 14 | return 0 15 | end 16 | 17 | function timer_event() 18 | end 19 | -------------------------------------------------------------------------------- /lfs/tests/integration/input_tail/singleline.log: -------------------------------------------------------------------------------- 1 | line 1 2 | line 2 3 | line 3 4 | -------------------------------------------------------------------------------- /lfs/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | rm -rf output 3 | mkdir -p load/input 4 | mkdir -p load/analysis 5 | mkdir -p load/output 6 | hindsight_cli ../hindsight.cfg 7 7 | rc=$?; if [[ $rc != 2 ]]; then exit $rc; fi 8 | -------------------------------------------------------------------------------- /libinjection/index.md: -------------------------------------------------------------------------------- 1 | # Lua libinjection Module 2 | 3 | https://github.com/client9/libinjection 4 | -------------------------------------------------------------------------------- /libinjection/tests/test.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | 7 | local libi = require "libinjection" 8 | 9 | local sqli = "1 UNION SELECT * FROM TABLE" 10 | 11 | state = libi.sqli_state() 12 | assert(state, "sqli_state returned nil") 13 | libi.sqli_init(state, sqli, string.len(sqli), 0) 14 | local v = libi.is_sqli(state) 15 | assert(v == 1, "is_sqli did not return 1") 16 | assert(state.fingerprint == "1UEok", string.format("incorrect fingerprint: %s", state.fingerprint)) 17 | 18 | local x1 = "');}" 19 | v = libi.xss(x1, string.len(x1)) 20 | assert(v == 1, "xss did not return 1") 21 | -------------------------------------------------------------------------------- /lpeg/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(lpeg LANGUAGES C) 7 | 8 | externalproject_add( 9 | ep_lpeg 10 | URL http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.2.tar.gz 11 | URL_MD5 d342571886f1abcb7afe6a83d024d583 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} -DPARENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_lpeg copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | include(sandbox_ep_test) 19 | -------------------------------------------------------------------------------- /lpeg/files.txt: -------------------------------------------------------------------------------- 1 | printf.c 2 | -------------------------------------------------------------------------------- /lpeg/index.md: -------------------------------------------------------------------------------- 1 | # Lua LPeg Module 2 | 3 | http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html 4 | -------------------------------------------------------------------------------- /lpeg/tests/phabricator.lua: -------------------------------------------------------------------------------- 1 | 2 | -- This Source Code Form is subject to the terms of the Mozilla Public 3 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 4 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | require "lpeg" 7 | require "string" 8 | 9 | local p = require "lpeg.phabricator" 10 | 11 | local asample = "[Wed, 08 Aug 2018 17:37:58 +0000] 106002 web.host 127.0.0.1 username PhabricatorConduitAPIController bugzilla.account.search /api/bugzilla.account.search - 200 57789" 12 | local asamplets = 1533749878 * 1e9 13 | 14 | function access() 15 | local v = p.access:match(asample) 16 | if not v then error(asample) end 17 | assert(v.timestamp == asamplets, string.format("%d", v.timestamp)) 18 | assert(v.ip == "127.0.0.1", v.ip) 19 | end 20 | 21 | access() 22 | -------------------------------------------------------------------------------- /lpeg/tests/printf_m1.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Module to test the printf.load_messages 7 | 8 | --]] 9 | 10 | -- Imports 11 | local M = {} 12 | setfenv(1, M) -- Remove external access to contain everything in the module 13 | 14 | printf_messages = { 15 | {"%d %d %d", "one", "two", "three"}, 16 | "printf_m2" 17 | } 18 | 19 | return M 20 | -------------------------------------------------------------------------------- /lpeg/tests/printf_m2.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Module to test the printf.load_messages 7 | 8 | --]] 9 | 10 | -- Imports 11 | local M = {} 12 | setfenv(1, M) -- Remove external access to contain everything in the module 13 | 14 | printf_messages = { 15 | {"%s %d %d %d", "status", "one", "two", "three"}, 16 | } 17 | 18 | return M 19 | -------------------------------------------------------------------------------- /lpeg/tests/printf_m3.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Module to test the printf.load_messages 7 | 8 | --]] 9 | 10 | -- Imports 11 | local M = {} 12 | setfenv(1, M) -- Remove external access to contain everything in the module 13 | 14 | printf_messages = { 15 | {"%s %d %d %d", "status", "one", "two", "three"}, 16 | {"%s", "@{:foo"}, 17 | } 18 | 19 | return M 20 | -------------------------------------------------------------------------------- /lpeg/tests/printf_m4.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Module to test the printf.load_messages 7 | 8 | --]] 9 | 10 | -- Imports 11 | local M = {} 12 | setfenv(1, M) -- Remove external access to contain everything in the module 13 | 14 | printf_messages = { 15 | {false, "status", "one", "two", "three"}, 16 | } 17 | 18 | return M 19 | -------------------------------------------------------------------------------- /lpeg/tests/test.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local files = { 6 | "common_log_format.lua", 7 | "date_time.lua", 8 | "escape_sequences.lua", 9 | "ip_address.lua", 10 | "logfmt.lua", 11 | "lpeg_heka.lua", 12 | "mysql.lua", 13 | "phabricator.lua", 14 | "postfix.lua", 15 | "printf.lua", 16 | "uri.lua" 17 | } 18 | 19 | for i,v in ipairs(files) do 20 | local f = assert(loadfile(v)) 21 | f() 22 | end 23 | -------------------------------------------------------------------------------- /lsb/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(lsb VERSION 1.1.1 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Utility modules for lua_sandbox") 8 | include(sandbox_module) 9 | -------------------------------------------------------------------------------- /lsb/tests/test.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local files = { 6 | "util.lua", 7 | "stats.lua", 8 | } 9 | 10 | for i,v in ipairs(files) do 11 | local f, err = loadfile(v) 12 | if not f then error(err) end 13 | f() 14 | end 15 | -------------------------------------------------------------------------------- /lyaml/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(lyaml LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_lyaml 10 | GIT_REPOSITORY https://github.com/gvvaughan/lyaml.git 11 | GIT_TAG 3164373b335af5e3a9583e9ba9e639899956bece # 6.2.5 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} -DPARENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_lyaml copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/tests ${CMAKE_CURRENT_BINARY_DIR} 18 | DEPENDEES install) 19 | -------------------------------------------------------------------------------- /lyaml/index.md: -------------------------------------------------------------------------------- 1 | # Lua YAML Module (lyaml) 2 | 3 | https://github.com/gvvaughan/lyaml 4 | -------------------------------------------------------------------------------- /maxminddb/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(maxminddb LANGUAGES C) 7 | 8 | externalproject_add( 9 | ep_maxminddb 10 | GIT_REPOSITORY https://github.com/fabled/lua-maxminddb.git 11 | GIT_TAG d81e9fc562af7779aa5e422dd2e302bd2c41dedd 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} -DPARENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | 16 | externalproject_add_step(ep_maxminddb copy_cpack 17 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 18 | DEPENDEES install) 19 | include(sandbox_ep_test) 20 | -------------------------------------------------------------------------------- /maxminddb/index.md: -------------------------------------------------------------------------------- 1 | # Lua MaxMindDB Module 2 | 3 | https://github.com/fabled/lua-maxminddb 4 | -------------------------------------------------------------------------------- /maxminddb/tests/GeoIP2-City-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/maxminddb/tests/GeoIP2-City-Test.mmdb -------------------------------------------------------------------------------- /maxminddb/tests/GeoIP2-ISP-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/maxminddb/tests/GeoIP2-ISP-Test.mmdb -------------------------------------------------------------------------------- /moz_ingest/tests/cities1000-test.txt: -------------------------------------------------------------------------------- 1 | 5803556 Milton Milton Mil'ton,Мильтон 47.24816 -122.3129 P PPL US WA 053 7670 93 93 America/Los_Angeles 2017-03-09 2 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_common/input.hpb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_ingest/tests/integration/decoder_common/input.hpb -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_common/run/analysis/verify_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder.lua" 2 | message_matcher = "Logger == 'moz_ingest' || Logger == 'common'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_common/run/input/generate_data.off: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_common/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_ingest/tests/integration/decoder_common/run/output/placeholder.txt -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_json/input.hpb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_ingest/tests/integration/decoder_json/input.hpb -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_json/namespaces/foo/bar/bar.1.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "properties": { 4 | "exampleString": { 5 | "description": "An example string.", 6 | "maxLength": 1024, 7 | "minLength": 1, 8 | "type": "string" 9 | } 10 | }, 11 | "required": [ 12 | "exampleString" 13 | ], 14 | "title": "foo", 15 | "type": "object" 16 | } 17 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_json/run/analysis/verify_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder.lua" 2 | message_matcher = "Logger != 'moz_ingest' && Logger != 'common'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_json/run/input/generate_data.off: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_json/run/input/test_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "test_decoder.lua" 2 | 3 | decoders_moz_ingest_common = { 4 | sub_decoders = { 5 | ["*"] = "decoders.moz_ingest.json" 6 | }, 7 | error_on_missing_sub_decoder = true, 8 | 9 | -- String used to specify GeoIP city database location on disk. 10 | city_db_file = "../../../../maxminddb/tests/GeoIP2-City-Test.mmdb", -- optional, if not specified no geoip lookup is performed 11 | 12 | -- number of items in the de-duping cuckoo filter 13 | cf_items = 1000, -- optional, if not provided de-duping is disabled 14 | 15 | -- interval size in minutes for cuckoo filter pruning 16 | cf_interval_size = 1 17 | } 18 | 19 | decoders_moz_ingest_json = { 20 | namespace_path = "namespaces", 21 | user_agent_transform = true, 22 | } 23 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/decoder_json/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_ingest/tests/integration/decoder_json/run/output/placeholder.txt -------------------------------------------------------------------------------- /moz_ingest/tests/integration/doctype_monitor/run/analysis/doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = 'moz_ingest_doctype_monitor.lua' 2 | docType = "main" 3 | message_matcher = "Fields[docType] == '" .. docType .. "' && Logger == 'telemetry'" 4 | ticker_interval = 60 5 | preserve_data = true 6 | output_limit = 0 7 | telemetry = true 8 | hierarchy = { 9 | "Fields[normalizedChannel]", 10 | } 11 | 12 | 13 | alert = { 14 | disabled = false, 15 | prefix = false, 16 | throttle = 0, 17 | modules = { 18 | email = {recipients = {"trink@mozilla.com"}}, 19 | }, 20 | 21 | thresholds = { 22 | release = { 23 | inactivity = 0, 24 | }, 25 | beta = { 26 | duplicates = 1, 27 | }, 28 | nightly = { 29 | ingestion_error = 1, 30 | }, 31 | Other = { 32 | capture_samples = 1, 33 | }, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/doctype_monitor/run/input/doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "doctype_monitor.lua" 2 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/doctype_monitor/run/output/dashboard.cfg: -------------------------------------------------------------------------------- 1 | filename = "heka_inject_payload.lua" 2 | message_matcher = "Type == 'inject_payload'" 3 | ticker_interval = 0 4 | read_queue = "analysis" 5 | 6 | output_dir = "dashboard" 7 | 8 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/doctype_monitor/run/output/doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "doctype_monitor.lua" 2 | message_matcher = "Type == 'alert'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_run_path = "run" 3 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 4 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 5 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 6 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 7 | hostname = "integration_test" 8 | 9 | analysis_defaults = {} 10 | input_defaults = {} 11 | output_defaults = {} 12 | -------------------------------------------------------------------------------- /moz_ingest/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /moz_ingest/tests/test.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | return 0 6 | -------------------------------------------------------------------------------- /moz_logging/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(moz-logging VERSION 0.0.7 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Infrastructure Logging Module") 8 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-lpeg (>= 1.0.8)") 9 | string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS}) 10 | include(sandbox_module) 11 | 12 | set(integration_tests 13 | decoder_json_heka 14 | decoder_json_heka_fields 15 | decoder_line_splitter 16 | ) 17 | 18 | foreach(test IN LISTS integration_tests) 19 | add_test(NAME ${PROJECT_NAME}_${test} 20 | COMMAND ../run.sh 21 | CONFIGURATIONS integration 22 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/integration/${test}) 23 | endforeach() 24 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_json_heka/run/analysis/verify_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder.lua" 2 | message_matcher = "TRUE" 3 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_json_heka/run/input/test_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "test_decoder.lua" 2 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_json_heka/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_logging/tests/integration/decoder_json_heka/run/output/placeholder.txt -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_json_heka_fields/run/analysis/verify_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder.lua" 2 | message_matcher = "TRUE" 3 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_json_heka_fields/run/input/test_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "test_decoder.lua" 2 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_json_heka_fields/run/input/test_decoder.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local tests = { 6 | {[[ 7 | {"Logger":"input1", "Items":[{}]} 8 | {"Logger":"input2", "Type":"type2"} 9 | {{invalid} 10 | {"foo":"bar", "Timestamp":123456789, "bstring":{"value":"binary", "value_type":1}} 11 | {"array":{"value":[1,2,3], "value_type":2, "representation":"count"}}]] 12 | , {Type = "default"}} 13 | } 14 | 15 | local dm = require("decoders.moz_logging.json_heka_fields").decode 16 | 17 | function process_message() 18 | for i,v in ipairs(tests) do 19 | dm(unpack(v)) 20 | end 21 | return 0 22 | end 23 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_json_heka_fields/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_logging/tests/integration/decoder_json_heka_fields/run/output/placeholder.txt -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_line_splitter/run/analysis/verify_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder.lua" 2 | message_matcher = "TRUE" 3 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_line_splitter/run/input/test_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "test_decoder.lua" 2 | 3 | decoders_moz_logging_line_splitter = { 4 | sub_decoder = "decoders.payload" 5 | } 6 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_line_splitter/run/input/test_decoder.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local tests = { 6 | {[[ 7 | line one 8 | line two 9 | line three]], {Type = "default"}} 10 | } 11 | 12 | local dm = require("decoders.moz_logging.line_splitter").decode 13 | 14 | function process_message() 15 | for i,v in ipairs(tests) do 16 | dm(unpack(v)) 17 | end 18 | return 0 19 | end 20 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/decoder_line_splitter/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_logging/tests/integration/decoder_line_splitter/run/output/placeholder.txt -------------------------------------------------------------------------------- /moz_logging/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_run_path = "run" 3 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 4 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 5 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 6 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 7 | hostname = "integration_test" 8 | 9 | analysis_defaults = {} 10 | input_defaults = {} 11 | output_defaults = {} 12 | -------------------------------------------------------------------------------- /moz_logging/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /moz_pioneer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(moz-pioneer VERSION 0.0.6 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Firefox Pioneer Data Processing") 8 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-moz-ingest (>= 0.0.11), ${PACKAGE_PREFIX}-moz-telemetry (>= 1.2.13), ${PACKAGE_PREFIX}-jose (>= 0.0.1), ${PACKAGE_PREFIX}-lpeg (>= 1.0.0), ${PACKAGE_PREFIX}-rjson (>= 1.1.4)") 9 | string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS}) 10 | include(sandbox_module) 11 | 12 | add_test(NAME ${PROJECT_NAME}_decoder_moz_ingest_pioneer 13 | COMMAND ../run.sh 14 | CONFIGURATIONS integration 15 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/integration/decoder_moz_ingest_pioneer) 16 | -------------------------------------------------------------------------------- /moz_pioneer/tests/integration/decoder_moz_ingest_pioneer/input.hpb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_pioneer/tests/integration/decoder_moz_ingest_pioneer/input.hpb -------------------------------------------------------------------------------- /moz_pioneer/tests/integration/decoder_moz_ingest_pioneer/run/analysis/verify_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder.lua" 2 | message_matcher = "Logger == 'telemetry'" 3 | -------------------------------------------------------------------------------- /moz_pioneer/tests/integration/decoder_moz_ingest_pioneer/run/input/generate_data.off: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /moz_pioneer/tests/integration/decoder_moz_ingest_pioneer/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_pioneer/tests/integration/decoder_moz_ingest_pioneer/run/output/placeholder.txt -------------------------------------------------------------------------------- /moz_pioneer/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_run_path = "run" 3 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 4 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 5 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 6 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 7 | hostname = "integration_test" 8 | 9 | analysis_defaults = {} 10 | input_defaults = {} 11 | output_defaults = {} 12 | -------------------------------------------------------------------------------- /moz_pioneer/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /moz_security/index.md: -------------------------------------------------------------------------------- 1 | # Mozilla Security Sandboxes and Modules 2 | 3 | ## Overview 4 | 5 | Contains various Mozilla Security sandboxes and modules for use with Hindsight 6 | and Lua Sandbox. 7 | 8 | ## Sandboxes 9 | 10 | Various sandboxes are available under [sandboxes](sandboxes/). 11 | 12 | ## Modules 13 | 14 | * hawk 15 | * iprepd 16 | * iputils 17 | * secmetrics 18 | -------------------------------------------------------------------------------- /moz_security/tests/hindsight/iprepd_submit/run/analysis/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_security/tests/hindsight/iprepd_submit/run/analysis/placeholder.txt -------------------------------------------------------------------------------- /moz_security/tests/hindsight/iprepd_submit/run/input/generate_data.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /moz_security/tests/hindsight/iprepd_submit/run/input/generate_data.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | 7 | local r = require "heka.iprepd" 8 | 9 | function process_message(cp) 10 | local iptab = {} 11 | for i=1,255,1 do 12 | iptab[#iptab+1] = {ip = string.format("192.168.0.%d", i), violation = "fxa:heavy_hitter_ip"} 13 | iptab[#iptab+1] = {ip = string.format("192.168.1.%d", i), violation = "fxa:heavy_hitter_ip"} 14 | iptab[#iptab+1] = {ip = string.format("192.168.2.%d", i), violation = "fxa:heavy_hitter_ip"} 15 | iptab[#iptab+1] = {ip = string.format("192.168.3.%d", i), violation = "fxa:heavy_hitter_ip"} 16 | end 17 | r.send(iptab) 18 | return 0 19 | end 20 | -------------------------------------------------------------------------------- /moz_security/tests/hindsight/iprepd_submit/run/output/iprepd.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_iprepd.lua" 2 | message_matcher = "Type == 'iprepd'" 3 | 4 | iprepd = { 5 | base_url = "http://localhost:8080/", 6 | id = "root", 7 | _key = "toor" 8 | } 9 | -------------------------------------------------------------------------------- /moz_security/tests/hindsight/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /moz_security/tests/hindsight/secmetrics/run/analysis/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_security/tests/hindsight/secmetrics/run/analysis/placeholder.txt -------------------------------------------------------------------------------- /moz_security/tests/hindsight/secmetrics/run/input/generate_data.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /moz_security/tests/hindsight/secmetrics/run/output/secmetrics.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_metrics.lua" 2 | message_matcher = "Type == 'secmetrics'" 3 | debug = true 4 | preserve_data = true 5 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_any/run/input/generate_auth.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_auth.lua" 2 | 3 | maxminddb_heka = { 4 | databases = { 5 | ["../../GeoIP2-City-Test.mmdb"] = { 6 | _city = {"city", "names", "en"}, 7 | _country = {"country", "iso_code"} 8 | } 9 | } 10 | } 11 | 12 | decoders_syslog = { 13 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 14 | printf_messages = { "lpeg.openssh_portable" }, 15 | sub_decoders = { 16 | sshd = { 17 | { "Accepted publickey for foobar from 10.11.12.13 port 4242 ssh2", 18 | { ssh_remote_ipaddr = "maxminddb.heka#add_geoip" } } 19 | }, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_any/run/output/auth_any_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "auth_any_verification.lua" 2 | message_matcher = "Logger =~ 'analysis.auth_any*' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_ip_geo/run/analysis/auth_ip_geo_1.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_auth_ip_geo.lua" 2 | message_matcher = "Logger == 'generate_auth_1' && Fields[programname] == 'sshd' && Fields[authmsg] == 'Accepted'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 1 5 | 6 | default_email = "foxsec-dump+OutOfHours@mozilla.com" 7 | user_email = "manatee-%s@moz-svc-ops.pagerduty.com" 8 | 9 | authhost_field = "Hostname" 10 | user_field = "Fields[user]" 11 | srcip_field = "Fields[ssh_remote_ipaddr]" 12 | geocity_field = "Fields[ssh_remote_ipaddr_city]" 13 | geocountry_field = "Fields[ssh_remote_ipaddr_country]" 14 | 15 | userspec = { 16 | q = { 17 | ip = { "127.0.0.1" } 18 | }, 19 | riker = { 20 | ip = { "192.168.1.0/24" }, 21 | geo = { "Toronto/CA" } 22 | }, 23 | worf = { 24 | geo = { "Milton/US" } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_ip_geo/run/analysis/auth_ip_geo_2.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_auth_ip_geo.lua" 2 | message_matcher = "Logger == 'generate_auth_2' && Fields[programname] == 'sshd' && Fields[authmsg] == 'Accepted'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 1 5 | 6 | default_email = "foxsec-dump+OutOfHours@mozilla.com" 7 | user_email = "manatee-%s@moz-svc-ops.pagerduty.com" 8 | drift_email = "captainkirk@mozilla.com" 9 | 10 | authhost_field = "Hostname" 11 | user_field = "Fields[user]" 12 | srcip_field = "Fields[ssh_remote_ipaddr]" 13 | geocity_field = "Fields[ssh_remote_ipaddr_city]" 14 | geocountry_field = "Fields[ssh_remote_ipaddr_country]" 15 | 16 | userspec = { 17 | q = { 18 | ip = { "127.0.0.1" } 19 | }, 20 | riker = { 21 | ip = { "192.168.1.0/24" }, 22 | geo = { "Toronto/CA" } 23 | }, 24 | worf = { 25 | geo = { "Milton/US" } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_ip_geo/run/analysis/auth_ip_geo_3.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_auth_ip_geo.lua" 2 | message_matcher = "Logger == 'generate_auth_3' && Fields[programname] == 'sshd' && Fields[authmsg] == 'Accepted'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 1 5 | 6 | default_email = "foxsec-dump+OutOfHours@mozilla.com" 7 | user_email = "manatee-%s@moz-svc-ops.pagerduty.com" 8 | 9 | authhost_field = "Hostname" 10 | user_field = "Fields[user]" 11 | srcip_field = "Fields[ssh_remote_ipaddr]" 12 | geocity_field = "Fields[ssh_remote_ipaddr_city]" 13 | geocountry_field = "Fields[ssh_remote_ipaddr_country]" 14 | 15 | userspec = { 16 | q = { 17 | ip = { "127.0.0.1" } 18 | }, 19 | riker = { 20 | ip = { "192.168.1.0/24" }, 21 | geo = { "Toronto/CA" } 22 | }, 23 | authipgeoany = { 24 | ip = { "10.0.0.1/32" }, 25 | geo = { "Milton/US" } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_ip_geo/run/input/generate_auth.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_auth.lua" 2 | 3 | maxminddb_heka = { 4 | databases = { 5 | ["../../GeoIP2-City-Test.mmdb"] = { 6 | _city = {"city", "names", "en"}, 7 | _country = {"country", "iso_code"} 8 | } 9 | } 10 | } 11 | 12 | decoders_syslog = { 13 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 14 | printf_messages = { "lpeg.openssh_portable" }, 15 | sub_decoders = { 16 | sshd = { 17 | { "Accepted publickey for foobar from 10.11.12.13 port 4242 ssh2", 18 | { ssh_remote_ipaddr = "maxminddb.heka#add_geoip" } } 19 | }, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_ip_geo/run/output/auth_ip_geo_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "auth_ip_geo_verification.lua" 2 | message_matcher = "Logger =~ 'analysis.auth_ip_geo*'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_lastx/run/input/generate_auth.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_auth.lua" 2 | 3 | maxminddb_heka = { 4 | databases = { 5 | ["../../GeoIP2-City-Test.mmdb"] = { 6 | _city = {"city", "names", "en"}, 7 | _country = {"country", "iso_code"} 8 | } 9 | } 10 | } 11 | 12 | decoders_syslog = { 13 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 14 | printf_messages = { "lpeg.openssh_portable" }, 15 | sub_decoders = { 16 | sshd = { 17 | { "Accepted publickey for foobar from 10.11.12.13 port 4242 ssh2", 18 | { ssh_remote_ipaddr = "maxminddb.heka#add_geoip" } } 19 | }, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_lastx/run/input/generate_auth_duo.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_auth_duo.lua" 2 | send_decode_failures = true 3 | 4 | maxminddb_heka = { 5 | databases = { 6 | ["../../GeoIP2-City-Test.mmdb"] = { 7 | _city = {"city", "names", "en"}, 8 | _country = {"country", "iso_code"} 9 | } 10 | } 11 | } 12 | 13 | decoders_syslog = { 14 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 15 | printf_messages = { "lpeg.openssh_portable" }, 16 | sub_decoders = { 17 | sshd = { 18 | { "Accepted publickey for foobar from 10.11.12.13 port 4242 ssh2", 19 | { ssh_remote_ipaddr = "maxminddb.heka#add_geoip" } } 20 | }, 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_lastx/run/input/generate_auth_multi.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_auth_multi.lua" 2 | 3 | maxminddb_heka = { 4 | databases = { 5 | ["../../GeoIP2-City-Test.mmdb"] = { 6 | _city = {"city", "names", "en"}, 7 | _country = {"country", "iso_code"} 8 | } 9 | } 10 | } 11 | 12 | decoders_syslog = { 13 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 14 | printf_messages = { "lpeg.openssh_portable" }, 15 | sub_decoders = { 16 | sshd = { 17 | { "Accepted publickey for foobar from 10.11.12.13 port 4242 ssh2", 18 | { ssh_remote_ipaddr = "maxminddb.heka#add_geoip" } } 19 | }, 20 | } 21 | } 22 | 23 | decoders_heka_table_to_fields = { 24 | module_name = "cjson", 25 | module_function = "decode", 26 | } 27 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_auth_lastx/run/output/auth_lastx_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "auth_lastx_verification.lua" 2 | message_matcher = "Logger =~ 'analysis.auth_lastx*' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_cloudtrail/run/input/generate_data.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "run/input/cloudtrail.log" 3 | decoder_module = "decoders.heka.table_to_fields" 4 | decoders_heka_table_to_fields = { 5 | module_name = "cjson", 6 | module_function = "decode", 7 | max_depth = 2 8 | } 9 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_cloudtrail/run/output/verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "verification.lua" 2 | message_matcher = "Logger =~ 'analysis.cloudtrail' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_duo/run/analysis/duo.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_duo.lua" 2 | message_matcher = "Logger == 'input.generate_data|duopull'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 2 5 | enable_metrics = true 6 | 7 | heka_secmetrics = { 8 | identifier = "duo" 9 | } 10 | 11 | bypass_create = true 12 | user_create = true 13 | auth_phone_fail = true 14 | auth_fraud = true 15 | admin_2fa_error = true 16 | integration_addup = true 17 | admin_addup = true 18 | anomalous_push = true 19 | 20 | alert = { 21 | modules = { 22 | email = { recipients = { "riker@mozilla.com" } } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_duo/run/input/generate_data.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "run/input/duo.log" 3 | decoder_module = "decoders.moz_logging.json_heka" 4 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_duo/run/output/verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "verification.lua" 2 | message_matcher = "Logger =~ 'analysis.duo' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/analysis/err.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_http_error_rate.lua" 2 | message_matcher = "Logger == 'input.err'" 3 | ticker_interval = 0 4 | enable_metrics = true 5 | 6 | heka_secmetrics = { 7 | identifier = "err" 8 | } 9 | 10 | id_field = "Fields[id]" 11 | code_field = "Fields[code]" 12 | error_threshold = 50 13 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/analysis/err_hawk.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_http_error_rate.lua" 2 | message_matcher = "Logger == 'input.err_hawk'" 3 | ticker_interval = 0 4 | enable_metrics = true 5 | 6 | heka_secmetrics = { 7 | identifier = "err_hawk" 8 | } 9 | 10 | id_field = "Fields[id]" 11 | code_field = "Fields[code]" 12 | error_threshold = 50 13 | 14 | send_iprepd = true 15 | violation_type = "fxa:client_error_rate" 16 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/input/generate_err.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_err.lua" 2 | logger = "input.err" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/input/generate_err_hawk.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_err.lua" 2 | logger = "input.err_hawk" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/output/err_hawk_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "err_hawk_verification.lua" 2 | message_matcher = "Logger == 'analysis.err_hawk' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/output/err_hawk_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Validate output of moz_security_http_error_rate, iprepd output 7 | --]] 8 | 9 | require "string" 10 | 11 | local violations = "[{\"violation\":\"fxa:client_error_rate\",\"ip\":\"10.0.2.10\"}," .. 12 | "{\"violation\":\"fxa:client_error_rate\",\"ip\":\"10.0.2.12\"}]" 13 | 14 | local cnt = 0 15 | 16 | function process_message() 17 | local v = read_message("Fields[violations]") or error("message missing violations field") 18 | if v ~= violations then 19 | error("message violation value did not match") 20 | end 21 | cnt = 1 22 | return 0 23 | end 24 | 25 | function timer_event() 26 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 27 | end 28 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/output/err_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "err_verification.lua" 2 | message_matcher = "Logger == 'analysis.err' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_error_rate/run/output/err_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Validate output of moz_security_http_error_rate 7 | --]] 8 | 9 | require "string" 10 | 11 | local result = [[error_threshold 50 12 | cms_size 3 13 | 10.0.2.10 75 14 | 10.0.2.12 500 15 | ]] 16 | 17 | local cnt = 0 18 | 19 | function process_message() 20 | local payload = read_message("Payload") 21 | assert(result == payload, payload) 22 | cnt = 1 23 | return 0 24 | end 25 | 26 | function timer_event() 27 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 28 | end 29 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/analysis/hh.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_hh.lua" 2 | message_matcher = "Logger == 'input.hh'" 3 | ticker_interval = 0 4 | enable_metrics = true 5 | 6 | heka_secmetrics = { 7 | identifier = "hh" 8 | } 9 | 10 | id_field = "Fields[id]" 11 | list_max_size = 10000 12 | 13 | threshold_cap = 2 14 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/analysis/hh_hawk.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_hh.lua" 2 | message_matcher = "Logger == 'input.hh_hawk'" 3 | ticker_interval = 0 4 | enable_metrics = true 5 | 6 | heka_secmetrics = { 7 | identifier = "hhhawk" 8 | } 9 | 10 | id_field = "Fields[id]" 11 | list_max_size = 10000 12 | 13 | threshold_cap = 2 14 | 15 | send_iprepd = true 16 | violation_type = "fxa:heavy_hitter_ip" 17 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/analysis/hh_min.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_hh.lua" 2 | message_matcher = "Logger == 'input.hh'" 3 | ticker_interval = 0 4 | 5 | id_field = "Fields[id]" 6 | list_max_size = 10000 7 | 8 | threshold_cap = 2 9 | threshold_min = 500 10 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/input/generate_hh.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_hh.lua" 2 | logger = "input.hh" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/input/generate_hh_hawk.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_hh.lua" 2 | logger = "input.hh_hawk" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/input/generate_hh_min.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_hh.lua" 2 | logger = "input.hh_min" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/output/hh_hawk_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "hh_hawk_verification.lua" 2 | message_matcher = "Logger == 'analysis.hh_hawk' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/output/hh_min_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "hh_min_verification.lua" 2 | message_matcher = "Logger == 'analysis.hh_min'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/output/hh_min_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Validate output of sampled heavy hitter analysis 7 | --]] 8 | 9 | require "string" 10 | 11 | local result = [[threshold 332.70378875 12 | list_size 1742 13 | event_count 193190 14 | low_threshold true 15 | ]] 16 | 17 | local cnt = 0 18 | 19 | function process_message() 20 | local payload = read_message("Payload") 21 | assert(result == payload, payload) 22 | cnt = 1 23 | return 0 24 | end 25 | 26 | 27 | function timer_event() 28 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 29 | end 30 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_hh/run/output/hh_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "hh_verification.lua" 2 | message_matcher = "Logger == 'analysis.hh' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_iprepd_alerts/run/analysis/iprepd_alerts.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_iprepd_alerts.lua" 2 | message_matcher = "Logger == 'input.generate_data|iprepd'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 2 5 | 6 | enable_metrics = true 7 | 8 | prefix = "hhfxa" 9 | alert = { 10 | modules = { 11 | irc = { target = "irc.server#channel" } 12 | } 13 | } 14 | 15 | heka_secmetrics = { 16 | identifier = "iprepd_alerts" 17 | } 18 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_iprepd_alerts/run/input/generate_data.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_iprepd_alerts/run/output/verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "verification.lua" 2 | message_matcher = "Logger =~ 'analysis.iprepd_alerts' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/analysis/pathsetfreq.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_pathsetfreq.lua" 2 | message_matcher = "Logger == 'input.generate_pathsetfreq'" 3 | ticker_interval = 0 4 | 5 | id_field = "Fields[remote_addr]" 6 | request_field = "Fields[request]" 7 | request_field_capture = "%S+%s+(%S+)" 8 | set_threshold = 5 9 | acceptable_variance = 5 10 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/analysis/pathsetfreq_hawk.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_pathsetfreq.lua" 2 | message_matcher = "Logger == 'input.generate_pathsetfreq'" 3 | ticker_interval = 0 4 | 5 | id_field = "Fields[remote_addr]" 6 | request_field = "Fields[request]" 7 | request_field_capture = "%S+%s+(%S+)" 8 | set_threshold = 5 9 | acceptable_variance = 5 10 | 11 | send_iprepd = true 12 | violation_type = "fxa:path_set_frequency" 13 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/analysis/pathsetfreq_nosingle.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_pathsetfreq.lua" 2 | message_matcher = "Logger == 'input.generate_pathsetfreq'" 3 | ticker_interval = 0 4 | 5 | id_field = "Fields[remote_addr]" 6 | request_field = "Fields[request]" 7 | request_field_capture = "%S+%s+(%S+)" 8 | no_single_set = true 9 | set_threshold = 5 10 | acceptable_variance = 5 11 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/input/generate_pathsetfreq.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "run/input/access.log" 3 | 4 | log_format = '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"' 5 | 6 | decoder_module = { 7 | { 8 | {"lpeg.common_log_format#build_nginx_grammar", log_format}, 9 | { 10 | http_user_agent = "lpeg.heka#add_normalized_user_agent", 11 | } 12 | } 13 | } 14 | 15 | lpeg_heka = { 16 | user_agent_normalized_field_name = "user_agent", 17 | user_agent_remove = false, 18 | } 19 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/output/pathsetfreq_hawk_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "pathsetfreq_hawk_verification.lua" 2 | message_matcher = "Logger == 'analysis.pathsetfreq_hawk'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/output/pathsetfreq_hawk_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | 7 | local violations = '[{"violation":"fxa:path_set_frequency","ip":"10.0.0.15"},' .. 8 | '{"violation":"fxa:path_set_frequency","ip":"10.0.0.18"},' .. 9 | '{"violation":"fxa:path_set_frequency","ip":"10.0.0.20"}]' 10 | 11 | local cnt = 0 12 | 13 | function process_message() 14 | local v = read_message("Fields[violations]") or error("message missing violations field") 15 | if v ~= violations then 16 | error("message violation value did not match") 17 | end 18 | cnt = 1 19 | return 0 20 | end 21 | 22 | function timer_event() 23 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 24 | end 25 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/output/pathsetfreq_nosingle_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "pathsetfreq_nosingle_verification.lua" 2 | message_matcher = "Logger == 'analysis.pathsetfreq_nosingle'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/output/pathsetfreq_nosingle_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | 7 | local result = "10.0.0.15 15\n" .. 8 | "10.0.0.20 6\n" 9 | 10 | local cnt = 0 11 | 12 | function process_message() 13 | local payload = read_message("Payload") 14 | assert(result == payload, payload) 15 | cnt = 1 16 | return 0 17 | end 18 | 19 | function timer_event() 20 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 21 | end 22 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/output/pathsetfreq_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "pathsetfreq_verification.lua" 2 | message_matcher = "Logger == 'analysis.pathsetfreq'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_pathsetfreq/run/output/pathsetfreq_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | 7 | local result = "10.0.0.15 15\n" .. 8 | "10.0.0.18 10\n" .. 9 | "10.0.0.20 6\n" 10 | 11 | local cnt = 0 12 | 13 | function process_message() 14 | local payload = read_message("Payload") 15 | assert(result == payload, payload) 16 | cnt = 1 17 | return 0 18 | end 19 | 20 | function timer_event() 21 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 22 | end 23 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_simuse/run/analysis/simuse_2.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_simuse.lua" 2 | message_matcher = "Logger == 'test_2'" 3 | ticker_interval = 0 4 | preserve_data = true 5 | process_message_inject_limit = 1 6 | acceptable_message_drift = 0 7 | 8 | selprinc_track = { "geocountry" } 9 | 10 | heka_selprinc = { 11 | events = { 12 | ssh = { 13 | select_field = "Fields[programname]", 14 | select_match = "^sshd$", 15 | subject_field = "Fields[user]", 16 | object_field = "Hostname", 17 | sourceip_field = "Fields[ssh_remote_ipaddr]", 18 | 19 | aux = { 20 | { "geocity", "Fields[ssh_remote_ipaddr_city]" }, 21 | { "geocountry", "Fields[ssh_remote_ipaddr_country]" } 22 | } 23 | } 24 | } 25 | } 26 | 27 | alert = { 28 | modules = { 29 | email = { recipients = { "riker@mozilla.com" } } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_simuse/run/analysis/simuse_3.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_simuse.lua" 2 | message_matcher = "Logger == 'test_3'" 3 | ticker_interval = 0 4 | preserve_data = true 5 | process_message_inject_limit = 1 6 | 7 | selprinc_track = { "geocountry" } 8 | 9 | heka_selprinc = { 10 | events = { 11 | ssh = { 12 | select_field = "Fields[programname]", 13 | select_match = "^sshd$", 14 | subject_field = "Fields[user]", 15 | object_field = "Hostname", 16 | sourceip_field = "Fields[ssh_remote_ipaddr]", 17 | 18 | aux = { 19 | { "geocity", "Fields[ssh_remote_ipaddr_city]" }, 20 | { "geocountry", "Fields[ssh_remote_ipaddr_country]" } 21 | } 22 | } 23 | } 24 | } 25 | 26 | alert = { 27 | modules = { 28 | email = { recipients = { "riker@mozilla.com" } } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_simuse/run/input/generate_data.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | 3 | maxminddb_heka = { 4 | databases = { 5 | ["../../GeoIP2-City-Test.mmdb"] = { 6 | _city = {"city", "names", "en"}, 7 | _country = {"country", "iso_code"} 8 | } 9 | } 10 | } 11 | 12 | decoders_syslog = { 13 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 14 | printf_messages = { "lpeg.openssh_portable" }, 15 | sub_decoders = { 16 | sshd = { 17 | { "Accepted publickey for foobar from 10.11.12.13 port 4242 ssh2", 18 | { ssh_remote_ipaddr = "maxminddb.heka#add_geoip" } } 19 | }, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_simuse/run/output/verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "verification.lua" 2 | message_matcher = "Logger =~ 'analysis.simuse*' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_sshd_login_monitor/run/analysis/sshd_login_monitor_1.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_sshd_login_monitor.lua" 2 | message_matcher = "Logger == 'generate_sshd_1' && Fields[programname] == 'sshd' && Fields[authmsg] == 'Accepted'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 1 5 | 6 | default_email = "foxsec-dump+OutOfHours@mozilla.com" 7 | user_email = "manatee-%s@moz-svc-ops.pagerduty.com" 8 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_sshd_login_monitor/run/analysis/sshd_login_monitor_2.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_sshd_login_monitor.lua" 2 | message_matcher = "Logger == 'generate_sshd_2' && Fields[programname] == 'sshd' && Fields[authmsg] == 'Accepted'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 1 5 | 6 | default_email = "foxsec-dump+OutOfHours@mozilla.com" 7 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_sshd_login_monitor/run/analysis/sshd_login_monitor_3.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_sshd_login_monitor.lua" 2 | message_matcher = "Logger == 'generate_sshd_3' && Fields[programname] == 'sshd' && Fields[authmsg] == 'Accepted'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 1 5 | 6 | default_email = "foxsec-dump+OutOfHours@mozilla.com" 7 | user_email = "manatee-%s@moz-svc-ops.pagerduty.com" 8 | drift_email = "captainkirk@mozilla.com" 9 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_sshd_login_monitor/run/analysis/sshd_login_monitor_4.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_sshd_login_monitor.lua" 2 | message_matcher = "Logger == 'generate_sshd_4' && Fields[programname] == 'sshd' && Fields[authmsg] == 'Accepted'" 3 | ticker_interval = 0 4 | process_message_inject_limit = 1 5 | 6 | default_email = "foxsec-dump+OutOfHours@mozilla.com" 7 | user_email = "manatee-%s@moz-svc-ops.pagerduty.com" 8 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_sshd_login_monitor/run/input/generate_sshd.cfg: -------------------------------------------------------------------------------- 1 | filename = "generate_sshd.lua" 2 | 3 | maxminddb_heka = { 4 | databases = { 5 | ["../../GeoIP2-City-Test.mmdb"] = { 6 | _city = {"city", "names", "en"}, 7 | _country = {"country", "iso_code"}, 8 | }, 9 | }, 10 | } 11 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_sshd_login_monitor/run/output/sshd_login_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "sshd_login_verification.lua" 2 | message_matcher = "Logger =~ 'analysis.sshd_login_monitor*'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_webinj/run/analysis/webinj.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_webinj.lua" 2 | message_matcher = "Logger == 'input.generate_webinj'" 3 | ticker_interval = 0 4 | enable_metrics = true 5 | 6 | heka_secmetrics = { 7 | identifier = "webinj" 8 | } 9 | 10 | id_field = "Fields[remote_addr]" 11 | request_field = "Fields[request]" 12 | request_field_capture = "%S+%s+(%S+)" 13 | list_max_size = 1000 14 | strip_nul = true 15 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_webinj/run/analysis/webinj_hawk.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_security_webinj.lua" 2 | message_matcher = "Logger == 'input.generate_webinj'" 3 | ticker_interval = 0 4 | enable_metrics = true 5 | 6 | heka_secmetrics = { 7 | identifier = "webinj_hawk" 8 | } 9 | 10 | id_field = "Fields[remote_addr]" 11 | request_field = "Fields[request]" 12 | request_field_capture = "%S+%s+(%S+)" 13 | list_max_size = 1000 14 | strip_nul = true 15 | 16 | send_iprepd = true 17 | xss_violation_type = "fxa:webinj_xss" 18 | sqli_violation_type = "fxa:webinj_sqli" 19 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_webinj/run/input/generate_webinj.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "run/input/access.log" 3 | 4 | log_format = '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"' 5 | 6 | decoder_module = { 7 | { 8 | {"lpeg.common_log_format#build_nginx_grammar", log_format}, 9 | { 10 | http_user_agent = "lpeg.heka#add_normalized_user_agent", 11 | } 12 | } 13 | } 14 | 15 | lpeg_heka = { 16 | user_agent_normalized_field_name = "user_agent", 17 | user_agent_remove = false, 18 | } 19 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_webinj/run/output/webinj_hawk_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "webinj_hawk_verification.lua" 2 | message_matcher = "Logger == 'analysis.webinj_hawk' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_webinj/run/output/webinj_hawk_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Validate output of libinjection sandbox, iprepd output 7 | --]] 8 | 9 | require "string" 10 | 11 | local violations = '[{"violation":"fxa:webinj_xss","ip":"10.0.0.15"},' .. 12 | '{"violation":"fxa:webinj_sqli","ip":"10.0.0.15"}]' 13 | 14 | local cnt = 0 15 | 16 | function process_message() 17 | local v = read_message("Fields[violations]") or error("message missing violations field") 18 | if v ~= violations then 19 | error("message violation value did not match") 20 | end 21 | cnt = 1 22 | return 0 23 | end 24 | 25 | function timer_event() 26 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 27 | end 28 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_webinj/run/output/webinj_verification.cfg: -------------------------------------------------------------------------------- 1 | filename = "webinj_verification.lua" 2 | message_matcher = "Logger == 'analysis.webinj' && Type != 'secmetrics'" 3 | -------------------------------------------------------------------------------- /moz_security/tests/integration/analysis_webinj/run/output/webinj_verification.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Validate output of libinjection sandbox 7 | --]] 8 | 9 | require "string" 10 | 11 | local result = "10.0.0.15 69 " .. 12 | "GET /sitemap.xml?query=%27%22%3Cscript%3Ealert%281%29%3B%3C%2Fscript%3E HTTP/1.1 " .. 13 | "GET /sitemap.xml?query=query%22%26timeout+%2FT+15%26%22 HTTP/1.1\n" 14 | 15 | local cnt = 0 16 | 17 | function process_message() 18 | local payload = read_message("Payload") 19 | assert(result == payload, payload) 20 | cnt = 1 21 | return 0 22 | end 23 | 24 | function timer_event() 25 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 26 | end 27 | -------------------------------------------------------------------------------- /moz_security/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_size = 1024 * 1024 * 1024 2 | output_path = "output" 3 | sandbox_run_path = "run" 4 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 5 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so;/usr/lib64/luasandbox/modules/?.so;" 6 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 7 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 8 | hostname = "integration_test" 9 | max_message_size = 1024 * 1024 10 | 11 | analysis_defaults = { 12 | instruction_limit = 0 13 | } 14 | input_defaults = { 15 | instruction_limit = 0 16 | } 17 | output_defaults = {} 18 | -------------------------------------------------------------------------------- /moz_security/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/decoder_telemetry/input.hpb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_telemetry/tests/integration/decoder_telemetry/input.hpb -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/decoder_telemetry/run/analysis/verify_decoder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder.lua" 2 | message_matcher = "Logger == 'telemetry' && ( Fields[docType] == 'saved-session' || Fields[docType]=='core' )" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/decoder_telemetry/run/input/generate_data.off: -------------------------------------------------------------------------------- 1 | filename = "generate_data.lua" 2 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/doctype_monitor/run/input/doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "doctype_monitor.lua" 2 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/doctype_monitor/run/output/dashboard.cfg: -------------------------------------------------------------------------------- 1 | filename = "heka_inject_payload.lua" 2 | message_matcher = "Type == 'inject_payload'" 3 | ticker_interval = 0 4 | read_queue = "analysis" 5 | 6 | output_dir = "dashboard" 7 | 8 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/doctype_monitor/run/output/doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "doctype_monitor.lua" 2 | message_matcher = "Logger == 'analysis.doctype_monitor' && Type == 'alert'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_size = 1024 * 1024 * 1024 2 | output_path = "output" 3 | sandbox_run_path = "run" 4 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 5 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 6 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 7 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 8 | hostname = "integration_test" 9 | max_message_size = 1024 * 1024 10 | 11 | analysis_defaults = {} 12 | input_defaults = {} 13 | output_defaults = {} 14 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/moz_telemetry_s3/moz_telemetry_s3.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 2, 3 | "dimensions": [ 4 | { 5 | "field_name": "docType", 6 | "allowed_values": "*" 7 | } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/moz_telemetry_s3/moz_telemetry_s3_experiments.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 2, 3 | "dimensions": [ 4 | { 5 | "field_name": "docType", 6 | "allowed_values": "*" 7 | }, 8 | { 9 | "field_name": "experimentId", 10 | "allowed_values": "*", 11 | "is_variable": true 12 | }, 13 | { 14 | "field_name": "experimentBranch", 15 | "allowed_values": "*", 16 | "is_variable": true 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/moz_telemetry_s3/run/analysis/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/moz_telemetry/tests/integration/moz_telemetry_s3/run/analysis/placeholder.txt -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/moz_telemetry_s3/run/input/moz_telemetry_s3.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_telemetry_s3.lua" 2 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_doctype_monitor/run/analysis/new_doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "moz_telemetry_new_doctype_monitor.lua" 2 | message_matcher = "Type == 'telemetry-new-doc-type'" 3 | ticker_interval = 60 4 | preserve_data = true 5 | threshold = 10 6 | known_doctypes = { "main" } 7 | 8 | alert = { 9 | disabled = false, 10 | prefix = false, 11 | threshold = 90, -- one alert every 90 minutes 12 | modules = { 13 | email = {recipients = {"amiyaguchi@mozilla.com"}}, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_doctype_monitor/run/input/new_doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "new_doctype_monitor.lua" 2 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_doctype_monitor/run/output/new_doctype_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "new_doctype_monitor.lua" 2 | message_matcher = "Logger == 'analysis.new_doctype_monitor' && Type == 'alert'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_doctype_monitor/run/output/new_doctype_monitor.lua: -------------------------------------------------------------------------------- 1 | require "string" 2 | 3 | -- TODO: Recieve more than one message, see `input/new_doctype_monitor.lua` 4 | local results = { 5 | "test_message_1", 6 | "test_message_2\ntest_message_3", 7 | "test_message_5" 8 | } 9 | 10 | local cnt = 0 11 | function process_message() 12 | cnt = cnt + 1 13 | local payload = read_message("Payload") 14 | assert(results[cnt] == payload, string.format("expected: %s actual: %s", results[cnt], payload)) 15 | return 0 16 | end 17 | 18 | 19 | function timer_event() 20 | assert(cnt == 1, string.format("%d out of 1 tests ran", cnt)) 21 | end 22 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_experiment_monitor/run/analysis/new_experiment_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = 'moz_telemetry_new_experiment_monitor.lua' 2 | ticker_interval = 60 3 | message_matcher = 'Type == "telemetry" && Fields[docType] == "main" && Fields[environment.experiments] != NIL' 4 | preserve_data = true 5 | alert = { 6 | modules = { 7 | email = {recipients = {'example@mozilla.com'}}, 8 | }, 9 | throttle = 1, 10 | disabled = false, 11 | prefix = false, 12 | } 13 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_experiment_monitor/run/input/generator.cfg: -------------------------------------------------------------------------------- 1 | filename = "generator.lua" 2 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_experiment_monitor/run/output/results.cfg: -------------------------------------------------------------------------------- 1 | filename = "results.lua" 2 | message_matcher = "Logger == 'analysis.new_experiment_monitor' && Type == 'alert'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/new_experiment_monitor/run/output/results.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Validates the moz_telemetry_new_experiment_monitor alerts 7 | --]] 8 | 9 | require "string" 10 | 11 | local cnt = 1 12 | function process_message() 13 | local payload = read_message("Payload") 14 | -- We expect two messages. Let's make sure we got them. 15 | if cnt == 1 then 16 | assert(payload:match("foo"), payload) 17 | elseif cnt == 2 then 18 | assert(payload:match("bar"), payload) 19 | end 20 | cnt = cnt + 1 21 | return 0 22 | end 23 | 24 | 25 | function timer_event() 26 | assert(cnt == 2, string.format("%d out of 2 expected alerts were received", cnt)) 27 | end 28 | -------------------------------------------------------------------------------- /moz_telemetry/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /openssl/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(openssl LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_openssl 10 | GIT_REPOSITORY https://github.com/zhaozg/lua-openssl.git 11 | GIT_TAG 2e8930c5e13b52705ba9c390110e84b1f4748ba9 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt && 14 | ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/config.h.in /src/config.h.in 15 | PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_LIST_DIR}/patch.diff 16 | ) 17 | externalproject_add_step(ep_openssl copy_cpack 18 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 19 | DEPENDEES install) 20 | -------------------------------------------------------------------------------- /openssl/config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef _CONFIG_H_ 2 | #define _CONFIG_H_ 3 | 4 | #cmakedefine HAVE_X509_ALGOR_CMP @HAVE_X509_ALGOR_CMP@ 5 | #cmakedefine HAVE_SSLV3_METHOD @HAVE_SSLV3_METHOD@ 6 | 7 | #endif 8 | -------------------------------------------------------------------------------- /openssl/index.md: -------------------------------------------------------------------------------- 1 | # Lua OpenSSL Module 2 | 3 | https://github.com/zhaozg/lua-openssl 4 | -------------------------------------------------------------------------------- /papertrail/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(papertrail VERSION 0.2.3 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Papertrail Log Ingestion") 8 | include(sandbox_module) 9 | -------------------------------------------------------------------------------- /papertrail/tests/hindsight/papertrail/run/analysis/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/papertrail/tests/hindsight/papertrail/run/analysis/placeholder.txt -------------------------------------------------------------------------------- /papertrail/tests/hindsight/papertrail/run/input/papertrail.cfg: -------------------------------------------------------------------------------- 1 | filename = "papertrail.lua" 2 | 3 | ticker_interval = 60 4 | 5 | limit = 10 6 | _key = "apikey" 7 | query = "ssh OR picard" 8 | 9 | decoder_module = "decoders.syslog" 10 | 11 | decoders_syslog = { 12 | template = "<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" 13 | } 14 | -------------------------------------------------------------------------------- /papertrail/tests/hindsight/papertrail/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/papertrail/tests/hindsight/papertrail/run/output/placeholder.txt -------------------------------------------------------------------------------- /papertrail/tests/hindsight/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /parquet/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.5) 6 | project(parquet VERSION 0.0.13 LANGUAGES C CXX) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Parquet Lua Module") 8 | 9 | find_package(parquet-cpp 1.3.1 REQUIRED CONFIG) 10 | 11 | set(MODULE_SRCS parquet.cpp parquet.def) 12 | set(INSTALL_MODULE_PATH ${INSTALL_IOMODULE_PATH}) 13 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "parquet-cpp (>= 1.3.1), ${PACKAGE_PREFIX}-lpeg (>= 1.0)") 14 | include(sandbox_module) 15 | target_link_libraries(parquet ${PARQUET-CPP_LIBRARIES}) 16 | -------------------------------------------------------------------------------- /parquet/parquet.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_parquet 3 | -------------------------------------------------------------------------------- /postgres/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(postgres LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_postgres 10 | GIT_REPOSITORY https://github.com/keplerproject/luasql.git 11 | GIT_TAG 5496d60185db0c4578e8abe0c74343e99b799311 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | PATCH_COMMAND patch src/ls_mysql.c < ${CMAKE_CURRENT_SOURCE_DIR}/patch.txt 15 | ) 16 | externalproject_add_step(ep_postgres copy_cpack 17 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 18 | DEPENDEES install) 19 | -------------------------------------------------------------------------------- /postgres/index.md: -------------------------------------------------------------------------------- 1 | # Lua Postgres Module 2 | 3 | * https://keplerproject.github.io/luasql/ 4 | * https://keplerproject.github.io/luasql/manual.html#postgres_extensions 5 | -------------------------------------------------------------------------------- /postgres/patch.txt: -------------------------------------------------------------------------------- 1 | --- postgres/ep_postgres-prefix/src/ep_postgres/src/ls_mysql.c 2020-04-29 12:39:59.568327879 -0700 2 | +++ ls_mysql.c 2020-04-29 12:45:07.260348905 -0700 3 | @@ -534,6 +534,7 @@ 4 | conn = mysql_init(NULL); 5 | if (conn == NULL) 6 | return luasql_faildirect(L, "error connecting: Out of memory."); 7 | + mysql_options(conn, MYSQL_SET_CHARSET_NAME, "utf8"); 8 | 9 | if (!mysql_real_connect(conn, host, username, password, 10 | sourcename, port, unix_socket, client_flag)) 11 | -------------------------------------------------------------------------------- /rjson/rjson.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_rjson 3 | -------------------------------------------------------------------------------- /sax/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(sax LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_sax 10 | GIT_REPOSITORY https://github.com/trink/symtseries.git 11 | GIT_TAG 62fbe4df396e68de637aca0cdb597e3593bab754 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_sax copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /sax/index.md: -------------------------------------------------------------------------------- 1 | # Lua SAX Module 2 | 3 | https://github.com/trink/symtseries/blob/master/README.md 4 | -------------------------------------------------------------------------------- /snappy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(snappy LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_snappy 10 | GIT_REPOSITORY https://github.com/forhappy/lua-snappy.git 11 | GIT_TAG 6b4f3f6736857d5c72aa6ed0ec566af6e222278d 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_snappy copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /snappy/index.md: -------------------------------------------------------------------------------- 1 | # Lua Snappy Module (DEPRECATED) 2 | 3 | https://github.com/forhappy/lua-snappy 4 | -------------------------------------------------------------------------------- /socket/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(socket LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_socket 10 | GIT_REPOSITORY https://github.com/diegonehab/luasocket 11 | GIT_TAG 316a9455b9cb4637fe6e62b20fbe05f5141fec54 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} -DPARENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_socket copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /socket/index.md: -------------------------------------------------------------------------------- 1 | # Lua Socket Module 2 | 3 | http://w3.impa.br/~diego/software/luasocket/ 4 | -------------------------------------------------------------------------------- /ssl/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(ssl LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_ssl 10 | GIT_REPOSITORY https://github.com/brunoos/luasec.git 11 | GIT_TAG c6704919bdc85f3324340bdb35c2795a02f7d625 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_ssl copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /ssl/index.md: -------------------------------------------------------------------------------- 1 | # Lua SSL Module (luasec) 2 | 3 | https://github.com/brunoos/luasec/wiki 4 | -------------------------------------------------------------------------------- /struct/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(struct LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_struct 10 | URL http://www.inf.puc-rio.br/~roberto/struct/struct-0.2.tar.gz 11 | URL_MD5 99384bf1f54457ec9f796ad0b539d19c 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_struct copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /struct/index.md: -------------------------------------------------------------------------------- 1 | # Lua Struct Module 2 | 3 | http://www.inf.puc-rio.br/~roberto/struct/ 4 | -------------------------------------------------------------------------------- /syslog/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(syslog VERSION 1.0.15 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Syslog parsers and collectors") 8 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-lpeg (>= 1.0.9), ${PACKAGE_PREFIX}-socket (>= 3.0)") 9 | string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS}) 10 | include(sandbox_module) 11 | 12 | add_test(NAME ${PROJECT_NAME}_tutorial 13 | COMMAND ../run.sh 14 | CONFIGURATIONS integration 15 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/integration/tutorial) 16 | -------------------------------------------------------------------------------- /syslog/modules/lpeg/linux/cron.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | -- Copyright 2015 Mathieu Parent 6 | 7 | --[[ 8 | # Linux CRON Grammar Module 9 | 10 | ## Variables 11 | ### LPEG Grammars 12 | * `syslog_grammar` 13 | --]] 14 | 15 | local l = require "lpeg" 16 | l.locale(l) 17 | local sl = require "lpeg.syslog" 18 | local pam = require "lpeg.linux.pam".syslog_grammar 19 | 20 | local M = {} 21 | setfenv(1, M) -- Remove external access to contain everything in the module 22 | 23 | syslog_grammar = l.Ct( 24 | l.P"(" 25 | * sl.capture_followed_by("cron_username", ") ") 26 | * sl.capture_followed_by("cron_event", " (") 27 | * sl.capture_followed_by("cron_detail", ")") 28 | * l.P(-1) 29 | ) 30 | + pam 31 | 32 | return M 33 | -------------------------------------------------------------------------------- /syslog/modules/lpeg/linux/groupadd.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | -- Copyright 2015 Mathieu Parent 6 | 7 | --[[ 8 | # Linux groupadd Grammar Module 9 | 10 | ## Variables 11 | ### LPEG Grammars 12 | * `syslog_grammar` 13 | --]] 14 | 15 | local l = require "lpeg" 16 | l.locale(l) 17 | local sl = require "lpeg.syslog" 18 | 19 | local tonumber = tonumber 20 | 21 | local M = {} 22 | setfenv(1, M) -- Remove external access to contain everything in the module 23 | 24 | syslog_grammar = l.Ct( 25 | ( 26 | l.P"new group: name=" 27 | * sl.capture_followed_by("group_name", ", GID=") 28 | * l.Cg(l.digit^1 / tonumber, "gid") 29 | * l.P(-1) 30 | ) 31 | ) 32 | 33 | return M 34 | -------------------------------------------------------------------------------- /syslog/modules/lpeg/linux/groupdel.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | -- Copyright 2015 Mathieu Parent 6 | 7 | --[[ 8 | # Linux groupdel Grammar Module 9 | 10 | ## Variables 11 | ### LPEG Grammars 12 | * `syslog_grammar` 13 | --]] 14 | 15 | local l = require "lpeg" 16 | l.locale(l) 17 | local sl = require "lpeg.syslog" 18 | 19 | local M = {} 20 | setfenv(1, M) -- Remove external access to contain everything in the module 21 | 22 | syslog_grammar = l.Ct( 23 | ( 24 | l.P"group '" 25 | * sl.capture_followed_by("group_name", "' removed") 26 | * (l.P" from " * l.Cg(l.P(1)^1, "group_dbname"))^-1 27 | ) 28 | ) 29 | 30 | return M 31 | -------------------------------------------------------------------------------- /syslog/modules/lpeg/sendmail.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # Sendmail module 7 | 8 | ## Variables 9 | ### LPEG Grammars 10 | * `grammar` 11 | --]] 12 | 13 | local l = require "lpeg" 14 | l.locale(l) -- we want l.alpha etc... 15 | 16 | local rawset = rawset 17 | 18 | local M = {} 19 | setfenv(1, M) -- Remove external access to contain everything in the module 20 | 21 | local sep = l.P", " 22 | local key = l.C(l.alpha^1) 23 | local value = l.C((l.P(1) - sep)^1) 24 | local statpair = l.Cg(l.C(l.P"stat") * "=" * l.C(l.P(1)^0)) 25 | local normalpair = l.Cg(key * "=" * value) * sep^-1 26 | local pair = statpair + normalpair 27 | 28 | local sendmailid = l.Cg(l.alnum^-8 * l.digit^6, "sendmailid") * l.P": " 29 | 30 | grammar = l.Cf(l.Ct(sendmailid) * pair^0, rawset) 31 | 32 | return M 33 | -------------------------------------------------------------------------------- /syslog/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_path = "output" 2 | sandbox_run_path = "run" 3 | analysis_lua_path = "../../../../common/?.lua;./modules/?.lua;/usr/lib/luasandbox/modules/?.lua" 4 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 5 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 6 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 7 | hostname = "integration_test" 8 | 9 | analysis_defaults = {} 10 | input_defaults = {} 11 | output_defaults = {} 12 | -------------------------------------------------------------------------------- /syslog/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/adhoc.log: -------------------------------------------------------------------------------- 1 | Jan 23 08:50:03 ubuntu foo[1234]: /tmp/input.tsv:23: invalid line 2 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/decoder_module.log: -------------------------------------------------------------------------------- 1 | Jan 23 08:50:02 ubuntu nginx[1234]: 127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" 2 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/grammar_module.log: -------------------------------------------------------------------------------- 1 | Feb 14 19:20:21 ubuntu someapp[3453]: foo=bar a=14 baz="hello kitty" cool%story=bro f %^asdf ip=216.160.83.56 2 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/modules/syslog_docs.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --[[ 6 | # syslog_doc demo module with match args 7 | 8 | ## Variables 9 | ### LPEG Grammars 10 | * `demo` - replaces hostname match with the provided CArg 11 | --]] 12 | 13 | -- Imports 14 | local l = require "lpeg" 15 | l.locale(l) 16 | 17 | local M = {} 18 | setfenv(1, M) -- Remove external access to contain everything in the module 19 | demo = l.Ct(l.Cg(l.Carg(1), "real_hostname")) 20 | return M 21 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_adhoc.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_adhoc.lua" 2 | message_matcher = "Logger == 'input.adhoc'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_decoder_module.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_decoder_module.lua" 2 | message_matcher = "Logger == 'input.decoder_module'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_function.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_function.lua" 2 | message_matcher = "Logger == 'input.function'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_grammar_args.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_grammar_args.lua" 2 | message_matcher = "Logger == 'input.grammar_args'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_grammar_args_nil.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_grammar_args_nil.lua" 2 | message_matcher = "Logger == 'input.grammar_args_nil'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_grammar_module.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_grammar_module.lua" 2 | message_matcher = "Logger == 'input.grammar_module'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_grammar_module_keep.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_grammar_module_keep.lua" 2 | message_matcher = "Logger == 'input.grammar_module_keep'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_grammar_transform.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_grammar_transform.lua" 2 | message_matcher = "Logger == 'input.grammar_transform'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_printf.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_printf.lua" 2 | message_matcher = "Logger == 'input.printf'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_printf_transform.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_printf_transform.lua" 2 | message_matcher = "Logger == 'input.printf_transform'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/analysis/verify_syslog.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_syslog.lua" 2 | message_matcher = "Logger == 'input.syslog'" 3 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/adhoc.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "adhoc.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | sub_decoders = { 10 | foo = { 11 | {{"%s:%lu: invalid line", "path", "linenum"}, nil}, 12 | }, 13 | }, 14 | } 15 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/decoder_module.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "decoder_module.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | sub_decoders = { 10 | nginx = "decoders.nginx.access", -- decoder module name 11 | }, 12 | } 13 | 14 | decoders_nginx_access = { 15 | log_format = '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"' 16 | } 17 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/function.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "decoder_module.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | log_format = '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"' 7 | 8 | decoders_syslog = { 9 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 10 | 11 | sub_decoders = { 12 | nginx = { 13 | {{"lpeg.common_log_format#build_nginx_grammar", log_format}, nil}, 14 | }, 15 | }, 16 | } 17 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/grammar_args.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "grammar_module.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | sub_decoders = { 10 | someapp = { 11 | {{"syslog_docs#demo", "www.example.com"}, nil}, 12 | }, 13 | }, 14 | } 15 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/grammar_args_nil.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "grammar_module.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | sub_decoders = { 10 | someapp = { 11 | {{"syslog_docs#demo"}, nil}, 12 | }, 13 | }, 14 | } 15 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/grammar_module.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "grammar_module.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | sub_decoders = { 10 | someapp = "lpeg.logfmt", 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/grammar_module_keep.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "grammar_module.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | keep_data = '@message', 9 | 10 | sub_decoders = { 11 | someapp = "lpeg.logfmt", 12 | }, 13 | } 14 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/grammar_transform.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "grammar_module.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | sub_decoders = { 10 | someapp = { 11 | {{"lpeg.logfmt"}, {ip = "maxminddb.heka#add_geoip"}}, 12 | }, 13 | }, 14 | } 15 | 16 | maxminddb_heka = { 17 | databases = { 18 | ["../../../../maxminddb/tests/GeoIP2-City-Test.mmdb"] = { 19 | _city = {"city", "names", "en"}, 20 | _country = {"country", "iso_code"} 21 | }, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/printf.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "syslog.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | printf_messages = { 10 | "lpeg.openssh_portable", 11 | }, 12 | 13 | sub_decoders = { 14 | sshd = { 15 | "Accepted publickey for foobar from 192.168.1.1 port 4567 ssh2", 16 | }, 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/printf_transform.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "syslog.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | 9 | printf_messages = { 10 | "lpeg.openssh_portable", 11 | }, 12 | 13 | sub_decoders = { 14 | sshd = { 15 | {"Accepted publickey for foobar from 10.11.12.13 port 4242 ssh2", {ssh_remote_ipaddr = "maxminddb.heka#add_geoip"}}, 16 | }, 17 | }, 18 | } 19 | 20 | maxminddb_heka = { 21 | databases = { 22 | ["../../../../maxminddb/tests/GeoIP2-City-Test.mmdb"] = { 23 | _city = {"city", "names", "en"}, 24 | _country = {"country", "iso_code"} 25 | }, 26 | }, 27 | } 28 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/input/syslog.cfg: -------------------------------------------------------------------------------- 1 | filename = "file.lua" 2 | input_filename = "syslog.log" 3 | send_decode_failures = true 4 | decoder_module = "decoders.syslog" 5 | 6 | decoders_syslog = { 7 | template = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%", 8 | } 9 | -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/syslog/tests/integration/tutorial/run/output/placeholder.txt -------------------------------------------------------------------------------- /syslog/tests/integration/tutorial/syslog.log: -------------------------------------------------------------------------------- 1 | Feb 13 14:25:19 ubuntu sshd[7192]: Accepted publickey for foobar from 216.160.83.56 port 4242 ssh2 2 | -------------------------------------------------------------------------------- /syslog/tests/linux/groupadd.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local grammar = require "lpeg.linux.groupadd".syslog_grammar 6 | local log 7 | local fields 8 | 9 | log = "new group: name=admin, GID=1234" 10 | fields = grammar:match(log) 11 | assert(fields.group_name == "admin", fields.group_name) 12 | assert(fields.gid == 1234, tostring(fields.gid)) 13 | -------------------------------------------------------------------------------- /syslog/tests/linux/groupdel.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local grammar = require "lpeg.linux.groupdel".syslog_grammar 6 | local log 7 | local fields 8 | 9 | log = "group 'admin' removed from dbname" 10 | fields = grammar:match(log) 11 | assert(fields.group_name == "admin", fields.group_name) 12 | assert(fields.group_dbname == "dbname", fields.group_dbname) 13 | -------------------------------------------------------------------------------- /syslog/tests/linux/login.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local grammar = require "lpeg.linux.login".syslog_grammar 6 | local log 7 | local fields 8 | 9 | log = "FAILED LOGIN (1) on '/dev/tty1' FOR 'root', Authentication failure" 10 | fields = grammar:match(log) 11 | assert(fields.failcount == 1, fields.failcount) 12 | assert(fields.tty == '/dev/tty1', fields.tty) 13 | assert(fields.user == 'root', fields.user) 14 | assert(fields.pam_error == 'Authentication failure', fields.pam_error) 15 | -------------------------------------------------------------------------------- /syslog/tests/test.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | assert(loadfile"syslog.lua")() 6 | 7 | assert(loadfile"bsd/filterlog.lua")() 8 | 9 | assert(loadfile"linux/cron.lua")() 10 | assert(loadfile"linux/dhclient.lua")() 11 | assert(loadfile"linux/dhcpd.lua")() 12 | assert(loadfile"linux/groupadd.lua")() 13 | assert(loadfile"linux/groupdel.lua")() 14 | assert(loadfile"linux/kernel.lua")() 15 | assert(loadfile"linux/login.lua")() 16 | assert(loadfile"linux/named.lua")() 17 | assert(loadfile"linux/pam.lua")() 18 | assert(loadfile"linux/puppet_agent.lua")() 19 | assert(loadfile"linux/sshd.lua")() -- todo deprecate 20 | assert(loadfile"linux/su.lua")() 21 | assert(loadfile"linux/sudo.lua")() 22 | assert(loadfile"linux/systemd_logind.lua")() 23 | assert(loadfile"linux/useradd.lua")() 24 | assert(loadfile"sendmail.lua")() 25 | -------------------------------------------------------------------------------- /systemd/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(systemd LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_systemd 10 | GIT_REPOSITORY https://github.com/daurnimator/lua-systemd.git 11 | GIT_TAG d64c914c1ece85f4f85f8b89e2111fb5dafa7372 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} -DPARENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_systemd copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /systemd/index.md: -------------------------------------------------------------------------------- 1 | # Lua systemd Module 2 | 3 | https://github.com/daurnimator/lua-systemd/ 4 | -------------------------------------------------------------------------------- /taskcluster/cloud_functions/mach_try_table_exports/requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-bigquery==1.21.0 2 | google-cloud-storage==1.21.0 3 | -------------------------------------------------------------------------------- /taskcluster/schemas/taskcluster/drain.1.bq.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "REQUIRED", 4 | "name": "platform", 5 | "type": "STRING" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "job_symbol", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "group_symbol", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "REQUIRED", 19 | "name": "template", 20 | "type": "STRING" 21 | } 22 | ] 23 | -------------------------------------------------------------------------------- /taskcluster/sql/datastudio/derived_missing_workertype_costs.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | kinds.provisionerId, 3 | kinds.workerType, 4 | COUNT(kinds.workerType) AS task_count 5 | FROM 6 | `moz-fx-data-taskclu-prod-8fbf.taskclusteretl.derived_kind_costs` AS kinds 7 | LEFT JOIN 8 | taskclusteretl.derived_daily_cost_per_workertype AS costs 9 | ON 10 | costs.workerType = kinds.workerType 11 | WHERE 12 | costs.workerType IS NULL 13 | AND kinds.cost IS NULL 14 | AND kinds.date > DATE_SUB(CURRENT_DATE(), INTERVAL 35 day) 15 | AND kinds.date < DATE_SUB(CURRENT_DATE(), INTERVAL 5 day) 16 | GROUP BY 17 | provisionerId, 18 | workerType 19 | -------------------------------------------------------------------------------- /taskcluster/sql/machtry_quantiles.sql: -------------------------------------------------------------------------------- 1 | WITH 2 | a AS ( 3 | SELECT 4 | APPROX_QUANTILES(total_seconds, 100 IGNORE NULLS) AS quants 5 | FROM 6 | `moz-fx-data-taskclu-prod-8fbf.taskclusteretl.derived_taskgroup_costs` 7 | WHERE 8 | project = 'try' 9 | AND cost IS NOT NULL 10 | AND tasks > 1 -- Ignore things that are likely only a decision task, due to rate of failures there 11 | AND date > DATE_SUB(@run_date, INTERVAL 60 day) 12 | AND 'reviewbot@noreply.mozilla.org' NOT IN UNNEST(owners) ) 13 | SELECT 14 | quant 15 | FROM 16 | a 17 | CROSS JOIN 18 | UNNEST(a.quants) AS quant 19 | ORDER BY 20 | quant ASC 21 | -------------------------------------------------------------------------------- /taskcluster/sql/task_duration_estimates.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | COUNT(started) as tasks_considered, 3 | ROUND(AVG(TIMESTAMP_DIFF(resolved, started, millisecond)) / 1000, 2) AS mean_duration_seconds, 4 | name 5 | FROM 6 | taskclusteretl.derived_task_summary timing 7 | JOIN ( 8 | SELECT 9 | taskId, 10 | metadata.name as name 11 | FROM 12 | taskclusteretl.task_definition ) defs 13 | ON 14 | timing.taskId = defs.taskId 15 | WHERE 16 | started >= CAST(DATE_SUB(@run_date, INTERVAL 30 day) AS timestamp) 17 | AND resolved < CAST(@run_time AS timestamp) 18 | GROUP by name 19 | -------------------------------------------------------------------------------- /taskcluster/test.lua: -------------------------------------------------------------------------------- 1 | s = "foo\t\t\tbar\ttest" 2 | for str in string.gmatch(s, "([^\t]*)") do 3 | print(".", str) 4 | end 5 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/hindsight.cfg: -------------------------------------------------------------------------------- 1 | output_size = 1024 * 1024 * 1024 2 | output_path = "output" 3 | sandbox_run_path = "run" 4 | analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua" 5 | analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so" 6 | io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua" 7 | io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so" 8 | hostname = "integration_test" 9 | max_message_size = 8 * 1024 * 1024 10 | 11 | analysis_defaults = {} 12 | input_defaults = {} 13 | output_defaults = {} 14 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf output 3 | hindsight_cli ../hindsight.cfg 7 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/C0PGASalTF6MZNoTg0eLGQ_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"C0PGASalTF6MZNoTg0eLGQ","provisionerId":"mobile-1","workerType":"b-linux-large","schedulerId":"mobile-level-1","taskGroupId":"UKJ5YAiQR16SSYRGAQwx_Q","deadline":"2020-01-29T15:32:30.795Z","expires":"2020-02-25T15:32:30.795Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-05e506b504c6a1106","takenUntil":"2020-01-28T16:18:24.127Z","scheduled":"2020-01-28T15:58:23.988Z","started":"2020-01-28T15:58:24.169Z","resolved":"2020-01-28T15:58:31.791Z"}]},"runId":0,"task":{"tags":{"os":"linux","createdForUser":"csadilek@users.noreply.github.com","worker-implementation":"docker-worker","kind":"complete","label":"complete-pr-2"}},"workerGroup":"aws","workerId":"i-05e506b504c6a1106","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/ZYFKUNR5RhmrBAq4Z4KB9g_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"ZYFKUNR5RhmrBAq4Z4KB9g","provisionerId":"gecko-3","workerType":"decision","schedulerId":"gecko-level-3","taskGroupId":"ZYFKUNR5RhmrBAq4Z4KB9g","deadline":"2020-01-28T17:00:00.000Z","expires":"2020-02-04T16:00:00.000Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-04b99c91f855bd0c4","takenUntil":"2020-01-28T16:20:42.071Z","scheduled":"2020-01-28T16:00:41.434Z","started":"2020-01-28T16:00:42.091Z","resolved":"2020-01-28T16:01:00.339Z"}]},"runId":0,"task":{"tags":{}},"workerGroup":"aws","workerId":"i-04b99c91f855bd0c4","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/android_sp-cm_Rap_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"USQ8K5YcQJKQgybb28cyXg","provisionerId":"proj-autophone","workerType":"gecko-t-bitbar-gw-perf-p2","schedulerId":"gecko-level-3","taskGroupId":"DRmWdkINS5KKmbffJ21nIA","deadline":"2020-01-25T10:32:39.866Z","expires":"2021-01-23T10:32:39.866Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"bitbar","workerId":"pixel2-47","takenUntil":"2020-01-24T11:37:02.889Z","scheduled":"2020-01-24T11:15:55.028Z","started":"2020-01-24T11:17:02.965Z","resolved":"2020-01-24T11:21:33.025Z"}]},"runId":0,"task":{"tags":{"kind":"test","os":"linux-bitbar","createdForUser":"cron@noreply.mozilla.org","retrigger":"true","label":"test-android-hw-p2-8-0-android-aarch64/pgo-raptor-speedometer-geckoview-cpu-memory-e10s","test-type":"raptor","worker-implementation":"generic-worker"}},"workerGroup":"bitbar","workerId":"pixel2-47","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/android_tp6m-c-2-vismet_Btime-fenix_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"bZPP4WKDR5Whwge70OcW9g","provisionerId":"gecko-t","workerType":"t-linux-xlarge","schedulerId":"gecko-level-3","taskGroupId":"Zv14iv51Sb6zPQLi-ceshA","deadline":"2020-01-24T21:45:29.065Z","expires":"2021-01-22T21:45:29.065Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-04ab3f0c135b3c762","takenUntil":"2020-01-23T23:57:16.903Z","scheduled":"2020-01-23T23:37:16.661Z","started":"2020-01-23T23:37:16.919Z","resolved":"2020-01-23T23:37:55.695Z"}]},"runId":0,"task":{"tags":{"kind":"visual-metrics-dep","worker-implementation":"docker-worker","createdForUser":"rmaries@mozilla.com","retrigger":"false","label":"test-vismet-android-hw-p2-8-0-android-aarch64/pgo-browsertime-tp6m-fenix-cold-youtube","os":"linux"}},"workerGroup":"aws","workerId":"i-04ab3f0c135b3c762","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/linux_B_Bpgo_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"SSXGXyVIRQGQqVKGym90mQ","provisionerId":"gecko-3","workerType":"b-linux","schedulerId":"gecko-level-3","taskGroupId":"FKvNIN4OSHqJmZCo7Shi3Q","deadline":"2020-01-24T16:20:40.106Z","expires":"2021-01-22T16:20:40.106Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-0241a66d65f1dd6a5","takenUntil":"2020-01-23T18:22:30.838Z","scheduled":"2020-01-23T17:15:00.445Z","started":"2020-01-23T17:16:21.063Z","resolved":"2020-01-23T18:03:41.981Z"}]},"runId":0,"task":{"tags":{"kind":"build","worker-implementation":"docker-worker","createdForUser":"ccoroiu@mozilla.com","retrigger":"false","label":"build-linux64-shippable/opt","os":"linux"}},"workerGroup":"aws","workerId":"i-0241a66d65f1dd6a5","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/linux_GTest_nil_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"CX96USXgR3CvJWA2ZsBYrA","provisionerId":"gecko-t","workerType":"t-linux-xlarge","schedulerId":"gecko-level-1","taskGroupId":"T4DsDmsWQGKg_-5YkiO-Vw","deadline":"2020-01-24T16:10:40.723Z","expires":"2020-02-06T16:10:40.723Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-0f272f3f5a47e63ba","takenUntil":"2020-01-23T18:15:13.558Z","scheduled":"2020-01-23T17:38:18.537Z","started":"2020-01-23T17:39:49.885Z","resolved":"2020-01-23T17:58:22.393Z"}]},"runId":0,"task":{"tags":{"kind":"test","worker-implementation":"docker-worker","createdForUser":"jvarga@mozilla.com","retrigger":"true","label":"test-linux64-shippable-qr/opt-gtest-1proc","os":"linux"}},"workerGroup":"aws","workerId":"i-0f272f3f5a47e63ba","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/linux_R1_R-1proc_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"a46tmlKvRFuopDLt8E7IEQ","provisionerId":"gecko-t","workerType":"t-linux-large","schedulerId":"gecko-level-3","taskGroupId":"fyLj3h7PSz2q6ZjfQ544zg","deadline":"2020-01-24T18:49:58.577Z","expires":"2021-01-22T18:49:58.577Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-0323222c23ed083b6","takenUntil":"2020-01-23T20:10:13.336Z","scheduled":"2020-01-23T19:34:49.624Z","started":"2020-01-23T19:34:49.931Z","resolved":"2020-01-23T20:00:16.794Z"}]},"runId":0,"task":{"tags":{"kind":"test","os":"linux","createdForUser":"ryanvm@gmail.com","retrigger":"true","label":"test-linux32/debug-reftest-1proc-1","test-type":"reftest","worker-implementation":"docker-worker"}},"workerGroup":"aws","workerId":"i-0323222c23ed083b6","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/linux_ach_p_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"KcwJfqv_Sqi6e_2KKFS1Ow","provisionerId":"gecko-3","workerType":"b-linux","schedulerId":"gecko-level-3","taskGroupId":"HqfENdOERUay9RRBzBq6Uw","deadline":"2020-01-24T22:02:23.124Z","expires":"2021-01-22T22:02:23.124Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-01499bc279bc4e4d7","takenUntil":"2020-01-24T00:02:53.123Z","scheduled":"2020-01-23T23:42:53.032Z","started":"2020-01-23T23:42:53.138Z","resolved":"2020-01-23T23:46:15.279Z"}]},"runId":0,"task":{"tags":{"createdForUser":"cron@noreply.mozilla.org","retrigger":"false","kind":"partials","label":"partials-ach-linux64-shippable/opt"}},"workerGroup":"aws","workerId":"i-01499bc279bc4e4d7","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/linux_ach_ps_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"GCqbg2tVRxiT2JaYrqev3g","provisionerId":"scriptworker-k8s","workerType":"gecko-3-signing","schedulerId":"gecko-level-3","taskGroupId":"HqfENdOERUay9RRBzBq6Uw","deadline":"2020-01-24T22:02:42.900Z","expires":"2021-01-22T22:02:42.900Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"gecko-3-signing","workerId":"gecko-3-signing-zgirrd9priurzsoxt0idg","takenUntil":"2020-01-24T00:07:29.847Z","scheduled":"2020-01-23T23:46:16.685Z","started":"2020-01-23T23:47:29.863Z","resolved":"2020-01-23T23:47:43.675Z"}]},"runId":0,"task":{"tags":{"createdForUser":"cron@noreply.mozilla.org","retrigger":"false","kind":"partials-signing","label":"partials-signing-ach-linux64-shippable/opt"}},"workerGroup":"gecko-3-signing","workerId":"gecko-3-signing-zgirrd9priurzsoxt0idg","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/osx_1_M_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"IViCwC-nTMi_gXPE3ckjPg","provisionerId":"releng-hardware","workerType":"gecko-t-osx-1014","schedulerId":"gecko-level-3","taskGroupId":"X6ZNelySTNuuZD5DrOt6Dw","deadline":"2020-01-24T16:56:37.899Z","expires":"2021-01-22T16:56:37.899Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"mdc2","workerId":"t-mojave-r7-077","takenUntil":"2020-01-23T18:15:26.391Z","scheduled":"2020-01-23T17:36:09.599Z","started":"2020-01-23T17:38:26.240Z","resolved":"2020-01-23T18:04:35.890Z"}]},"runId":0,"task":{"tags":{"kind":"test","os":"macosx","createdForUser":"ccoroiu@mozilla.com","retrigger":"true","label":"test-macosx1014-64/debug-mochitest-e10s-1","test-type":"mochitest","worker-implementation":"generic-worker"}},"workerGroup":"mdc2","workerId":"t-mojave-r7-077","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/osx_tp6-c_Rap_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"HbxirNOCRYiM1yUmLBMeYQ","provisionerId":"releng-hardware","workerType":"gecko-t-osx-1014","schedulerId":"gecko-level-3","taskGroupId":"QVAR0sWvSduH7Ub2uOCB8A","deadline":"2020-04-16T11:35:06.003Z","expires":"2021-04-15T11:35:06.003Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"mdc2","workerId":"t-mojave-r7-107","takenUntil":"2020-04-15T15:07:47.350Z","scheduled":"2020-04-15T14:46:46.305Z","started":"2020-04-15T14:47:47.369Z","resolved":"2020-04-15T14:54:59.350Z"}]},"runId":0,"task":{"tags":{"kind":"test","os":"macosx","createdForUser":"nchevobbe@mozilla.com","retrigger":"true","label":"test-macosx1014-64-shippable/opt-raptor-tp6-9-firefox-cold-e10s","test-type":"raptor","worker-implementation":"generic-worker"}},"workerGroup":"mdc2","workerId":"t-mojave-r7-107","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_artifact_list.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_artifact_list.lua" 2 | message_matcher = "Type == 'artifact_list'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_error.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_error.lua" 2 | message_matcher = "Type =~ 'error'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_perfherder.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_perfherder.lua" 2 | message_matcher = "Type == 'perfherder'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_project.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_project.lua" 2 | message_matcher = "Fields[level] == 0" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_resolution_definition_cnts.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_resolution_definition_cnts.lua" 2 | message_matcher = "Type == 'pulse_task' || Type == 'task_definition'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_resolution_definition_cnts.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | require "table" 7 | 8 | local ecnts = {task_definition = 14, pulse_task = 14} 9 | local acnts = {} 10 | 11 | function process_message() 12 | local t = read_message("Type") 13 | local cnt = acnts[t] 14 | if not cnt then 15 | acnts[t] = 1 16 | else 17 | acnts[t] = cnt + 1 18 | end 19 | return 0 20 | end 21 | 22 | function timer_event(ns) 23 | for k,ecnt in pairs(ecnts) do 24 | local acnt = acnts[k] 25 | if ecnt ~= acnt then error(string.format("%s messages received %d expected: %d", k, acnt, ecnt)) end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_resource_monitor.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_resource_monitor.lua" 2 | message_matcher = "Type == 'resource_monitor'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_resource_monitor.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | require "cjson" 7 | 8 | local cnt = 0 9 | 10 | local expected = {231097} 11 | 12 | function process_message() 13 | cnt = cnt + 1 14 | local s = read_message("Payload") 15 | local r = #s 16 | local e = expected[cnt] 17 | if r ~= e then error(string.format("bytes received %d expected: %d", r, e)) end 18 | return 0 19 | end 20 | 21 | function timer_event(ns) 22 | if #expected ~= cnt then error(string.format("received %d expected: %d", cnt, #expected)) end 23 | end 24 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/analysis/verify_timing.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_timing.lua" 2 | message_matcher = "Type == 'timing'" 3 | ticker_interval = 0 4 | output_limit = 8 * 1000 * 1000 5 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/input/tc.cfg: -------------------------------------------------------------------------------- 1 | filename = "tc.lua" 2 | instruction_limit = 0 3 | memory_limit = 0 4 | output_limit = 8 * 1024 * 1024 5 | 6 | decoders_taskcluster_live_backing_log = { 7 | integration_test = true 8 | } 9 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/taskcluster/tests/integration/taskcluster/run/output/placeholder.txt -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_AuIAPvWhSiyp0eUh5vAdCw_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ], 10 | "continuationToken": "more" 11 | } 12 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_AuIAPvWhSiyp0eUh5vAdCw_runs_0_artifacts_continuationToken_more: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/more.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ], 10 | "continuationToken": "more1" 11 | } 12 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_AuIAPvWhSiyp0eUh5vAdCw_runs_0_artifacts_continuationToken_more1: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/more1.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_B02j0uS8SOGRfB4TrE2q4w_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_B02j0uS8SOGRfB4TrE2q4w_runs_1_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_C0PGASalTF6MZNoTg0eLGQ_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_CX96USXgR3CvJWA2ZsBYrA_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_CZ9BVge4QdKrJ3ZWdVbsTQ_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_GCqbg2tVRxiT2JaYrqev3g_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_HbxirNOCRYiM1yUmLBMeYQ_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | }, 9 | { 10 | "storageType": "s3", 11 | "name": "public/test_info/perfherder-data.json", 12 | "expires": "2020-07-29T20:55:54.564Z", 13 | "contentType": "application/json" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_HbxirNOCRYiM1yUmLBMeYQ_runs_0_artifacts_public_logs_live_backing.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/taskcluster/tests/integration/taskcluster/task_HbxirNOCRYiM1yUmLBMeYQ_runs_0_artifacts_public_logs_live_backing.log -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_IViCwC-nTMi_gXPE3ckjPg_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_IViCwC-nTMi_gXPE3ckjPg_runs_0_artifacts_public_logs_live_backing.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/taskcluster/tests/integration/taskcluster/task_IViCwC-nTMi_gXPE3ckjPg_runs_0_artifacts_public_logs_live_backing.log -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_KcwJfqv_Sqi6e_2KKFS1Ow_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_SSXGXyVIRQGQqVKGym90mQ_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_ZYFKUNR5RhmrBAq4Z4KB9g_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_a46tmlKvRFuopDLt8E7IEQ_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_aZ19zQ0SS8OLeFnQoHSwNA_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_aZ19zQ0SS8OLeFnQoHSwNA_runs_0_artifacts_public_logs_live_backing.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/taskcluster/tests/integration/taskcluster/task_aZ19zQ0SS8OLeFnQoHSwNA_runs_0_artifacts_public_logs_live_backing.log -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/task_bZPP4WKDR5Whwge70OcW9g_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/tests.txt: -------------------------------------------------------------------------------- 1 | win_x_T_pulse.json 2 | osx_1_M_pulse.json 3 | android_sp-cm_Rap_pulse.json 4 | linux_R1_R-1proc_pulse.json 5 | linux_GTest_nil_pulse.json 6 | linux_B_Bpgo_pulse.json 7 | linux_ach_p_pulse.json 8 | win_X1_X_pulse.json 9 | ZYFKUNR5RhmrBAq4Z4KB9g_pulse.json 10 | C0PGASalTF6MZNoTg0eLGQ_pulse.json 11 | win_sy-tp6_SY_pulse.json 12 | android_tp6m-c-2-vismet_Btime-fenix_pulse.json 13 | win_tp6-c_Btime-Prof_pulse.json 14 | osx_tp6-c_Rap_pulse.json 15 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/win_X1_X_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"AuIAPvWhSiyp0eUh5vAdCw","provisionerId":"gecko-t","workerType":"t-win10-64","schedulerId":"gecko-level-1","taskGroupId":"bldEIJ3tQzCnNXTSkeUhkw","deadline":"2020-01-24T16:08:27.358Z","expires":"2020-02-06T16:08:27.358Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-0eceefdf6123bd08e","takenUntil":"2020-01-23T18:12:20.753Z","scheduled":"2020-01-23T17:30:00.669Z","started":"2020-01-23T17:35:19.531Z","resolved":"2020-01-23T17:58:54.486Z"}]},"runId":0,"task":{"tags":{"kind":"test","worker-implementation":"generic-worker","createdForUser":"jvarga@mozilla.com","retrigger":"true","label":"test-windows10-64/debug-xpcshell-e10s-1","os":"windows"}},"workerGroup":"aws","workerId":"i-0eceefdf6123bd08e","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/win_tp6-c_Btime-Prof_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"aZ19zQ0SS8OLeFnQoHSwNA","provisionerId":"releng-hardware","workerType":"gecko-t-win10-64-hw","schedulerId":"gecko-level-1","taskGroupId":"a0LjdlBqScCLTP6xQEsZgw","deadline":"2020-01-24T17:17:35.677Z","expires":"2020-02-06T17:17:35.677Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"mdc2","workerId":"T-W1064-MS-321","takenUntil":"2020-01-23T21:59:24.652Z","scheduled":"2020-01-23T19:23:28.732Z","started":"2020-01-23T21:39:24.672Z","resolved":"2020-01-23T21:43:17.435Z"}]},"runId":0,"task":{"tags":{"kind":"test","os":"windows","createdForUser":"rstewart@mozilla.com","retrigger":"true","label":"test-windows10-64-shippable/opt-browsertime-tp6-profiling-firefox-cold-amazon-e10s","test-type":"raptor","worker-implementation":"generic-worker"}},"workerGroup":"mdc2","workerId":"T-W1064-MS-321","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster/win_x_T_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"CZ9BVge4QdKrJ3ZWdVbsTQ","provisionerId":"gecko-t","workerType":"t-win7-32","schedulerId":"gecko-level-3","taskGroupId":"FPYC9v-JQNOk25Zr7XWFrQ","deadline":"2020-01-24T15:42:33.279Z","expires":"2021-01-22T15:42:33.279Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-00fadafaca08ac0f8","takenUntil":"2020-01-23T18:13:38.245Z","scheduled":"2020-01-23T17:53:37.656Z","started":"2020-01-23T17:53:38.261Z","resolved":"2020-01-23T18:06:52.424Z"}]},"runId":0,"task":{"tags":{"kind":"test","os":"windows","createdForUser":"ccoroiu@mozilla.com","retrigger":"true","label":"test-windows7-32-shippable/opt-talos-xperf-e10s","test-type":"talos","worker-implementation":"generic-worker"}},"workerGroup":"aws","workerId":"i-00fadafaca08ac0f8","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/C0PGASalTF6MZNoTg0eLGQ_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"C0PGASalTF6MZNoTg0eLGQ","provisionerId":"mobile-1","workerType":"b-linux-large","schedulerId":"mobile-level-1","taskGroupId":"UKJ5YAiQR16SSYRGAQwx_Q","deadline":"2020-01-29T15:32:30.795Z","expires":"2020-02-25T15:32:30.795Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-05e506b504c6a1106","takenUntil":"2020-01-28T16:18:24.127Z","scheduled":"2020-01-28T15:58:23.988Z","started":"2020-01-28T15:58:24.169Z","resolved":"2020-01-28T15:58:31.791Z"}]},"runId":0,"task":{"tags":{"os":"linux","createdForUser":"csadilek@users.noreply.github.com","worker-implementation":"docker-worker","kind":"complete","label":"complete-pr-2"}},"workerGroup":"aws","workerId":"i-05e506b504c6a1106","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/USQ8K5YcQJKQgybb28cyXg_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"USQ8K5YcQJKQgybb28cyXg","provisionerId":"proj-autophone","workerType":"gecko-t-bitbar-gw-perf-p2","schedulerId":"gecko-level-3","taskGroupId":"DRmWdkINS5KKmbffJ21nIA","deadline":"2020-01-25T10:32:39.866Z","expires":"2021-01-23T10:32:39.866Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"bitbar","workerId":"pixel2-47","takenUntil":"2020-01-24T11:37:02.889Z","scheduled":"2020-01-24T11:15:55.028Z","started":"2020-01-24T11:17:02.965Z","resolved":"2020-01-24T11:21:33.025Z"}]},"runId":0,"task":{"tags":{"kind":"test","os":"linux-bitbar","createdForUser":"cron@noreply.mozilla.org","retrigger":"true","label":"test-android-hw-p2-8-0-android-aarch64/pgo-raptor-speedometer-geckoview-cpu-memory-e10s","test-type":"raptor","worker-implementation":"generic-worker"}},"workerGroup":"bitbar","workerId":"pixel2-47","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/ZYFKUNR5RhmrBAq4Z4KB9g_pulse.json: -------------------------------------------------------------------------------- 1 | {"status":{"taskId":"ZYFKUNR5RhmrBAq4Z4KB9g","provisionerId":"gecko-3","workerType":"decision","schedulerId":"gecko-level-3","taskGroupId":"ZYFKUNR5RhmrBAq4Z4KB9g","deadline":"2020-01-28T17:00:00.000Z","expires":"2020-02-04T16:00:00.000Z","retriesLeft":5,"state":"completed","runs":[{"runId":0,"state":"completed","reasonCreated":"scheduled","reasonResolved":"completed","workerGroup":"aws","workerId":"i-04b99c91f855bd0c4","takenUntil":"2020-01-28T16:20:42.071Z","scheduled":"2020-01-28T16:00:41.434Z","started":"2020-01-28T16:00:42.091Z","resolved":"2020-01-28T16:01:00.339Z"}]},"runId":0,"task":{"tags":{}},"workerGroup":"aws","workerId":"i-04b99c91f855bd0c4","version":1} -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/run/analysis/verify_backfill_cnts.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_backfill_cnts.lua" 2 | message_matcher = "Logger == 'input.backfill'" 3 | ticker_interval = 0 4 | 5 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/run/analysis/verify_backfill_cnts.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | require "string" 6 | require "table" 7 | 8 | local ecnts = {task_definition = 1, perfherder = 2, timing = 9, artifact_list = 1, resource_monitor = 1} 9 | local acnts = {} 10 | 11 | function process_message() 12 | local t = read_message("Type") 13 | local cnt = acnts[t] 14 | if not cnt then 15 | acnts[t] = 1 16 | else 17 | acnts[t] = cnt + 1 18 | end 19 | return 0 20 | end 21 | 22 | function timer_event(ns) 23 | for k,ecnt in pairs(ecnts) do 24 | local acnt = acnts[k] 25 | if ecnt ~= acnt then error(string.format("%s messages received %d expected: %d", k, acnt, ecnt)) end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/run/input/backfill.cfg: -------------------------------------------------------------------------------- 1 | filename = "taskcluster_backfill.lua" 2 | 3 | instruction_limit = 0 4 | memory_limit = 0 5 | output_limit = 8 * 1024 * 1024 6 | integration_test = true 7 | 8 | decoders_taskcluster_live_backing_log = { 9 | integration_test = integration_test 10 | } 11 | 12 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/run/input/tc.cfg: -------------------------------------------------------------------------------- 1 | filename = "tc.lua" 2 | instruction_limit = 0 3 | memory_limit = 0 4 | output_limit = 8 * 1024 * 1024 5 | 6 | decoders_taskcluster_live_backing_log = { 7 | integration_test = true 8 | } 9 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/run/output/verify_error.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_error.lua" 2 | message_matcher = "Type =~ '^error%.curl%.' && Fields[data] != NIL" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/task_BF_C0PGASalTF6MZNoTg0eLGQ_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/task_ZYFKUNR5RhmrBAq4Z4KB9g_runs_0_artifacts: -------------------------------------------------------------------------------- 1 | { 2 | "artifacts": [ 3 | { 4 | "storageType": "s3", 5 | "name": "public/logs/live_backing.log", 6 | "expires": "2020-07-29T20:55:54.564Z", 7 | "contentType": "text/plain; charset=utf-8" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/taskcluster_backfill/tests.txt: -------------------------------------------------------------------------------- 1 | C0PGASalTF6MZNoTg0eLGQ_pulse.json 2 | ZYFKUNR5RhmrBAq4Z4KB9g_pulse.json 3 | USQ8K5YcQJKQgybb28cyXg_pulse.json 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/worker_metrics/run/analysis/verify.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify.lua" 2 | message_matcher = "Type == 'worker_metrics'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/worker_metrics/run/analysis/verify_error.cfg: -------------------------------------------------------------------------------- 1 | filename = "verify_error.lua" 2 | message_matcher = "Type =~ 'error'" 3 | ticker_interval = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/worker_metrics/run/input/worker_metrics.cfg: -------------------------------------------------------------------------------- 1 | filename = "worker_metrics.lua" 2 | instruction_limit = 0 3 | memory_limit = 0 4 | -------------------------------------------------------------------------------- /taskcluster/tests/integration/worker_metrics/run/output/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/lua_sandbox_extensions/88ca3c5acd2df9e0a667643a92925df4acfd2af6/taskcluster/tests/integration/worker_metrics/run/output/placeholder.txt -------------------------------------------------------------------------------- /xxhash/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0) 6 | project(xxhash VERSION 0.0.1 LANGUAGES C) 7 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "xxhash Module") 8 | set(MODULE_SRCS xxhash.c ../common/xxhash.c xxhash.def) 9 | include(sandbox_module) 10 | -------------------------------------------------------------------------------- /xxhash/xxhash.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | luaopen_xxhash 3 | -------------------------------------------------------------------------------- /zlib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | cmake_minimum_required(VERSION 3.0 FATAL_ERROR) 6 | project(zlib LANGUAGES NONE) 7 | 8 | externalproject_add( 9 | ep_zlib 10 | GIT_REPOSITORY https://github.com/brimworks/lua-zlib.git 11 | GIT_TAG 80d48b3d7fb9dfa764e081b0aa6201f8779a44f8 12 | CMAKE_ARGS ${EP_CMAKE_ARGS} 13 | UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt.${PROJECT_NAME} /CMakeLists.txt 14 | ) 15 | externalproject_add_step(ep_zlib copy_cpack 16 | COMMAND ${CMAKE_COMMAND} -E copy /${PROJECT_NAME}.cpack ${CMAKE_BINARY_DIR} 17 | DEPENDEES install) 18 | -------------------------------------------------------------------------------- /zlib/index.md: -------------------------------------------------------------------------------- 1 | # Lua ZLIB Module 2 | 3 | https://github.com/brimworks/lua-zlib 4 | --------------------------------------------------------------------------------