├── .gitchangelog.rc ├── .github └── workflows │ └── ail_framework_test.yml ├── .gitignore ├── .gitmodules ├── HOWTO.md ├── LICENSE ├── README.md ├── SECURITY.md ├── bin ├── AIL_Init.py ├── LAUNCH.sh ├── Update-conf.py ├── Update.py ├── core │ ├── D4_client.py │ ├── Sync_importer.py │ ├── Sync_manager.py │ ├── Sync_module.py │ ├── ail_2_ail.py │ ├── ail_2_ail_client.py │ ├── ail_2_ail_server.py │ └── screen.py ├── crawlers │ ├── Crawler.py │ └── blacklist.txt ├── exporter │ ├── MISPExporter.py │ ├── MailExporter.py │ ├── TheHiveExporter.py │ ├── WebHookExporter.py │ ├── abstract_exporter.py │ └── default_mail_template.py ├── importer │ ├── FeederImporter.py │ ├── FileImporter.py │ ├── MispImport.py │ ├── PystemonImporter.py │ ├── ZMQImporter.py │ ├── abstract_importer.py │ └── feeders │ │ ├── BgpMonitor.py │ │ ├── Default.py │ │ ├── Discord.py │ │ ├── Jabber.py │ │ ├── Matrix.py │ │ ├── Telegram.py │ │ ├── Twitter.py │ │ ├── Urlextract.py │ │ └── abstract_chats_feeder.py ├── lib │ ├── ConfigLoader.py │ ├── Config_DB.py │ ├── Duplicate.py │ ├── Investigations.py │ ├── Language.py │ ├── MispModules.py │ ├── Statistics.py │ ├── Tag.py │ ├── Tracker.py │ ├── ail_api.py │ ├── ail_config.py │ ├── ail_core.py │ ├── ail_files.py │ ├── ail_logger.py │ ├── ail_orgs.py │ ├── ail_queues.py │ ├── ail_stats.py │ ├── ail_updates.py │ ├── ail_users.py │ ├── btc_ail.py │ ├── chats_viewer.py │ ├── correlations_engine.py │ ├── crawlers.py │ ├── d4.py │ ├── data_retention_engine.py │ ├── exceptions.py │ ├── images_engine.py │ ├── index_whoosh.py │ ├── item_basic.py │ ├── module_extractor.py │ ├── objects │ │ ├── BarCodes.py │ │ ├── ChatSubChannels.py │ │ ├── ChatThreads.py │ │ ├── Chats.py │ │ ├── CookiesNames.py │ │ ├── CryptoCurrencies.py │ │ ├── Cves.py │ │ ├── Decodeds.py │ │ ├── DomHashs.py │ │ ├── Domains.py │ │ ├── Etags.py │ │ ├── Favicons.py │ │ ├── FilesNames.py │ │ ├── GTrackers.py │ │ ├── HHHashs.py │ │ ├── Images.py │ │ ├── Items.py │ │ ├── Mails.py │ │ ├── Messages.py │ │ ├── Ocrs.py │ │ ├── Pgps.py │ │ ├── QrCodes.py │ │ ├── Screenshots.py │ │ ├── Titles.py │ │ ├── Usernames.py │ │ ├── UsersAccount.py │ │ ├── abstract_chat_object.py │ │ ├── abstract_daterange_object.py │ │ ├── abstract_object.py │ │ ├── abstract_subtype_object.py │ │ └── ail_objects.py │ ├── regex_helper.py │ ├── relationships_engine.py │ ├── search_engine.py │ ├── telegram.py │ └── timeline_engine.py ├── modules │ ├── ApiKey.py │ ├── CEDetector.py │ ├── Categ.py │ ├── CodeReader.py │ ├── Credential.py │ ├── CreditCards.py │ ├── Cryptocurrencies.py │ ├── CveModule.py │ ├── Decoder.py │ ├── DomClassifier.py │ ├── Duplicates.py │ ├── Exif.py │ ├── Global.py │ ├── Hosts.py │ ├── IPAddress.py │ ├── Iban.py │ ├── Indexer.py │ ├── Keys.py │ ├── Languages.py │ ├── LibInjection.py │ ├── MISP_Thehive_Auto_Push.py │ ├── Mail.py │ ├── Mixer.py │ ├── ModuleStats.py │ ├── OcrExtractor.py │ ├── Onion.py │ ├── Pasties.py │ ├── PgpDump.py │ ├── Phone.py │ ├── README.md │ ├── SQLInjectionDetection.py │ ├── SentimentAnalysis.py │ ├── SubmitPaste.py │ ├── Tags.py │ ├── Telegram.py │ ├── TemplateModule.py │ ├── Tools.py │ ├── TrackingId.py │ ├── Urls.py │ ├── __init__.py │ └── abstract_module.py ├── packages │ ├── Date.py │ ├── Hash.py │ ├── Import_helper.py │ ├── __init__.py │ ├── git_status.py │ ├── lib_refine.py │ └── lib_words.py ├── trackers │ ├── Retro_Hunt.py │ ├── Tracker_Regex.py │ ├── Tracker_Term.py │ ├── Tracker_Typo_Squatting.py │ ├── Tracker_Yara.py │ └── yara │ │ └── custom-rules │ │ └── test.yar └── update-background.py ├── configs ├── 6379.conf ├── 6380.conf ├── 6381.conf ├── 6382.conf ├── 6383.conf ├── core.cfg.sample ├── d4client_passiveDNS_conf │ ├── destination │ ├── key │ ├── snaplen │ ├── source │ ├── type │ └── version ├── docker │ └── splash_onion │ │ └── etc │ │ └── splash │ │ └── proxy-profiles │ │ └── default.ini ├── keys │ ├── mispKEYS.py.sample │ ├── theHiveKEYS.py.sample │ └── virusTotalKEYS.py.sample ├── logging.json ├── modules.cfg └── update.cfg.sample ├── doc ├── README.md ├── ail_modules_queues.png ├── api.md ├── presentation │ ├── ail-pass-the-salt.pdf │ ├── ail-training-december-2018.pdf │ ├── ail-training-luxembourg-2018.pdf │ ├── ail-training-november.pdf │ ├── source-training │ │ ├── ail-training.pdf │ │ ├── ail-training.tex │ │ ├── beamercolorthemechameleon.sty │ │ ├── beamerinnerthemefancy.sty │ │ ├── beamerouterthemedecolines.sty │ │ ├── beamerthemeTorino.sty │ │ ├── clean.sh │ │ ├── cmds_summary │ │ │ ├── beamercolorthemechameleon.sty │ │ │ ├── beamerinnerthemefancy.sty │ │ │ ├── beamerouterthemedecolines.sty │ │ │ ├── beamerthemeTorino.sty │ │ │ ├── circl-introduction.listing │ │ │ ├── circl-introduction.tex │ │ │ └── content-introduction.tex │ │ ├── images │ │ │ ├── MISP.png │ │ │ ├── ail-correlation.png │ │ │ ├── ail-project.png │ │ │ ├── ail_01.png │ │ │ ├── ail_02.png │ │ │ ├── ail_03.png │ │ │ ├── ail_04.png │ │ │ ├── ail_06.png │ │ │ ├── ail_07.png │ │ │ ├── ail_08.png │ │ │ ├── ail_09.png │ │ │ ├── ail_10.png │ │ │ ├── ail_11.png │ │ │ ├── ail_12.png │ │ │ ├── ail_13.png │ │ │ ├── ail_14.png │ │ │ ├── ail_15.png │ │ │ ├── ail_16.png │ │ │ ├── ail_17.png │ │ │ ├── ail_crf_10min.png │ │ │ ├── ail_empty_1min.png │ │ │ ├── ail_pyst_10min.png │ │ │ ├── alert.png │ │ │ ├── circl-small.png │ │ │ ├── crawled-ddos.png │ │ │ ├── dancing.png │ │ │ ├── decoded-dos.png │ │ │ ├── flip-table.png │ │ │ ├── git-pass-table.png │ │ │ ├── git-pass.png │ │ │ ├── module-data-flow-alert.png │ │ │ ├── module-data-flow-tags-misp-feeder.png │ │ │ ├── module-data-flow-tags.png │ │ │ ├── module-data-flow.png │ │ │ ├── moduleManager.pmg │ │ │ ├── moduleManager.png │ │ │ ├── muscle.png │ │ │ ├── one-does-not-simply.jpeg │ │ │ ├── pastebin.png │ │ │ ├── pastebinOnion_2.png │ │ │ ├── pastebinOnion_3.png │ │ │ ├── pastebinOnion_4.png │ │ │ ├── pastebinOnion_5.png │ │ │ ├── pastebinOnion_6.png │ │ │ ├── pastes-ex-joke.png │ │ │ ├── pastes-ex.png │ │ │ ├── pub_1.png │ │ │ ├── recon-paste.png │ │ │ ├── sentiment.png │ │ │ ├── stream_exemp_cred.png │ │ │ ├── term-tracker-create.png │ │ │ ├── term-tracker.png │ │ │ ├── tr-46-1.png │ │ │ ├── tr-46.png │ │ │ └── uncle-sam.jpg │ │ ├── logo-circl.pdf │ │ └── screenshot │ │ │ ├── DashboardAIL.png │ │ │ ├── WordtrendingAIL.png │ │ │ ├── ail-1.png │ │ │ ├── ail-2.png │ │ │ ├── ail-3.png │ │ │ ├── ail-4.png │ │ │ ├── ail-5.png │ │ │ ├── ail-6.png │ │ │ ├── bitcoin-misp.png │ │ │ ├── browse-important.png │ │ │ ├── create-event-base64.png │ │ │ ├── create-misp-event-base64.png │ │ │ ├── dashboard.png │ │ │ ├── domain-misp.png │ │ │ ├── galaxies_list.png │ │ │ ├── galaxy_tag_edit.png │ │ │ ├── misp-export.png │ │ │ ├── module_information.png │ │ │ ├── paste_submit.png │ │ │ ├── paste_submitted.png │ │ │ ├── paste_tags_edit.png │ │ │ ├── sentiment.png │ │ │ ├── tag_auto_export.png │ │ │ ├── tag_delete_confirm.png │ │ │ ├── tags.png │ │ │ ├── tags2.png │ │ │ ├── tags_search.png │ │ │ ├── terms-manager.png │ │ │ ├── terms-plot.png │ │ │ ├── terms-top.png │ │ │ ├── trending-module.png │ │ │ └── trending-web.png │ └── workshop-sha2017.pdf └── screenshots │ ├── ail-bitcoinmixer.png │ ├── ail-internal.png │ ├── ail-lacus.png │ ├── ail-overview.png │ ├── chat_protocols.png │ ├── correlation_decoded_image.png │ ├── crawler-cookie-edit.png │ ├── crawler-cookiejar-all.png │ ├── crawler-cookiejar-create.png │ ├── crawler-cookiejar-domain-crawled.png │ ├── crawler-cookiejar-edit.png │ ├── crawler-manual-crawler.png │ ├── crawler_nb_captures.png │ ├── crawler_nb_captures_edit.png │ ├── dashboard0.png │ ├── decodeds_dashboard.png │ ├── domain_circl.png │ ├── galaxies_list.png │ ├── galaxy_tag_edit.png │ ├── hashedfile-graph.png │ ├── investigation_mixer.png │ ├── lacus_config.png │ ├── lacus_config_edit.png │ ├── misp_export.png │ ├── retro_hunt.png │ ├── sentiment.png │ ├── tag_delete_confirm.png │ ├── tags.png │ ├── tags_misp_auto.png │ ├── tags_search.png │ ├── tags_search_items.png │ ├── term-tracker.png │ ├── tracker_create.png │ ├── tracker_yara.png │ ├── trending-module.png │ ├── trending-web.png │ ├── ui_submit.png │ └── webshells.gif ├── files ├── ApiKey ├── Credential ├── CreditCards ├── Cve ├── Mail ├── Onion ├── SourceCode ├── Urls ├── make_Credential_list.sh ├── protocolsfile ├── tldsfile ├── top_pwd_clear └── wordfile ├── install_virtualenv.sh ├── installing_deps.sh ├── logs └── .keepdir ├── other_installers ├── LXD │ ├── .gitignore │ ├── INSTALL.sh │ ├── README.md │ ├── build │ │ ├── ailbuilder.py │ │ ├── build.sh │ │ ├── conf │ │ │ ├── lacus.service │ │ │ ├── sign.json.template │ │ │ └── tracker.json.template │ │ └── systemd │ │ │ ├── ailbuilder.service │ │ │ ├── setup.sh │ │ │ └── update.sh │ └── systemd │ │ └── lacus.service ├── ansible │ ├── .gitignore │ ├── Dockerfile.testing │ ├── README.md │ ├── deploy.sh │ ├── deploy.yml │ ├── deployLocal.yml │ ├── group_vars │ │ └── ail.yml │ └── roles │ │ └── ail-host │ │ ├── files │ │ ├── ail-flask.service │ │ ├── ail-leveldb.service │ │ ├── ail-logging.service │ │ ├── ail-queues.service │ │ ├── ail-redis.service │ │ ├── ail-scripts.service │ │ ├── startFlask.sh │ │ ├── startLogging.sh │ │ ├── startQueues.sh │ │ ├── startRedis.sh │ │ ├── startRedisLevelDB.sh │ │ ├── startScripts.sh │ │ └── staticBackup │ │ │ ├── css │ │ │ ├── bootstrap-rtl.css │ │ │ ├── bootstrap-rtl.min.css │ │ │ ├── bootstrap.css │ │ │ ├── bootstrap.min.css │ │ │ ├── dataTables.bootstrap.css │ │ │ ├── jquery-ui.min.css │ │ │ ├── plugins │ │ │ │ └── morris.css │ │ │ ├── sb-admin-2.css │ │ │ ├── sb-admin-rtl.css │ │ │ ├── sb-admin.css │ │ │ └── timeline.css │ │ │ ├── font-awesome │ │ │ ├── css │ │ │ │ ├── font-awesome.css │ │ │ │ └── font-awesome.min.css │ │ │ ├── fonts │ │ │ │ ├── FontAwesome.otf │ │ │ │ ├── fontawesome-webfont.eot │ │ │ │ ├── fontawesome-webfont.svg │ │ │ │ ├── fontawesome-webfont.ttf │ │ │ │ └── fontawesome-webfont.woff │ │ │ ├── less │ │ │ │ ├── bordered-pulled.less │ │ │ │ ├── core.less │ │ │ │ ├── fixed-width.less │ │ │ │ ├── font-awesome.less │ │ │ │ ├── icons.less │ │ │ │ ├── larger.less │ │ │ │ ├── list.less │ │ │ │ ├── mixins.less │ │ │ │ ├── path.less │ │ │ │ ├── rotated-flipped.less │ │ │ │ ├── spinning.less │ │ │ │ ├── stacked.less │ │ │ │ └── variables.less │ │ │ └── scss │ │ │ │ ├── _bordered-pulled.scss │ │ │ │ ├── _core.scss │ │ │ │ ├── _fixed-width.scss │ │ │ │ ├── _icons.scss │ │ │ │ ├── _larger.scss │ │ │ │ ├── _list.scss │ │ │ │ ├── _mixins.scss │ │ │ │ ├── _path.scss │ │ │ │ ├── _rotated-flipped.scss │ │ │ │ ├── _spinning.scss │ │ │ │ ├── _stacked.scss │ │ │ │ ├── _variables.scss │ │ │ │ └── font-awesome.scss │ │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ └── glyphicons-halflings-regular.woff2 │ │ │ ├── image │ │ │ └── AIL.png │ │ │ └── js │ │ │ ├── bootstrap.js │ │ │ ├── bootstrap.min.js │ │ │ ├── dataTables.bootstrap.js │ │ │ ├── dygraph-combined.js │ │ │ ├── jquery-ui.min.js │ │ │ ├── jquery.canvasjs.min.js │ │ │ ├── jquery.dataTables.min.js │ │ │ ├── jquery.flot.js │ │ │ ├── jquery.flot.pie.js │ │ │ ├── jquery.flot.stack.js │ │ │ ├── jquery.flot.time.js │ │ │ ├── jquery.js │ │ │ ├── jquery.sparkline.min.js │ │ │ └── plugins │ │ │ ├── flot │ │ │ ├── excanvas.min.js │ │ │ ├── flot-data.js │ │ │ ├── jquery.flot.js │ │ │ ├── jquery.flot.pie.js │ │ │ ├── jquery.flot.resize.js │ │ │ └── jquery.flot.tooltip.min.js │ │ │ └── morris │ │ │ ├── morris-data.js │ │ │ ├── morris.js │ │ │ ├── morris.min.js │ │ │ └── raphael.min.js │ │ └── tasks │ │ └── main.yml ├── centos_installing_deps.sh └── docker │ ├── .dockerignore │ ├── Dockerfile │ ├── README.md │ ├── docker-compose.yml │ ├── docker_start.sh │ └── pystemon │ ├── config.cfg │ ├── install.sh │ ├── proxies.txt │ └── pystemon.yaml ├── requirements.txt ├── reset_AIL.sh ├── samples └── 2021 │ └── 01 │ └── 01 │ ├── api_keys.gz │ ├── categ.gz │ ├── credit_cards.gz │ ├── domain_classifier.gz │ ├── keys.gz │ ├── keys_certificat_sample.gz │ └── onion.gz ├── tests ├── __init__.py ├── test_api.py ├── test_modules.py └── zmq_test.py ├── tools ├── crawler_add_task.py ├── file_dir_importer.py ├── gen_cert │ ├── README.md │ ├── ext3.cnf │ ├── gen_cert.sh │ ├── gen_root.sh │ └── san.cnf ├── manual_update_db.py └── reprocess_objects.py ├── update ├── bin │ ├── Update_Redis.sh │ └── ail_updater.py ├── default_update │ ├── Update.py │ └── Update.sh ├── v5.0 │ └── DB_KVROCKS_MIGRATION.py ├── v5.2 │ ├── Update.py │ ├── Update.sh │ └── compress_har.py ├── v5.3 │ ├── Update.py │ └── Update.sh ├── v5.4 │ ├── Update.py │ └── Update.sh ├── v5.5 │ ├── Update.py │ └── Update.sh ├── v5.7 │ ├── Update.py │ └── Update.sh ├── v5.8 │ ├── Update.py │ └── Update.sh ├── v5.9 │ ├── Update.py │ ├── Update.sh │ └── reprocess_dom_hash.py ├── v6.0.1 │ ├── Update.py │ └── Update.sh ├── v6.0 │ ├── Update.py │ └── Update.sh ├── v6.1 │ ├── Update.py │ └── Update.sh └── v6.2 │ ├── Update.py │ └── Update.sh └── var └── www ├── Flask_config.py ├── Flask_server.py ├── Role_Manager.py ├── blueprints ├── ail_2_ail_sync.py ├── api_rest.py ├── chats_explorer.py ├── correlation.py ├── crawler_splash.py ├── dashboard.py ├── hunters.py ├── import_export.py ├── investigations_b.py ├── languages_ui.py ├── objects_barcode.py ├── objects_cookie_name.py ├── objects_cve.py ├── objects_decoded.py ├── objects_dom_hash.py ├── objects_etag.py ├── objects_favicon.py ├── objects_file_name.py ├── objects_gtracker.py ├── objects_hhhash.py ├── objects_image.py ├── objects_item.py ├── objects_mail.py ├── objects_ocr.py ├── objects_qrcode.py ├── objects_subtypes.py ├── objects_title.py ├── old_endpoints.py ├── root.py ├── search_b.py ├── settings_b.py ├── tags_ui.py └── ui_submit.py ├── create_default_user.py ├── static ├── codemirror │ └── codemirror-yara │ │ ├── index.html │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── src │ │ └── main.js │ │ └── vite.config.js ├── css │ ├── ail-project.css │ ├── dygraph_gallery.css │ ├── switch_checkbox.css │ └── tags.css ├── csv │ └── .keepdir ├── image │ ├── ail-icon.png │ ├── ail-project-inverted.png │ ├── ail-project.png │ ├── ail.png │ ├── d4-logo.png │ ├── loading.gif │ ├── misp-logo.png │ ├── thehive-logo.png │ ├── thehive_icon.png │ └── tm.png └── js │ ├── FlexGauge.js │ ├── codemirror-yara.es.js │ ├── d3 │ ├── barchart_stack.js │ ├── chord_directed_diagram.js │ ├── graphlinesgroup.js │ ├── heatmap_week_hour.js │ ├── pie_chart.js │ ├── sparklines.js │ └── timeline_basic.js │ ├── helper.js │ ├── indexjavascript.js │ ├── moduleTrending.js │ ├── plot-graph.js │ ├── sentiment_plot.js │ ├── sentiment_trending.js │ ├── tags.js │ └── trendingchart.js ├── templates ├── ail_2_ail │ ├── add_ail_server.html │ ├── add_sync_queue.html │ ├── ail_2_ail_dashboard.html │ ├── ail_servers.html │ ├── edit_ail_server.html │ ├── edit_sync_queue.html │ ├── register_queue.html │ ├── sync_queues.html │ ├── view_ail_server.html │ └── view_sync_queue.html ├── change_password.html ├── chats_explorer │ ├── ChatMessage.html │ ├── ChatMessages.html │ ├── SubChannelMessages.html │ ├── ThreadMessages.html │ ├── basic_card_chat.html │ ├── basic_card_image.html │ ├── block_language_stats.html │ ├── block_message.html │ ├── block_obj_time_search.html │ ├── block_translation.html │ ├── card_chat.html │ ├── card_chat_subchannel.html │ ├── card_image.html │ ├── card_user_account.html │ ├── chat_instance.html │ ├── chat_monitoring_requests.html │ ├── chat_participants.html │ ├── chat_viewer.html │ ├── chats_instance.html │ ├── chats_networks.html │ ├── chats_protocols.html │ ├── pagination.html │ ├── request_chat_monitoring.html │ ├── user_account.html │ └── user_chat_messages.html ├── correlation │ ├── legend_graph_correlation.html │ ├── metadata_card_cookie_name.html │ ├── metadata_card_cryptocurrency.html │ ├── metadata_card_cve.html │ ├── metadata_card_decoded.html │ ├── metadata_card_dom_hash.html │ ├── metadata_card_domain.html │ ├── metadata_card_etag.html │ ├── metadata_card_favicon.html │ ├── metadata_card_filename.html │ ├── metadata_card_gtracker.html │ ├── metadata_card_hhhash.html │ ├── metadata_card_item.html │ ├── metadata_card_mail.html │ ├── metadata_card_pgp.html │ ├── metadata_card_screenshot.html │ ├── metadata_card_title.html │ ├── metadata_card_username.html │ ├── show_correlation.html │ └── show_relationship.html ├── crawler │ ├── crawler_disabled.html │ ├── crawler_splash │ │ ├── add_cookie.html │ │ ├── add_cookiejar.html │ │ ├── add_cookies_block.html │ │ ├── all_cookiejar.html │ │ ├── cookie_edit_block.html │ │ ├── cookies_card_block.html │ │ ├── crawler_blacklist.html │ │ ├── crawler_manual.html │ │ ├── crawler_schedule_uuid.html │ │ ├── crawler_scheduler_dashboard.html │ │ ├── dashboard_crawler.html │ │ ├── domain_explorer.html │ │ ├── domains_daterange.html │ │ ├── edit_cookie.html │ │ ├── explorer_vanity_clusters.html │ │ ├── explorer_vanity_domains.html │ │ ├── last_crawled.html │ │ ├── settings_crawler.html │ │ ├── settings_edit_crawlers_to_launch.html │ │ ├── settings_edit_lacus_crawler.html │ │ ├── showDomain.html │ │ ├── show_cookiejar.html │ │ └── table_cookiejar.html │ ├── menu_sidebar.html │ └── show_domains_by_daterange.html ├── dashboard │ ├── dashboard.html │ └── update_modal.html ├── decoded │ ├── decodeds_dashboard.html │ ├── show_helper_cryptocurrency.html │ └── show_helper_pgpdump.html ├── domains │ ├── block_domains_name_search.html │ ├── block_languages_search.html │ ├── card_img_domain.html │ ├── domains_filter_languages.html │ ├── domains_result_list.html │ └── filter_domains.html ├── error │ ├── 403.html │ └── 404.html ├── hunter │ ├── add_retro_hunt_task.html │ ├── menu_sidebar.html │ ├── messages_report.html │ ├── retro_hunt_tasks.html │ ├── show_retro_hunt.html │ ├── tracker_add.html │ ├── tracker_show.html │ ├── trackersManagement.html │ └── trackers_dashboard.html ├── import_export │ ├── block_add_user_object_to_export.html │ ├── block_add_user_object_to_export_small.html │ ├── block_create_misp_event.html │ ├── block_to_export_input.html │ ├── export_object.html │ └── import_object.html ├── investigations │ ├── add_investigation.html │ ├── investigations.html │ └── view_investigation.html ├── item_submit │ └── menu_sidebar.html ├── login.html ├── modals │ ├── add_tags.html │ ├── create_hive_case.html │ ├── edit_tag.html │ ├── investigations_register_obj.html │ ├── show_min_item.html │ └── tracker_remove_object.html ├── nav_bar.html ├── objects │ ├── barcode │ │ ├── BarcodeDaterange.html │ │ ├── ShowBarcode.html │ │ └── card_barcode.html │ ├── block_obj_button.html │ ├── block_object_footer_small.html │ ├── cookie-name │ │ ├── CookieNameDaterange.html │ │ ├── block_cookie_name_search.html │ │ └── search_cookie_name_result.html │ ├── cve │ │ ├── CveDaterange.html │ │ └── block_cve_search.html │ ├── dom-hash │ │ └── DomHashDaterange.html │ ├── etag │ │ └── EtagDaterange.html │ ├── favicon │ │ └── FaviconDaterange.html │ ├── file-name │ │ ├── FileNameDaterange.html │ │ ├── block_file_name_search.html │ │ └── search_file_name_result.html │ ├── gtracker │ │ ├── GtrackerDaterange.html │ │ ├── block_gtrackers_search.html │ │ └── search_gtracker_result.html │ ├── hhhash │ │ └── HHHashDaterange.html │ ├── image │ │ ├── ImageDaterange.html │ │ ├── block_blur_img_slider.html │ │ ├── block_domain_ollama.html │ │ ├── block_img_ollama.html │ │ └── block_img_ollama_small.html │ ├── item │ │ ├── show_item.html │ │ └── show_item_min.html │ ├── mail │ │ ├── MailDaterange.html │ │ ├── block_mails_search.html │ │ └── search_mail_result.html │ ├── obj_svg_block.html │ ├── objs_dashboard.html │ ├── ocr │ │ ├── OcrDaterange.html │ │ ├── ShowOcr.html │ │ └── card_ocr.html │ ├── qrcode │ │ ├── QrcodeDaterange.html │ │ ├── ShowQrcode.html │ │ └── card_qrcode.html │ ├── subtypes_objs_dashboard.html │ ├── title │ │ ├── TitleDaterange.html │ │ ├── block_titles_search.html │ │ └── search_title_result.html │ ├── tooltip_ail_objects.html │ └── username │ │ ├── block_usernames_search.html │ │ └── search_usernames_result.html ├── pagination.html ├── search │ ├── block_chats_search.html │ ├── block_crawled_search.html │ ├── pagination.html │ ├── search_chats.html │ ├── search_crawled.html │ ├── search_dashboard.html │ └── search_messages_result.html ├── settings │ ├── ail_configs.html │ ├── block_password_requirements.html │ ├── create_org.html │ ├── create_user.html │ ├── menu_sidebar.html │ ├── misp_create_instance.html │ ├── modules.html │ ├── orgs_list.html │ ├── passive_dns.html │ ├── settings_index.html │ ├── user_hotp.html │ ├── user_profile.html │ ├── users_list.html │ └── view_organisation.html ├── setup_otp.html ├── sidebars │ ├── sidebar_objects.html │ └── sidebar_search.html ├── submit │ └── submit_items.html ├── tags │ ├── block_obj_tags_search.html │ ├── block_tags_selector.html │ ├── galaxies.html │ ├── galaxy.html │ ├── galaxy_tag.html │ ├── menu_sidebar.html │ ├── search_obj_by_tags.html │ ├── tags_auto_push.html │ ├── taxonomies.html │ └── taxonomy.html └── verify_otp.html └── update_thirdparty.sh /.github/workflows/ail_framework_test.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: CI 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | push: 9 | branches: [ master, dev ] 10 | pull_request: 11 | branches: [ master, dev ] 12 | 13 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 14 | jobs: 15 | # This workflow contains a single job called "build" 16 | ail_test: 17 | # The type of runner that the job will run on 18 | runs-on: ubuntu-latest 19 | 20 | strategy: 21 | matrix: 22 | python-version: ['3.7', '3.8', '3.9', '3.10'] 23 | 24 | 25 | # Steps represent a sequence of tasks that will be executed as part of the job 26 | steps: 27 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 28 | - uses: actions/checkout@v3 29 | with: 30 | submodules: 'recursive' 31 | fetch-depth: 500 32 | 33 | 34 | # Runs a single command using the runners shell 35 | - name: Install AIL 36 | run: bash installing_deps.sh 37 | 38 | # Runs a set of commands using the runners shell 39 | - name: Launch AIL 40 | run: | 41 | pushd bin 42 | bash LAUNCH.sh -l 43 | popd 44 | 45 | # Runs a set of commands using the runners shell 46 | - name: Run tests 47 | run: | 48 | pushd bin 49 | bash LAUNCH.sh -t 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Temp files 2 | *.swp 3 | *.pyc 4 | *.swo 5 | .idea 6 | .coverage 7 | 8 | # Install Dirs 9 | AILENV 10 | redis 11 | ardb 12 | kvrocks 13 | DATA_KVROCKS 14 | faup 15 | tlsh 16 | Blooms 17 | PASTES 18 | CRAWLED_SCREENSHOT 19 | IMAGES 20 | FAVICONS 21 | BASE64 22 | HASHS 23 | DATA_ARDB 24 | indexdir/ 25 | logs/ 26 | old/ 27 | pgpdump/ 28 | temp/ 29 | 30 | 31 | DEFAULT_PASSWORD 32 | 33 | # Webstuff 34 | var/www/static/ 35 | !var/www/static/css/dygraph_gallery.css 36 | !var/www/static/js/indexjavascript.js 37 | !var/www/static/js/moduleTrending.js 38 | !var/www/static/js/plot-graph.js 39 | !var/www/static/js/trendingchart.js 40 | var/www/templates/header.html 41 | var/www/submitted 42 | var/www/server.crt 43 | var/www/server.key 44 | 45 | # Local config 46 | configs/keys 47 | configs/core.cfg 48 | configs/core.cfg.backup 49 | configs/update.cfg 50 | update/current_version 51 | files 52 | configs/d4client_passiveDNS_conf/uuid 53 | 54 | # Trackers 55 | bin/trackers/yara/custom-rules/* 56 | 57 | # Helper 58 | tools/gen_cert/rootCA.* 59 | tools/gen_cert/server.* 60 | 61 | 62 | # Pystemon archives 63 | pystemon/archives 64 | 65 | # installed files 66 | nltk_data/ 67 | doc/all_modules.txt 68 | # auto generated 69 | doc/module-data-flow.png 70 | doc/data-flow.png 71 | doc/ail_queues.dot 72 | doc/ail_queues.svg 73 | doc/statistics 74 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "bin/trackers/yara/ail-yara-rules"] 2 | path = bin/trackers/yara/ail-yara-rules 3 | url = https://github.com/ail-project/ail-yara-rules.git 4 | [submodule "files/misp-taxonomies"] 5 | path = files/misp-taxonomies 6 | url = https://github.com/MISP/misp-taxonomies.git 7 | [submodule "files/misp-galaxy"] 8 | path = files/misp-galaxy 9 | url = https://github.com/MISP/misp-galaxy.git 10 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting security vulnerabilities for AIL or related AIL project repositories 4 | 5 | Reporting security vulnerabilities is of great importance for us, as AIL is used by different security operators and organisations. 6 | 7 | In the case of a security vulnerability report, we ask the reporter to send it directly to [CIRCL](https://www.circl.lu/contact/), if possible encrypted with the following GnuPG key: **CA57 2205 C002 4E06 BA70 BE89 EAAD CFFC 22BD 4CD5**. We usually fix reported and confirmed security vulnerabilities in less than 48 hours, followed by a software release containing the fixes within the following days. 8 | 9 | If you report security vulnerabilities, do not forget to **tell us if and how you want to be acknowledged** and if you already requested CVE(s). Otherwise, we will request the CVE(s) directly. 10 | 11 | As one of the critical user-bases of AIL project consists of the CSIRT community, it is our duty to clearly state which bug could be abused and have a security impact on a AIL instance. CVE assignment is performed even for minor bugs suspected of having a security impact. This allows every user with AIL instances set up in their environments to understand which bugs could impact their security. 12 | 13 | We firmly believe that, even though unfortunately it is often not regarded as common practice in our industry, being as transparent as possible about vulnerabilities, no matter how minor, is of crucial importance. At AIL Project, we care about the security of our users and prefer to have a high number of published CVEs rather than sweeping some of them under the rug. 14 | -------------------------------------------------------------------------------- /bin/AIL_Init.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | AIL Init 5 | ============================ 6 | 7 | Init DB + Clear Stats 8 | 9 | """ 10 | 11 | import os 12 | import sys 13 | import logging.config 14 | 15 | sys.path.append(os.environ['AIL_BIN']) 16 | ################################## 17 | # Import Project packages 18 | ################################## 19 | from lib import ail_core 20 | from lib import ail_queues 21 | from lib import ail_logger 22 | 23 | logging.config.dictConfig(ail_logger.get_config(name='modules')) 24 | logger = logging.getLogger() 25 | 26 | if __name__ == "__main__": 27 | ail_queues.save_queue_digraph() 28 | ail_queues.clear_modules_queues_stats() 29 | 30 | # Send module state to logs 31 | ail_uuid = ail_core.get_ail_uuid() 32 | logger.warning(f"AIL {ail_uuid} started") 33 | -------------------------------------------------------------------------------- /bin/core/D4_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | """ 5 | The D4_Client Module 6 | ============================ 7 | 8 | The D4_Client modules send all DNS records to a D4 Server. 9 | Data produced by D4 sensors are ingested into 10 | a Passive DNS server which can be queried later to search for the Passive DNS records. 11 | """ 12 | 13 | import os 14 | import sys 15 | import time 16 | 17 | sys.path.append(os.environ['AIL_BIN']) 18 | ################################## 19 | # Import Project packages 20 | ################################# 21 | from modules.abstract_module import AbstractModule 22 | from lib import d4 23 | 24 | # # TODO: launch me in core screen 25 | # # TODO: check if already launched in core screen 26 | 27 | class D4Client(AbstractModule): 28 | """ 29 | D4Client module for AIL framework 30 | """ 31 | 32 | def __init__(self): 33 | super(D4Client, self).__init__() 34 | 35 | self.d4_client = d4.create_d4_client() 36 | self.last_refresh = time.time() 37 | self.last_config_check = time.time() 38 | 39 | # Send module state to logs 40 | self.logger.info(f'Module {self.module_name} initialized') 41 | 42 | def compute(self, dns_record): 43 | # Refresh D4 Client 44 | if self.last_config_check < int(time.time()) - 30: 45 | print('refresh rrrr') 46 | if self.last_refresh < d4.get_config_last_update_time(): 47 | self.d4_client = d4.create_d4_client() 48 | self.last_refresh = time.time() 49 | print('D4 Client: config updated') 50 | self.last_config_check = time.time() 51 | 52 | if self.d4_client: 53 | # Send DNS Record to D4Server 54 | self.d4_client.send_manual_data(dns_record) 55 | 56 | 57 | if __name__ == '__main__': 58 | module = D4Client() 59 | module.run() 60 | -------------------------------------------------------------------------------- /bin/core/Sync_manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | import time 7 | 8 | sys.path.append(os.environ['AIL_BIN']) 9 | ################################## 10 | # Import Project packages 11 | ################################## 12 | from core import ail_2_ail 13 | 14 | # # TODO: launch me in core screen 15 | 16 | if __name__ == '__main__': 17 | 18 | Client_Manager = ail_2_ail.AIL2AILClientManager() 19 | 20 | while True: 21 | command = Client_Manager.get_manager_command() 22 | if command: 23 | Client_Manager.execute_manager_command(command) 24 | else: 25 | time.sleep(5) 26 | -------------------------------------------------------------------------------- /bin/crawlers/blacklist.txt: -------------------------------------------------------------------------------- 1 | www.facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion 2 | facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion 3 | keybase5wmilwokqirssclfnsqrjdsi7jdir5wy7y7iu3tanwmtp6oid.onion 4 | -------------------------------------------------------------------------------- /bin/exporter/abstract_exporter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | Importer Class 5 | ================ 6 | 7 | Import Content 8 | 9 | """ 10 | import os 11 | import sys 12 | 13 | from abc import ABC, abstractmethod 14 | 15 | 16 | # sys.path.append(os.environ['AIL_BIN']) 17 | ################################## 18 | # Import Project packages 19 | ################################## 20 | # from ConfigLoader import ConfigLoader 21 | 22 | class AbstractExporter(ABC): 23 | def __init__(self): 24 | """ 25 | Init Module 26 | """ 27 | # Module name if provided else instance className 28 | self.name = self._name() 29 | 30 | def _name(self): 31 | """ 32 | Returns the instance class name (ie. the Exporter Name) 33 | """ 34 | return self.__class__.__name__ 35 | 36 | @abstractmethod 37 | def export(self, *args, **kwargs): 38 | """Importer function""" 39 | pass 40 | # res = self.export(*args, **kwargs) 41 | # if self.next_exporter: 42 | # self.next_exporter.exporter(res) 43 | 44 | 45 | -------------------------------------------------------------------------------- /bin/exporter/default_mail_template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | default_subject = 'AIL services credentials' 5 | 6 | def get_default_template(user_id, password): 7 | return f'''Dear AIL user, 8 | 9 | Here are your credentials to access the AIL services. 10 | 11 | Afterward, you can use the following credentials and setup for your MFA: 12 | 13 | Login: {user_id} 14 | Password: {password} 15 | 16 | We hope this information is helpful. 17 | 18 | Best regards,''' 19 | -------------------------------------------------------------------------------- /bin/importer/feeders/BgpMonitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The Bgp Monitor Feeder Importer Module 5 | ================ 6 | 7 | Process Bgp Monitor JSON 8 | 9 | """ 10 | import os 11 | import sys 12 | 13 | sys.path.append(os.environ['AIL_BIN']) 14 | ################################## 15 | # Import Project packages 16 | ################################## 17 | from importer.feeders.Default import DefaultFeeder 18 | from lib.objects.Items import Item 19 | 20 | 21 | class BgpMonitorFeeder(DefaultFeeder): 22 | """BgpMonitorFeeder Feeder functions""" 23 | 24 | def __init__(self, json_data): 25 | super().__init__(json_data) 26 | self.name = 'bgp_monitor' 27 | 28 | def process_meta(self): 29 | """ 30 | Process JSON meta filed. 31 | """ 32 | # DIRTY FIX 33 | tag = 'infoleak:automatic-detection=bgp_monitor' 34 | item = Item(self.get_item_id()) 35 | item.add_tag(tag) 36 | return set() 37 | -------------------------------------------------------------------------------- /bin/importer/feeders/Discord.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The Telegram Feeder Importer Module 5 | ================ 6 | 7 | Process Telegram JSON 8 | 9 | """ 10 | import os 11 | import sys 12 | import datetime 13 | 14 | sys.path.append(os.environ['AIL_BIN']) 15 | ################################## 16 | # Import Project packages 17 | ################################## 18 | from importer.feeders.abstract_chats_feeder import AbstractChatFeeder 19 | from lib.ConfigLoader import ConfigLoader 20 | from lib.objects import ail_objects 21 | from lib.objects.Chats import Chat 22 | from lib.objects import Messages 23 | from lib.objects import UsersAccount 24 | from lib.objects.Usernames import Username 25 | 26 | import base64 27 | 28 | class DiscordFeeder(AbstractChatFeeder): 29 | 30 | def __init__(self, json_data): 31 | super().__init__('discord', json_data) 32 | 33 | # def get_obj(self):. 34 | # obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) 35 | # obj_id = f'message:telegram:{obj_id}' 36 | # self.obj = ail_objects.get_obj_from_global_id(obj_id) 37 | # return self.obj 38 | 39 | -------------------------------------------------------------------------------- /bin/importer/feeders/Jabber.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The Jabber Feeder Importer Module 5 | ================ 6 | 7 | Process Jabber JSON 8 | 9 | """ 10 | import os 11 | import sys 12 | import time 13 | 14 | sys.path.append(os.environ['AIL_BIN']) 15 | ################################## 16 | # Import Project packages 17 | ################################## 18 | from importer.feeders.Default import DefaultFeeder 19 | from lib.objects.Usernames import Username 20 | from lib.objects.Items import Item 21 | 22 | 23 | class JabberFeeder(DefaultFeeder): 24 | """Jabber Feeder functions""" 25 | 26 | def __init__(self, json_data): 27 | super().__init__(json_data) 28 | self.name = 'jabber' 29 | 30 | # define item id 31 | def get_item_id(self): 32 | date = time.strptime(self.json_data['meta']['jabber:ts'], "%Y-%m-%dT%H:%M:%S.%f") 33 | date_str = time.strftime("%Y/%m/%d", date) 34 | item_id = str(self.json_data['meta']['jabber:id']) 35 | item_id = os.path.join('jabber', date_str, item_id) 36 | self.item_id = f'{item_id}.gz' 37 | return self.item_id 38 | 39 | def process_meta(self): # TODO replace me by message 40 | """ 41 | Process JSON meta field. 42 | """ 43 | # jabber_id = str(self.json_data['meta']['jabber:id']) 44 | # item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ############################################## 45 | to = str(self.json_data['meta']['jabber:to']) 46 | fr = str(self.json_data['meta']['jabber:from']) 47 | 48 | item = Item(self.item_id) 49 | date = item.get_date() 50 | 51 | user_to = Username(to, 'jabber') 52 | user_fr = Username(fr, 'jabber') 53 | user_to.add(date, item) 54 | user_fr.add(date, item) 55 | return set() 56 | -------------------------------------------------------------------------------- /bin/importer/feeders/Matrix.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The Telegram Feeder Importer Module 5 | ================ 6 | 7 | Process Telegram JSON 8 | 9 | """ 10 | import os 11 | import sys 12 | import datetime 13 | 14 | sys.path.append(os.environ['AIL_BIN']) 15 | ################################## 16 | # Import Project packages 17 | ################################## 18 | from importer.feeders.abstract_chats_feeder import AbstractChatFeeder 19 | from lib.ConfigLoader import ConfigLoader 20 | from lib.objects import ail_objects 21 | from lib.objects.Chats import Chat 22 | from lib.objects import Messages 23 | from lib.objects import UsersAccount 24 | from lib.objects.Usernames import Username 25 | 26 | import base64 27 | 28 | class MatrixFeeder(AbstractChatFeeder): 29 | 30 | def __init__(self, json_data): 31 | super().__init__('matrix', json_data) 32 | 33 | # def get_obj(self):. 34 | # obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) 35 | # obj_id = f'message:telegram:{obj_id}' 36 | # self.obj = ail_objects.get_obj_from_global_id(obj_id) 37 | # return self.obj 38 | 39 | -------------------------------------------------------------------------------- /bin/importer/feeders/Telegram.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The Telegram Feeder Importer Module 5 | ================ 6 | 7 | Process Telegram JSON 8 | 9 | """ 10 | import os 11 | import sys 12 | import datetime 13 | 14 | sys.path.append(os.environ['AIL_BIN']) 15 | ################################## 16 | # Import Project packages 17 | ################################## 18 | from importer.feeders.abstract_chats_feeder import AbstractChatFeeder 19 | from lib.ConfigLoader import ConfigLoader 20 | from lib.objects import ail_objects 21 | from lib.objects.Chats import Chat 22 | from lib.objects import Messages 23 | from lib.objects import UsersAccount 24 | from lib.objects.Usernames import Username 25 | 26 | import base64 27 | 28 | class TelegramFeeder(AbstractChatFeeder): 29 | 30 | def __init__(self, json_data): 31 | super().__init__('telegram', json_data) 32 | 33 | # def get_obj(self):. 34 | # obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) 35 | # obj_id = f'message:telegram:{obj_id}' 36 | # self.obj = ail_objects.get_obj_from_global_id(obj_id) 37 | # return self.obj 38 | 39 | -------------------------------------------------------------------------------- /bin/importer/feeders/Twitter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The Twitter Feeder Importer Module 5 | ================ 6 | 7 | Process Twitter JSON 8 | 9 | """ 10 | import os 11 | import sys 12 | import datetime 13 | 14 | sys.path.append(os.environ['AIL_BIN']) 15 | ################################## 16 | # Import Project packages 17 | ################################## 18 | from importer.feeders.Default import DefaultFeeder 19 | from lib.objects.Usernames import Username 20 | from lib.objects.Items import Item 21 | 22 | class TwitterFeeder(DefaultFeeder): 23 | 24 | def __init__(self, json_data): 25 | super().__init__(json_data) 26 | self.name = 'twitter' 27 | 28 | # define item id 29 | def get_item_id(self): 30 | # TODO twitter timestamp message date 31 | date = datetime.date.today().strftime("%Y/%m/%d") 32 | item_id = str(self.json_data['meta']['twitter:tweet_id']) 33 | item_id = os.path.join('twitter', date, item_id) 34 | self.item_id = f'{item_id}.gz' 35 | return self.item_id 36 | 37 | def process_meta(self): 38 | ''' 39 | Process JSON meta field. 40 | ''' 41 | # tweet_id = str(self.json_data['meta']['twitter:tweet_id']) 42 | # item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################ 43 | item = Item(self.item_id) 44 | date = item.get_date() 45 | user = str(self.json_data['meta']['twitter:id']) 46 | username = Username(user, 'twitter') 47 | username.add(date, item) 48 | return set() 49 | -------------------------------------------------------------------------------- /bin/importer/feeders/Urlextract.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The JSON Receiver Module 5 | ================ 6 | 7 | Recieve Json Items (example: Twitter feeder) 8 | 9 | """ 10 | import os 11 | import sys 12 | import datetime 13 | import uuid 14 | 15 | sys.path.append(os.environ['AIL_BIN']) 16 | ################################## 17 | # Import Project packages 18 | ################################## 19 | from importer.feeders.Default import DefaultFeeder 20 | from lib.objects.Items import Item 21 | 22 | 23 | class UrlextractFeeder(DefaultFeeder): 24 | 25 | def __init__(self, json_data): 26 | super().__init__(json_data) 27 | self.name = 'urlextract' 28 | 29 | # define item id 30 | def get_item_id(self): 31 | date = datetime.date.today().strftime("%Y/%m/%d") 32 | item_id = str(self.json_data['meta']['twitter:url-extracted']) 33 | item_id = item_id.split('//') 34 | if len(item_id) > 1: 35 | item_id = ''.join(item_id[1:]) 36 | else: 37 | item_id = item_id[0] 38 | item_id = item_id.replace('/', '_') 39 | # limit ID length 40 | if len(item_id) > 215: 41 | item_id = item_id[:215] 42 | item_id = f'{item_id}{str(uuid.uuid4())}.gz' 43 | self.item_id = os.path.join('urlextract', date, item_id) 44 | return self.item_id 45 | 46 | def process_meta(self): 47 | """ 48 | Process JSON meta field. 49 | """ 50 | # ADD Other parents here 51 | parent_id = None 52 | if self.json_data['meta'].get('parent:twitter:tweet_id'): 53 | parent_id = str(self.json_data['meta']['parent:twitter:tweet_id']) 54 | 55 | if parent_id: 56 | item = Item(self.item_id) 57 | item.set_parent(parent_id) 58 | 59 | return set() 60 | 61 | -------------------------------------------------------------------------------- /bin/lib/exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | # from pymisp import PyMISPError 5 | 6 | # SIGNAL ALARM 7 | class TimeoutException(Exception): 8 | pass 9 | 10 | class AILError(Exception): 11 | def __init__(self, message): 12 | super(AILError, self).__init__(message) 13 | self.message = message 14 | 15 | class UpdateInvestigationError(AILError): 16 | pass 17 | 18 | class NewTagError(AILError): 19 | pass 20 | 21 | class ModuleQueueError(AILError): 22 | pass 23 | 24 | class MISPConnectionError(AILError): 25 | pass 26 | 27 | class AILObjectUnknown(AILError): 28 | pass 29 | 30 | class OnionFilteringError(AILError): 31 | pass 32 | -------------------------------------------------------------------------------- /bin/modules/CveModule.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The CVE Module 5 | ====================== 6 | 7 | This module is consuming the Redis-list created by the Categ module. 8 | 9 | It apply CVE regexes on paste content and warn if a reference to a CVE is spotted. 10 | 11 | """ 12 | 13 | ################################## 14 | # Import External packages 15 | ################################## 16 | import os 17 | import re 18 | import sys 19 | 20 | sys.path.append(os.environ['AIL_BIN']) 21 | ################################## 22 | # Import Project packages 23 | ################################## 24 | from modules.abstract_module import AbstractModule 25 | from lib.objects import Cves 26 | from lib.objects.Items import Item 27 | 28 | 29 | class CveModule(AbstractModule): 30 | """ 31 | CveModule for AIL framework 32 | """ 33 | 34 | def __init__(self): 35 | super(CveModule, self).__init__() 36 | 37 | # regex to find CVE 38 | self.reg_cve = re.compile(r'CVE-[1-2]\d{1,4}-\d{1,5}') 39 | 40 | # Waiting time in seconds between to message processed 41 | self.pending_seconds = 1 42 | 43 | # Send module state to logs 44 | self.logger.info(f'Module {self.module_name} initialized') 45 | 46 | def compute(self, message): 47 | item = self.get_obj() 48 | item_id = item.get_id() 49 | 50 | cves = self.regex_findall(self.reg_cve, item_id, item.get_content()) 51 | if cves: 52 | # print(cves) 53 | date = item.get_date() 54 | for cve_id in cves: 55 | cve = Cves.Cve(cve_id) 56 | cve.add(date, item) 57 | 58 | print(f'{self.obj.get_global_id()} contains CVEs {cves}') 59 | 60 | tag = 'infoleak:automatic-detection="cve"' 61 | # Send to Tags Queue 62 | self.add_message_to_queue(message=tag, queue='Tags') 63 | 64 | 65 | if __name__ == '__main__': 66 | 67 | module = CveModule() 68 | module.run() 69 | -------------------------------------------------------------------------------- /bin/modules/Indexer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | """ 5 | The Indexer Module 6 | ============================ 7 | 8 | AIL Indexer 9 | 10 | """ 11 | ################################## 12 | # Import External packages 13 | ################################## 14 | import os 15 | import sys 16 | import time 17 | 18 | 19 | sys.path.append(os.environ['AIL_BIN']) 20 | ################################## 21 | # Import Project packages 22 | ################################## 23 | from modules.abstract_module import AbstractModule 24 | from lib.ConfigLoader import ConfigLoader 25 | from lib import search_engine 26 | 27 | 28 | class Indexer(AbstractModule): 29 | """ 30 | Indexer module for AIL framework 31 | """ 32 | 33 | def __init__(self): 34 | """ 35 | Init Instance 36 | """ 37 | super(Indexer, self).__init__() 38 | 39 | config_loader = ConfigLoader() 40 | 41 | self.is_enabled_meilisearch = search_engine.is_meilisearch_enabled() 42 | 43 | def compute(self, message): 44 | obj = self.get_obj() 45 | if self.is_enabled_meilisearch and obj: 46 | if self.obj.type == 'message': 47 | search_engine.index_message(obj) 48 | 49 | 50 | if __name__ == '__main__': 51 | module = Indexer() 52 | module.run() 53 | -------------------------------------------------------------------------------- /bin/modules/Languages.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_BIN']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from modules.abstract_module import AbstractModule 12 | from lib.objects.Domains import Domain 13 | from lib.objects.Items import Item 14 | # from lib.ConfigLoader import ConfigLoader 15 | 16 | class Languages(AbstractModule): 17 | """ 18 | Languages module for AIL framework 19 | """ 20 | 21 | def __init__(self): 22 | super(Languages, self).__init__() 23 | 24 | # Send module state to logs 25 | self.logger.info(f'Module {self.module_name} initialized') 26 | 27 | def compute(self, message): 28 | obj = self.get_obj() 29 | 30 | if obj.type == 'item': 31 | if obj.is_crawled(): 32 | domain = Domain(obj.get_domain()) 33 | for lang in obj.get_languages(min_probability=0.8, force_gcld3=True): 34 | print(lang) 35 | domain.add_language(lang) 36 | # Detect Chat Message Language 37 | elif obj.type == 'message': 38 | lang = obj.detect_language() 39 | print(self.obj.id, lang) 40 | 41 | 42 | if __name__ == '__main__': 43 | module = Languages() 44 | module.run() 45 | -------------------------------------------------------------------------------- /bin/modules/ModuleStats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | This module makes statistics for some modules and providers 5 | 6 | """ 7 | 8 | ################################## 9 | # Import External packages # 10 | ################################## 11 | import os 12 | import sys 13 | 14 | sys.path.append(os.environ['AIL_BIN']) 15 | ################################## 16 | # Import Project packages # 17 | ################################## 18 | from modules.abstract_module import AbstractModule 19 | from lib.objects.Items import Item 20 | from lib import Statistics 21 | 22 | 23 | class ModuleStats(AbstractModule): 24 | """ 25 | Module Statistics module for AIL framework 26 | """ 27 | 28 | def __init__(self): 29 | 30 | super(ModuleStats, self).__init__() 31 | 32 | # Waiting time in seconds between to message processed 33 | self.pending_seconds = 20 34 | 35 | def compute(self, message): 36 | 37 | # MODULE STATS 38 | if len(message.split(';')) > 1: 39 | module_name, num, keyword, date = message.split(';') 40 | Statistics.update_module_stats(module_name, num, keyword, date) 41 | # ITEM STATS 42 | else: 43 | item_id = message 44 | item = Item(item_id) 45 | source = item.get_source() 46 | date = item.get_date() 47 | size = item.get_size() 48 | Statistics.update_item_stats_size_nb(item_id, source, size, date) 49 | 50 | 51 | if __name__ == '__main__': 52 | 53 | module = ModuleStats() 54 | module.run() 55 | -------------------------------------------------------------------------------- /bin/modules/README.md: -------------------------------------------------------------------------------- 1 | AIL MODULES 2 | === 3 | -------------------------------------------------------------------------------- /bin/modules/Tags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | """ 5 | The Tags Module 6 | ================================ 7 | 8 | This module add tags to an object. 9 | 10 | """ 11 | 12 | ################################## 13 | # Import External packages 14 | ################################## 15 | import os 16 | import sys 17 | 18 | sys.path.append(os.environ['AIL_BIN']) 19 | ################################## 20 | # Import Project packages 21 | ################################## 22 | from modules.abstract_module import AbstractModule 23 | 24 | class Tags(AbstractModule): 25 | """ 26 | Tags module for AIL framework 27 | """ 28 | 29 | def __init__(self): 30 | super(Tags, self).__init__() 31 | 32 | # Waiting time in seconds between to message processed 33 | self.pending_seconds = 10 34 | 35 | # Send module state to logs 36 | self.logger.info(f'Module {self.module_name} initialized') 37 | 38 | def compute(self, message): 39 | obj = self.obj 40 | tag = message 41 | 42 | # Create a new tag 43 | obj.add_tag(tag) 44 | print(f'{self.obj.get_global_id()}: Tagged {tag}') 45 | 46 | # Forward message to channel 47 | self.add_message_to_queue(message=tag, queue='Tag_feed') 48 | 49 | if __name__ == '__main__': 50 | module = Tags() 51 | module.run() 52 | -------------------------------------------------------------------------------- /bin/modules/TemplateModule.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | """ 4 | The Template Module 5 | ====================== 6 | 7 | This module is a template for Template for new modules 8 | 9 | """ 10 | 11 | ################################## 12 | # Import External packages 13 | ################################## 14 | import os 15 | import sys 16 | import time 17 | 18 | sys.path.append(os.environ['AIL_BIN']) 19 | ################################## 20 | # Import Project packages 21 | ################################## 22 | from modules.abstract_module import AbstractModule 23 | # from lib.objects.Items import Item 24 | 25 | class Template(AbstractModule): 26 | """ 27 | Template module for AIL framework 28 | """ 29 | 30 | def __init__(self): 31 | super(Template, self).__init__() 32 | 33 | # Pending time between two computation (computeNone) in seconds, 10 by default 34 | # self.pending_seconds = 10 35 | 36 | # logs 37 | self.logger.info(f'Module {self.module_name} initialized') 38 | 39 | # def computeNone(self): 40 | # """ 41 | # Do something when there is no message in the queue. Optional 42 | # """ 43 | # self.logger.debug("No message in queue") 44 | 45 | def compute(self, message): 46 | """ 47 | Compute a message in queue / process the message (item_id, ...) 48 | """ 49 | self.logger.debug("Compute message in queue") 50 | # # if message is an item_id: 51 | # item = Item(message) 52 | # content = item.get_content() 53 | 54 | 55 | if __name__ == '__main__': 56 | module = Template() 57 | module.run() 58 | -------------------------------------------------------------------------------- /bin/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/bin/modules/__init__.py -------------------------------------------------------------------------------- /bin/packages/Hash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import hashlib 4 | import crcmod 5 | import mmh3 6 | import ssdeep 7 | import tlsh 8 | 9 | 10 | class Hash(object): 11 | """docstring for Hash""" 12 | def __init__(self, name): 13 | self.name = name 14 | 15 | def __str__(self): 16 | return "{0}".format(self.name) 17 | 18 | def _get_hash_name(self): 19 | return self.name 20 | 21 | def _set_hash_name(self, name): 22 | self.p_hash_name = name 23 | 24 | def Calculate(self, string): 25 | if self.name == "md5": 26 | hash = hashlib.md5(string).hexdigest() 27 | 28 | elif self.name == "sha1": 29 | hash = hashlib.sha1(string).hexdigest() 30 | 31 | elif self.name == "crc": 32 | crc32 = crcmod.Crc(0x104c11db7, initCrc=0, xorOut=0xFFFFFFFF) 33 | crc32.update(string) 34 | hash = crc32.hexdigest() 35 | 36 | elif self.name == "murmur": 37 | hash = mmh3.hash(string) 38 | 39 | elif self.name == "ssdeep": 40 | hash = ssdeep.hash(string) 41 | 42 | elif self.name == "tlsh": 43 | hash = tlsh.hash(string) 44 | 45 | return hash 46 | -------------------------------------------------------------------------------- /bin/packages/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/bin/packages/__init__.py -------------------------------------------------------------------------------- /bin/trackers/yara/custom-rules/test.yar: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Test Rule 4 | */ 5 | 6 | rule certificatestest 7 | { 8 | strings: 9 | $ssh_priv = "BEGIN RSA PRIVATE KEY" wide ascii nocase 10 | $pem_cert = "BEGIN CERTIFICATE" wide ascii nocase 11 | 12 | condition: 13 | any of them 14 | } 15 | -------------------------------------------------------------------------------- /configs/d4client_passiveDNS_conf/destination: -------------------------------------------------------------------------------- 1 | d4pdns.circl.lu:4443 2 | -------------------------------------------------------------------------------- /configs/d4client_passiveDNS_conf/key: -------------------------------------------------------------------------------- 1 | ail passivedns sensor key 2 | -------------------------------------------------------------------------------- /configs/d4client_passiveDNS_conf/snaplen: -------------------------------------------------------------------------------- 1 | 4096 2 | -------------------------------------------------------------------------------- /configs/d4client_passiveDNS_conf/source: -------------------------------------------------------------------------------- 1 | stdin 2 | -------------------------------------------------------------------------------- /configs/d4client_passiveDNS_conf/type: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /configs/d4client_passiveDNS_conf/version: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /configs/docker/splash_onion/etc/splash/proxy-profiles/default.ini: -------------------------------------------------------------------------------- 1 | [proxy] 2 | host=172.17.0.1 3 | port=9050 4 | type=SOCKS5 5 | -------------------------------------------------------------------------------- /configs/keys/mispKEYS.py.sample: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | misp_url = 'https://10.1.0.143' 5 | misp_key = 'c5VRXJahYVux3rkPULbbILTNmAwlXU8Eas9zvl36' # The MISP auth key can be found on the MISP web interface under the automation section 6 | misp_verifycert = True 7 | -------------------------------------------------------------------------------- /configs/keys/theHiveKEYS.py.sample: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | the_hive_url = 'http://10.1.0.145:9000' 5 | the_hive_key = 'KfiPGz3Bi8E5styWbC2eU4WiTKDGdf22' # The Hive auth key can be found on the The Hive web interface under the User Management 6 | the_hive_verifycert = True 7 | -------------------------------------------------------------------------------- /configs/keys/virusTotalKEYS.py.sample: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | vt_key = 'YOUR_API_KEY' #virus total API key 5 | -------------------------------------------------------------------------------- /configs/logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "disable_existing_loggers": false, 4 | "formatters": { 5 | "simple": { 6 | "format": "%(asctime)s %(name)s %(levelname)s:%(message)s" 7 | } 8 | }, 9 | "handlers": { 10 | "stdout": { 11 | "class": "logging.StreamHandler", 12 | "level": "INFO", 13 | "stream": "ext://sys.stdout" 14 | }, 15 | "file": { 16 | "class": "logging.handlers.RotatingFileHandler", 17 | "level": "WARNING", 18 | "formatter": "simple", 19 | "filename": "logs/flask_warning.log", 20 | "mode": "a", 21 | "maxBytes": 1000000, 22 | "backupCount": 5 23 | } 24 | }, 25 | "root": { 26 | "level": "DEBUG", 27 | "handlers": [ 28 | "stdout", 29 | "file" 30 | ] 31 | } 32 | } 33 | 34 | -------------------------------------------------------------------------------- /configs/update.cfg.sample: -------------------------------------------------------------------------------- 1 | [Update] 2 | auto_update = True 3 | upstream = upstream 4 | update-fork = False 5 | -------------------------------------------------------------------------------- /doc/ail_modules_queues.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/ail_modules_queues.png -------------------------------------------------------------------------------- /doc/presentation/ail-pass-the-salt.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/ail-pass-the-salt.pdf -------------------------------------------------------------------------------- /doc/presentation/ail-training-december-2018.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/ail-training-december-2018.pdf -------------------------------------------------------------------------------- /doc/presentation/ail-training-luxembourg-2018.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/ail-training-luxembourg-2018.pdf -------------------------------------------------------------------------------- /doc/presentation/ail-training-november.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/ail-training-november.pdf -------------------------------------------------------------------------------- /doc/presentation/source-training/ail-training.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/ail-training.pdf -------------------------------------------------------------------------------- /doc/presentation/source-training/beamercolorthemechameleon.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2007 by Marco Barisione 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | 8 | \mode 9 | 10 | %\definecolor{chameleongreen1}{RGB}{129,138,143} %grey CIRCL 11 | \definecolor{chameleongreen1}{RGB}{213,43,30} 12 | \definecolor{chameleongreen2}{RGB}{213,43,30} %red CIRCL 13 | \definecolor{chameleongreen3}{RGB}{30,30,30} %black CIRCL 14 | \definecolor{chameleongreen4}{RGB}{129,138,143} 15 | 16 | \setbeamercolor*{palette primary}{fg=white,bg=chameleongreen2} 17 | \setbeamercolor*{palette secondary}{fg=white,bg=chameleongreen3} 18 | \setbeamercolor*{palette tertiary}{fg=white,bg=chameleongreen4} 19 | \setbeamercolor*{palette quaternary}{fg=white,bg=chameleongreen1} 20 | 21 | \setbeamercolor*{titlelike}{bg=chameleongreen3} 22 | \setbeamercolor*{frametitle}{bg=black,fg=black} 23 | \setbeamercolor*{part title}{bg=black,fg=black} 24 | \setbeamercolor*{item}{fg=chameleongreen3} 25 | 26 | \setbeamercolor*{separation line}{} 27 | \setbeamercolor*{fine separation line}{} 28 | 29 | \mode 30 | 31 | -------------------------------------------------------------------------------- /doc/presentation/source-training/beamerthemeTorino.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2007 by Marco Barisione 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | 8 | \mode 9 | 10 | \DeclareOptionBeamer{alternativetitlepage}[true]{\PassOptionsToPackage{alternativetitlepage=#1}{beamerinnerthemefancy}} 11 | \DeclareOptionBeamer{titlepagelogo}{\PassOptionsToPackage{titlepagelogo=#1}{beamerinnerthemefancy}} 12 | \DeclareOptionBeamer{bullet}{\PassOptionsToPackage{bullet=#1}{beamerinnerthemefancy}} 13 | \DeclareOptionBeamer{pageofpages}{\PassOptionsToPackage{pageofpages=#1}{beamerouterthemedecolines}} 14 | \DeclareOptionBeamer{titleline}[true]{\PassOptionsToPackage{titleline=#1}{beamerouterthemedecolines}} 15 | \DeclareOptionBeamer{watermark}{\PassOptionsToPackage{watermark=#1}{beamerouterthemedecolines}} 16 | \DeclareOptionBeamer{watermarkheight}{\PassOptionsToPackage{watermarkheight=#1}{beamerouterthemedecolines}} 17 | \DeclareOptionBeamer{watermarkheightmult}{\PassOptionsToPackage{watermarkheightmult=#1}{beamerouterthemedecolines}} 18 | 19 | \ProcessOptionsBeamer 20 | 21 | \useinnertheme{fancy} 22 | \useoutertheme{decolines} 23 | \usecolortheme{chameleon} 24 | 25 | \setbeamertemplate{navigation symbols}{} 26 | 27 | \mode 28 | 29 | -------------------------------------------------------------------------------- /doc/presentation/source-training/clean.sh: -------------------------------------------------------------------------------- 1 | rm *.aux *.listing *.log *.nav *.out *.snm *.toc *.vrb 2 | -------------------------------------------------------------------------------- /doc/presentation/source-training/cmds_summary/beamercolorthemechameleon.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2007 by Marco Barisione 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | 8 | \mode 9 | 10 | %\definecolor{chameleongreen1}{RGB}{129,138,143} %grey CIRCL 11 | \definecolor{chameleongreen1}{RGB}{213,43,30} 12 | \definecolor{chameleongreen2}{RGB}{213,43,30} %red CIRCL 13 | \definecolor{chameleongreen3}{RGB}{30,30,30} %black CIRCL 14 | \definecolor{chameleongreen4}{RGB}{129,138,143} 15 | 16 | \setbeamercolor*{palette primary}{fg=white,bg=chameleongreen2} 17 | \setbeamercolor*{palette secondary}{fg=white,bg=chameleongreen3} 18 | \setbeamercolor*{palette tertiary}{fg=white,bg=chameleongreen4} 19 | \setbeamercolor*{palette quaternary}{fg=white,bg=chameleongreen1} 20 | 21 | \setbeamercolor*{titlelike}{bg=chameleongreen3} 22 | \setbeamercolor*{frametitle}{bg=black,fg=black} 23 | \setbeamercolor*{part title}{bg=black,fg=black} 24 | \setbeamercolor*{item}{fg=chameleongreen3} 25 | 26 | \setbeamercolor*{separation line}{} 27 | \setbeamercolor*{fine separation line}{} 28 | 29 | \mode 30 | 31 | -------------------------------------------------------------------------------- /doc/presentation/source-training/cmds_summary/beamerthemeTorino.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2007 by Marco Barisione 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | 8 | \mode 9 | 10 | \DeclareOptionBeamer{alternativetitlepage}[true]{\PassOptionsToPackage{alternativetitlepage=#1}{beamerinnerthemefancy}} 11 | \DeclareOptionBeamer{titlepagelogo}{\PassOptionsToPackage{titlepagelogo=#1}{beamerinnerthemefancy}} 12 | \DeclareOptionBeamer{bullet}{\PassOptionsToPackage{bullet=#1}{beamerinnerthemefancy}} 13 | \DeclareOptionBeamer{pageofpages}{\PassOptionsToPackage{pageofpages=#1}{beamerouterthemedecolines}} 14 | \DeclareOptionBeamer{titleline}[true]{\PassOptionsToPackage{titleline=#1}{beamerouterthemedecolines}} 15 | \DeclareOptionBeamer{watermark}{\PassOptionsToPackage{watermark=#1}{beamerouterthemedecolines}} 16 | \DeclareOptionBeamer{watermarkheight}{\PassOptionsToPackage{watermarkheight=#1}{beamerouterthemedecolines}} 17 | \DeclareOptionBeamer{watermarkheightmult}{\PassOptionsToPackage{watermarkheightmult=#1}{beamerouterthemedecolines}} 18 | 19 | \ProcessOptionsBeamer 20 | 21 | \useinnertheme{fancy} 22 | \useoutertheme{decolines} 23 | \usecolortheme{chameleon} 24 | 25 | \setbeamertemplate{navigation symbols}{} 26 | 27 | \mode 28 | 29 | -------------------------------------------------------------------------------- /doc/presentation/source-training/cmds_summary/circl-introduction.listing: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/cmds_summary/circl-introduction.listing -------------------------------------------------------------------------------- /doc/presentation/source-training/cmds_summary/content-introduction.tex: -------------------------------------------------------------------------------- 1 | % DO NOT COMPILE THIS FILE DIRECTLY! 2 | % This is included by the other .tex files. 3 | 4 | \begin{frame}[t,plain] 5 | %\titlepage 6 | \end{frame} 7 | 8 | \section{Useful commands} 9 | \begin{frame} 10 | \frametitle{Useful commands} 11 | 12 | \begin{tcblisting}{colback=black!85,coltext=green,listing only, title=Setting up AIL-Framework from source, fonttitle=\bfseries} 13 | git clone https://github.com/ail-project/ail-framework.git 14 | cd AIL-framework 15 | ./installing_deps.sh 16 | cd var/www/ 17 | ./update_thirdparty.sh 18 | \end{tcblisting} 19 | 20 | % \begin{tcblisting}{colback=black!85,coltext=green,listing only, 21 | % title=Accessing the environment and starting AIL, fonttitle=\bfseries} 22 | %# Activate the virtualenv 23 | %. ./AILENV/bin/activate 24 | % 25 | %# Launch the system 26 | %cd bin/ 27 | %./LAUNCH 28 | % # check options 1->5 29 | % 30 | %# Start web interface 31 | %cd var/www/ 32 | %./Flask_server.py 33 | % # -> Browse http://localhost:7000/ 34 | %\end{tcblisting} 35 | % 36 | % \lstset{style=default} 37 | % \begin{lstlisting} 38 | %Web interface (default network settings): 39 | % http://192.168.56.51:7000/ 40 | %Shell/SSH: 41 | % ail/Password1234 42 | %\end{lstlisting} 43 | 44 | \end{frame} 45 | -------------------------------------------------------------------------------- /doc/presentation/source-training/images/MISP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/MISP.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail-correlation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail-correlation.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail-project.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_01.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_02.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_03.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_04.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_06.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_07.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_07.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_08.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_08.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_09.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_09.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_10.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_11.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_12.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_13.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_14.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_15.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_16.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_17.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_crf_10min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_crf_10min.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_empty_1min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_empty_1min.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/ail_pyst_10min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/ail_pyst_10min.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/alert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/alert.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/circl-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/circl-small.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/crawled-ddos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/crawled-ddos.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/dancing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/dancing.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/decoded-dos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/decoded-dos.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/flip-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/flip-table.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/git-pass-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/git-pass-table.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/git-pass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/git-pass.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/module-data-flow-alert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/module-data-flow-alert.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/module-data-flow-tags-misp-feeder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/module-data-flow-tags-misp-feeder.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/module-data-flow-tags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/module-data-flow-tags.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/module-data-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/module-data-flow.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/moduleManager.pmg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/moduleManager.pmg -------------------------------------------------------------------------------- /doc/presentation/source-training/images/moduleManager.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/moduleManager.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/muscle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/muscle.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/one-does-not-simply.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/one-does-not-simply.jpeg -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastebin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastebin.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastebinOnion_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastebinOnion_2.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastebinOnion_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastebinOnion_3.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastebinOnion_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastebinOnion_4.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastebinOnion_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastebinOnion_5.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastebinOnion_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastebinOnion_6.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastes-ex-joke.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastes-ex-joke.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pastes-ex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pastes-ex.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/pub_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/pub_1.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/recon-paste.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/recon-paste.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/sentiment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/sentiment.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/stream_exemp_cred.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/stream_exemp_cred.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/term-tracker-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/term-tracker-create.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/term-tracker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/term-tracker.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/tr-46-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/tr-46-1.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/tr-46.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/tr-46.png -------------------------------------------------------------------------------- /doc/presentation/source-training/images/uncle-sam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/images/uncle-sam.jpg -------------------------------------------------------------------------------- /doc/presentation/source-training/logo-circl.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/logo-circl.pdf -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/DashboardAIL.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/DashboardAIL.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/WordtrendingAIL.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/WordtrendingAIL.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/ail-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/ail-1.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/ail-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/ail-2.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/ail-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/ail-3.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/ail-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/ail-4.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/ail-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/ail-5.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/ail-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/ail-6.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/bitcoin-misp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/bitcoin-misp.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/browse-important.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/browse-important.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/create-event-base64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/create-event-base64.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/create-misp-event-base64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/create-misp-event-base64.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/dashboard.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/domain-misp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/domain-misp.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/galaxies_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/galaxies_list.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/galaxy_tag_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/galaxy_tag_edit.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/misp-export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/misp-export.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/module_information.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/module_information.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/paste_submit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/paste_submit.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/paste_submitted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/paste_submitted.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/paste_tags_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/paste_tags_edit.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/sentiment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/sentiment.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/tag_auto_export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/tag_auto_export.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/tag_delete_confirm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/tag_delete_confirm.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/tags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/tags.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/tags2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/tags2.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/tags_search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/tags_search.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/terms-manager.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/terms-manager.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/terms-plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/terms-plot.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/terms-top.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/terms-top.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/trending-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/trending-module.png -------------------------------------------------------------------------------- /doc/presentation/source-training/screenshot/trending-web.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/source-training/screenshot/trending-web.png -------------------------------------------------------------------------------- /doc/presentation/workshop-sha2017.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/presentation/workshop-sha2017.pdf -------------------------------------------------------------------------------- /doc/screenshots/ail-bitcoinmixer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/ail-bitcoinmixer.png -------------------------------------------------------------------------------- /doc/screenshots/ail-internal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/ail-internal.png -------------------------------------------------------------------------------- /doc/screenshots/ail-lacus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/ail-lacus.png -------------------------------------------------------------------------------- /doc/screenshots/ail-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/ail-overview.png -------------------------------------------------------------------------------- /doc/screenshots/chat_protocols.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/chat_protocols.png -------------------------------------------------------------------------------- /doc/screenshots/correlation_decoded_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/correlation_decoded_image.png -------------------------------------------------------------------------------- /doc/screenshots/crawler-cookie-edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler-cookie-edit.png -------------------------------------------------------------------------------- /doc/screenshots/crawler-cookiejar-all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler-cookiejar-all.png -------------------------------------------------------------------------------- /doc/screenshots/crawler-cookiejar-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler-cookiejar-create.png -------------------------------------------------------------------------------- /doc/screenshots/crawler-cookiejar-domain-crawled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler-cookiejar-domain-crawled.png -------------------------------------------------------------------------------- /doc/screenshots/crawler-cookiejar-edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler-cookiejar-edit.png -------------------------------------------------------------------------------- /doc/screenshots/crawler-manual-crawler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler-manual-crawler.png -------------------------------------------------------------------------------- /doc/screenshots/crawler_nb_captures.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler_nb_captures.png -------------------------------------------------------------------------------- /doc/screenshots/crawler_nb_captures_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/crawler_nb_captures_edit.png -------------------------------------------------------------------------------- /doc/screenshots/dashboard0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/dashboard0.png -------------------------------------------------------------------------------- /doc/screenshots/decodeds_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/decodeds_dashboard.png -------------------------------------------------------------------------------- /doc/screenshots/domain_circl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/domain_circl.png -------------------------------------------------------------------------------- /doc/screenshots/galaxies_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/galaxies_list.png -------------------------------------------------------------------------------- /doc/screenshots/galaxy_tag_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/galaxy_tag_edit.png -------------------------------------------------------------------------------- /doc/screenshots/hashedfile-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/hashedfile-graph.png -------------------------------------------------------------------------------- /doc/screenshots/investigation_mixer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/investigation_mixer.png -------------------------------------------------------------------------------- /doc/screenshots/lacus_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/lacus_config.png -------------------------------------------------------------------------------- /doc/screenshots/lacus_config_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/lacus_config_edit.png -------------------------------------------------------------------------------- /doc/screenshots/misp_export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/misp_export.png -------------------------------------------------------------------------------- /doc/screenshots/retro_hunt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/retro_hunt.png -------------------------------------------------------------------------------- /doc/screenshots/sentiment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/sentiment.png -------------------------------------------------------------------------------- /doc/screenshots/tag_delete_confirm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/tag_delete_confirm.png -------------------------------------------------------------------------------- /doc/screenshots/tags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/tags.png -------------------------------------------------------------------------------- /doc/screenshots/tags_misp_auto.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/tags_misp_auto.png -------------------------------------------------------------------------------- /doc/screenshots/tags_search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/tags_search.png -------------------------------------------------------------------------------- /doc/screenshots/tags_search_items.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/tags_search_items.png -------------------------------------------------------------------------------- /doc/screenshots/term-tracker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/term-tracker.png -------------------------------------------------------------------------------- /doc/screenshots/tracker_create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/tracker_create.png -------------------------------------------------------------------------------- /doc/screenshots/tracker_yara.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/tracker_yara.png -------------------------------------------------------------------------------- /doc/screenshots/trending-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/trending-module.png -------------------------------------------------------------------------------- /doc/screenshots/trending-web.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/trending-web.png -------------------------------------------------------------------------------- /doc/screenshots/ui_submit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/ui_submit.png -------------------------------------------------------------------------------- /doc/screenshots/webshells.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/doc/screenshots/webshells.gif -------------------------------------------------------------------------------- /files/ApiKey: -------------------------------------------------------------------------------- 1 | amazon 2 | amazonaws 3 | amzn 4 | aws 5 | googleapis 6 | -------------------------------------------------------------------------------- /files/CreditCards: -------------------------------------------------------------------------------- 1 | mastercard 2 | MA 3 | visa 4 | jcb 5 | discover 6 | americanexpress 7 | amex 8 | dankort 9 | cogeban 10 | banrisul 11 | interac 12 | discovercard 13 | unionpay 14 | CB 15 | rupay 16 | verve 17 | Mir 18 | QIWI 19 | WebMoney 20 | -------------------------------------------------------------------------------- /files/Cve: -------------------------------------------------------------------------------- 1 | CVE 2 | -------------------------------------------------------------------------------- /files/Mail: -------------------------------------------------------------------------------- 1 | gmail 2 | hotmail 3 | yahoo 4 | orange 5 | seznam 6 | yandex 7 | gmx 8 | laposte 9 | wanadoo 10 | numericable 11 | darty 12 | bouygues 13 | aol 14 | outlook 15 | hushmail 16 | yandex 17 | runbox 18 | rediff 19 | rackspace 20 | mail 21 | email 22 | msn 23 | inbox 24 | zoho 25 | @ 26 | -------------------------------------------------------------------------------- /files/Onion: -------------------------------------------------------------------------------- 1 | onion 2 | i2p 3 | -------------------------------------------------------------------------------- /files/SourceCode: -------------------------------------------------------------------------------- 1 | CVE 2 | exploit 3 | vulnerability 4 | payload 5 | uname 6 | chmod 7 | adduser 8 | base64_decode 9 | gzinflate 10 | -------------------------------------------------------------------------------- /files/Urls: -------------------------------------------------------------------------------- 1 | http 2 | https 3 | www 4 | -------------------------------------------------------------------------------- /files/make_Credential_list.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | while read line; do 6 | echo $line 7 | echo -n $line | md5sum | cut -d' ' -f 1 8 | echo -n $line | sha1sum | cut -d' ' -f 1 9 | echo -n $line | sha256sum | cut -d' ' -f 1 10 | done < top_pwd_clear > Credential 11 | -------------------------------------------------------------------------------- /files/protocolsfile: -------------------------------------------------------------------------------- 1 | afs 2 | file 3 | ftp 4 | z39.50 5 | z39.50r 6 | z39.50s 7 | vemmi 8 | urn 9 | nfs 10 | dict 11 | acap 12 | rtspu 13 | rtsp 14 | rtsps 15 | tip 16 | pop 17 | cid 18 | mid 19 | data 20 | thismessage 21 | service 22 | shttp 23 | fax 24 | modem 25 | tv 26 | sip 27 | sips 28 | go 29 | icap 30 | h323 31 | ipp 32 | xmlrpc.beep 33 | xmlrpc.beeps 34 | tftp 35 | mupdate 36 | pres 37 | im 38 | mtqp 39 | tel 40 | iris 41 | iris.beep 42 | crid 43 | snmp 44 | tag 45 | wais 46 | prospero 47 | soap.beep 48 | soap.beeps 49 | telnet 50 | gopher 51 | cap 52 | info 53 | dns 54 | ldap 55 | dav 56 | opaquelocktoken 57 | msrp 58 | msrps 59 | dtn 60 | imap 61 | xmpp 62 | iax 63 | news 64 | nntp 65 | snews 66 | sms 67 | rsync 68 | sieve 69 | geo 70 | mailto 71 | jms 72 | mailserver 73 | ipn 74 | tn3270 75 | ws 76 | wss 77 | xcon 78 | xcon-userid 79 | about 80 | aaa 81 | aaas 82 | session 83 | ni 84 | nih 85 | reload 86 | ham 87 | stun 88 | stuns 89 | turn 90 | turns 91 | http 92 | https 93 | coap 94 | coaps 95 | rtmfp 96 | ipps 97 | pkcs11 98 | acct 99 | example 100 | vnc 101 | -------------------------------------------------------------------------------- /files/top_pwd_clear: -------------------------------------------------------------------------------- 1 | 000000 2 | 111111 3 | 123123 4 | 1234 5 | 12345 6 | 123456 7 | 1234567 8 | 12345678 9 | 123456789 10 | 1234567890 11 | 1qaz2wsx 12 | abc123 13 | adobe123 14 | azerty 15 | baseball 16 | dragon 17 | football 18 | iloveyou 19 | letmein 20 | login 21 | master 22 | monkey 23 | passw0rd 24 | password 25 | password1 26 | photoshop 27 | princess 28 | qwerty 29 | qwertyuiop 30 | shadow 31 | solo 32 | starwars 33 | sunshine 34 | trustno1 35 | welcome 36 | -------------------------------------------------------------------------------- /files/wordfile: -------------------------------------------------------------------------------- 1 | password 2 | oppetrol 3 | bitcoin 4 | opsavegaza 5 | AnonGhost 6 | Israel 7 | Gaza 8 | creditcard 9 | tracker 10 | compromise 11 | malware 12 | cert 13 | flaw 14 | exploit 15 | 0day 16 | leak 17 | database 18 | terrorist 19 | anonymous 20 | opsyria 21 | OpHackingCup 22 | OpIsrael 23 | OpBigBrother 24 | dump 25 | forex 26 | vulnerable 27 | phishing 28 | account 29 | stolen 30 | wikileaks 31 | gmail 32 | hotmail 33 | hackingteam 34 | finfisher 35 | gamma-international 36 | COTTONMOUTH 37 | COTTONMOUTH-I 38 | COTTONMOUTH-II 39 | COTTONMOUTH-III 40 | DISHFIRE 41 | TURMOIL 42 | SCREAMINGHARPY 43 | SECUREINSIGHT 44 | SHAREDTAFFY 45 | WAGONBED 46 | WILLOWVIXEN 47 | -------------------------------------------------------------------------------- /install_virtualenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # halt on errors 4 | set -e 5 | 6 | ## bash debug mode toggle below 7 | #set -x 8 | 9 | if [ -z "$VIRTUAL_ENV" ]; then 10 | 11 | virtualenv -p python3 AILENV 12 | 13 | echo export AIL_HOME=$(pwd) >> ./AILENV/bin/activate 14 | echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate 15 | echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate 16 | echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate 17 | echo export AIL_KVROCKS=$(pwd)/kvrocks/src/ >> ./AILENV/bin/activate 18 | 19 | fi 20 | 21 | 22 | # activate virtual environment 23 | . ./AILENV/bin/activate 24 | 25 | 26 | pip3 install -U pip 27 | pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client' 28 | pip3 install -U -r requirements.txt 29 | 30 | # Pyfaup 31 | pushd faup/src/lib/bindings/python/ 32 | python3 setup.py install 33 | popd 34 | 35 | # Py tlsh 36 | pushd tlsh/py_ext 37 | python3 setup.py build 38 | python3 setup.py install 39 | 40 | # Download the necessary NLTK corpora and sentiment vader 41 | HOME=$(pwd) python3 -m textblob.download_corpora 42 | python3 -m nltk.downloader vader_lexicon 43 | python3 -m nltk.downloader punkt 44 | popd 45 | 46 | pushd ${AIL_FLASK} 47 | ./update_thirdparty.sh 48 | popd 49 | -------------------------------------------------------------------------------- /logs/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/logs/.keepdir -------------------------------------------------------------------------------- /other_installers/LXD/.gitignore: -------------------------------------------------------------------------------- 1 | build/conf/sign.json 2 | build/conf/tracker.json 3 | images/ 4 | 5 | -------------------------------------------------------------------------------- /other_installers/LXD/build/conf/lacus.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=lacus service 3 | After=network.target 4 | 5 | [Service] 6 | User=root 7 | Group=root 8 | Type=forking 9 | WorkingDirectory=/root/lacus 10 | Environment="PATH=/root/.local/bin/poetry:/usr/bin" 11 | Environment="LACUS_HOME=/root/lacus" 12 | ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start" 13 | ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop" 14 | StandardOutput=append:/var/log/lacus_message.log 15 | StandardError=append:/var/log/lacus_error.log 16 | 17 | 18 | [Install] 19 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /other_installers/LXD/build/conf/sign.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "NAME": "admin", 3 | "EMAIL": "admin@admin.test", 4 | "COMMENT": "Key for signing images", 5 | "EXPIRE_DATE": 0, 6 | "PASSPHRASE": "admin" 7 | } -------------------------------------------------------------------------------- /other_installers/LXD/build/conf/tracker.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "check_interval": 600, 3 | "outputdir": "/opt/ailbuilder/images", 4 | "sign": true, 5 | "github": [ 6 | { 7 | "name": "AIL", 8 | "id": "ail-project/ail-framework", 9 | "mode": "commits", 10 | "args": [ 11 | "--ail", 12 | "--ail-name", 13 | "AIL" 14 | ] 15 | }, 16 | { 17 | "name": "Lacus", 18 | "id": "ail-project/lacus", 19 | "mode": "commits", 20 | "args": [ 21 | "--lacus", 22 | "--lacus-name", 23 | "Lacus" 24 | ] 25 | } 26 | ], 27 | "apt": [] 28 | } -------------------------------------------------------------------------------- /other_installers/LXD/build/systemd/ailbuilder.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Service for building AIL and Lacus LXD images 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | User=ailbuilder 8 | ExecStart=/usr/bin/python3 /opt/ailbuilder/build/ailbuilder.py 9 | Restart=on-failure 10 | Environment=PYTHONUNBUFFERED=1 11 | 12 | [Install] 13 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /other_installers/LXD/build/systemd/update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SERVICE_FILE="ailbuilder.service" 4 | SERVICE_PATH="/etc/systemd/system/" 5 | MISP_AIRGAP_PATH="/opt/ailbuilder" 6 | DIR="$(dirname "$0")" 7 | BUILD_DIR="${DIR}/../../build" 8 | 9 | 10 | log() { 11 | echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" 12 | } 13 | echo "Updating ailbuilder service ..." 14 | 15 | if systemctl is-active --quiet ailbuilder; then 16 | systemctl stop ailbuilder || { log "Failed to stop service"; exit 1; } 17 | systemctl disable ailbuilder || { log "Failed to disable service"; exit 1; } 18 | fi 19 | 20 | if [[ -f "$SERVICE_FILE" ]]; then 21 | cp "${SERVICE_FILE}" "${SERVICE_PATH}" || { log "Failed to copy service file"; exit 1; } 22 | else 23 | log "Service file $SERVICE_FILE not found" 24 | exit 1 25 | fi 26 | 27 | if [[ -d "$BUILD_DIR" ]]; then 28 | cp -r "$BUILD_DIR" "$MISP_AIRGAP_PATH/" || { log "Failed to copy build directory"; exit 1; } 29 | else 30 | log "Build directory $BUILD_DIR does not exist" 31 | exit 1 32 | fi 33 | 34 | systemctl daemon-reload || { log "Failed to reload systemd"; exit 1; } 35 | systemctl enable ailbuilder || { log "Failed to enable service"; exit 1; } 36 | systemctl start ailbuilder || { log "Failed to start service"; exit 1; } 37 | -------------------------------------------------------------------------------- /other_installers/LXD/systemd/lacus.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=lacus service 3 | After=network.target 4 | 5 | [Service] 6 | User=root 7 | Group=root 8 | Type=forking 9 | WorkingDirectory=/root/lacus 10 | Environment="PATH=/root/.local/bin/poetry:/usr/bin" 11 | Environment="LACUS_HOME=/root/lacus" 12 | ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start" 13 | ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop" 14 | StandardOutput=append:/var/log/lacus_message.log 15 | StandardError=append:/var/log/lacus_error.log 16 | 17 | 18 | [Install] 19 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /other_installers/ansible/.gitignore: -------------------------------------------------------------------------------- 1 | inventories/ 2 | **inventories/ 3 | *inventories/ 4 | -------------------------------------------------------------------------------- /other_installers/ansible/Dockerfile.testing: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | 3 | # Install Ansible 4 | RUN apt-get -y update && \ 5 | apt-get -y install \ 6 | software-properties-common && \ 7 | apt-add-repository ppa:ansible/ansible && \ 8 | apt-get update && \ 9 | apt-get -y install ansible 10 | 11 | # Add the playbook 12 | ADD . /tmp/AIL-framework-ansible 13 | 14 | # Run the AIL role on localhost 15 | RUN ansible-playbook /tmp/AIL-framework-ansible/deployLocal.yml -c local 16 | -------------------------------------------------------------------------------- /other_installers/ansible/deploy.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | ansible-playbook deploy.yml -i inventories/develop 4 | -------------------------------------------------------------------------------- /other_installers/ansible/deploy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: ail 3 | remote_user: "{{remote_user}}" 4 | roles: 5 | - ail-host 6 | -------------------------------------------------------------------------------- /other_installers/ansible/deployLocal.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: 127.0.0.1 3 | remote_user: root 4 | roles: 5 | - ail-host 6 | -------------------------------------------------------------------------------- /other_installers/ansible/group_vars/ail.yml: -------------------------------------------------------------------------------- 1 | --- 2 | remote_user: root 3 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/ail-flask.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIL-Framework: Flask service 3 | Requires=ail-redis.service ail-scripts.service 4 | After=ail-redis.service ail-scripts.service 5 | 6 | [Service] 7 | User=ail 8 | Group=ail 9 | ExecStartPre=/bin/sleep 10 10 | ExecStart=/bin/bash -c "/opt/AIL-scripts/startFlask.sh" 11 | [Install] 12 | WantedBy=reboot.target 13 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/ail-leveldb.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIL-Framework: LevelDB services 3 | Requires=ail-redis.service 4 | After=ail-redis.service 5 | 6 | [Service] 7 | User=ail 8 | Group=ail 9 | ExecStartPre=/bin/sleep 10 10 | ExecStart=/bin/bash -c "/opt/AIL-scripts/startRedisLevelDB.sh" 11 | [Install] 12 | WantedBy=reboot.target 13 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/ail-logging.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIL-Framework: LevelDB services 3 | Requires=ail-redis.service ail-leveldb.service 4 | After=ail-redis.servic ail-leveldb.service 5 | 6 | [Service] 7 | User=ail 8 | Group=ail 9 | ExecStartPre=/bin/sleep 10 10 | ExecStart=/bin/bash -c "/opt/AIL-scripts/startLogging.sh" 11 | 12 | [Install] 13 | WantedBy=reboot.target 14 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/ail-queues.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIL-Framework: Queue service 3 | Requires=ail-redis.service ail-logging.service 4 | After=ail-redis.service ail-logging.service 5 | 6 | [Service] 7 | Restart=always 8 | RestartSec=5 9 | User=ail 10 | Group=ail 11 | ExecStartPre=/bin/sleep 10 12 | ExecStart=/bin/bash -c "/opt/AIL-scripts/startQueues.sh" 13 | 14 | [Install] 15 | WantedBy=reboot.target 16 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/ail-redis.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIL-Framework: Redis services 3 | 4 | [Service] 5 | Restart=always 6 | RestartSec=5 7 | User=ail 8 | Group=ail 9 | LimitNOFILE=65536 10 | ExecStart=/bin/bash -c "/opt/AIL-scripts/startRedis.sh" 11 | 12 | [Install] 13 | WantedBy=reboot.target 14 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/ail-scripts.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIL-Framework: Scripts service 3 | Requires=ail-redis.service ail-logging.service 4 | After=ail-redis.service ail-logging.service 5 | 6 | [Service] 7 | Restart=always 8 | RestartSec=5 9 | User=ail 10 | Group=ail 11 | ExecStartPre=/bin/sleep 10 12 | ExecStart=/bin/bash -c "/opt/AIL-scripts/startScripts.sh" 13 | 14 | [Install] 15 | WantedBy=reboot.target 16 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/startFlask.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /opt/AIL-framework/AILENV/bin/activate 4 | 5 | cd /opt/AIL-framework/var/www 6 | /usr/bin/python2 /opt/AIL-framework/var/www/Flask_server.py 7 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/startLogging.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /opt/AIL-framework/AILENV/bin/activate 4 | /bin/mkdir -p /var/log/AIL 5 | /bin/bash -c "/usr/local/bin/log_subscriber -p 6380 -c Queuing -l /var/log/AIL&" 6 | /usr/local/bin/log_subscriber -p 6380 -c Script -l /var/log/AIL 7 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/startQueues.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /opt/AIL-framework/bin 4 | source /opt/AIL-framework/AILENV/bin/activate 5 | /usr/bin/python /opt/AIL-framework/bin/launch_queues.py 6 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/startRedis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /opt/AIL-framework/configs && \ 4 | /bin/bash -c "/usr/bin/redis-server /opt/AIL-framework/configs/6379.conf &" && \ 5 | /bin/bash -c "/usr/bin/redis-server /opt/AIL-framework/configs/6380.conf &" && \ 6 | /bin/bash -c "/usr/bin/redis-server /opt/AIL-framework/configs/6381.conf &" && \ 7 | /bin/bash -c "/usr/bin/redis-server /opt/AIL-framework/configs/6382.conf" 8 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/startRedisLevelDB.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | numberOfDBs=2 4 | currentYear=`/bin/date +%Y` 5 | # -1 because the last process has to be spawned in foreground 6 | let "untilYearLoop = currentYear - 1" 7 | let "startYear = untilYearLoop-numberOfDBs" 8 | 9 | for year in $(seq $startYear $untilYearLoop); do 10 | mkdir -p /opt/AIL-framework/LEVEL_DB_DATA/$year 11 | /opt/redis-leveldb/redis-leveldb -d -H 127.0.0.1 -D /opt/AIL-framework/LEVEL_DB_DATA/$year -P $year -M 13 & 12 | done 13 | 14 | # Spawn the last instance 15 | mkdir -p /opt/AIL-framework/LEVEL_DB_DATA/2018 16 | /opt/redis-leveldb/redis-leveldb -H 127.0.0.1 -D /opt/AIL-framework/LEVEL_DB_DATA/$currentYear -P $currentYear -M 13 17 | 18 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/startScripts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Wait for all redis instances to start 4 | while true; do redis-cli -p 6379 PING && break; sleep 1; done 5 | while true; do redis-cli -p 6380 PING && break; sleep 1; done 6 | while true; do redis-cli -p 6381 PING && break; sleep 1; done 7 | while true; do redis-cli -p 6382 PING && break; sleep 1; done 8 | 9 | # Wait for redis to initialize - otherwise CurveManageTopSets.py 10 | # might crash 11 | sleep 10 12 | 13 | source /opt/AIL-framework/AILENV/bin/activate 14 | cd /opt/AIL-framework/bin/ 15 | 16 | # Note: Before adding a script, test if it works and that all 17 | # dependencies are available. Otherwise the whole service will fail. 18 | 19 | /opt/AIL-framework/bin/ModuleInformation.py -k 0 -c 1 & 20 | /opt/AIL-framework/bin/Mixer.py & 21 | /opt/AIL-framework/bin/Global.py & 22 | /opt/AIL-framework/bin/Duplicates.py & 23 | /opt/AIL-framework/bin/Attributes.py & 24 | /opt/AIL-framework/bin/Lines.py & 25 | /opt/AIL-framework/bin/DomClassifier.py & 26 | /opt/AIL-framework/bin/Categ.py & 27 | /opt/AIL-framework/bin/Tokenize.py & 28 | /opt/AIL-framework/bin/CreditCards.py & 29 | /opt/AIL-framework/bin/Onion.py & 30 | /opt/AIL-framework/bin/Mail.py & 31 | /opt/AIL-framework/bin/Web.py & 32 | /opt/AIL-framework/bin/Credential.py & 33 | /opt/AIL-framework/bin/Curve.py & 34 | 35 | # This crashes if redis isn't ready - execute it in a loop 36 | bash -c "while true; do /opt/AIL-framework/bin/CurveManageTopSets.py; done" & 37 | 38 | /opt/AIL-framework/bin/Indexer.py & 39 | /opt/AIL-framework/bin/Keys.py & 40 | /opt/AIL-framework/bin/Phone.py & 41 | /opt/AIL-framework/bin/Release.py & 42 | /opt/AIL-framework/bin/Cve.py & 43 | /opt/AIL-framework/bin/WebStats.py & 44 | /opt/AIL-framework/bin/ModuleStats.py & 45 | /opt/AIL-framework/bin/SQLInjectionDetection.py & 46 | /opt/AIL-framework/bin/alertHandler.py & 47 | /opt/AIL-framework/bin/RegexForTermsFrequency.py & 48 | /opt/AIL-framework/bin/SetForTermsFrequency.py & 49 | /opt/AIL-framework/bin/SentimentAnalysis.py 50 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css: -------------------------------------------------------------------------------- 1 | .morris-hover{position:absolute;z-index:1000}.morris-hover.morris-default-style{border-radius:10px;padding:6px;color:#666;background:rgba(255,255,255,0.8);border:solid 2px rgba(230,230,230,0.8);font-family:sans-serif;font-size:12px;text-align:center}.morris-hover.morris-default-style .morris-hover-row-label{font-weight:bold;margin:0.25em 0} 2 | .morris-hover.morris-default-style .morris-hover-point{white-space:nowrap;margin:0.1em 0} 3 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css: -------------------------------------------------------------------------------- 1 | 2 | @media (min-width: 768px){ 3 | #wrapper {padding-right: 225px; padding-left: 0;} 4 | .side-nav{right: 0;left: auto;} 5 | } -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less: -------------------------------------------------------------------------------- 1 | // Bordered & Pulled 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-border { 5 | padding: .2em .25em .15em; 6 | border: solid .08em @fa-border-color; 7 | border-radius: .1em; 8 | } 9 | 10 | .pull-right { float: right; } 11 | .pull-left { float: left; } 12 | 13 | .@{fa-css-prefix} { 14 | &.pull-left { margin-right: .3em; } 15 | &.pull-right { margin-left: .3em; } 16 | } 17 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less: -------------------------------------------------------------------------------- 1 | // Base Class Definition 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix} { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less: -------------------------------------------------------------------------------- 1 | // Fixed Width Icons 2 | // ------------------------- 3 | .@{fa-css-prefix}-fw { 4 | width: (18em / 14); 5 | text-align: center; 6 | } 7 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.2.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */ 5 | 6 | @import "variables.less"; 7 | @import "mixins.less"; 8 | @import "path.less"; 9 | @import "core.less"; 10 | @import "larger.less"; 11 | @import "fixed-width.less"; 12 | @import "list.less"; 13 | @import "bordered-pulled.less"; 14 | @import "spinning.less"; 15 | @import "rotated-flipped.less"; 16 | @import "stacked.less"; 17 | @import "icons.less"; 18 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less: -------------------------------------------------------------------------------- 1 | // Icon Sizes 2 | // ------------------------- 3 | 4 | /* makes the font 33% larger relative to the icon container */ 5 | .@{fa-css-prefix}-lg { 6 | font-size: (4em / 3); 7 | line-height: (3em / 4); 8 | vertical-align: -15%; 9 | } 10 | .@{fa-css-prefix}-2x { font-size: 2em; } 11 | .@{fa-css-prefix}-3x { font-size: 3em; } 12 | .@{fa-css-prefix}-4x { font-size: 4em; } 13 | .@{fa-css-prefix}-5x { font-size: 5em; } 14 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less: -------------------------------------------------------------------------------- 1 | // List Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-ul { 5 | padding-left: 0; 6 | margin-left: @fa-li-width; 7 | list-style-type: none; 8 | > li { position: relative; } 9 | } 10 | .@{fa-css-prefix}-li { 11 | position: absolute; 12 | left: -@fa-li-width; 13 | width: @fa-li-width; 14 | top: (2em / 14); 15 | text-align: center; 16 | &.@{fa-css-prefix}-lg { 17 | left: (-@fa-li-width + (4em / 14)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less: -------------------------------------------------------------------------------- 1 | // Mixins 2 | // -------------------------- 3 | 4 | .fa-icon() { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | 13 | .fa-icon-rotate(@degrees, @rotation) { 14 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=@rotation); 15 | -webkit-transform: rotate(@degrees); 16 | -ms-transform: rotate(@degrees); 17 | transform: rotate(@degrees); 18 | } 19 | 20 | .fa-icon-flip(@horiz, @vert, @rotation) { 21 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=@rotation, mirror=1); 22 | -webkit-transform: scale(@horiz, @vert); 23 | -ms-transform: scale(@horiz, @vert); 24 | transform: scale(@horiz, @vert); 25 | } 26 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less: -------------------------------------------------------------------------------- 1 | /* FONT PATH 2 | * -------------------------- */ 3 | 4 | @font-face { 5 | font-family: 'FontAwesome'; 6 | src: url('@{fa-font-path}/fontawesome-webfont.eot?v=@{fa-version}'); 7 | src: url('@{fa-font-path}/fontawesome-webfont.eot?#iefix&v=@{fa-version}') format('embedded-opentype'), 8 | url('@{fa-font-path}/fontawesome-webfont.woff?v=@{fa-version}') format('woff'), 9 | url('@{fa-font-path}/fontawesome-webfont.ttf?v=@{fa-version}') format('truetype'), 10 | url('@{fa-font-path}/fontawesome-webfont.svg?v=@{fa-version}#fontawesomeregular') format('svg'); 11 | // src: url('@{fa-font-path}/FontAwesome.otf') format('opentype'); // used when developing fonts 12 | font-weight: normal; 13 | font-style: normal; 14 | } 15 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less: -------------------------------------------------------------------------------- 1 | // Rotated & Flipped Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-rotate-90 { .fa-icon-rotate(90deg, 1); } 5 | .@{fa-css-prefix}-rotate-180 { .fa-icon-rotate(180deg, 2); } 6 | .@{fa-css-prefix}-rotate-270 { .fa-icon-rotate(270deg, 3); } 7 | 8 | .@{fa-css-prefix}-flip-horizontal { .fa-icon-flip(-1, 1, 0); } 9 | .@{fa-css-prefix}-flip-vertical { .fa-icon-flip(1, -1, 2); } 10 | 11 | // Hook for IE8-9 12 | // ------------------------- 13 | 14 | :root .@{fa-css-prefix}-rotate-90, 15 | :root .@{fa-css-prefix}-rotate-180, 16 | :root .@{fa-css-prefix}-rotate-270, 17 | :root .@{fa-css-prefix}-flip-horizontal, 18 | :root .@{fa-css-prefix}-flip-vertical { 19 | filter: none; 20 | } 21 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less: -------------------------------------------------------------------------------- 1 | // Spinning Icons 2 | // -------------------------- 3 | 4 | .@{fa-css-prefix}-spin { 5 | -webkit-animation: fa-spin 2s infinite linear; 6 | animation: fa-spin 2s infinite linear; 7 | } 8 | 9 | @-webkit-keyframes fa-spin { 10 | 0% { 11 | -webkit-transform: rotate(0deg); 12 | transform: rotate(0deg); 13 | } 14 | 100% { 15 | -webkit-transform: rotate(359deg); 16 | transform: rotate(359deg); 17 | } 18 | } 19 | 20 | @keyframes fa-spin { 21 | 0% { 22 | -webkit-transform: rotate(0deg); 23 | transform: rotate(0deg); 24 | } 25 | 100% { 26 | -webkit-transform: rotate(359deg); 27 | transform: rotate(359deg); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less: -------------------------------------------------------------------------------- 1 | // Stacked Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-stack { 5 | position: relative; 6 | display: inline-block; 7 | width: 2em; 8 | height: 2em; 9 | line-height: 2em; 10 | vertical-align: middle; 11 | } 12 | .@{fa-css-prefix}-stack-1x, .@{fa-css-prefix}-stack-2x { 13 | position: absolute; 14 | left: 0; 15 | width: 100%; 16 | text-align: center; 17 | } 18 | .@{fa-css-prefix}-stack-1x { line-height: inherit; } 19 | .@{fa-css-prefix}-stack-2x { font-size: 2em; } 20 | .@{fa-css-prefix}-inverse { color: @fa-inverse; } 21 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss: -------------------------------------------------------------------------------- 1 | // Bordered & Pulled 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-border { 5 | padding: .2em .25em .15em; 6 | border: solid .08em $fa-border-color; 7 | border-radius: .1em; 8 | } 9 | 10 | .pull-right { float: right; } 11 | .pull-left { float: left; } 12 | 13 | .#{$fa-css-prefix} { 14 | &.pull-left { margin-right: .3em; } 15 | &.pull-right { margin-left: .3em; } 16 | } 17 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss: -------------------------------------------------------------------------------- 1 | // Base Class Definition 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix} { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss: -------------------------------------------------------------------------------- 1 | // Fixed Width Icons 2 | // ------------------------- 3 | .#{$fa-css-prefix}-fw { 4 | width: (18em / 14); 5 | text-align: center; 6 | } 7 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss: -------------------------------------------------------------------------------- 1 | // Icon Sizes 2 | // ------------------------- 3 | 4 | /* makes the font 33% larger relative to the icon container */ 5 | .#{$fa-css-prefix}-lg { 6 | font-size: (4em / 3); 7 | line-height: (3em / 4); 8 | vertical-align: -15%; 9 | } 10 | .#{$fa-css-prefix}-2x { font-size: 2em; } 11 | .#{$fa-css-prefix}-3x { font-size: 3em; } 12 | .#{$fa-css-prefix}-4x { font-size: 4em; } 13 | .#{$fa-css-prefix}-5x { font-size: 5em; } 14 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss: -------------------------------------------------------------------------------- 1 | // List Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-ul { 5 | padding-left: 0; 6 | margin-left: $fa-li-width; 7 | list-style-type: none; 8 | > li { position: relative; } 9 | } 10 | .#{$fa-css-prefix}-li { 11 | position: absolute; 12 | left: -$fa-li-width; 13 | width: $fa-li-width; 14 | top: (2em / 14); 15 | text-align: center; 16 | &.#{$fa-css-prefix}-lg { 17 | left: -$fa-li-width + (4em / 14); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss: -------------------------------------------------------------------------------- 1 | // Mixins 2 | // -------------------------- 3 | 4 | @mixin fa-icon() { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | 13 | @mixin fa-icon-rotate($degrees, $rotation) { 14 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=#{$rotation}); 15 | -webkit-transform: rotate($degrees); 16 | -ms-transform: rotate($degrees); 17 | transform: rotate($degrees); 18 | } 19 | 20 | @mixin fa-icon-flip($horiz, $vert, $rotation) { 21 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=#{$rotation}); 22 | -webkit-transform: scale($horiz, $vert); 23 | -ms-transform: scale($horiz, $vert); 24 | transform: scale($horiz, $vert); 25 | } 26 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss: -------------------------------------------------------------------------------- 1 | /* FONT PATH 2 | * -------------------------- */ 3 | 4 | @font-face { 5 | font-family: 'FontAwesome'; 6 | src: url('#{$fa-font-path}/fontawesome-webfont.eot?v=#{$fa-version}'); 7 | src: url('#{$fa-font-path}/fontawesome-webfont.eot?#iefix&v=#{$fa-version}') format('embedded-opentype'), 8 | url('#{$fa-font-path}/fontawesome-webfont.woff?v=#{$fa-version}') format('woff'), 9 | url('#{$fa-font-path}/fontawesome-webfont.ttf?v=#{$fa-version}') format('truetype'), 10 | url('#{$fa-font-path}/fontawesome-webfont.svg?v=#{$fa-version}#fontawesomeregular') format('svg'); 11 | //src: url('#{$fa-font-path}/FontAwesome.otf') format('opentype'); // used when developing fonts 12 | font-weight: normal; 13 | font-style: normal; 14 | } 15 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss: -------------------------------------------------------------------------------- 1 | // Rotated & Flipped Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-rotate-90 { @include fa-icon-rotate(90deg, 1); } 5 | .#{$fa-css-prefix}-rotate-180 { @include fa-icon-rotate(180deg, 2); } 6 | .#{$fa-css-prefix}-rotate-270 { @include fa-icon-rotate(270deg, 3); } 7 | 8 | .#{$fa-css-prefix}-flip-horizontal { @include fa-icon-flip(-1, 1, 0); } 9 | .#{$fa-css-prefix}-flip-vertical { @include fa-icon-flip(1, -1, 2); } 10 | 11 | // Hook for IE8-9 12 | // ------------------------- 13 | 14 | :root .#{$fa-css-prefix}-rotate-90, 15 | :root .#{$fa-css-prefix}-rotate-180, 16 | :root .#{$fa-css-prefix}-rotate-270, 17 | :root .#{$fa-css-prefix}-flip-horizontal, 18 | :root .#{$fa-css-prefix}-flip-vertical { 19 | filter: none; 20 | } 21 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss: -------------------------------------------------------------------------------- 1 | // Spinning Icons 2 | // -------------------------- 3 | 4 | .#{$fa-css-prefix}-spin { 5 | -webkit-animation: fa-spin 2s infinite linear; 6 | animation: fa-spin 2s infinite linear; 7 | } 8 | 9 | @-webkit-keyframes fa-spin { 10 | 0% { 11 | -webkit-transform: rotate(0deg); 12 | transform: rotate(0deg); 13 | } 14 | 100% { 15 | -webkit-transform: rotate(359deg); 16 | transform: rotate(359deg); 17 | } 18 | } 19 | 20 | @keyframes fa-spin { 21 | 0% { 22 | -webkit-transform: rotate(0deg); 23 | transform: rotate(0deg); 24 | } 25 | 100% { 26 | -webkit-transform: rotate(359deg); 27 | transform: rotate(359deg); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss: -------------------------------------------------------------------------------- 1 | // Stacked Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-stack { 5 | position: relative; 6 | display: inline-block; 7 | width: 2em; 8 | height: 2em; 9 | line-height: 2em; 10 | vertical-align: middle; 11 | } 12 | .#{$fa-css-prefix}-stack-1x, .#{$fa-css-prefix}-stack-2x { 13 | position: absolute; 14 | left: 0; 15 | width: 100%; 16 | text-align: center; 17 | } 18 | .#{$fa-css-prefix}-stack-1x { line-height: inherit; } 19 | .#{$fa-css-prefix}-stack-2x { font-size: 2em; } 20 | .#{$fa-css-prefix}-inverse { color: $fa-inverse; } 21 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.2.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */ 5 | 6 | @import "variables"; 7 | @import "mixins"; 8 | @import "path"; 9 | @import "core"; 10 | @import "larger"; 11 | @import "fixed-width"; 12 | @import "list"; 13 | @import "bordered-pulled"; 14 | @import "spinning"; 15 | @import "rotated-flipped"; 16 | @import "stacked"; 17 | @import "icons"; 18 | -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /other_installers/ansible/roles/ail-host/files/staticBackup/image/AIL.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/other_installers/ansible/roles/ail-host/files/staticBackup/image/AIL.png -------------------------------------------------------------------------------- /other_installers/docker/.dockerignore: -------------------------------------------------------------------------------- 1 | *.md 2 | -------------------------------------------------------------------------------- /other_installers/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | ARG tz_buildtime=Europe/Rome 3 | ENV TZ=$tz_buildtime 4 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 5 | 6 | # Make sure that all updates are in place 7 | RUN apt-get clean && apt-get update -y && apt-get upgrade -y \ 8 | && apt-get dist-upgrade -y && apt-get autoremove -y 9 | 10 | # Install needed packages 11 | RUN apt-get install git python3-dev build-essential \ 12 | libffi-dev libssl-dev libfuzzy-dev wget sudo -y 13 | 14 | # Adding sudo command 15 | RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo 16 | RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers 17 | 18 | # Installing AIL dependencies 19 | RUN mkdir /opt/AIL 20 | ADD . /opt/AIL 21 | WORKDIR /opt/AIL 22 | RUN ./installing_deps.sh 23 | WORKDIR /opt/AIL 24 | 25 | # Default to UTF-8 file.encoding 26 | ENV LANG C.UTF-8 27 | ENV AIL_HOME /opt/AIL 28 | ENV AIL_BIN ${AIL_HOME}/bin 29 | ENV AIL_FLASK ${AIL_HOME}/var/www 30 | ENV AIL_REDIS ${AIL_HOME}/redis/src 31 | ENV AIL_ARDB ${AIL_HOME}/ardb/src 32 | ENV AIL_VENV ${AIL_HOME}/AILENV 33 | 34 | ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 35 | 36 | RUN ./pystemon/install.sh 37 | RUN pip install -r /opt/pystemon/requirements.txt 38 | 39 | COPY docker_start.sh /docker_start.sh 40 | ENTRYPOINT ["/bin/bash", "docker_start.sh"] 41 | -------------------------------------------------------------------------------- /other_installers/docker/README.md: -------------------------------------------------------------------------------- 1 | Docker Quick Start (Ubuntu 18.04 LTS) 2 | ------------ 3 | 4 | :warning: 5 | This Docker is not maintained at the moment. 6 | If you are interested to contribute, please submit a Pull Request 7 | 8 | 9 | 1. Install Docker 10 | ```bash 11 | sudo su 12 | apt-get install -y curl 13 | curl https://get.docker.com | /bin/bash 14 | ``` 15 | 16 | 2. Type these commands to build the Docker image: 17 | ```bash 18 | git clone https://github.com/ail-project/ail-framework.git 19 | cd AIL-framework 20 | cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ 21 | cp ./configs/update.cfg.sample ./configs/update.cfg 22 | vim/nano ./configs/update.cfg (set auto_update to False) 23 | docker build --build-arg tz_buildtime=YOUR_GEO_AREA/YOUR_CITY -t ail-framework . 24 | ``` 25 | 3. To start AIL on port 7000, type the following command below: 26 | ``` 27 | docker run -p 7000:7000 ail-framework 28 | ``` 29 | 30 | 4. To debug the running container, type the following command and note the container name or identifier: 31 | ```bash 32 | docker ps 33 | ``` 34 | 35 | After getting the name or identifier type the following commands: 36 | ```bash 37 | docker exec -it CONTAINER_NAME_OR_IDENTIFIER bash 38 | cd /opt/ail 39 | ``` 40 | 41 | Install using Ansible 42 | --------------------- 43 | 44 | Please check the [Ansible readme](ansible/README.md). 45 | 46 | -------------------------------------------------------------------------------- /other_installers/docker/docker_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | signalListener() { 3 | "$@" & 4 | pid="$!" 5 | trap "echo 'Stopping'; kill -SIGTERM $pid" SIGINT SIGTERM 6 | 7 | while kill -0 $pid > /dev/null 2>&1; do 8 | wait 9 | done 10 | } 11 | 12 | 13 | source ./AILENV/bin/activate 14 | cd bin 15 | ./LAUNCH.sh -l 16 | ./LAUNCH.sh -c 17 | ./LAUNCH.sh -f 18 | 19 | signalListener tail -f /dev/null $@ 20 | 21 | ./LAUNCH.sh -k 22 | -------------------------------------------------------------------------------- /other_installers/docker/pystemon/install.sh: -------------------------------------------------------------------------------- 1 | git clone https://github.com/cvandeplas/pystemon.git /opt/pystemon 2 | 3 | apt-get install -y python-pip python-requests python-yaml python-redis 4 | 5 | pip install beautifulsoup4 6 | 7 | BASEDIR=$(dirname "$0") 8 | cp $BASEDIR/config.cfg /opt/AIL/bin/packages/ 9 | cp $BASEDIR/pystemon.yaml /opt/pystemon/ 10 | -------------------------------------------------------------------------------- /other_installers/docker/pystemon/proxies.txt: -------------------------------------------------------------------------------- 1 | http://127.0.0.1:8080 2 | -------------------------------------------------------------------------------- /samples/2021/01/01/api_keys.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/samples/2021/01/01/api_keys.gz -------------------------------------------------------------------------------- /samples/2021/01/01/categ.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/samples/2021/01/01/categ.gz -------------------------------------------------------------------------------- /samples/2021/01/01/credit_cards.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/samples/2021/01/01/credit_cards.gz -------------------------------------------------------------------------------- /samples/2021/01/01/domain_classifier.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/samples/2021/01/01/domain_classifier.gz -------------------------------------------------------------------------------- /samples/2021/01/01/keys.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/samples/2021/01/01/keys.gz -------------------------------------------------------------------------------- /samples/2021/01/01/keys_certificat_sample.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/samples/2021/01/01/keys_certificat_sample.gz -------------------------------------------------------------------------------- /samples/2021/01/01/onion.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/samples/2021/01/01/onion.gz -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/tests/__init__.py -------------------------------------------------------------------------------- /tests/zmq_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of AIL framework - Analysis Information Leak framework 5 | # 6 | # 7 | # Python script to test if the ZMQ feed works as expected 8 | # 9 | 10 | import sys 11 | import zmq 12 | 13 | port = "5556" 14 | 15 | context = zmq.Context() 16 | socket = context.socket(zmq.SUB) 17 | socket.connect ("tcp://crf.circl.lu:%s" % port) 18 | 19 | # 101 Name of the pastes only 20 | # 102 Full pastes in raw base64(gz) 21 | 22 | topicfilter = "102" 23 | socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter) 24 | 25 | while True: 26 | message = socket.recv() 27 | print('b1') 28 | print (message) 29 | if topicfilter == "102": 30 | topic, paste, messagedata = message.split() 31 | print(paste, messagedata) 32 | else: 33 | print (message) 34 | -------------------------------------------------------------------------------- /tools/file_dir_importer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | DIR/File Importer Helper 5 | ================ 6 | 7 | Import Content 8 | 9 | """ 10 | 11 | import argparse 12 | import os 13 | import sys 14 | 15 | sys.path.append(os.environ['AIL_BIN']) 16 | ################################## 17 | # Import Project packages 18 | ################################## 19 | from importer import FileImporter 20 | 21 | 22 | if __name__ == "__main__": 23 | parser = argparse.ArgumentParser(description='Directory or file importer') 24 | parser.add_argument('-d', '--directory', type=str, help='Root directory to import') 25 | parser.add_argument('-f', '--file', type=str, help='File to import') 26 | args = parser.parse_args() 27 | 28 | if not args.directory and not args.file: 29 | parser.print_help() 30 | sys.exit(0) 31 | 32 | if args.directory: 33 | dir_path = args.directory 34 | dir_importer = FileImporter.DirImporter() 35 | dir_importer.importer(dir_path) 36 | 37 | if args.file: 38 | file_path = args.file 39 | file_importer = FileImporter.FileImporter() 40 | file_importer.importer(file_path) 41 | -------------------------------------------------------------------------------- /tools/gen_cert/README.md: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | These scripts are useful to generate the server's self-signed certificate. 4 | ## Root Certification Authority ## 5 | `gen_root.sh` 6 | This will create a rootCA.crt to verify the server's cert. 7 | ## Server certificate ## 8 | ### Configuration Files ### 9 | Look into these files if you plan to verify the certificate: 10 | * san.cnf holds basic information about the certificate. 11 | * ext.3 holds in particular the 'subjectAltNames** option that is for the verification to succeed. 12 | 13 | ### Generation ### 14 | `gen_root.sh` 15 | -------------------------------------------------------------------------------- /tools/gen_cert/ext3.cnf: -------------------------------------------------------------------------------- 1 | keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment 2 | subjectAltName = @alt_names 3 | 4 | [alt_names] 5 | IP.1 = 127.0.0.1 6 | -------------------------------------------------------------------------------- /tools/gen_cert/gen_cert.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Create Server key 3 | openssl genrsa -out server.key 4096 4 | # Create the Server Signing Request - non interactive, config in san.cnf 5 | openssl req -sha256 -new -key server.key -out server.csr -config san.cnf 6 | # Create the server certificate by rootCA, with ext3 subjectAltName in ext3.cnf 7 | openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 500 -sha256 -extfile ext3.cnf 8 | # Concat in pem 9 | cat server.crt server.key > server.pem 10 | -------------------------------------------------------------------------------- /tools/gen_cert/gen_root.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Create Root key 3 | openssl genrsa -out rootCA.key 4096 4 | # Create and Sign the Root CA Certificate 5 | openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.crt -config san.cnf 6 | -------------------------------------------------------------------------------- /tools/gen_cert/san.cnf: -------------------------------------------------------------------------------- 1 | [req] 2 | default_bits = 4096 3 | prompt = no 4 | default_md = sha256 5 | distinguished_name = dn 6 | 7 | [dn] 8 | C=LU 9 | L=Luxembourg 10 | O=AIL-Default 11 | CN = localhost 12 | -------------------------------------------------------------------------------- /tools/manual_update_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | import argparse 7 | 8 | # # TODO: check max update 9 | max_update = 3.5 10 | 11 | def check_version(version): 12 | if version[0] == 'v' and '.' in version: 13 | try: 14 | res = float(version[1:]) 15 | if res >= 1 and res <= max_update: 16 | return True 17 | except: 18 | pass 19 | print(f'ERROR: invalid version/tag: {version}') 20 | return False 21 | 22 | if __name__ == '__main__': 23 | 24 | parser = argparse.ArgumentParser(description='Trigger backgroud update') 25 | parser.add_argument('-v', '--version', help='update version (tag) ex: v3.5', type=str, dest='version', required=True, default=None) 26 | args = parser.parse_args() 27 | 28 | if args.version is None: 29 | parser.print_help() 30 | sys.exit(0) 31 | version = args.version 32 | if not check_version(version): 33 | sys.exit(0) 34 | 35 | update_db_dir = os.path.join(os.environ['AIL_HOME'], 'update', version) 36 | update_db_script = os.path.join(update_db_dir, 'Update.py') 37 | if not os.path.isfile(update_db_script): 38 | # # TODO: launch default update 39 | print('DB Up To Date') 40 | else: 41 | # import Updater clas 42 | sys.path.append(update_db_dir) 43 | from Update import Updater 44 | updater = Updater(version) 45 | updater.run_update() 46 | -------------------------------------------------------------------------------- /update/bin/Update_Redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_BIN. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | echo "Killing all screens ..." 14 | bash -c "bash ${AIL_BIN}/LAUNCH.sh -k" 15 | echo "" 16 | echo "Updating Redis ..." 17 | pushd $AIL_HOME/redis 18 | git pull || exit 1 19 | git checkout 5.0 || exit 1 20 | make || exit 1 21 | popd 22 | echo "Redis Updated" 23 | echo "" 24 | 25 | exit 0 26 | -------------------------------------------------------------------------------- /update/bin/ail_updater.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | import time 7 | 8 | sys.path.append(os.environ['AIL_BIN']) 9 | ################################## 10 | # Import Project packages 11 | ################################## 12 | from lib import ail_updates 13 | from lib.ConfigLoader import ConfigLoader 14 | from packages.git_status import clear_git_meta_cache 15 | 16 | class AIL_Updater(object): 17 | """docstring for AIL_Updater.""" 18 | 19 | def __init__(self, new_version): 20 | self.version = new_version 21 | self.start_time = time.time() 22 | 23 | self.config = ConfigLoader() 24 | self.r_serv = self.config.get_db_conn("Kvrocks_DB") 25 | 26 | self.f_version = float(self.version[1:]) 27 | self.current_f_version = ail_updates.get_ail_float_version() 28 | clear_git_meta_cache() 29 | 30 | def update(self): 31 | """ 32 | AIL DB update 33 | """ 34 | pass 35 | 36 | def end_update(self): 37 | """ 38 | Update DB version 39 | """ 40 | ail_updates.add_ail_update(self.version) 41 | 42 | def run_update(self): 43 | self.update() 44 | self.end_update() 45 | -------------------------------------------------------------------------------- /update/default_update/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | import time 7 | import argparse 8 | 9 | sys.path.append(os.environ['AIL_BIN']) 10 | ################################## 11 | # Import Project packages 12 | ################################## 13 | from lib import ail_updates 14 | 15 | if __name__ == '__main__': 16 | parser = argparse.ArgumentParser(description='AIL default update') 17 | parser.add_argument('-t', help='version tag', type=str, dest='tag', required=True) 18 | args = parser.parse_args() 19 | 20 | if not args.tag: 21 | parser.print_help() 22 | sys.exit(0) 23 | 24 | # remove space 25 | update_tag = args.tag.replace(' ', '') 26 | if not ail_updates.check_version(update_tag): 27 | parser.print_help() 28 | print(f'Error: Invalid update tag {update_tag}') 29 | sys.exit(0) 30 | 31 | start_deb = time.time() 32 | 33 | ail_updates.add_ail_update(update_tag) 34 | -------------------------------------------------------------------------------- /update/default_update/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$1" ] 4 | then 5 | echo "No tags version supplied" 6 | fi 7 | 8 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 9 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 10 | [ -z "$AIL_KVROCKS" ] && echo "Needs the env var AIL_KVROCKS. Run the script from the virtual environment." && exit 1; 11 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_BIN. Run the script from the virtual environment." && exit 1; 12 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 13 | 14 | export PATH=$AIL_HOME:$PATH 15 | export PATH=$AIL_REDIS:$PATH 16 | export PATH=AIL_KVROCKS:$PATH 17 | export PATH=$AIL_BIN:$PATH 18 | export PATH=$AIL_FLASK:$PATH 19 | 20 | GREEN="\\033[1;32m" 21 | DEFAULT="\\033[0;39m" 22 | 23 | echo -e $GREEN"Shutting down AIL Script ..."$DEFAULT 24 | bash ${AIL_BIN}/LAUNCH.sh -ks 25 | wait 26 | 27 | echo "" 28 | bash ${AIL_BIN}/LAUNCH.sh -lkv 29 | wait 30 | 31 | echo "" 32 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 33 | echo "" 34 | python ${AIL_HOME}/update/default_update/Update.py "-t $1" 35 | wait 36 | echo "" 37 | echo "" 38 | 39 | echo "" 40 | echo -e $GREEN"Killing Script ..."$DEFAULT 41 | bash ${AIL_BIN}/LAUNCH.sh -ks 42 | wait 43 | 44 | echo "" 45 | echo -e $GREEN"Update thirdparty ..."$DEFAULT 46 | bash ${AIL_BIN}/LAUNCH.sh -t 47 | wait 48 | 49 | 50 | echo "" 51 | 52 | exit 0 53 | -------------------------------------------------------------------------------- /update/v5.2/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib import ail_updates 13 | 14 | class Updater(AIL_Updater): 15 | """default Updater.""" 16 | 17 | def __init__(self, version): 18 | super(Updater, self).__init__(version) 19 | 20 | 21 | if __name__ == '__main__': 22 | updater = Updater('v5.2') 23 | updater.run_update() 24 | ail_updates.add_background_update('v5.2') 25 | -------------------------------------------------------------------------------- /update/v5.2/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | echo "" 24 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 25 | echo "" 26 | python ${AIL_HOME}/update/v5.2/Update.py 27 | wait 28 | echo "" 29 | echo "" 30 | 31 | exit 0 32 | -------------------------------------------------------------------------------- /update/v5.2/compress_har.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import gzip 5 | import os 6 | import sys 7 | 8 | sys.path.append(os.environ['AIL_BIN']) 9 | ################################## 10 | # Import Project packages 11 | ################################## 12 | from lib import ail_updates 13 | from lib import crawlers 14 | 15 | if __name__ == '__main__': 16 | update = ail_updates.AILBackgroundUpdate('v5.2') 17 | HAR_DIR = crawlers.HAR_DIR 18 | hars_ids = crawlers.get_all_har_ids() 19 | update.set_nb_to_update(len(hars_ids)) 20 | n = 0 21 | for har_id in hars_ids: 22 | crawlers._gzip_har(har_id) 23 | update.inc_nb_updated() 24 | if n % 100 == 0: 25 | update.update_progress() 26 | 27 | crawlers._gzip_all_hars() 28 | -------------------------------------------------------------------------------- /update/v5.3/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib import ail_updates 13 | 14 | class Updater(AIL_Updater): 15 | """default Updater.""" 16 | 17 | def __init__(self, version): 18 | super(Updater, self).__init__(version) 19 | 20 | 21 | if __name__ == '__main__': 22 | updater = Updater('v5.3') 23 | updater.run_update() 24 | 25 | -------------------------------------------------------------------------------- /update/v5.3/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -k 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | echo "" 24 | echo -e $GREEN"Updating python packages ..."$DEFAULT 25 | echo "" 26 | pip install -U gcld3 27 | pip install -U libretranslatepy 28 | pip install -U xxhash 29 | pip install -U DomainClassifier 30 | 31 | echo "" 32 | echo -e $GREEN"Updating KVROCKS ..."$DEFAULT 33 | echo "" 34 | pushd ${AIL_HOME}/kvrocks 35 | git pull 36 | ./x.py build -j 4 37 | popd 38 | 39 | bash ${AIL_BIN}/LAUNCH.sh -lrv 40 | bash ${AIL_BIN}/LAUNCH.sh -lkv 41 | 42 | echo "" 43 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 44 | echo "" 45 | python ${AIL_HOME}/update/v5.3/Update.py 46 | wait 47 | echo "" 48 | echo "" 49 | 50 | exit 0 51 | -------------------------------------------------------------------------------- /update/v5.4/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib import ail_updates 13 | from lib import chats_viewer 14 | 15 | class Updater(AIL_Updater): 16 | """default Updater.""" 17 | 18 | def __init__(self, version): 19 | super(Updater, self).__init__(version) 20 | 21 | 22 | if __name__ == '__main__': 23 | chats_viewer.fix_correlations_subchannel_message() 24 | updater = Updater('v5.4') 25 | updater.run_update() 26 | 27 | -------------------------------------------------------------------------------- /update/v5.4/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | echo "" 24 | echo -e $GREEN"Updating python packages ..."$DEFAULT 25 | echo "" 26 | pip install -U pyail 27 | pip install -U pylacus 28 | pip install -U git+https://github.com/ail-project/demoji 29 | pip install -U lexilang 30 | 31 | 32 | bash ${AIL_BIN}/LAUNCH.sh -lrv 33 | bash ${AIL_BIN}/LAUNCH.sh -lkv 34 | 35 | echo "" 36 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 37 | echo "" 38 | python ${AIL_HOME}/update/v5.4/Update.py 39 | wait 40 | echo "" 41 | echo "" 42 | 43 | exit 0 44 | -------------------------------------------------------------------------------- /update/v5.5/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib import ail_updates 13 | from lib import chats_viewer 14 | 15 | class Updater(AIL_Updater): 16 | """default Updater.""" 17 | 18 | def __init__(self, version): 19 | super(Updater, self).__init__(version) 20 | 21 | 22 | if __name__ == '__main__': 23 | chats_viewer.fix_correlations_subchannel_message() 24 | updater = Updater('v5.5') 25 | updater.run_update() 26 | 27 | -------------------------------------------------------------------------------- /update/v5.5/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | echo "" 24 | echo -e $GREEN"Updating python packages ..."$DEFAULT 25 | echo "" 26 | pip install -U easyocr 27 | 28 | 29 | bash ${AIL_BIN}/LAUNCH.sh -lrv 30 | bash ${AIL_BIN}/LAUNCH.sh -lkv 31 | 32 | echo "" 33 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 34 | echo "" 35 | python ${AIL_HOME}/update/v5.5/Update.py 36 | wait 37 | echo "" 38 | echo "" 39 | 40 | exit 0 41 | -------------------------------------------------------------------------------- /update/v5.7/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | bash ${AIL_BIN}/LAUNCH.sh -lrv 24 | bash ${AIL_BIN}/LAUNCH.sh -lkv 25 | 26 | echo "" 27 | echo -e $GREEN"Updating python packages ..."$DEFAULT 28 | echo "" 29 | pip install -U pyotp 30 | pip install -U segno 31 | 32 | echo "" 33 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 34 | echo "" 35 | python ${AIL_HOME}/update/v5.7/Update.py 36 | wait 37 | echo "" 38 | echo "" 39 | 40 | exit 0 41 | -------------------------------------------------------------------------------- /update/v5.8/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | 13 | class Updater(AIL_Updater): 14 | """default Updater.""" 15 | 16 | def __init__(self, version): 17 | super(Updater, self).__init__(version) 18 | 19 | 20 | if __name__ == '__main__': 21 | updater = Updater('v5.8') 22 | updater.run_update() 23 | -------------------------------------------------------------------------------- /update/v5.8/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | # Install LIB 21 | sudo apt-get install python3-opencv 22 | sudo apt-get install libzbar0 23 | 24 | 25 | # SUBMODULES # 26 | git submodule update 27 | 28 | bash ${AIL_BIN}/LAUNCH.sh -lrv 29 | bash ${AIL_BIN}/LAUNCH.sh -lkv 30 | 31 | echo "" 32 | echo -e $GREEN"Updating python packages ..."$DEFAULT 33 | echo "" 34 | pip install -U pyzbar 35 | pip install -U qreader 36 | 37 | echo "" 38 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 39 | echo "" 40 | python ${AIL_HOME}/update/v5.8/Update.py 41 | wait 42 | echo "" 43 | echo "" 44 | 45 | exit 0 46 | -------------------------------------------------------------------------------- /update/v5.9/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib import ail_updates 13 | 14 | class Updater(AIL_Updater): 15 | """default Updater.""" 16 | 17 | def __init__(self, version): 18 | super(Updater, self).__init__(version) 19 | 20 | 21 | if __name__ == '__main__': 22 | updater = Updater('v5.9') 23 | updater.run_update() 24 | ail_updates.add_background_update('v5.9') 25 | -------------------------------------------------------------------------------- /update/v5.9/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | echo "" 24 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 25 | echo "" 26 | python ${AIL_HOME}/update/v5.9/Update.py 27 | wait 28 | echo "" 29 | echo "" 30 | 31 | exit 0 32 | -------------------------------------------------------------------------------- /update/v5.9/reprocess_dom_hash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import gzip 5 | import os 6 | import sys 7 | 8 | sys.path.append(os.environ['AIL_BIN']) 9 | ################################## 10 | # Import Project packages 11 | ################################## 12 | from lib import ail_updates 13 | from lib.objects import ail_objects 14 | from lib.objects import DomHashs 15 | from lib.objects.Domains import Domain 16 | 17 | if __name__ == '__main__': 18 | update = ail_updates.AILBackgroundUpdate('v5.9') 19 | n = 0 20 | nb_items = ail_objects.card_obj_iterator('item', filters={'sources': ['crawled']}) 21 | update.set_nb_to_update(nb_items) 22 | 23 | for item in ail_objects.obj_iterator('item', filters={'sources': ['crawled']}): 24 | dom = item.get_domain() 25 | domain = Domain(dom) 26 | i_content = item.get_content() 27 | if domain.exists() and i_content: 28 | date = item.get_date() 29 | # DOM-HASH 30 | dom_hash = DomHashs.create(i_content) 31 | dom_hash.add(date, item) 32 | dom_hash.add_correlation('domain', '', domain.id) 33 | 34 | print(domain.id, item.id, dom_hash.id) 35 | 36 | update.inc_nb_updated() 37 | n += 1 38 | if n % 100 == 0: 39 | update.update_progress() 40 | -------------------------------------------------------------------------------- /update/v6.0.1/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | echo "" 24 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 25 | echo "" 26 | python ${AIL_HOME}/update/v6.0.1/Update.py 27 | wait 28 | echo "" 29 | echo "" 30 | 31 | exit 0 32 | -------------------------------------------------------------------------------- /update/v6.0/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib.ConfigLoader import ConfigLoader 13 | from lib import ail_updates 14 | 15 | class Updater(AIL_Updater): 16 | """default Updater.""" 17 | 18 | def __init__(self, version): 19 | super(Updater, self).__init__(version) 20 | 21 | 22 | if __name__ == '__main__': 23 | config_loader = ConfigLoader() 24 | r_queues = config_loader.get_redis_conn("Redis_Queues") 25 | config_loader = None 26 | r_queues.delete('modules') 27 | updater = Updater('v6.0') 28 | updater.run_update() 29 | -------------------------------------------------------------------------------- /update/v6.0/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -ks 18 | wait 19 | 20 | echo -e $GREEN"Updating UI resources..."$DEFAULT 21 | bash ${AIL_BIN}/LAUNCH.sh -ut 22 | wait 23 | 24 | echo -e $GREEN"Updating python requirement..."$DEFAULT 25 | pip install -U flask-sock 26 | 27 | # SUBMODULES # 28 | git submodule update 29 | 30 | echo "" 31 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 32 | echo "" 33 | python ${AIL_HOME}/update/v6.0/Update.py 34 | wait 35 | echo "" 36 | echo "" 37 | 38 | exit 0 39 | -------------------------------------------------------------------------------- /update/v6.1/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib import ail_updates 13 | 14 | class Updater(AIL_Updater): 15 | """default Updater.""" 16 | 17 | def __init__(self, version): 18 | super(Updater, self).__init__(version) 19 | 20 | 21 | if __name__ == '__main__': 22 | updater = Updater('v6.1') 23 | updater.run_update() 24 | 25 | -------------------------------------------------------------------------------- /update/v6.1/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -k 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | bash ${AIL_BIN}/LAUNCH.sh -lrv 24 | bash ${AIL_BIN}/LAUNCH.sh -lkv 25 | 26 | echo "" 27 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 28 | echo "" 29 | python ${AIL_HOME}/update/v6.1/Update.py 30 | wait 31 | echo "" 32 | echo "" 33 | 34 | exit 0 35 | -------------------------------------------------------------------------------- /update/v6.2/Update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_HOME']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from update.bin.ail_updater import AIL_Updater 12 | from lib.ConfigLoader import ConfigLoader 13 | 14 | config_loader = ConfigLoader() 15 | r_lang = config_loader.get_db_conn("Kvrocks_Languages") 16 | config_loader = None 17 | 18 | class Updater(AIL_Updater): 19 | """default Updater.""" 20 | 21 | def __init__(self, version): 22 | super(Updater, self).__init__(version) 23 | 24 | 25 | if __name__ == '__main__': 26 | updater = Updater('v6.2') 27 | r_lang.flushdb() 28 | updater.run_update() 29 | -------------------------------------------------------------------------------- /update/v6.2/Update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; 4 | [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; 5 | [ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; 6 | [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; 7 | 8 | export PATH=$AIL_HOME:$PATH 9 | export PATH=$AIL_REDIS:$PATH 10 | export PATH=$AIL_BIN:$PATH 11 | export PATH=$AIL_FLASK:$PATH 12 | 13 | GREEN="\\033[1;32m" 14 | DEFAULT="\\033[0;39m" 15 | 16 | echo -e $GREEN"Shutting down AIL ..."$DEFAULT 17 | bash ${AIL_BIN}/LAUNCH.sh -k 18 | wait 19 | 20 | # SUBMODULES # 21 | git submodule update 22 | 23 | bash ${AIL_BIN}/LAUNCH.sh -lrv 24 | bash ${AIL_BIN}/LAUNCH.sh -lkv 25 | 26 | echo -e $GREEN"Updating python Lexilang..."$DEFAULT 27 | pip uninstall -y lexilang 28 | pip install -U git+https://github.com/ail-project/LexiLang 29 | 30 | echo -e $GREEN"Updating python MeiliSearch..."$DEFAULT 31 | pip install -U meilisearch 32 | 33 | echo "" 34 | echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT 35 | echo "" 36 | python ${AIL_HOME}/update/v6.2/Update.py 37 | wait 38 | echo "" 39 | echo "" 40 | 41 | exit 0 42 | -------------------------------------------------------------------------------- /var/www/blueprints/dashboard.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | ''' 5 | Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... 6 | ''' 7 | 8 | import os 9 | import sys 10 | import time 11 | 12 | import json # TODO REMOVE ME 13 | 14 | from flask import render_template, Response, request, Blueprint 15 | from flask_login import login_required, current_user 16 | 17 | # Import Role_Manager 18 | from Role_Manager import login_admin, login_read_only 19 | 20 | sys.path.append(os.environ['AIL_BIN']) 21 | ################################## 22 | # Import Project packages 23 | ################################## 24 | from lib import ail_stats 25 | 26 | # TODO STATS 27 | # from lib import ail_updates 28 | 29 | # ============ BLUEPRINT ============ 30 | dashboard = Blueprint('dashboard', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/dashboard')) 31 | 32 | # ============ VARIABLES ============ 33 | bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] 34 | 35 | # ============ FUNCTIONS ============ 36 | 37 | # ============= ROUTES ============== 38 | 39 | @dashboard.route("/", methods=['GET']) 40 | @login_required 41 | @login_read_only 42 | def index(): 43 | user_org = current_user.get_org() 44 | user_id = current_user.get_user_id() 45 | nb_objects = ail_stats.get_nb_objs_dashboard() 46 | feeders_dashboard = ail_stats.get_feeders_dashboard_full() 47 | crawlers_stats = ail_stats.get_crawlers_stats() 48 | trackers = ail_stats.get_tracked_objs_dashboard(user_org, user_id) 49 | tagged_objs = ail_stats.get_tagged_objs_dashboard() 50 | return render_template("dashboard.html", feeders_dashboard=feeders_dashboard, 51 | nb_objects=nb_objects, trackers=trackers, tagged_objs=tagged_objs, 52 | bootstrap_label=bootstrap_label, crawlers_stats=crawlers_stats) 53 | -------------------------------------------------------------------------------- /var/www/blueprints/old_endpoints.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | ''' 5 | Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... 6 | ''' 7 | 8 | import os 9 | import sys 10 | 11 | from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response 12 | from flask_login import login_required 13 | 14 | # Import Role_Manager 15 | from Role_Manager import login_admin, login_read_only 16 | 17 | # ============ BLUEPRINT ============ 18 | old_endpoints = Blueprint('old_endpoints', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates')) 19 | 20 | # ============ VARIABLES ============ 21 | 22 | 23 | 24 | # ============ FUNCTIONS ============ 25 | 26 | 27 | # ============= ROUTES ============== 28 | @old_endpoints.route("/showsavedpaste/") 29 | @login_required 30 | @login_read_only 31 | def showsavedpaste(): 32 | item_id = request.args.get('paste', '') 33 | return redirect(url_for('objects_item.showItem', id=item_id)) 34 | -------------------------------------------------------------------------------- /var/www/create_default_user.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding:UTF-8 -* 3 | 4 | import os 5 | import sys 6 | 7 | sys.path.append(os.environ['AIL_BIN']) 8 | ################################## 9 | # Import Project packages 10 | ################################## 11 | from lib import ail_orgs 12 | from lib import ail_users 13 | 14 | if __name__ == "__main__": 15 | 16 | user_id = 'admin@admin.test' 17 | password = ail_users.gen_password() 18 | 19 | # create role_list 20 | ail_users._create_roles() 21 | 22 | if not ail_users.exists_user(user_id): 23 | # Create Default Org 24 | org = ail_orgs.create_default_org() 25 | ail_users.create_user(user_id, password=password, admin_id='admin@admin.test', org_uuid=org.get_uuid(), role='admin') 26 | # EDIT Password 27 | else: 28 | ail_users.edit_user('admin@admin.test', user_id, password=password, chg_passwd=True) 29 | 30 | token = ail_users.get_default_admin_token() 31 | 32 | default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD') 33 | to_write_str = '# Password Generated by default\n# This file is deleted after the first login\n#\nemail=admin@admin.test\npassword=' 34 | to_write_str = f'{to_write_str}{password}\nAPI_Key={token}\n' 35 | with open(default_passwd_file, 'w') as f: 36 | f.write(to_write_str) 37 | 38 | print(f'new user created: {user_id}') 39 | print(f'password: {password}') 40 | print(f'token: {token}') 41 | -------------------------------------------------------------------------------- /var/www/static/codemirror/codemirror-yara/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Codemirror Yara 7 | 8 | 9 |

Codemirro Yara

10 |
11 | 12 | 27 |
28 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /var/www/static/codemirror/codemirror-yara/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "codemirror-yara", 3 | "private": true, 4 | "version": "0.0.0", 5 | "files": [ 6 | "dist" 7 | ], 8 | "module": "./dist/codemirror-yara.es.js", 9 | "exports": { 10 | ".": { 11 | "import": "./dist/codemirror-yara.es.js" 12 | } 13 | }, 14 | "scripts": { 15 | "dev": "vite", 16 | "build": "vite build", 17 | "preview": "vite preview" 18 | }, 19 | "devDependencies": { 20 | "vite": "^6.3.5" 21 | }, 22 | "dependencies": { 23 | "codemirror": "^6.0.1" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /var/www/static/codemirror/codemirror-yara/vite.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const { defineConfig } = require('vite') 3 | 4 | module.exports = defineConfig({ 5 | build: { 6 | target: 'es2015', 7 | lib: { 8 | entry: path.resolve(__dirname, 'src/main.js'), 9 | name: 'codemirror-yara', 10 | fileName: (format) => `codemirror-yara.${format}.js`, 11 | formats: ['es'] 12 | } 13 | } 14 | }); -------------------------------------------------------------------------------- /var/www/static/css/ail-project.css: -------------------------------------------------------------------------------- 1 | .hg-text{ 2 | padding-top: 0.2em; 3 | padding-bottom: 0.2em; 4 | padding-right: 0.15em; 5 | padding-left: 0.15em; 6 | background-color: #2e5; 7 | } 8 | 9 | .blue { 10 | color: #0088cc !important; 11 | } 12 | 13 | .bold { 14 | font-weight: bold; 15 | } 16 | 17 | .object_node_icon { 18 | font-size: 16px; 19 | pointer-events: none; 20 | } 21 | 22 | .text-collapse { 23 | overflow: hidden; 24 | display: block; 25 | max-height: 3em; 26 | line-height: 1.5em; 27 | transition: max-height 0.3s ease-in-out; 28 | } 29 | -------------------------------------------------------------------------------- /var/www/static/css/dygraph_gallery.css: -------------------------------------------------------------------------------- 1 | #WordTrending .dygraph-legend > span { display: none; } 2 | #WordTrending .dygraph-legend > span.highlight { display: inline; } 3 | 4 | .dygraph-title { font-size: 42px; text-shadow: 2px 2px 2px gray; } 5 | 6 | .dygraph-label { font-size: 18px; text-shadow: -2px 2px 2px gray; } 7 | -------------------------------------------------------------------------------- /var/www/static/csv/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/csv/.keepdir -------------------------------------------------------------------------------- /var/www/static/image/ail-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/ail-icon.png -------------------------------------------------------------------------------- /var/www/static/image/ail-project-inverted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/ail-project-inverted.png -------------------------------------------------------------------------------- /var/www/static/image/ail-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/ail-project.png -------------------------------------------------------------------------------- /var/www/static/image/ail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/ail.png -------------------------------------------------------------------------------- /var/www/static/image/d4-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/d4-logo.png -------------------------------------------------------------------------------- /var/www/static/image/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/loading.gif -------------------------------------------------------------------------------- /var/www/static/image/misp-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/misp-logo.png -------------------------------------------------------------------------------- /var/www/static/image/thehive-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/thehive-logo.png -------------------------------------------------------------------------------- /var/www/static/image/thehive_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/thehive_icon.png -------------------------------------------------------------------------------- /var/www/static/image/tm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ail-project/ail-framework/9b112b0742ef832195c39d9d24232d79ac55b861/var/www/static/image/tm.png -------------------------------------------------------------------------------- /var/www/static/js/d3/sparklines.js: -------------------------------------------------------------------------------- 1 | const sparkline = (container_id, data, options) => { 2 | 3 | const defaults = { 4 | style: { 5 | stroke: "rgb(0, 0, 0)", 6 | strokeWidth: 2 7 | }, 8 | margin: {top:3, right:3, bottom:3, left:3}, 9 | width: 100, 10 | height: 60 11 | }; 12 | 13 | options = $.extend(true, defaults, options); 14 | 15 | 16 | let width_spark = options.width - options.margin.left - options.margin.right; 17 | let height_spark = options.height - options.margin.top - options.margin.bottom; 18 | 19 | let maxX = data.length; 20 | let maxY = d3.max(data, function(d) { return d } ); 21 | 22 | let x = d3.scaleLinear() 23 | .range([0, width_spark]) 24 | .domain([0,maxX]); 25 | 26 | let y = d3.scaleLinear() 27 | .range([height_spark, 0]) 28 | .domain([0,maxY]); 29 | 30 | let line = d3.line() 31 | .x(function(d, i) {return x(i)}) 32 | .y(function(d) {return y(d)}); 33 | 34 | let res = d3.select( "#"+container_id ).append('svg') 35 | .attr('width', options.width) 36 | .attr('height', options.height) 37 | .append('g') 38 | .attr("transform", "translate("+options.margin.left+","+options.margin.top+")") 39 | .append('path') 40 | .datum(data) 41 | .attr('d', line) 42 | .style("fill", "none") 43 | .style("stroke", options.style.stroke) 44 | .style("stroke-width", options.style.strokeWidth); 45 | 46 | return res 47 | 48 | } 49 | -------------------------------------------------------------------------------- /var/www/templates/chats_explorer/block_language_stats.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /var/www/templates/chats_explorer/block_translation.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 | Translation 6 |
7 |
8 | 19 |
20 |
21 | 25 |
26 |
27 |
28 |
29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /var/www/templates/crawler/crawler_disabled.html: -------------------------------------------------------------------------------- 1 | {%if not is_manager_connected['status']%} 2 | 30 | 31 | {%endif%} 32 | -------------------------------------------------------------------------------- /var/www/templates/crawler/crawler_splash/cookie_edit_block.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 |
6 |
7 |
8 | 9 |
10 | 11 |
12 |
13 |
14 | 15 |
16 | 17 |
18 |
19 |
20 | 21 |
22 | 23 |
24 |
25 | 26 |
27 | 28 | 31 |
32 |
33 | 34 | 37 |
38 | -------------------------------------------------------------------------------- /var/www/templates/crawler/crawler_splash/cookies_card_block.html: -------------------------------------------------------------------------------- 1 | {% for dict_cookie in l_elem %} 2 | 3 | {% if loop.index0 % 4 == 0 %} 4 |
5 | {% endif %} 6 | 7 |
8 |
9 |
10 |
11 | 12 | 13 | 14 |
15 |
16 | 17 | 18 | 19 |
20 |
21 | 22 |
23 |
24 | 
{{dict_cookie['data']}}
25 |
26 |
27 | 28 | {% if loop.index0 % 4 == 3 %} 29 |
30 | {% endif %} 31 | 32 | {% endfor %} 33 | 34 | {% if l_elem|length % 4 != 0 %} 35 | 36 | {% endif %} 37 | -------------------------------------------------------------------------------- /var/www/templates/crawler/crawler_splash/settings_edit_crawlers_to_launch.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | AIL-Framework 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | {% include 'nav_bar.html' %} 21 | 22 |
23 |
24 | 25 | {% include 'crawler/menu_sidebar.html' %} 26 | 27 |
28 | 29 |
30 |
31 | 32 |
33 |

Number of Concurrent Crawlers to Launch:

34 | 35 | 36 | 37 |
38 |
39 |
40 | 41 |
42 |
43 |
44 | 45 | 46 | 47 | 48 | 56 | -------------------------------------------------------------------------------- /var/www/templates/crawler/crawler_splash/table_cookiejar.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {% for dict_cookiejar in all_cookiejar %} 14 | 15 | 16 | 21 | 26 | 37 | 42 | 43 | 44 | {% endfor %} 45 | 46 |
DescriptionDateUUIDLevelOrgUser
{{dict_cookiejar['description']}} 17 | {%if dict_cookiejar['date']%} 18 | {{dict_cookiejar['date'][0:4]}}/{{dict_cookiejar['date'][4:6]}}/{{dict_cookiejar['date'][6:8]}} 19 | {%endif%} 20 | 22 | 23 | {{ dict_cookiejar['uuid']}} 24 | 25 | 27 | {% if 'level' in dict_cookiejar %} 28 | {% if dict_cookiejar['level'] == 0 %} 29 | User 30 | {% elif dict_cookiejar['level'] == 1 %} 31 | Global 32 | {% elif dict_cookiejar['level'] == 2 %} 33 | Community 34 | {% endif %} 35 | {% endif %} 36 | 38 | {% if dict_cookiejar['org'] %} 39 | {{dict_cookiejar['org_name']}}
{{ dict_cookiejar['org'] }} 40 | {% endif %} 41 | 		{{dict_cookiejar['user']}}
47 | -------------------------------------------------------------------------------- /var/www/templates/decoded/show_helper_cryptocurrency.html: -------------------------------------------------------------------------------- 1 |
    2 |
    • 3 |
  • 4 |

    Double click on a node to open Cryptocurrency/Paste

    5 | 6 | Current Cryptocurrency
    7 | 8 | Cryptocurrency
    9 | 10 | Pastes 11 |

    12 |
    • 13 |
  • 14 | Cryptocurrency Types: 15 |
    • 16 |
  • 17 | Bitcoin
    18 | Monero
    19 |
    • 20 |
21 | -------------------------------------------------------------------------------- /var/www/templates/error/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 404 - AIL 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | {% include 'nav_bar.html' %} 16 | 17 |
18 |
19 |
20 |

404 Not Found

21 |
22 |
23 |
24 |
25 |
26 |
27 | 
28 |         ,d8       ,a8888a,              ,d8
29 |       ,d888     ,8P"'  `"Y8,          ,d888
30 |     ,d8" 88    ,8P        Y8,       ,d8" 88
31 |   ,d8"   88    88          88     ,d8"   88
32 | ,d8"     88    88          88   ,d8"     88
33 | 8888888888888  `8b        d8'   8888888888888
34 |          88     `8ba,  ,ad8'             88
35 |          88       "Y8888P"               88
36 |
37 |
38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /var/www/templates/import_export/block_add_user_object_to_export.html: -------------------------------------------------------------------------------- 1 | 2 | Add to 3 | 4 | Export 5 | 6 | -------------------------------------------------------------------------------- /var/www/templates/import_export/block_add_user_object_to_export_small.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 | 5 | 6 |
7 |
8 | 9 | 10 | 11 |
12 |
13 | -------------------------------------------------------------------------------- /var/www/templates/import_export/block_to_export_input.html: -------------------------------------------------------------------------------- 1 |
2 | 11 | 12 | 13 | 14 |
15 | {%if obj_type=="Object type..."%} 16 |     Please select an object type. 17 | {%else%} 18 |              Unknown or Invalid ID. 19 | {%endif%} 20 |
21 |
22 | -------------------------------------------------------------------------------- /var/www/templates/item_submit/menu_sidebar.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 7 | 8 | 13 |
14 | -------------------------------------------------------------------------------- /var/www/templates/modals/tracker_remove_object.html: -------------------------------------------------------------------------------- 1 | 28 | 29 | 40 | -------------------------------------------------------------------------------- /var/www/templates/objects/block_obj_button.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 5 | {{ nb }} 6 | 7 |
8 |
{{ name }}
9 | -------------------------------------------------------------------------------- /var/www/templates/objects/block_object_footer_small.html: -------------------------------------------------------------------------------- 1 |
2 | {#
#} 3 | {# #} 4 | {# #} 5 | {# #} 6 | {#
#} 7 |
8 | 9 | 10 | 11 |
12 |
13 | -------------------------------------------------------------------------------- /var/www/templates/objects/cookie-name/block_cookie_name_search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
Titles Search:
4 |
5 |
6 | 7 | 11 | 12 | 13 |
14 |
15 | 16 | 17 |
18 |
19 |
20 |
-------------------------------------------------------------------------------- /var/www/templates/objects/cve/block_cve_search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
CVE Search:
4 |
5 |
6 | 7 | 8 | 9 |
10 |
11 |
12 |
-------------------------------------------------------------------------------- /var/www/templates/objects/file-name/block_file_name_search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
File Names Search:
4 |
5 |
6 | 7 | 8 | 9 |
10 |
11 | 12 | 13 |
14 |
15 |
16 |
-------------------------------------------------------------------------------- /var/www/templates/objects/gtracker/block_gtrackers_search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
GTracking Search:
4 |
5 |
6 | 7 | 8 | 9 |
10 |
11 |
12 |
-------------------------------------------------------------------------------- /var/www/templates/objects/image/block_blur_img_slider.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 | 9 |
10 |
11 | 12 | 13 |
14 |
15 | 19 |
20 |
21 |
22 |
23 | 24 | 25 | -------------------------------------------------------------------------------- /var/www/templates/objects/image/block_domain_ollama.html: -------------------------------------------------------------------------------- 1 | {% if ollama_enabled or ollama_description %} 2 |
3 |
4 | {% if ollama_description %} 5 |

{{ ollama_description }}

6 | {% elif ollama_enabled %} 7 | 8 | 📝Image Description 9 | 10 | {% endif %} 11 |
12 |
13 | {% endif %} -------------------------------------------------------------------------------- /var/www/templates/objects/image/block_img_ollama.html: -------------------------------------------------------------------------------- 1 | {% if ollama_enabled or ollama_description %} 2 |
3 |
4 | {% if ollama_description %} 5 |

{{ ollama_description }}

6 | {% elif ollama_enabled %} 7 | 8 | 📝Image Description 9 | 10 | {% endif %} 11 |
12 |
13 | {% endif %} -------------------------------------------------------------------------------- /var/www/templates/objects/image/block_img_ollama_small.html: -------------------------------------------------------------------------------- 1 | {% if ollama_enabled or ollama_description %} 2 | 3 | {% if ollama_description %} 4 | 5 | {% elif ollama_enabled %} 6 | 7 | 📝 8 | 9 | {% endif %} 10 | 11 | {% endif %} -------------------------------------------------------------------------------- /var/www/templates/objects/mail/block_mails_search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
Mails Search:
4 |
5 |
6 | 7 | 8 | 9 |
10 |
11 |
12 |
-------------------------------------------------------------------------------- /var/www/templates/objects/obj_svg_block.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {{ icon }} 5 | 6 | -------------------------------------------------------------------------------- /var/www/templates/objects/title/block_titles_search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
Titles Search:
4 |
5 |
6 | 7 | 11 | 12 | 13 |
14 |
15 | 16 | 17 |
18 |
19 |
20 |
-------------------------------------------------------------------------------- /var/www/templates/objects/username/block_usernames_search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
Usernames Search:
4 |
5 |
6 | 7 | 12 | 13 | 14 |
15 |
16 | 17 | 18 |
19 |
20 |
21 |
-------------------------------------------------------------------------------- /var/www/templates/settings/block_password_requirements.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
Password Requirements
6 |
    7 |
  • 8 | Minimal length 9 | 10 10 |
    • 11 |
  • 12 | Upper characters: A-Z 13 | 1 14 |
    • 15 |
  • 16 | Lower characters: a-z 17 | 1 18 |
    • 19 |
  • 20 | Digits: 0-9 21 | 2 22 |
    • 23 |
  • 24 | Maximum length 25 | 100 26 |
    • 27 |
28 |
29 | 30 | --------------------------------------------------------------------------------