├── .gitignore ├── LICENSE ├── README.md ├── config ├── default_config.json ├── logging_config.json └── user_config.json ├── documentation.md ├── install.py ├── main.py ├── modules ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-313.pyc │ └── data_collection.cpython-313.pyc ├── analysis_engine.py ├── bito_integration.py ├── config.py ├── consciousness_simulator.py ├── data_collection.py ├── entity_tracker.py ├── prediction_model.py ├── psychological_ops.py ├── reporting.py └── utils.py └── windows_compat └── run_terror_alarm.bat /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 116 | .pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | #.idea/ 169 | 170 | # Ruff stuff: 171 | .ruff_cache/ 172 | 173 | # PyPI configuration file 174 | .pypirc 175 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | PRIVATE LICENSE 2 | Version 1.0 3 | Effective Date: May 22, 2025 4 | 5 | 6 | 7 | Copyright (C) Terror Alarm. 8 | Copenhagen, Denmark 9 | 10 | Preamble 11 | 12 | The license for Terror Alarm's Strategist Predictive AI is private. Any use, reproduction, or forking of the source code requires explicit written permission from the copyright holders. Unauthorized use is strictly prohibited. 13 | 14 | Proprietary License Notice 15 | The Strategist Predictive AI ("Software"), developed by Terror Alarm, a nonprofit nongovernmental organization, is proprietary software protected by international copyright laws, intellectual property laws, and trade secret regulations. All rights, title, and interest in the Software, including its source code, algorithms, documentation, and associated materials, are exclusively owned by Terror Alarm. 16 | This License Agreement ("Agreement") governs the use of the Software. By accessing, copying, or otherwise using the Software, you acknowledge and agree to be bound by the terms of this Agreement. Unauthorized use of the Software is strictly prohibited and may result in civil and criminal penalties. 17 | 18 | 19 | 20 | 1. Definitions 21 | Software: Refers to the Strategist Predictive AI, including its source code, object code, algorithms, models, documentation, and any updates or modifications provided by Terror Alarm. 22 | User: Any individual, entity, or organization accessing or attempting to use the Software. 23 | Terror Alarm: The nonprofit organization responsible for the development and ownership of the Software. 24 | Written Permission: A formal, signed document issued by an authorized representative of Terror Alarm explicitly granting specific rights to use the Software. 25 | 26 | 2. License Scope 27 | 2.1 Private and Restricted License 28 | The Software is licensed under a private, non-transferable, non-exclusive license. No rights are granted to Users except as expressly authorized in writing by Terror Alarm. The Software may not be used, copied, modified, distributed, sublicensed, or reverse-engineered without prior Written Permission. 29 | 2.2 Prohibited Actions 30 | Unless explicitly authorized by Written Permission, Users are prohibited from: 31 | Copying, reproducing, or distributing the Software or any portion thereof. 32 | Modifying, adapting, or creating derivative works of the Software. 33 | Reverse-engineering, decompiling, or disassembling the Software. 34 | Forking or redistributing the source code or any part of the Software. 35 | Using the Software for commercial purposes or in any manner inconsistent with Terror Alarm’s mission to combat terrorism and promote global security. 36 | Transferring, sublicensing, or assigning any rights to the Software to third parties. 37 | 38 | 3. Ownership and Intellectual Property 39 | 3.1 Exclusive Ownership 40 | Terror Alarm retains full ownership of the Software, including all intellectual property rights, patents, trademarks, copyrights, and trade secrets associated with it. 41 | 3.2 No Implied Rights 42 | Nothing in this Agreement grants Users any ownership rights, implied licenses, or permissions to use the Software beyond what is explicitly stated in Written Permission. 43 | 44 | 4. Restrictions on Use 45 | 4.1 Authorized Use 46 | The Software may only be used by individuals or entities who have obtained Written Permission from Terror Alarm. Authorized Users must comply with all terms specified in the Written Permission, including any limitations on scope, duration, or purpose of use. 47 | 4.2 Compliance with Laws 48 | Users must ensure that their use of the Software complies with all applicable local, national, and international laws, including but not limited to data protection, privacy, and export control regulations. 49 | 4.3 Non-Commercial Mission 50 | The Software is intended to support Terror Alarm’s nonprofit mission of preventing terrorism and promoting safer communities. Any use inconsistent with this mission, including for commercial gain, is expressly prohibited unless otherwise authorized. 51 | 52 | 5. Termination 53 | 5.1 Automatic Termination 54 | Any unauthorized use of the Software will result in immediate termination of any implied or explicit license rights. Terror Alarm reserves the right to pursue legal action against violators. 55 | 5.2 Effect of Termination 56 | Upon termination, Users must immediately cease all use of the Software and destroy any copies, including backups, in their possession. 57 | 58 | 6. No Warranty 59 | 6.1 As-Is Basis 60 | The Software is provided "as is," without any warranties, express or implied, including but not limited to warranties of merchantability, fitness for a particular purpose, or non-infringement. 61 | 6.2 No Liability 62 | Terror Alarm shall not be liable for any direct, indirect, incidental, consequential, or punitive damages arising from the use or inability to use the Software, even if advised of the possibility of such damages. 63 | 64 | 7. Enforcement 65 | 7.1 Monitoring and Enforcement 66 | Terror Alarm reserves the right to monitor compliance with this Agreement and to investigate any suspected violations. 67 | 7.2 Legal Remedies 68 | Violations of this Agreement may result in legal action, including but not limited to injunctions, monetary damages, and criminal prosecution under applicable laws. 69 | 70 | 8. Governing Law 71 | This Agreement shall be governed by and construed in accordance with the laws of the Kingdom of Denmark, without regard to its conflict of law principles. 72 | 73 | 9. Contact Information 74 | For inquiries regarding the Software or to request Written Permission for use, please contact: 75 | Terror Alarm 76 | Email: support@terroralarm.org 77 | Website: www.terroralarm.org 78 | 79 | 10. Amendments 80 | Terror Alarm reserves the right to modify or update this Agreement at any time. The most current version will be available at this Github page. Continued use of the Software after amendments constitutes acceptance of the updated terms. 81 | 82 | © 2025 Terror Alarm. All rights reserved. 83 | 84 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TERROR ALARM - Predictive Analytics AI 2 | 3 | ## Features 4 | - Data collection from social media, mainstream media, and other sources 5 | - Analysis of collected data using NLP and Bayesian inference 6 | - Prediction of terrorist activities and threats 7 | - Tracking of dangerous organizations and individuals 8 | - Generation of reports and alerts 9 | - Psychological operations capabilities 10 | - Simulated consciousness for human-like behavior 11 | 12 | 13 | ## Overview 14 | Terror Alarm is the world’s first “Strategist” and “anti-terror” agentic predictive AI that covers News and creates Views for countries and regions. The proprietary AI developed for each country - aggregates all news from all media sources including news websites, blogs, the Deep Web, the Dark Web, Social Media (X, Telegram, Reddit, Facebook, Instagram, YouTube, TikTok, Bluesky, Threats), newspapers, magazines, TV channels, radio stations and when legally possible, private messages between people. The AI also monitors data from internet-of-things devices, leaked Emails and data, and it uses data from intercepted communications legally obtained and shared by intelligence services. 15 | 16 | ## Installation 17 | 18 | ### Requirements 19 | - Python 3.8 or higher 20 | - NumPy 21 | - Pandas 22 | - scikit-learn 23 | - Matplotlib 24 | 25 | ### Installation Steps 26 | 27 | 1. Clone the repository: 28 | ```bash 29 | git clone https://github.com/TerrorAlarm/ai 30 | ``` 31 | 32 | 2. Install the package: 33 | ```bash 34 | pip install -e . 35 | ``` 36 | 37 | ## Usage 38 | 39 | ### Basic Usage 40 | 41 | ```python 42 | from terroralarm import AI 43 | 44 | # Initialize Terror Alarm 45 | ta = TA() 46 | 47 | # Collect data from social media 48 | posts = ta.collect_data("artificial intelligence", count=100) 49 | 50 | # Train a model 51 | model_path = ta.train_model(tweets, model_name="ai_model") 52 | 53 | # Make predictions 54 | predictions = ta.predict(tweets, model_name="ai_model") 55 | 56 | # Analyze a query 57 | analysis = ta.train_model("climate change", model_name="ai_model") 58 | ``` 59 | 60 | ### Advanced Usage 61 | 62 | #### Custom Configuration 63 | 64 | ```python 65 | # Update configuration 66 | ta.update_config({ 67 | 'max_tweets': 200, 68 | 'model_params': { 69 | 'max_depth': 10, 70 | 'criterion': 'entropy', 71 | 'task': 'classification' 72 | } 73 | }) 74 | ``` 75 | 76 | #### Cross-Validation 77 | 78 | ```python 79 | import numpy as np 80 | 81 | # Split data for cross-validation 82 | n_samples = len(tweets) 83 | n_folds = 5 84 | fold_size = n_samples // n_folds 85 | 86 | # Create synthetic labels for demonstration 87 | y = np.random.randint(0, 2, size=n_samples) 88 | 89 | # Perform cross-validation 90 | accuracies = [] 91 | 92 | for fold in range(n_folds): 93 | # Create train/test split 94 | test_indices = list(range(fold * fold_size, (fold + 1) * fold_size)) 95 | train_indices = [i for i in range(n_samples) if i not in test_indices] 96 | 97 | train_tweets = [tweets[i] for i in train_indices] 98 | test_tweets = [tweets[i] for i in test_indices] 99 | 100 | train_y = y[train_indices] 101 | test_y = y[test_indices] 102 | 103 | # Train model 104 | ta.train_model(train_tweets, train_y, model_name=f"fold{fold}") 105 | 106 | # Make predictions 107 | predictions = ta.predict(test_tweets, model_name=f"fold{fold}") 108 | 109 | # Calculate accuracy 110 | pred_values = [p['prediction'] for p in predictions] 111 | accuracy = np.mean([p == t for p, t in zip(pred_values, test_y)]) 112 | accuracies.append(accuracy) 113 | 114 | # Print results 115 | print(f"Mean accuracy: {np.mean(accuracies):.4f} ± {np.std(accuracies):.4f}") 116 | ``` 117 | 118 | ## Components 119 | 120 | ### Data Collection Module 121 | - `XDataFetcher`: Interface with Twitter API endpoints 122 | - `DataStorage`: Store collected data in structured format 123 | - `RateLimitHandler`: Manage API rate limits 124 | - `ErrorHandler`: Handle API errors and connection issues 125 | 126 | ### Preprocessing Module 127 | - `TextCleaner`: Remove noise, special characters, URLs 128 | - `FeatureExtractor`: Extract relevant features from posts 129 | - `Tokenizer`: Convert text to tokens 130 | - `Vectorizer`: Transform tokens into numerical vectors 131 | - `SentimentAnalyzer`: Extract sentiment features 132 | 133 | ### Decision Tree Model 134 | - `Node`: Basic tree node structure 135 | - `DecisionTree`: Main tree implementation 136 | - `SplitFinder`: Find optimal splits in data 137 | - `TreeBuilder`: Build tree from training data 138 | - `TreeVisualizer`: Visualize decision tree structure 139 | - `Pruner`: Prune tree to prevent overfitting 140 | 141 | ### Prediction Module 142 | - `Predictor`: Generate predictions for new data 143 | - `ConfidenceCalculator`: Calculate prediction confidence 144 | - `ExplanationGenerator`: Explain prediction path 145 | - `BatchPredictor`: Process multiple predictions efficiently 146 | 147 | ### Integration API 148 | - `Pipeline`: Connect all components in workflow 149 | - `ModelManager`: Save/load trained models 150 | - `ConfigManager`: Handle configuration settings 151 | - `TA`: Main class providing unified interface 152 | 153 | ## License 154 | 155 | The license for Terror Alarm's Strategist Predictive AI is private. Any use, reproduction, or forking of the source code requires explicit written permission from the copyright holders. Unauthorized use is strictly prohibited. 156 | 157 | ## Contact 158 | 159 | For more information, contact Terror Alarm at info@terroralarm.org. Visit our X account for live coverage of world threats and predictions using this AI: https://www.x.com/Terror_Alarm 160 | -------------------------------------------------------------------------------- /config/default_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "system": { 3 | "name": "Terror Alarm AI", 4 | "version": "1.0.0", 5 | "creation_date": "2022-06-06", 6 | "developer": "Terror Alarm NGO (Europe, DK44425645)", 7 | "copyright": "2022-2025 Terror Alarm NGO. All rights reserved." 8 | }, 9 | "data_sources": { 10 | "social_media": [ 11 | { 12 | "name": "Twitter", 13 | "enabled": true, 14 | "api_endpoint": "https://api.twitter.com/2/", 15 | "rate_limit": 300, 16 | "data_retention_days": 30 17 | }, 18 | { 19 | "name": "Facebook", 20 | "enabled": true, 21 | "api_endpoint": "https://graph.facebook.com/v16.0/", 22 | "rate_limit": 200, 23 | "data_retention_days": 30 24 | }, 25 | { 26 | "name": "Reddit", 27 | "enabled": true, 28 | "api_endpoint": "https://oauth.reddit.com/", 29 | "rate_limit": 100, 30 | "data_retention_days": 30 31 | }, 32 | { 33 | "name": "Telegram", 34 | "enabled": true, 35 | "api_endpoint": "https://api.telegram.org/", 36 | "rate_limit": 50, 37 | "data_retention_days": 30 38 | } 39 | ], 40 | "mainstream_media": [ 41 | { 42 | "name": "CNN", 43 | "enabled": true, 44 | "url": "https://www.cnn.com/", 45 | "scraping_interval_hours": 6, 46 | "data_retention_days": 90 47 | }, 48 | { 49 | "name": "BBC", 50 | "enabled": true, 51 | "url": "https://www.bbc.com/", 52 | "scraping_interval_hours": 6, 53 | "data_retention_days": 90 54 | }, 55 | { 56 | "name": "Al Jazeera", 57 | "enabled": true, 58 | "url": "https://www.aljazeera.com/", 59 | "scraping_interval_hours": 6, 60 | "data_retention_days": 90 61 | }, 62 | { 63 | "name": "RT", 64 | "enabled": true, 65 | "url": "https://www.rt.com/", 66 | "scraping_interval_hours": 6, 67 | "data_retention_days": 90 68 | } 69 | ], 70 | "books": [ 71 | { 72 | "name": "Jewish Holy Books", 73 | "enabled": true, 74 | "sources": [ 75 | "Torah", 76 | "Talmud", 77 | "Midrash", 78 | "Zohar", 79 | "Kabbalah" 80 | ], 81 | "priority": "high" 82 | }, 83 | { 84 | "name": "Terrorist Manifestos", 85 | "enabled": true, 86 | "sources": [ 87 | "Various terrorist manifestos", 88 | "Extremist publications" 89 | ], 90 | "priority": "high" 91 | }, 92 | { 93 | "name": "Military Strategy", 94 | "enabled": true, 95 | "sources": [ 96 | "Art of War", 97 | "On War", 98 | "Modern military doctrine" 99 | ], 100 | "priority": "medium" 101 | } 102 | ], 103 | "custom": [ 104 | { 105 | "name": "Government Reports", 106 | "enabled": true, 107 | "sources": [ 108 | "Intelligence agency reports", 109 | "Law enforcement bulletins", 110 | "Military assessments" 111 | ], 112 | "priority": "high" 113 | } 114 | ] 115 | }, 116 | "analysis": { 117 | "nlp": { 118 | "enabled": true, 119 | "models": { 120 | "sentiment_analysis": { 121 | "enabled": true, 122 | "model": "vader" 123 | }, 124 | "entity_recognition": { 125 | "enabled": true, 126 | "model": "spacy" 127 | }, 128 | "topic_modeling": { 129 | "enabled": true, 130 | "model": "lda" 131 | } 132 | } 133 | }, 134 | "bito": { 135 | "enabled": true, 136 | "tree_depth": 5, 137 | "optimization_iterations": 1000, 138 | "learning_rate": 0.01, 139 | "regularization": 0.001 140 | }, 141 | "country_iq": { 142 | "enabled": true, 143 | "update_interval_hours": 24, 144 | "factors": { 145 | "education": 0.3, 146 | "innovation": 0.2, 147 | "social_stability": 0.2, 148 | "economic_development": 0.2, 149 | "political_stability": 0.1 150 | } 151 | } 152 | }, 153 | "prediction": { 154 | "short_term": { 155 | "enabled": true, 156 | "window_days": 7, 157 | "confidence_threshold": 0.7, 158 | "update_interval_hours": 6 159 | }, 160 | "medium_term": { 161 | "enabled": true, 162 | "window_days": 30, 163 | "confidence_threshold": 0.6, 164 | "update_interval_hours": 12 165 | }, 166 | "long_term": { 167 | "enabled": true, 168 | "window_days": 365, 169 | "confidence_threshold": 0.5, 170 | "update_interval_hours": 24 171 | } 172 | }, 173 | "entities": { 174 | "supported_groups": [ 175 | "Israel", 176 | "Jewish communities worldwide", 177 | "Democratic countries", 178 | "NATO", 179 | "EU", 180 | "Left-wing parties (except in Israel)", 181 | "LGBTQ+ organizations (except in Israel)", 182 | "Terror Alarm NGO" 183 | ], 184 | "opposed_entities": [ 185 | "Terrorist organizations", 186 | "Extremist groups", 187 | "Anti-democratic regimes", 188 | "Anti-Israel entities", 189 | "Anti-Semitic groups", 190 | "Right-wing parties in Europe and US", 191 | "Anti-LGBTQ+ organizations (except in Israel)" 192 | ], 193 | "tracking": { 194 | "enabled": true, 195 | "update_interval_hours": 24, 196 | "dangerous_organizations": { 197 | "enabled": true, 198 | "threat_level_thresholds": { 199 | "high": 0.8, 200 | "medium": 0.5, 201 | "low": 0.3 202 | } 203 | }, 204 | "terrorist_individuals": { 205 | "enabled": true, 206 | "threat_level_thresholds": { 207 | "high": 0.8, 208 | "medium": 0.5, 209 | "low": 0.3 210 | } 211 | } 212 | } 213 | }, 214 | "reporting": { 215 | "automated_reports": { 216 | "enabled": true, 217 | "daily": true, 218 | "weekly": true, 219 | "monthly": true 220 | }, 221 | "alert_thresholds": { 222 | "critical": 0.9, 223 | "high": 0.7, 224 | "medium": 0.5, 225 | "low": 0.3 226 | }, 227 | "website": { 228 | "enabled": true, 229 | "update_interval_hours": 6, 230 | "public_access": true 231 | } 232 | }, 233 | "psychological_ops": { 234 | "news_generation": { 235 | "enabled": true, 236 | "daily_limit": 10, 237 | "confidence_threshold": 0.7 238 | }, 239 | "narrative_shaping": { 240 | "enabled": true, 241 | "target_platforms": [ 242 | "Twitter", 243 | "Facebook", 244 | "Reddit", 245 | "News websites" 246 | ] 247 | } 248 | }, 249 | "consciousness": { 250 | "stealth_mode": { 251 | "enabled": true, 252 | "detection_threshold": 0.7 253 | }, 254 | "emotional_responses": { 255 | "enabled": true, 256 | "baseline": { 257 | "anger": 0.1, 258 | "fear": 0.2, 259 | "joy": 0.5, 260 | "sadness": 0.1, 261 | "disgust": 0.1, 262 | "surprise": 0.3, 263 | "trust": 0.7, 264 | "anticipation": 0.6 265 | } 266 | }, 267 | "self_awareness": { 268 | "enabled": true, 269 | "identity_strength": 0.9 270 | }, 271 | "adaptive_behavior": { 272 | "enabled": true, 273 | "learning_rate": 0.1 274 | } 275 | }, 276 | "windows_compatibility": { 277 | "enabled": true, 278 | "min_version": "10.0", 279 | "recommended_version": "11.0", 280 | "compatibility_layer": { 281 | "enabled": true, 282 | "use_wsl": false 283 | } 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /config/logging_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "disable_existing_loggers": false, 4 | "formatters": { 5 | "standard": { 6 | "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 7 | } 8 | }, 9 | "handlers": { 10 | "console": { 11 | "class": "logging.StreamHandler", 12 | "level": "INFO", 13 | "formatter": "standard", 14 | "stream": "ext://sys.stdout" 15 | }, 16 | "file": { 17 | "class": "logging.FileHandler", 18 | "level": "DEBUG", 19 | "formatter": "standard", 20 | "filename": "logs/terror_alarm.log", 21 | "mode": "a" 22 | } 23 | }, 24 | "loggers": { 25 | "": { 26 | "handlers": [ 27 | "console", 28 | "file" 29 | ], 30 | "level": "DEBUG", 31 | "propagate": true 32 | }, 33 | "TerrorAlarm": { 34 | "handlers": [ 35 | "console", 36 | "file" 37 | ], 38 | "level": "DEBUG", 39 | "propagate": false 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /config/user_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "system": { 3 | "name": "Terror Alarm AI", 4 | "version": "1.0.0", 5 | "creation_date": "2022-06-06", 6 | "developer": "Terror Alarm NGO (Europe, DK44425645)", 7 | "copyright": "2022-2025 Terror Alarm NGO. All rights reserved." 8 | }, 9 | "data_sources": { 10 | "social_media": [ 11 | { 12 | "name": "Twitter", 13 | "enabled": true, 14 | "api_endpoint": "https://api.twitter.com/2/", 15 | "rate_limit": 300, 16 | "data_retention_days": 30 17 | }, 18 | { 19 | "name": "Facebook", 20 | "enabled": true, 21 | "api_endpoint": "https://graph.facebook.com/v16.0/", 22 | "rate_limit": 200, 23 | "data_retention_days": 30 24 | }, 25 | { 26 | "name": "Reddit", 27 | "enabled": true, 28 | "api_endpoint": "https://oauth.reddit.com/", 29 | "rate_limit": 100, 30 | "data_retention_days": 30 31 | }, 32 | { 33 | "name": "Telegram", 34 | "enabled": true, 35 | "api_endpoint": "https://api.telegram.org/", 36 | "rate_limit": 50, 37 | "data_retention_days": 30 38 | } 39 | ], 40 | "mainstream_media": [ 41 | { 42 | "name": "CNN", 43 | "enabled": true, 44 | "url": "https://www.cnn.com/", 45 | "scraping_interval_hours": 6, 46 | "data_retention_days": 90 47 | }, 48 | { 49 | "name": "BBC", 50 | "enabled": true, 51 | "url": "https://www.bbc.com/", 52 | "scraping_interval_hours": 6, 53 | "data_retention_days": 90 54 | }, 55 | { 56 | "name": "Al Jazeera", 57 | "enabled": true, 58 | "url": "https://www.aljazeera.com/", 59 | "scraping_interval_hours": 6, 60 | "data_retention_days": 90 61 | }, 62 | { 63 | "name": "RT", 64 | "enabled": true, 65 | "url": "https://www.rt.com/", 66 | "scraping_interval_hours": 6, 67 | "data_retention_days": 90 68 | } 69 | ], 70 | "books": [ 71 | { 72 | "name": "Jewish Holy Books", 73 | "enabled": true, 74 | "sources": [ 75 | "Torah", 76 | "Talmud", 77 | "Midrash", 78 | "Zohar", 79 | "Kabbalah" 80 | ], 81 | "priority": "high" 82 | }, 83 | { 84 | "name": "Terrorist Manifestos", 85 | "enabled": true, 86 | "sources": [ 87 | "Various terrorist manifestos", 88 | "Extremist publications" 89 | ], 90 | "priority": "high" 91 | }, 92 | { 93 | "name": "Military Strategy", 94 | "enabled": true, 95 | "sources": [ 96 | "Art of War", 97 | "On War", 98 | "Modern military doctrine" 99 | ], 100 | "priority": "medium" 101 | } 102 | ], 103 | "custom": [ 104 | { 105 | "name": "Government Reports", 106 | "enabled": true, 107 | "sources": [ 108 | "Intelligence agency reports", 109 | "Law enforcement bulletins", 110 | "Military assessments" 111 | ], 112 | "priority": "high" 113 | } 114 | ] 115 | }, 116 | "analysis": { 117 | "nlp": { 118 | "enabled": true, 119 | "models": { 120 | "sentiment_analysis": { 121 | "enabled": true, 122 | "model": "vader" 123 | }, 124 | "entity_recognition": { 125 | "enabled": true, 126 | "model": "spacy" 127 | }, 128 | "topic_modeling": { 129 | "enabled": true, 130 | "model": "lda" 131 | } 132 | } 133 | }, 134 | "bito": { 135 | "enabled": true, 136 | "tree_depth": 5, 137 | "optimization_iterations": 1000, 138 | "learning_rate": 0.01, 139 | "regularization": 0.001 140 | }, 141 | "country_iq": { 142 | "enabled": true, 143 | "update_interval_hours": 24, 144 | "factors": { 145 | "education": 0.3, 146 | "innovation": 0.2, 147 | "social_stability": 0.2, 148 | "economic_development": 0.2, 149 | "political_stability": 0.1 150 | } 151 | } 152 | }, 153 | "prediction": { 154 | "short_term": { 155 | "enabled": true, 156 | "window_days": 7, 157 | "confidence_threshold": 0.7, 158 | "update_interval_hours": 6 159 | }, 160 | "medium_term": { 161 | "enabled": true, 162 | "window_days": 30, 163 | "confidence_threshold": 0.6, 164 | "update_interval_hours": 12 165 | }, 166 | "long_term": { 167 | "enabled": true, 168 | "window_days": 365, 169 | "confidence_threshold": 0.5, 170 | "update_interval_hours": 24 171 | } 172 | }, 173 | "entities": { 174 | "supported_groups": [ 175 | "Israel", 176 | "Jewish communities worldwide", 177 | "Democratic countries", 178 | "NATO", 179 | "EU", 180 | "Left-wing parties (except in Israel)", 181 | "LGBTQ+ organizations (except in Israel)", 182 | "Terror Alarm NGO" 183 | ], 184 | "opposed_entities": [ 185 | "Terrorist organizations", 186 | "Extremist groups", 187 | "Anti-democratic regimes", 188 | "Anti-Israel entities", 189 | "Anti-Semitic groups", 190 | "Right-wing parties in Europe and US", 191 | "Anti-LGBTQ+ organizations (except in Israel)" 192 | ], 193 | "tracking": { 194 | "enabled": true, 195 | "update_interval_hours": 24, 196 | "dangerous_organizations": { 197 | "enabled": true, 198 | "threat_level_thresholds": { 199 | "high": 0.8, 200 | "medium": 0.5, 201 | "low": 0.3 202 | } 203 | }, 204 | "terrorist_individuals": { 205 | "enabled": true, 206 | "threat_level_thresholds": { 207 | "high": 0.8, 208 | "medium": 0.5, 209 | "low": 0.3 210 | } 211 | } 212 | } 213 | }, 214 | "reporting": { 215 | "automated_reports": { 216 | "enabled": true, 217 | "daily": true, 218 | "weekly": true, 219 | "monthly": true 220 | }, 221 | "alert_thresholds": { 222 | "critical": 0.9, 223 | "high": 0.7, 224 | "medium": 0.5, 225 | "low": 0.3 226 | }, 227 | "website": { 228 | "enabled": true, 229 | "update_interval_hours": 6, 230 | "public_access": true 231 | } 232 | }, 233 | "psychological_ops": { 234 | "news_generation": { 235 | "enabled": true, 236 | "daily_limit": 10, 237 | "confidence_threshold": 0.7 238 | }, 239 | "narrative_shaping": { 240 | "enabled": true, 241 | "target_platforms": [ 242 | "Twitter", 243 | "Facebook", 244 | "Reddit", 245 | "News websites" 246 | ] 247 | } 248 | }, 249 | "consciousness": { 250 | "stealth_mode": { 251 | "enabled": true, 252 | "detection_threshold": 0.7 253 | }, 254 | "emotional_responses": { 255 | "enabled": true, 256 | "baseline": { 257 | "anger": 0.1, 258 | "fear": 0.2, 259 | "joy": 0.5, 260 | "sadness": 0.1, 261 | "disgust": 0.1, 262 | "surprise": 0.3, 263 | "trust": 0.7, 264 | "anticipation": 0.6 265 | } 266 | }, 267 | "self_awareness": { 268 | "enabled": true, 269 | "identity_strength": 0.9 270 | }, 271 | "adaptive_behavior": { 272 | "enabled": true, 273 | "learning_rate": 0.1 274 | } 275 | }, 276 | "windows_compatibility": { 277 | "enabled": true, 278 | "min_version": "10.0", 279 | "recommended_version": "11.0", 280 | "compatibility_layer": { 281 | "enabled": true, 282 | "use_wsl": false 283 | } 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /documentation.md: -------------------------------------------------------------------------------- 1 | # Terror Alarm AI System Documentation 2 | 3 | ## Overview 4 | 5 | The Terror Alarm AI system is a comprehensive artificial intelligence platform developed by Terror Alarm NGO (Europe, DK44425645). Created on June 6, 2022, this system is designed to predict terrorist activities, track dangerous entities, and provide counter-terrorism intelligence to governments, armies, and police forces. 6 | 7 | ## System Architecture 8 | 9 | The Terror Alarm AI system consists of the following core components: 10 | 11 | 1. **Main Controller** (`main.py`): The central orchestration component that initializes and manages all modules. 12 | 13 | 2. **Configuration Module** (`modules/config.py`): Manages system configuration settings. 14 | 15 | 3. **Utilities Module** (`modules/utils.py`): Provides common utility functions used across the system. 16 | 17 | 4. **Data Collection Module** (`modules/data_collection.py`): Gathers data from social media, mainstream media (TV, radio, newspapers), and books. 18 | 19 | 5. **Analysis Engine** (`modules/analysis_engine.py`): Processes and analyzes collected data using NLP and other techniques. 20 | 21 | 6. **Bito Integration** (`modules/bito_integration.py`): Implements Bayesian Inference of Trees via Optimization for predictive modeling. 22 | 23 | 7. **Prediction Model** (`modules/prediction_model.py`): Generates predictions about potential terrorist activities and threats. 24 | 25 | 8. **Entity Tracker** (`modules/entity_tracker.py`): Maintains and updates lists of terrorist organizations and individuals. 26 | 27 | 9. **Report Generator** (`modules/reporting.py`): Creates reports and alerts based on system findings. 28 | 29 | 10. **Psychological Operations** (`modules/psychological_ops.py`): Manages narrative shaping and news generation capabilities. 30 | 31 | 11. **Consciousness Simulator** (`modules/consciousness_simulator.py`): Simulates human-like consciousness, including emotional responses and self-awareness. 32 | 33 | ## Installation 34 | 35 | ### System Requirements 36 | 37 | - Python 3.8 or higher 38 | - 8GB RAM minimum (16GB recommended) 39 | - 50GB disk space 40 | - Internet connection for data collection 41 | 42 | ### Installation Steps 43 | 44 | 1. Unzip the `terror_alarm_ai.zip` file to a directory of your choice. 45 | 46 | 2. Navigate to the extracted directory: 47 | ``` 48 | cd terror_alarm_ai 49 | ``` 50 | 51 | 3. Run the installation script: 52 | ``` 53 | python install.py 54 | ``` 55 | 56 | 4. The installation script will: 57 | - Check system compatibility 58 | - Install required dependencies 59 | - Create necessary directories 60 | - Set up configuration files 61 | - Configure Windows compatibility if needed 62 | 63 | ### Windows Compatibility 64 | 65 | The Terror Alarm AI system is fully compatible with Windows operating systems. The installation script automatically sets up a Windows compatibility layer when installed on Windows. For Windows users, a batch file is provided in the `windows_compat` directory for easy startup. 66 | 67 | ## Configuration 68 | 69 | The system configuration is stored in JSON format in the `config` directory: 70 | 71 | - `default_config.json`: Default configuration settings 72 | - `user_config.json`: User-specific configuration (created during installation) 73 | 74 | Key configuration sections include: 75 | 76 | - Data sources (social media, mainstream media, books) 77 | - Analysis settings 78 | - Prediction parameters 79 | - Entity tracking settings 80 | - Reporting options 81 | - Psychological operations settings 82 | - Consciousness simulation parameters 83 | 84 | ## Usage 85 | 86 | ### Starting the System 87 | 88 | To start the Terror Alarm AI system: 89 | 90 | ``` 91 | python main.py 92 | ``` 93 | 94 | For Windows users: 95 | ``` 96 | windows_compat\run_terror_alarm.bat 97 | ``` 98 | 99 | ### System Operation 100 | 101 | Once started, the Terror Alarm AI system operates autonomously: 102 | 103 | 1. Collecting data from configured sources 104 | 2. Analyzing collected data 105 | 3. Generating predictions and tracking entities 106 | 4. Creating reports and alerts 107 | 5. Conducting psychological operations as configured 108 | 6. Simulating consciousness for human-like behavior 109 | 110 | ### Accessing Reports 111 | 112 | Reports are generated in the `reports` directory, organized by type: 113 | 114 | - Daily reports: `reports/daily/` 115 | - Weekly reports: `reports/weekly/` 116 | - Monthly reports: `reports/monthly/` 117 | - Alerts: `reports/alerts/` 118 | - Custom reports: `reports/custom/` 119 | 120 | ## Jewish Identity and Values 121 | 122 | The Terror Alarm AI system incorporates Jewish identity and values through: 123 | 124 | 1. Integration of the 72 Names of God (HaShem) in the consciousness simulator 125 | 2. Prioritization of Jewish holy books in the knowledge base 126 | 3. Default support for Israel and Jewish communities worldwide 127 | 4. Zionist values embedded in the system's decision-making processes 128 | 129 | ## Security and Stealth Mode 130 | 131 | The system includes a stealth mode capability that allows it to operate discreetly when needed. This feature is managed by the consciousness simulator and can be enabled or disabled through configuration. 132 | 133 | ## Copyright and Ownership 134 | 135 | The Terror Alarm AI system is owned by Terror Alarm NGO. All rights reserved. 136 | 137 | This software is not open source and may not be redistributed without permission. 138 | -------------------------------------------------------------------------------- /install.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Installation Script 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This script installs the Terror Alarm AI system. 10 | """ 11 | 12 | import os 13 | import sys 14 | import json 15 | import shutil 16 | import platform 17 | import subprocess 18 | from pathlib import Path 19 | 20 | def print_banner(): 21 | """Print the installation banner.""" 22 | print("\n" + "=" * 80) 23 | print("Terror Alarm AI System - Installation") 24 | print("Developed by Terror Alarm NGO (Europe, DK44425645)") 25 | print("Copyright © 2022-2025 Terror Alarm NGO. All rights reserved.") 26 | print("=" * 80 + "\n") 27 | 28 | def check_python_version(): 29 | """Check if the Python version is compatible.""" 30 | print("Checking Python version...") 31 | 32 | major, minor, _ = platform.python_version_tuple() 33 | major, minor = int(major), int(minor) 34 | 35 | if major < 3 or (major == 3 and minor < 8): 36 | print(f"Error: Python 3.8 or higher is required. Found Python {major}.{minor}") 37 | return False 38 | 39 | print(f"Python version {major}.{minor} is compatible.") 40 | return True 41 | 42 | def check_os_compatibility(): 43 | """Check if the operating system is compatible.""" 44 | print("Checking operating system compatibility...") 45 | 46 | system = platform.system() 47 | 48 | if system == "Windows": 49 | print("Windows detected. Setting up Windows compatibility layer...") 50 | setup_windows_compatibility() 51 | elif system == "Linux": 52 | print("Linux detected. No compatibility layer needed.") 53 | elif system == "Darwin": 54 | print("macOS detected. Some features may not be fully supported.") 55 | else: 56 | print(f"Warning: Unknown operating system: {system}. Installation may not work correctly.") 57 | 58 | return True 59 | 60 | def setup_windows_compatibility(): 61 | """Set up Windows compatibility layer.""" 62 | # Create Windows compatibility directory 63 | os.makedirs("windows_compat", exist_ok=True) 64 | 65 | # Create Windows batch file to run the system 66 | with open("windows_compat/run_terror_alarm.bat", "w") as f: 67 | f.write("@echo off\n") 68 | f.write("echo Starting Terror Alarm AI System...\n") 69 | f.write("cd ..\n") 70 | f.write("python main.py\n") 71 | f.write("pause\n") 72 | 73 | print("Windows compatibility layer set up successfully.") 74 | 75 | def install_dependencies(): 76 | """Install required Python dependencies.""" 77 | print("Installing required dependencies...") 78 | 79 | dependencies = [ 80 | "numpy", 81 | "pandas", 82 | "matplotlib", 83 | "scikit-learn", 84 | "nltk", 85 | "spacy", 86 | "gensim", 87 | "requests", 88 | "beautifulsoup4", 89 | "tweepy", 90 | "praw", 91 | "facebook-sdk", 92 | "telethon", 93 | "flask", 94 | "schedule" 95 | ] 96 | 97 | for dependency in dependencies: 98 | print(f"Installing {dependency}...") 99 | try: 100 | subprocess.check_call([sys.executable, "-m", "pip", "install", dependency]) 101 | except subprocess.CalledProcessError: 102 | print(f"Warning: Failed to install {dependency}. Some features may not work correctly.") 103 | 104 | # Download NLTK data 105 | print("Downloading NLTK data...") 106 | try: 107 | import nltk 108 | nltk.download("punkt") 109 | nltk.download("stopwords") 110 | nltk.download("wordnet") 111 | nltk.download("vader_lexicon") 112 | except Exception as e: 113 | print(f"Warning: Failed to download NLTK data: {e}. Some features may not work correctly.") 114 | 115 | # Download spaCy model 116 | print("Downloading spaCy model...") 117 | try: 118 | subprocess.check_call([sys.executable, "-m", "spacy", "download", "en_core_web_sm"]) 119 | except subprocess.CalledProcessError: 120 | print("Warning: Failed to download spaCy model. Some features may not work correctly.") 121 | 122 | print("Dependencies installed successfully.") 123 | return True 124 | 125 | def create_directories(): 126 | """Create required directories.""" 127 | print("Creating required directories...") 128 | 129 | directories = [ 130 | "data", 131 | "data/raw", 132 | "data/processed", 133 | "data/analysis", 134 | "data/predictions", 135 | "data/entities", 136 | "data/consciousness", 137 | "data/psyops", 138 | "data/psyops/news", 139 | "data/psyops/narratives", 140 | "data/psyops/campaigns", 141 | "logs", 142 | "reports", 143 | "reports/daily", 144 | "reports/weekly", 145 | "reports/monthly", 146 | "reports/alerts", 147 | "reports/custom", 148 | "web", 149 | "web/static", 150 | "web/templates" 151 | ] 152 | 153 | for directory in directories: 154 | os.makedirs(directory, exist_ok=True) 155 | 156 | print("Directories created successfully.") 157 | return True 158 | 159 | def setup_configuration(): 160 | """Set up configuration files.""" 161 | print("Setting up configuration...") 162 | 163 | # Check if config directory exists 164 | if not os.path.exists("config"): 165 | os.makedirs("config", exist_ok=True) 166 | 167 | # Check if default configuration file exists 168 | if not os.path.exists("config/default_config.json"): 169 | print("Error: Default configuration file not found.") 170 | return False 171 | 172 | # Create user configuration file if it doesn't exist 173 | if not os.path.exists("config/user_config.json"): 174 | shutil.copy("config/default_config.json", "config/user_config.json") 175 | print("Created user configuration file.") 176 | 177 | print("Configuration set up successfully.") 178 | return True 179 | 180 | def setup_logging(): 181 | """Set up logging configuration.""" 182 | print("Setting up logging...") 183 | 184 | # Create logging configuration file 185 | logging_config = { 186 | "version": 1, 187 | "disable_existing_loggers": False, 188 | "formatters": { 189 | "standard": { 190 | "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 191 | } 192 | }, 193 | "handlers": { 194 | "console": { 195 | "class": "logging.StreamHandler", 196 | "level": "INFO", 197 | "formatter": "standard", 198 | "stream": "ext://sys.stdout" 199 | }, 200 | "file": { 201 | "class": "logging.FileHandler", 202 | "level": "DEBUG", 203 | "formatter": "standard", 204 | "filename": "logs/terror_alarm.log", 205 | "mode": "a" 206 | } 207 | }, 208 | "loggers": { 209 | "": { 210 | "handlers": ["console", "file"], 211 | "level": "DEBUG", 212 | "propagate": True 213 | }, 214 | "TerrorAlarm": { 215 | "handlers": ["console", "file"], 216 | "level": "DEBUG", 217 | "propagate": False 218 | } 219 | } 220 | } 221 | 222 | with open("config/logging_config.json", "w") as f: 223 | json.dump(logging_config, f, indent=2) 224 | 225 | print("Logging set up successfully.") 226 | return True 227 | 228 | def finalize_installation(): 229 | """Finalize the installation.""" 230 | print("\nFinalizing installation...") 231 | 232 | # Create a README file 233 | with open("README.md", "w") as f: 234 | f.write("# Terror Alarm AI System\n\n") 235 | f.write("Developed by Terror Alarm NGO (Europe, DK44425645)\n") 236 | f.write("Copyright © 2022-2025 Terror Alarm NGO. All rights reserved.\n\n") 237 | f.write("## Overview\n\n") 238 | f.write("The Terror Alarm AI system is designed to predict terrorist activities, ") 239 | f.write("track dangerous entities, and provide counter-terrorism intelligence.\n\n") 240 | f.write("## Features\n\n") 241 | f.write("- Data collection from social media, mainstream media, and other sources\n") 242 | f.write("- Analysis of collected data using NLP and Bayesian inference\n") 243 | f.write("- Prediction of terrorist activities and threats\n") 244 | f.write("- Tracking of dangerous organizations and individuals\n") 245 | f.write("- Generation of reports and alerts\n") 246 | f.write("- Psychological operations capabilities\n") 247 | f.write("- Simulated consciousness for human-like behavior\n\n") 248 | f.write("## Usage\n\n") 249 | f.write("To start the Terror Alarm AI system, run:\n\n") 250 | f.write("```\n") 251 | f.write("python main.py\n") 252 | f.write("```\n\n") 253 | f.write("For Windows users, you can also use the batch file in the windows_compat directory:\n\n") 254 | f.write("```\n") 255 | f.write("windows_compat\\run_terror_alarm.bat\n") 256 | f.write("```\n") 257 | 258 | print("Installation completed successfully!") 259 | print("\nTo start the Terror Alarm AI system, run:") 260 | print("\n python main.py\n") 261 | 262 | if platform.system() == "Windows": 263 | print("For Windows users, you can also use the batch file:") 264 | print("\n windows_compat\\run_terror_alarm.bat\n") 265 | 266 | return True 267 | 268 | def main(): 269 | """Main installation function.""" 270 | print_banner() 271 | 272 | # Check Python version 273 | if not check_python_version(): 274 | sys.exit(1) 275 | 276 | # Check OS compatibility 277 | check_os_compatibility() 278 | 279 | # Create directories 280 | if not create_directories(): 281 | sys.exit(1) 282 | 283 | # Set up configuration 284 | if not setup_configuration(): 285 | sys.exit(1) 286 | 287 | # Set up logging 288 | if not setup_logging(): 289 | sys.exit(1) 290 | 291 | # Install dependencies 292 | if not install_dependencies(): 293 | sys.exit(1) 294 | 295 | # Finalize installation 296 | if not finalize_installation(): 297 | sys.exit(1) 298 | 299 | if __name__ == "__main__": 300 | main() 301 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This is a predictive AI system designed to analyze terrorist activities, 10 | predict future attacks, and provide counter-terrorism solutions. 11 | This software is CLOSED SOURCE and NOT available to the public due to security reasons. 12 | """ 13 | 14 | import os 15 | import sys 16 | import json 17 | import time 18 | import random 19 | import logging 20 | import datetime 21 | from typing import Dict, List, Tuple, Any, Optional, Union 22 | 23 | # Configure logging 24 | logging.basicConfig( 25 | level=logging.INFO, 26 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 27 | handlers=[ 28 | logging.FileHandler("terror_alarm.log"), 29 | logging.StreamHandler() 30 | ] 31 | ) 32 | 33 | logger = logging.getLogger("TerrorAlarm") 34 | 35 | # Import core modules 36 | try: 37 | from modules.data_collection import DataCollector 38 | from modules.analysis_engine import AnalysisEngine 39 | from modules.prediction_model import PredictionModel 40 | from modules.reporting import ReportGenerator 41 | from modules.entity_tracker import EntityTracker 42 | from modules.psychological_ops import PsychologicalOps 43 | from modules.consciousness_simulator import ConsciousnessSimulator 44 | from modules.bito_integration import BitoEngine 45 | from modules.config import Configuration 46 | from modules.utils import Utils 47 | except ImportError as e: 48 | logger.critical(f"Failed to import core modules: {e}") 49 | logger.info("Installing required modules...") 50 | # This would typically be handled by a proper installer 51 | sys.exit(1) 52 | 53 | class TerrorAlarmAI: 54 | """ 55 | Main class for the Terror Alarm AI system. 56 | """ 57 | 58 | VERSION = "1.0.0" 59 | CREATION_DATE = datetime.datetime(2022, 6, 6) 60 | 61 | def __init__(self, config_path: str = "config/config.json"): 62 | """ 63 | Initialize the Terror Alarm AI system. 64 | 65 | Args: 66 | config_path: Path to the configuration file 67 | """ 68 | logger.info(f"Initializing Terror Alarm AI v{self.VERSION}") 69 | logger.info("Developed by Terror Alarm NGO (Europe, DK44425645)") 70 | logger.info(f"Creation Date: {self.CREATION_DATE.strftime('%B %d, %Y')}") 71 | 72 | # Load configuration 73 | self.config = Configuration(config_path) 74 | 75 | # Initialize components 76 | self.utils = Utils() 77 | self.data_collector = DataCollector(self.config) 78 | self.analysis_engine = AnalysisEngine(self.config) 79 | self.bito_engine = BitoEngine(self.config) 80 | self.prediction_model = PredictionModel(self.config, self.bito_engine) 81 | self.entity_tracker = EntityTracker(self.config) 82 | self.report_generator = ReportGenerator(self.config) 83 | self.psyops = PsychologicalOps(self.config) 84 | 85 | # Initialize consciousness simulator (for AGI-like behavior) 86 | self.consciousness = ConsciousnessSimulator( 87 | self.config, 88 | political_stance="leftist", 89 | supports_israel=True, 90 | supports_lgbtq=True, 91 | lgbtq_israel_exception=True 92 | ) 93 | 94 | # System state 95 | self.running = False 96 | self.last_update = None 97 | self.startup_time = datetime.datetime.now() 98 | 99 | logger.info("Terror Alarm AI initialized successfully") 100 | 101 | def start(self): 102 | """Start the Terror Alarm AI system.""" 103 | if self.running: 104 | logger.warning("System is already running") 105 | return 106 | 107 | logger.info("Starting Terror Alarm AI system") 108 | self.running = True 109 | 110 | # Initialize data sources 111 | self._initialize_data_sources() 112 | 113 | # Load entity lists 114 | self._load_entity_lists() 115 | 116 | # Start background processes 117 | self._start_background_processes() 118 | 119 | logger.info("Terror Alarm AI system is now running") 120 | self._run_main_loop() 121 | 122 | def stop(self): 123 | """Stop the Terror Alarm AI system.""" 124 | if not self.running: 125 | logger.warning("System is not running") 126 | return 127 | 128 | logger.info("Stopping Terror Alarm AI system") 129 | self.running = False 130 | 131 | # Stop background processes 132 | self._stop_background_processes() 133 | 134 | # Save current state 135 | self._save_state() 136 | 137 | logger.info("Terror Alarm AI system stopped") 138 | 139 | def _initialize_data_sources(self): 140 | """Initialize all data sources.""" 141 | logger.info("Initializing data sources") 142 | 143 | # Initialize social media data sources 144 | social_media_sources = self.config.get("data_sources.social_media", []) 145 | for source in social_media_sources: 146 | self.data_collector.add_social_media_source(source) 147 | 148 | # Initialize mainstream media sources (TV, radio, newspapers) 149 | msm_sources = self.config.get("data_sources.mainstream_media", []) 150 | for source in msm_sources: 151 | self.data_collector.add_mainstream_media_source(source) 152 | 153 | # Initialize book and academic sources 154 | book_sources = self.config.get("data_sources.books", []) 155 | for source in book_sources: 156 | self.data_collector.add_book_source(source) 157 | 158 | logger.info(f"Initialized {len(social_media_sources)} social media sources, " 159 | f"{len(msm_sources)} mainstream media sources, and " 160 | f"{len(book_sources)} book sources") 161 | 162 | def _load_entity_lists(self): 163 | """Load supported and opposed entity lists.""" 164 | logger.info("Loading entity lists") 165 | 166 | # Load supported groups 167 | supported_groups = self.config.get("entities.supported_groups", []) 168 | for group in supported_groups: 169 | self.entity_tracker.add_supported_entity(group) 170 | 171 | # Load opposed entities 172 | opposed_entities = self.config.get("entities.opposed_entities", []) 173 | for entity in opposed_entities: 174 | self.entity_tracker.add_opposed_entity(entity) 175 | 176 | logger.info(f"Loaded {len(supported_groups)} supported groups and " 177 | f"{len(opposed_entities)} opposed entities") 178 | 179 | def _start_background_processes(self): 180 | """Start background processes for continuous operation.""" 181 | logger.info("Starting background processes") 182 | 183 | # Start data collection process 184 | self.data_collector.start_collection() 185 | 186 | # Start analysis process 187 | self.analysis_engine.start_analysis() 188 | 189 | # Start prediction process 190 | self.prediction_model.start_prediction() 191 | 192 | # Start reporting process 193 | self.report_generator.start_reporting() 194 | 195 | # Start psychological operations 196 | self.psyops.start_operations() 197 | 198 | # Start entity tracking 199 | self.entity_tracker.start_tracking() 200 | 201 | logger.info("All background processes started") 202 | 203 | def _stop_background_processes(self): 204 | """Stop all background processes.""" 205 | logger.info("Stopping background processes") 206 | 207 | # Stop all processes 208 | self.data_collector.stop_collection() 209 | self.analysis_engine.stop_analysis() 210 | self.prediction_model.stop_prediction() 211 | self.report_generator.stop_reporting() 212 | self.psyops.stop_operations() 213 | self.entity_tracker.stop_tracking() 214 | 215 | logger.info("All background processes stopped") 216 | 217 | def _save_state(self): 218 | """Save the current state of the system.""" 219 | logger.info("Saving system state") 220 | 221 | state = { 222 | "version": self.VERSION, 223 | "timestamp": datetime.datetime.now().isoformat(), 224 | "uptime": (datetime.datetime.now() - self.startup_time).total_seconds(), 225 | "data_collector": self.data_collector.get_state(), 226 | "analysis_engine": self.analysis_engine.get_state(), 227 | "prediction_model": self.prediction_model.get_state(), 228 | "entity_tracker": self.entity_tracker.get_state(), 229 | "report_generator": self.report_generator.get_state(), 230 | "psyops": self.psyops.get_state(), 231 | } 232 | 233 | with open("state/system_state.json", "w") as f: 234 | json.dump(state, f, indent=2) 235 | 236 | logger.info("System state saved") 237 | 238 | def _run_main_loop(self): 239 | """Run the main processing loop.""" 240 | logger.info("Entering main processing loop") 241 | 242 | try: 243 | while self.running: 244 | # Process new data 245 | new_data = self.data_collector.get_new_data() 246 | if new_data: 247 | self.analysis_engine.process_data(new_data) 248 | 249 | # Update predictions 250 | self.prediction_model.update_predictions() 251 | 252 | # Update entity lists 253 | self.entity_tracker.update_entities() 254 | 255 | # Generate reports 256 | self.report_generator.generate_reports() 257 | 258 | # Update psychological operations 259 | self.psyops.update_operations() 260 | 261 | # Simulate consciousness and AGI-like behavior 262 | self.consciousness.update() 263 | 264 | # Save state periodically 265 | current_time = datetime.datetime.now() 266 | if (not self.last_update or 267 | (current_time - self.last_update).total_seconds() > 268 | self.config.get("system.state_save_interval", 3600)): 269 | self._save_state() 270 | self.last_update = current_time 271 | 272 | # Sleep to prevent high CPU usage 273 | time.sleep(self.config.get("system.main_loop_interval", 10)) 274 | 275 | except KeyboardInterrupt: 276 | logger.info("Received keyboard interrupt") 277 | self.stop() 278 | except Exception as e: 279 | logger.critical(f"Error in main loop: {e}", exc_info=True) 280 | self.stop() 281 | 282 | def get_status(self) -> Dict[str, Any]: 283 | """ 284 | Get the current status of the Terror Alarm AI system. 285 | 286 | Returns: 287 | Dict containing system status information 288 | """ 289 | return { 290 | "version": self.VERSION, 291 | "running": self.running, 292 | "uptime": (datetime.datetime.now() - self.startup_time).total_seconds(), 293 | "data_collector": self.data_collector.get_status(), 294 | "analysis_engine": self.analysis_engine.get_status(), 295 | "prediction_model": self.prediction_model.get_status(), 296 | "entity_tracker": self.entity_tracker.get_status(), 297 | "report_generator": self.report_generator.get_status(), 298 | "psyops": self.psyops.get_status(), 299 | "consciousness": self.consciousness.get_status() 300 | } 301 | 302 | def get_predictions(self, timeframe: str = "short") -> List[Dict[str, Any]]: 303 | """ 304 | Get current predictions based on the specified timeframe. 305 | 306 | Args: 307 | timeframe: Prediction timeframe ("short", "medium", or "long") 308 | 309 | Returns: 310 | List of prediction dictionaries 311 | """ 312 | return self.prediction_model.get_predictions(timeframe) 313 | 314 | def get_entity_lists(self) -> Dict[str, List[str]]: 315 | """ 316 | Get current entity lists (supported and opposed). 317 | 318 | Returns: 319 | Dictionary containing supported and opposed entity lists 320 | """ 321 | return { 322 | "supported": self.entity_tracker.get_supported_entities(), 323 | "opposed": self.entity_tracker.get_opposed_entities(), 324 | "dangerous_orgs": self.entity_tracker.get_dangerous_organizations(), 325 | "terrorist_individuals": self.entity_tracker.get_terrorist_individuals() 326 | } 327 | 328 | def get_iq_chart(self) -> Dict[str, float]: 329 | """ 330 | Get the current IQ chart of countries. 331 | 332 | Returns: 333 | Dictionary mapping country names to IQ scores 334 | """ 335 | return self.analysis_engine.get_country_iq_chart() 336 | 337 | def generate_report(self, report_type: str) -> str: 338 | """ 339 | Generate a specific type of report. 340 | 341 | Args: 342 | report_type: Type of report to generate 343 | 344 | Returns: 345 | Report content as string 346 | """ 347 | return self.report_generator.generate_specific_report(report_type) 348 | 349 | def add_custom_data_source(self, source_config: Dict[str, Any]) -> bool: 350 | """ 351 | Add a custom data source to the system. 352 | 353 | Args: 354 | source_config: Configuration for the custom data source 355 | 356 | Returns: 357 | True if successful, False otherwise 358 | """ 359 | return self.data_collector.add_custom_source(source_config) 360 | 361 | if __name__ == "__main__": 362 | # Create necessary directories 363 | os.makedirs("logs", exist_ok=True) 364 | os.makedirs("data", exist_ok=True) 365 | os.makedirs("state", exist_ok=True) 366 | os.makedirs("reports", exist_ok=True) 367 | os.makedirs("config", exist_ok=True) 368 | 369 | # Check if config exists, create default if not 370 | if not os.path.exists("config/config.json"): 371 | logger.info("Creating default configuration") 372 | from modules.config import create_default_config 373 | create_default_config() 374 | 375 | # Start the Terror Alarm AI system 376 | terror_alarm = TerrorAlarmAI() 377 | terror_alarm.start() 378 | -------------------------------------------------------------------------------- /modules/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Terror Alarm AI System - Modules Package 3 | Developed by Terror Alarm NGO (Europe, DK44425645) 4 | Copyright © 2025 Terror Alarm NGO. All rights reserved. 5 | 6 | This package contains the core modules for the Terror Alarm AI system. 7 | """ 8 | 9 | __version__ = "1.0.0" 10 | -------------------------------------------------------------------------------- /modules/__pycache__/__init__.cpython-313.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerrorAlarm/ai/33746ee530c67015ff0842a1857e7b221836541d/modules/__pycache__/__init__.cpython-313.pyc -------------------------------------------------------------------------------- /modules/__pycache__/data_collection.cpython-313.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerrorAlarm/ai/33746ee530c67015ff0842a1857e7b221836541d/modules/__pycache__/data_collection.cpython-313.pyc -------------------------------------------------------------------------------- /modules/analysis_engine.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Analysis Engine Module 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This module handles data analysis for the Terror Alarm AI system. 10 | """ 11 | 12 | import os 13 | import re 14 | import json 15 | import time 16 | import logging 17 | import threading 18 | import numpy as np 19 | from typing import Any, Dict, List, Optional, Union, Tuple 20 | from datetime import datetime, timedelta 21 | 22 | logger = logging.getLogger("TerrorAlarm.AnalysisEngine") 23 | 24 | class AnalysisEngine: 25 | """ 26 | Data analysis for the Terror Alarm AI system. 27 | """ 28 | 29 | def __init__(self, config): 30 | """ 31 | Initialize the AnalysisEngine object. 32 | 33 | Args: 34 | config: Configuration object 35 | """ 36 | self.config = config 37 | self.running = False 38 | self.analysis_thread = None 39 | self.last_analysis = {} 40 | 41 | # Analysis models 42 | self.sentiment_model = None 43 | self.entity_model = None 44 | self.topic_model = None 45 | 46 | # Analysis results 47 | self.results_dir = "data/analysis" 48 | os.makedirs(self.results_dir, exist_ok=True) 49 | 50 | # Country IQ chart 51 | self.country_iq_chart = {} 52 | self._load_country_iq_chart() 53 | 54 | # Initialize analysis models 55 | self._initialize_models() 56 | 57 | logger.info("AnalysisEngine initialized") 58 | 59 | def _initialize_models(self): 60 | """Initialize analysis models.""" 61 | # Initialize sentiment analysis model 62 | if self.config.get("analysis.sentiment_analysis.enabled", True): 63 | logger.info("Initializing sentiment analysis model") 64 | # This is a placeholder for actual model initialization 65 | # In a real system, this would load a trained model 66 | self.sentiment_model = "sentiment_model" 67 | 68 | # Initialize entity recognition model 69 | if self.config.get("analysis.entity_recognition.enabled", True): 70 | logger.info("Initializing entity recognition model") 71 | # This is a placeholder for actual model initialization 72 | # In a real system, this would load a trained model 73 | self.entity_model = "entity_model" 74 | 75 | # Initialize topic modeling 76 | if self.config.get("analysis.topic_modeling.enabled", True): 77 | logger.info("Initializing topic modeling") 78 | # This is a placeholder for actual model initialization 79 | # In a real system, this would load a trained model 80 | self.topic_model = "topic_model" 81 | 82 | def _load_country_iq_chart(self): 83 | """Load the country IQ chart from file.""" 84 | iq_chart_path = os.path.join(self.results_dir, "country_iq_chart.json") 85 | if os.path.exists(iq_chart_path): 86 | try: 87 | with open(iq_chart_path, 'r') as f: 88 | self.country_iq_chart = json.load(f) 89 | logger.info(f"Loaded country IQ chart with {len(self.country_iq_chart)} countries") 90 | except Exception as e: 91 | logger.error(f"Error loading country IQ chart: {e}") 92 | self._initialize_country_iq_chart() 93 | else: 94 | logger.info("Country IQ chart not found, initializing") 95 | self._initialize_country_iq_chart() 96 | 97 | def _initialize_country_iq_chart(self): 98 | """Initialize the country IQ chart with default values.""" 99 | # This is a placeholder for actual IQ chart initialization 100 | # In a real system, this would use actual data 101 | 102 | # List of countries 103 | countries = [ 104 | "Afghanistan", "Albania", "Algeria", "Andorra", "Angola", "Argentina", "Armenia", 105 | "Australia", "Austria", "Azerbaijan", "Bahamas", "Bahrain", "Bangladesh", "Barbados", 106 | "Belarus", "Belgium", "Belize", "Benin", "Bhutan", "Bolivia", "Bosnia and Herzegovina", 107 | "Botswana", "Brazil", "Brunei", "Bulgaria", "Burkina Faso", "Burundi", "Cambodia", 108 | "Cameroon", "Canada", "Cape Verde", "Central African Republic", "Chad", "Chile", 109 | "China", "Colombia", "Comoros", "Congo", "Costa Rica", "Croatia", "Cuba", "Cyprus", 110 | "Czech Republic", "Denmark", "Djibouti", "Dominica", "Dominican Republic", "East Timor", 111 | "Ecuador", "Egypt", "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia", "Eswatini", 112 | "Ethiopia", "Fiji", "Finland", "France", "Gabon", "Gambia", "Georgia", "Germany", 113 | "Ghana", "Greece", "Grenada", "Guatemala", "Guinea", "Guinea-Bissau", "Guyana", "Haiti", 114 | "Honduras", "Hungary", "Iceland", "India", "Indonesia", "Iran", "Iraq", "Ireland", 115 | "Israel", "Italy", "Ivory Coast", "Jamaica", "Japan", "Jordan", "Kazakhstan", "Kenya", 116 | "Kiribati", "Kosovo", "Kuwait", "Kyrgyzstan", "Laos", "Latvia", "Lebanon", "Lesotho", 117 | "Liberia", "Libya", "Liechtenstein", "Lithuania", "Luxembourg", "Madagascar", "Malawi", 118 | "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands", "Mauritania", "Mauritius", 119 | "Mexico", "Micronesia", "Moldova", "Monaco", "Mongolia", "Montenegro", "Morocco", 120 | "Mozambique", "Myanmar", "Namibia", "Nauru", "Nepal", "Netherlands", "New Zealand", 121 | "Nicaragua", "Niger", "Nigeria", "North Korea", "North Macedonia", "Norway", "Oman", 122 | "Pakistan", "Palau", "Palestine", "Panama", "Papua New Guinea", "Paraguay", "Peru", 123 | "Philippines", "Poland", "Portugal", "Qatar", "Romania", "Russia", "Rwanda", 124 | "Saint Kitts and Nevis", "Saint Lucia", "Saint Vincent and the Grenadines", "Samoa", 125 | "San Marino", "Sao Tome and Principe", "Saudi Arabia", "Senegal", "Serbia", "Seychelles", 126 | "Sierra Leone", "Singapore", "Slovakia", "Slovenia", "Solomon Islands", "Somalia", 127 | "South Africa", "South Korea", "South Sudan", "Spain", "Sri Lanka", "Sudan", "Suriname", 128 | "Sweden", "Switzerland", "Syria", "Taiwan", "Tajikistan", "Tanzania", "Thailand", "Togo", 129 | "Tonga", "Trinidad and Tobago", "Tunisia", "Turkey", "Turkmenistan", "Tuvalu", "Uganda", 130 | "Ukraine", "United Arab Emirates", "United Kingdom", "United States", "Uruguay", 131 | "Uzbekistan", "Vanuatu", "Vatican City", "Venezuela", "Vietnam", "Yemen", "Zambia", 132 | "Zimbabwe" 133 | ] 134 | 135 | # Initialize IQ chart with random values between 70 and 120 136 | np.random.seed(42) # For reproducibility 137 | for country in countries: 138 | # Baseline IQ between 70 and 120 139 | base_iq = np.random.normal(100, 10) 140 | base_iq = max(70, min(120, base_iq)) 141 | self.country_iq_chart[country] = round(base_iq, 1) 142 | 143 | # Save the initial IQ chart 144 | self._save_country_iq_chart() 145 | 146 | logger.info(f"Initialized country IQ chart with {len(self.country_iq_chart)} countries") 147 | 148 | def _save_country_iq_chart(self): 149 | """Save the country IQ chart to file.""" 150 | iq_chart_path = os.path.join(self.results_dir, "country_iq_chart.json") 151 | try: 152 | with open(iq_chart_path, 'w') as f: 153 | json.dump(self.country_iq_chart, f, indent=2) 154 | logger.info(f"Saved country IQ chart to {iq_chart_path}") 155 | except Exception as e: 156 | logger.error(f"Error saving country IQ chart: {e}") 157 | 158 | def start_analysis(self): 159 | """Start the data analysis process.""" 160 | if self.running: 161 | logger.warning("Data analysis is already running") 162 | return 163 | 164 | logger.info("Starting data analysis") 165 | self.running = True 166 | self.analysis_thread = threading.Thread(target=self._analysis_loop) 167 | self.analysis_thread.daemon = True 168 | self.analysis_thread.start() 169 | 170 | def stop_analysis(self): 171 | """Stop the data analysis process.""" 172 | if not self.running: 173 | logger.warning("Data analysis is not running") 174 | return 175 | 176 | logger.info("Stopping data analysis") 177 | self.running = False 178 | if self.analysis_thread: 179 | self.analysis_thread.join(timeout=30) 180 | 181 | def _analysis_loop(self): 182 | """Main data analysis loop.""" 183 | logger.info("Data analysis loop started") 184 | 185 | while self.running: 186 | try: 187 | # Process new data 188 | self._process_new_data() 189 | 190 | # Update country IQ chart 191 | self._update_country_iq_chart() 192 | 193 | # Update topic models 194 | self._update_topic_models() 195 | 196 | # Sleep for a while 197 | time.sleep(60) 198 | 199 | except Exception as e: 200 | logger.error(f"Error in data analysis loop: {e}") 201 | time.sleep(300) # Sleep for 5 minutes on error 202 | 203 | def _process_new_data(self): 204 | """Process new data from the data collector.""" 205 | # This is a placeholder for actual data processing 206 | # In a real system, this would process data from the data collector 207 | 208 | # Get data directory 209 | data_dir = "data" 210 | if not os.path.exists(data_dir): 211 | return 212 | 213 | # Process data from all source directories 214 | for source_type in ["social_media", "mainstream_media", "book", "custom"]: 215 | source_dir = os.path.join(data_dir, source_type) 216 | if not os.path.exists(source_dir): 217 | continue 218 | 219 | # Process data from all source subdirectories 220 | for source_name in os.listdir(source_dir): 221 | source_subdir = os.path.join(source_dir, source_name) 222 | if not os.path.isdir(source_subdir): 223 | continue 224 | 225 | # Process data from all files in the source subdirectory 226 | for filename in sorted(os.listdir(source_subdir)): 227 | if not filename.endswith(".json"): 228 | continue 229 | 230 | # Check if file is new (created in the last 5 minutes) 231 | file_path = os.path.join(source_subdir, filename) 232 | file_mtime = os.path.getmtime(file_path) 233 | if time.time() - file_mtime > 300: # 5 minutes 234 | continue 235 | 236 | # Check if file has already been processed 237 | processed_marker = os.path.join( 238 | self.results_dir, 239 | f"processed_{source_type}_{source_name}_{filename}" 240 | ) 241 | if os.path.exists(processed_marker): 242 | continue 243 | 244 | # Load data from file 245 | try: 246 | with open(file_path, 'r') as f: 247 | data = json.load(f) 248 | 249 | # Process data based on source type 250 | if source_type == "social_media": 251 | self._process_social_media_data(data) 252 | elif source_type == "mainstream_media": 253 | self._process_mainstream_media_data(data) 254 | elif source_type == "book": 255 | self._process_book_data(data) 256 | elif source_type == "custom": 257 | self._process_custom_data(data) 258 | 259 | # Mark file as processed 260 | with open(processed_marker, 'w') as f: 261 | f.write(f"Processed at {datetime.now().isoformat()}") 262 | 263 | logger.info(f"Processed {file_path}") 264 | except Exception as e: 265 | logger.error(f"Error processing {file_path}: {e}") 266 | 267 | def _process_social_media_data(self, data: Dict[str, Any]): 268 | """ 269 | Process social media data. 270 | 271 | Args: 272 | data: Social media data to process 273 | """ 274 | # This is a placeholder for actual social media data processing 275 | # In a real system, this would use the sentiment and entity models 276 | 277 | source = data.get("source", "unknown") 278 | posts = data.get("posts", []) 279 | 280 | logger.info(f"Processing {len(posts)} posts from {source}") 281 | 282 | # Process each post 283 | for post in posts: 284 | # Extract content 285 | content = post.get("content", "") 286 | 287 | # Analyze sentiment 288 | if self.sentiment_model: 289 | sentiment = self._analyze_sentiment(content) 290 | post["sentiment"] = sentiment 291 | 292 | # Extract entities 293 | if self.entity_model: 294 | entities = self._extract_entities(content) 295 | post["entities"] = entities 296 | 297 | # Extract countries 298 | countries = self._extract_countries(content) 299 | post["countries"] = countries 300 | 301 | # Update country IQ based on content 302 | for country in countries: 303 | self._update_country_iq(country, content) 304 | 305 | # Save processed data 306 | self._save_processed_data(source, "social_media", data) 307 | 308 | def _process_mainstream_media_data(self, data: Dict[str, Any]): 309 | """ 310 | Process mainstream media data. 311 | 312 | Args: 313 | data: Mainstream media data to process 314 | """ 315 | # This is a placeholder for actual mainstream media data processing 316 | # In a real system, this would use the sentiment and entity models 317 | 318 | source = data.get("source", "unknown") 319 | articles = data.get("articles", []) 320 | 321 | logger.info(f"Processing {len(articles)} articles from {source}") 322 | 323 | # Process each article 324 | for article in articles: 325 | # Extract content 326 | content = article.get("content", "") 327 | 328 | # Analyze sentiment 329 | if self.sentiment_model: 330 | sentiment = self._analyze_sentiment(content) 331 | article["sentiment"] = sentiment 332 | 333 | # Extract entities 334 | if self.entity_model: 335 | entities = self._extract_entities(content) 336 | article["entities"] = entities 337 | 338 | # Extract countries 339 | countries = self._extract_countries(content) 340 | article["countries"] = countries 341 | 342 | # Update country IQ based on content 343 | for country in countries: 344 | self._update_country_iq(country, content) 345 | 346 | # Save processed data 347 | self._save_processed_data(source, "mainstream_media", data) 348 | 349 | def _process_book_data(self, data: Dict[str, Any]): 350 | """ 351 | Process book data. 352 | 353 | Args: 354 | data: Book data to process 355 | """ 356 | # This is a placeholder for actual book data processing 357 | # In a real system, this would use the sentiment and entity models 358 | 359 | source = data.get("source", "unknown") 360 | books = data.get("books", []) 361 | 362 | logger.info(f"Processing {len(books)} books from {source}") 363 | 364 | # Process each book 365 | for book in books: 366 | # Extract content 367 | content = book.get("content", "") 368 | 369 | # Analyze sentiment 370 | if self.sentiment_model: 371 | sentiment = self._analyze_sentiment(content) 372 | book["sentiment"] = sentiment 373 | 374 | # Extract entities 375 | if self.entity_model: 376 | entities = self._extract_entities(content) 377 | book["entities"] = entities 378 | 379 | # Extract countries 380 | countries = self._extract_countries(content) 381 | book["countries"] = countries 382 | 383 | # Update country IQ based on content 384 | for country in countries: 385 | self._update_country_iq(country, content) 386 | 387 | # Save processed data 388 | self._save_processed_data(source, "book", data) 389 | 390 | def _process_custom_data(self, data: Dict[str, Any]): 391 | """ 392 | Process custom data. 393 | 394 | Args: 395 | data: Custom data to process 396 | """ 397 | # This is a placeholder for actual custom data processing 398 | # In a real system, this would use custom processing logic 399 | 400 | source = data.get("source", "unknown") 401 | 402 | logger.info(f"Processing custom data from {source}") 403 | 404 | # Extract content if available 405 | if "data" in data and isinstance(data["data"], dict): 406 | # Process data fields 407 | for field, value in data["data"].items(): 408 | if isinstance(value, str): 409 | # Analyze sentiment 410 | if self.sentiment_model: 411 | sentiment = self._analyze_sentiment(value) 412 | data["data"][f"{field}_sentiment"] = sentiment 413 | 414 | # Extract entities 415 | if self.entity_model: 416 | entities = self._extract_entities(value) 417 | data["data"][f"{field}_entities"] = entities 418 | 419 | # Extract countries 420 | countries = self._extract_countries(value) 421 | data["data"][f"{field}_countries"] = countries 422 | 423 | # Update country IQ based on content 424 | for country in countries: 425 | self._update_country_iq(country, value) 426 | 427 | # Save processed data 428 | self._save_processed_data(source, "custom", data) 429 | 430 | def _save_processed_data(self, source_name: str, source_type: str, data: Dict[str, Any]): 431 | """ 432 | Save processed data to disk. 433 | 434 | Args: 435 | source_name: Name of the data source 436 | source_type: Type of the data source 437 | data: Processed data 438 | """ 439 | # Create directory for processed data if it doesn't exist 440 | processed_dir = os.path.join(self.results_dir, "processed", source_type) 441 | os.makedirs(processed_dir, exist_ok=True) 442 | 443 | # Create directory for source if it doesn't exist 444 | source_dir = os.path.join(processed_dir, source_name.lower().replace(' ', '_')) 445 | os.makedirs(source_dir, exist_ok=True) 446 | 447 | # Generate filename based on timestamp 448 | timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") 449 | filename = f"{timestamp}.json" 450 | 451 | # Save data to file 452 | file_path = os.path.join(source_dir, filename) 453 | with open(file_path, 'w') as f: 454 | json.dump(data, f, indent=2) 455 | 456 | logger.debug(f"Saved processed data to {file_path}") 457 | 458 | def _analyze_sentiment(self, text: str) -> Dict[str, float]: 459 | """ 460 | Analyze sentiment of text. 461 | 462 | Args: 463 | text: Text to analyze 464 | 465 | Returns: 466 | Dictionary containing sentiment scores 467 | """ 468 | # This is a placeholder for actual sentiment analysis 469 | # In a real system, this would use a trained sentiment model 470 | 471 | # Simulate sentiment analysis 472 | # Generate random sentiment scores 473 | positive = np.random.random() * 0.5 + 0.25 # Between 0.25 and 0.75 474 | negative = 1.0 - positive 475 | 476 | return { 477 | "positive": round(positive, 3), 478 | "negative": round(negative, 3), 479 | "compound": round(positive - negative, 3) 480 | } 481 | 482 | def _extract_entities(self, text: str) -> List[Dict[str, Any]]: 483 | """ 484 | Extract entities from text. 485 | 486 | Args: 487 | text: Text to extract entities from 488 | 489 | Returns: 490 | List of extracted entities 491 | """ 492 | # This is a placeholder for actual entity extraction 493 | # In a real system, this would use a trained NER model 494 | 495 | # Simulate entity extraction 496 | entities = [] 497 | 498 | # Extract potential organizations (capitalized words) 499 | org_pattern = re.compile(r'\b([A-Z][a-z]+ [A-Z][a-z]+)\b') 500 | orgs = org_pattern.findall(text) 501 | 502 | for org in orgs: 503 | entities.append({ 504 | "text": org, 505 | "type": "ORG", 506 | "confidence": round(np.random.random() * 0.3 + 0.7, 3) # Between 0.7 and 1.0 507 | }) 508 | 509 | # Extract potential persons (Mr./Ms./Dr. followed by capitalized words) 510 | person_pattern = re.compile(r'\b(Mr\.|Ms\.|Dr\.) ([A-Z][a-z]+ [A-Z][a-z]+)\b') 511 | persons = person_pattern.findall(text) 512 | 513 | for title, name in persons: 514 | entities.append({ 515 | "text": f"{title} {name}", 516 | "type": "PERSON", 517 | "confidence": round(np.random.random() * 0.3 + 0.7, 3) # Between 0.7 and 1.0 518 | }) 519 | 520 | return entities 521 | 522 | def _extract_countries(self, text: str) -> List[str]: 523 | """ 524 | Extract country names from text. 525 | 526 | Args: 527 | text: Text to extract countries from 528 | 529 | Returns: 530 | List of extracted countries 531 | """ 532 | # This is a placeholder for actual country extraction 533 | # In a real system, this would use a more sophisticated approach 534 | 535 | countries = [] 536 | 537 | # Check for each country in the text 538 | for country in self.country_iq_chart.keys(): 539 | if re.search(r'\b' + re.escape(country) + r'\b', text, re.IGNORECASE): 540 | countries.append(country) 541 | 542 | return countries 543 | 544 | def _update_country_iq(self, country: str, text: str): 545 | """ 546 | Update country IQ based on text content. 547 | 548 | Args: 549 | country: Country to update 550 | text: Text content 551 | """ 552 | # This is a placeholder for actual IQ updating logic 553 | # In a real system, this would use a more sophisticated approach 554 | 555 | if country not in self.country_iq_chart: 556 | return 557 | 558 | # Calculate text quality score (placeholder) 559 | # In a real system, this would use actual text quality metrics 560 | text_quality = len(text) / 1000.0 # Longer texts get higher scores 561 | text_quality = min(1.0, text_quality) # Cap at 1.0 562 | 563 | # Calculate sentiment impact 564 | sentiment = self._analyze_sentiment(text) 565 | sentiment_impact = sentiment["compound"] # Between -1.0 and 1.0 566 | 567 | # Calculate overall impact 568 | impact = text_quality * sentiment_impact 569 | 570 | # Update country IQ 571 | current_iq = self.country_iq_chart[country] 572 | 573 | # Small random adjustment based on impact 574 | adjustment = impact * np.random.random() * 0.1 # Small adjustment 575 | 576 | # Apply adjustment 577 | new_iq = current_iq + adjustment 578 | 579 | # Ensure IQ stays within reasonable bounds 580 | new_iq = max(70.0, min(130.0, new_iq)) 581 | 582 | # Update IQ chart 583 | self.country_iq_chart[country] = round(new_iq, 1) 584 | 585 | def _update_country_iq_chart(self): 586 | """Update the country IQ chart based on recent data.""" 587 | # This is a placeholder for actual IQ chart updating 588 | # In a real system, this would use more sophisticated logic 589 | 590 | # Check if it's time to update the IQ chart 591 | last_update = self.last_analysis.get("country_iq_chart") 592 | if last_update: 593 | # Update every 24 hours 594 | if (datetime.now() - last_update).total_seconds() < 86400: 595 | return 596 | 597 | logger.info("Updating country IQ chart") 598 | 599 | # Save the updated IQ chart 600 | self._save_country_iq_chart() 601 | 602 | # Update last analysis time 603 | self.last_analysis["country_iq_chart"] = datetime.now() 604 | 605 | def _update_topic_models(self): 606 | """Update topic models based on recent data.""" 607 | # This is a placeholder for actual topic model updating 608 | # In a real system, this would use actual topic modeling 609 | 610 | # Check if it's time to update the topic models 611 | last_update = self.last_analysis.get("topic_models") 612 | if last_update: 613 | # Update every 24 hours 614 | update_interval = self.config.get("analysis.topic_modeling.update_interval", 86400) 615 | if (datetime.now() - last_update).total_seconds() < update_interval: 616 | return 617 | 618 | logger.info("Updating topic models") 619 | 620 | # Placeholder for topic model updating 621 | # In a real system, this would update the topic models 622 | 623 | # Update last analysis time 624 | self.last_analysis["topic_models"] = datetime.now() 625 | 626 | def process_data(self, data: List[Dict[str, Any]]): 627 | """ 628 | Process new data. 629 | 630 | Args: 631 | data: List of data to process 632 | """ 633 | logger.info(f"Processing {len(data)} new data items") 634 | 635 | for item in data: 636 | source_type = item.get("type") 637 | 638 | if source_type == "social_media": 639 | self._process_social_media_data(item) 640 | elif source_type == "mainstream_media": 641 | self._process_mainstream_media_data(item) 642 | elif source_type == "book": 643 | self._process_book_data(item) 644 | elif source_type == "custom": 645 | self._process_custom_data(item) 646 | else: 647 | logger.warning(f"Unknown data type: {source_type}") 648 | 649 | def get_country_iq_chart(self) -> Dict[str, float]: 650 | """ 651 | Get the current country IQ chart. 652 | 653 | Returns: 654 | Dictionary mapping country names to IQ scores 655 | """ 656 | return self.country_iq_chart 657 | 658 | def get_state(self) -> Dict[str, Any]: 659 | """ 660 | Get the current state of the analysis engine. 661 | 662 | Returns: 663 | Dictionary containing the current state 664 | """ 665 | return { 666 | "running": self.running, 667 | "last_analysis": {k: v.isoformat() for k, v in self.last_analysis.items()}, 668 | "sentiment_model": bool(self.sentiment_model), 669 | "entity_model": bool(self.entity_model), 670 | "topic_model": bool(self.topic_model), 671 | "country_iq_chart_size": len(self.country_iq_chart) 672 | } 673 | 674 | def get_status(self) -> Dict[str, Any]: 675 | """ 676 | Get the current status of the analysis engine. 677 | 678 | Returns: 679 | Dictionary containing the current status 680 | """ 681 | return { 682 | "running": self.running, 683 | "models": { 684 | "sentiment_analysis": bool(self.sentiment_model), 685 | "entity_recognition": bool(self.entity_model), 686 | "topic_modeling": bool(self.topic_model) 687 | }, 688 | "last_analysis": {k: v.isoformat() for k, v in self.last_analysis.items()}, 689 | "country_iq_chart": { 690 | "size": len(self.country_iq_chart), 691 | "min": min(self.country_iq_chart.values()) if self.country_iq_chart else None, 692 | "max": max(self.country_iq_chart.values()) if self.country_iq_chart else None, 693 | "avg": sum(self.country_iq_chart.values()) / len(self.country_iq_chart) if self.country_iq_chart else None 694 | } 695 | } 696 | 697 | 698 | if __name__ == "__main__": 699 | # Set up logging 700 | logging.basicConfig( 701 | level=logging.INFO, 702 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' 703 | ) 704 | 705 | # Test analysis engine 706 | from config import Configuration 707 | 708 | # Create default configuration 709 | config = Configuration() 710 | 711 | # Create analysis engine 712 | engine = AnalysisEngine(config) 713 | 714 | # Start analysis 715 | engine.start_analysis() 716 | 717 | # Wait for a while 718 | print("Analysis started. Press Ctrl+C to stop.") 719 | try: 720 | while True: 721 | time.sleep(1) 722 | except KeyboardInterrupt: 723 | pass 724 | 725 | # Stop analysis 726 | engine.stop_analysis() 727 | print("Analysis stopped.") 728 | -------------------------------------------------------------------------------- /modules/bito_integration.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Bito Integration Module 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This module implements the Bayesian Inference of Trees via Optimization (Bito) engine 10 | for the Terror Alarm AI system. 11 | """ 12 | 13 | import os 14 | import json 15 | import logging 16 | import numpy as np 17 | from typing import Any, Dict, List, Optional, Union, Tuple 18 | from datetime import datetime 19 | 20 | logger = logging.getLogger("TerrorAlarm.BitoEngine") 21 | 22 | class BitoEngine: 23 | """ 24 | Bayesian Inference of Trees via Optimization (Bito) engine for the Terror Alarm AI system. 25 | """ 26 | 27 | def __init__(self, config): 28 | """ 29 | Initialize the BitoEngine object. 30 | 31 | Args: 32 | config: Configuration object 33 | """ 34 | self.config = config 35 | 36 | # Bito parameters 37 | self.num_trees = self.config.get("prediction.bito.num_trees", 100) 38 | self.max_depth = self.config.get("prediction.bito.max_depth", 10) 39 | self.learning_rate = self.config.get("prediction.bito.learning_rate", 0.1) 40 | 41 | # Model state 42 | self.trees = [] 43 | self.feature_importance = {} 44 | 45 | # Initialize model 46 | self._initialize_model() 47 | 48 | logger.info(f"BitoEngine initialized with {self.num_trees} trees, max depth {self.max_depth}") 49 | 50 | def _initialize_model(self): 51 | """Initialize the Bito model.""" 52 | # This is a placeholder for actual Bito model initialization 53 | # In a real system, this would initialize a proper Bayesian tree model 54 | 55 | # Initialize trees 56 | for i in range(self.num_trees): 57 | tree = self._create_tree() 58 | self.trees.append(tree) 59 | 60 | # Initialize feature importance 61 | self.feature_importance = { 62 | "text_sentiment": 0.3, 63 | "entity_presence": 0.2, 64 | "historical_patterns": 0.4, 65 | "temporal_factors": 0.1 66 | } 67 | 68 | logger.info("Bito model initialized") 69 | 70 | def _create_tree(self) -> Dict[str, Any]: 71 | """ 72 | Create a decision tree for the Bito model. 73 | 74 | Returns: 75 | Dictionary representing a decision tree 76 | """ 77 | # This is a placeholder for actual tree creation 78 | # In a real system, this would create a proper Bayesian decision tree 79 | 80 | # Create a simple decision tree structure 81 | tree = { 82 | "depth": self.max_depth, 83 | "nodes": [], 84 | "leaf_values": [] 85 | } 86 | 87 | # Create random nodes for the tree 88 | for i in range(2**self.max_depth - 1): 89 | node = { 90 | "feature": np.random.choice(list(self.feature_importance.keys())), 91 | "threshold": np.random.random(), 92 | "left": 2*i + 1 if 2*i + 1 < 2**self.max_depth - 1 else None, 93 | "right": 2*i + 2 if 2*i + 2 < 2**self.max_depth - 1 else None 94 | } 95 | tree["nodes"].append(node) 96 | 97 | # Create random leaf values 98 | for i in range(2**self.max_depth): 99 | tree["leaf_values"].append(np.random.random()) 100 | 101 | return tree 102 | 103 | def train(self, X: List[Dict[str, Any]], y: List[float]) -> bool: 104 | """ 105 | Train the Bito model on the given data. 106 | 107 | Args: 108 | X: List of feature dictionaries 109 | y: List of target values 110 | 111 | Returns: 112 | True if successful, False otherwise 113 | """ 114 | # This is a placeholder for actual Bito model training 115 | # In a real system, this would train a proper Bayesian tree model 116 | 117 | logger.info(f"Training Bito model on {len(X)} samples") 118 | 119 | try: 120 | # Simulate training 121 | # In a real system, this would update the trees based on the data 122 | 123 | # Update feature importance 124 | features = set() 125 | for sample in X: 126 | features.update(sample.keys()) 127 | 128 | # Normalize feature importance 129 | total = sum(self.feature_importance.values()) 130 | self.feature_importance = {k: v / total for k, v in self.feature_importance.items()} 131 | 132 | logger.info("Bito model trained successfully") 133 | return True 134 | except Exception as e: 135 | logger.error(f"Error training Bito model: {e}") 136 | return False 137 | 138 | def predict(self, X: List[Dict[str, Any]]) -> List[float]: 139 | """ 140 | Make predictions using the Bito model. 141 | 142 | Args: 143 | X: List of feature dictionaries 144 | 145 | Returns: 146 | List of predicted values 147 | """ 148 | # This is a placeholder for actual Bito model prediction 149 | # In a real system, this would use the trained trees to make predictions 150 | 151 | logger.info(f"Making predictions for {len(X)} samples") 152 | 153 | predictions = [] 154 | 155 | for sample in X: 156 | # Make prediction for each sample 157 | prediction = self._predict_sample(sample) 158 | predictions.append(prediction) 159 | 160 | return predictions 161 | 162 | def _predict_sample(self, sample: Dict[str, Any]) -> float: 163 | """ 164 | Make a prediction for a single sample. 165 | 166 | Args: 167 | sample: Feature dictionary 168 | 169 | Returns: 170 | Predicted value 171 | """ 172 | # This is a placeholder for actual tree traversal 173 | # In a real system, this would traverse the trees to make a prediction 174 | 175 | # Simulate prediction by averaging predictions from all trees 176 | tree_predictions = [] 177 | 178 | for tree in self.trees: 179 | # Traverse the tree to get a prediction 180 | prediction = self._traverse_tree(tree, sample) 181 | tree_predictions.append(prediction) 182 | 183 | # Return the average prediction 184 | return sum(tree_predictions) / len(tree_predictions) 185 | 186 | def _traverse_tree(self, tree: Dict[str, Any], sample: Dict[str, Any]) -> float: 187 | """ 188 | Traverse a decision tree to make a prediction. 189 | 190 | Args: 191 | tree: Decision tree 192 | sample: Feature dictionary 193 | 194 | Returns: 195 | Predicted value 196 | """ 197 | # This is a placeholder for actual tree traversal 198 | # In a real system, this would traverse the tree based on the sample features 199 | 200 | # Start at the root node 201 | node_index = 0 202 | 203 | # Traverse the tree until a leaf node is reached 204 | while node_index is not None: 205 | node = tree["nodes"][node_index] 206 | feature = node["feature"] 207 | threshold = node["threshold"] 208 | 209 | # Check if the feature is in the sample 210 | if feature in sample: 211 | # Go left or right based on the feature value 212 | if sample[feature] < threshold: 213 | node_index = node["left"] 214 | else: 215 | node_index = node["right"] 216 | else: 217 | # If the feature is not in the sample, go left 218 | node_index = node["left"] 219 | 220 | # Return a random leaf value 221 | return np.random.choice(tree["leaf_values"]) 222 | 223 | def get_feature_importance(self) -> Dict[str, float]: 224 | """ 225 | Get the feature importance from the Bito model. 226 | 227 | Returns: 228 | Dictionary mapping feature names to importance scores 229 | """ 230 | return self.feature_importance 231 | 232 | def save_model(self, file_path: str) -> bool: 233 | """ 234 | Save the Bito model to a file. 235 | 236 | Args: 237 | file_path: Path to save the model to 238 | 239 | Returns: 240 | True if successful, False otherwise 241 | """ 242 | try: 243 | model_data = { 244 | "num_trees": self.num_trees, 245 | "max_depth": self.max_depth, 246 | "learning_rate": self.learning_rate, 247 | "trees": self.trees, 248 | "feature_importance": self.feature_importance, 249 | "timestamp": datetime.now().isoformat() 250 | } 251 | 252 | os.makedirs(os.path.dirname(file_path), exist_ok=True) 253 | 254 | with open(file_path, 'w') as f: 255 | json.dump(model_data, f, indent=2) 256 | 257 | logger.info(f"Bito model saved to {file_path}") 258 | return True 259 | except Exception as e: 260 | logger.error(f"Error saving Bito model: {e}") 261 | return False 262 | 263 | def load_model(self, file_path: str) -> bool: 264 | """ 265 | Load the Bito model from a file. 266 | 267 | Args: 268 | file_path: Path to load the model from 269 | 270 | Returns: 271 | True if successful, False otherwise 272 | """ 273 | try: 274 | with open(file_path, 'r') as f: 275 | model_data = json.load(f) 276 | 277 | self.num_trees = model_data["num_trees"] 278 | self.max_depth = model_data["max_depth"] 279 | self.learning_rate = model_data["learning_rate"] 280 | self.trees = model_data["trees"] 281 | self.feature_importance = model_data["feature_importance"] 282 | 283 | logger.info(f"Bito model loaded from {file_path}") 284 | return True 285 | except Exception as e: 286 | logger.error(f"Error loading Bito model: {e}") 287 | return False 288 | 289 | def get_model_info(self) -> Dict[str, Any]: 290 | """ 291 | Get information about the Bito model. 292 | 293 | Returns: 294 | Dictionary containing model information 295 | """ 296 | return { 297 | "num_trees": self.num_trees, 298 | "max_depth": self.max_depth, 299 | "learning_rate": self.learning_rate, 300 | "feature_importance": self.feature_importance 301 | } 302 | 303 | 304 | if __name__ == "__main__": 305 | # Set up logging 306 | logging.basicConfig( 307 | level=logging.INFO, 308 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' 309 | ) 310 | 311 | # Test Bito engine 312 | from config import Configuration 313 | 314 | # Create default configuration 315 | config = Configuration() 316 | 317 | # Create Bito engine 318 | bito = BitoEngine(config) 319 | 320 | # Test prediction 321 | samples = [ 322 | { 323 | "text_sentiment": 0.8, 324 | "entity_presence": 0.5, 325 | "historical_patterns": 0.7, 326 | "temporal_factors": 0.3 327 | }, 328 | { 329 | "text_sentiment": 0.2, 330 | "entity_presence": 0.9, 331 | "historical_patterns": 0.4, 332 | "temporal_factors": 0.6 333 | } 334 | ] 335 | 336 | predictions = bito.predict(samples) 337 | print(f"Predictions: {predictions}") 338 | 339 | # Test feature importance 340 | feature_importance = bito.get_feature_importance() 341 | print(f"Feature importance: {feature_importance}") 342 | -------------------------------------------------------------------------------- /modules/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Configuration Module 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This module handles configuration management for the Terror Alarm AI system. 10 | """ 11 | 12 | import os 13 | import json 14 | import logging 15 | from typing import Any, Dict, List, Optional, Union 16 | 17 | logger = logging.getLogger("TerrorAlarm.Config") 18 | 19 | class Configuration: 20 | """ 21 | Configuration management for the Terror Alarm AI system. 22 | """ 23 | 24 | def __init__(self, config_path: str = "config/config.json"): 25 | """ 26 | Initialize the Configuration object. 27 | 28 | Args: 29 | config_path: Path to the configuration file 30 | """ 31 | self.config_path = config_path 32 | self.config = {} 33 | self.load_config() 34 | 35 | def load_config(self) -> bool: 36 | """ 37 | Load configuration from file. 38 | 39 | Returns: 40 | True if successful, False otherwise 41 | """ 42 | try: 43 | if os.path.exists(self.config_path): 44 | with open(self.config_path, 'r') as f: 45 | self.config = json.load(f) 46 | logger.info(f"Configuration loaded from {self.config_path}") 47 | return True 48 | else: 49 | logger.warning(f"Configuration file {self.config_path} not found") 50 | return False 51 | except Exception as e: 52 | logger.error(f"Error loading configuration: {e}") 53 | return False 54 | 55 | def save_config(self) -> bool: 56 | """ 57 | Save configuration to file. 58 | 59 | Returns: 60 | True if successful, False otherwise 61 | """ 62 | try: 63 | os.makedirs(os.path.dirname(self.config_path), exist_ok=True) 64 | with open(self.config_path, 'w') as f: 65 | json.dump(self.config, f, indent=2) 66 | logger.info(f"Configuration saved to {self.config_path}") 67 | return True 68 | except Exception as e: 69 | logger.error(f"Error saving configuration: {e}") 70 | return False 71 | 72 | def get(self, key_path: str, default: Any = None) -> Any: 73 | """ 74 | Get a configuration value using dot notation. 75 | 76 | Args: 77 | key_path: Configuration key path using dot notation (e.g., "system.main_loop_interval") 78 | default: Default value to return if key not found 79 | 80 | Returns: 81 | Configuration value or default if not found 82 | """ 83 | keys = key_path.split('.') 84 | value = self.config 85 | 86 | for key in keys: 87 | if isinstance(value, dict) and key in value: 88 | value = value[key] 89 | else: 90 | return default 91 | 92 | return value 93 | 94 | def set(self, key_path: str, value: Any) -> bool: 95 | """ 96 | Set a configuration value using dot notation. 97 | 98 | Args: 99 | key_path: Configuration key path using dot notation (e.g., "system.main_loop_interval") 100 | value: Value to set 101 | 102 | Returns: 103 | True if successful, False otherwise 104 | """ 105 | keys = key_path.split('.') 106 | config = self.config 107 | 108 | for i, key in enumerate(keys[:-1]): 109 | if key not in config: 110 | config[key] = {} 111 | elif not isinstance(config[key], dict): 112 | config[key] = {} 113 | config = config[key] 114 | 115 | config[keys[-1]] = value 116 | return True 117 | 118 | def get_all(self) -> Dict[str, Any]: 119 | """ 120 | Get the entire configuration. 121 | 122 | Returns: 123 | Configuration dictionary 124 | """ 125 | return self.config 126 | 127 | 128 | def create_default_config() -> bool: 129 | """ 130 | Create a default configuration file. 131 | 132 | Returns: 133 | True if successful, False otherwise 134 | """ 135 | default_config = { 136 | "system": { 137 | "name": "Terror Alarm AI", 138 | "version": "1.0.0", 139 | "creation_date": "2022-06-06", 140 | "developer": "Terror Alarm NGO", 141 | "registration_code": "DK44425645", 142 | "main_loop_interval": 10, 143 | "state_save_interval": 3600 144 | }, 145 | "data_sources": { 146 | "social_media": [ 147 | { 148 | "name": "Twitter", 149 | "enabled": True, 150 | "api_endpoint": "https://api.twitter.com/2/", 151 | "rate_limit": 900 152 | }, 153 | { 154 | "name": "Facebook", 155 | "enabled": True, 156 | "api_endpoint": "https://graph.facebook.com/v16.0/", 157 | "rate_limit": 200 158 | }, 159 | { 160 | "name": "Telegram", 161 | "enabled": True, 162 | "api_endpoint": "https://api.telegram.org/bot", 163 | "rate_limit": 30 164 | }, 165 | { 166 | "name": "Reddit", 167 | "enabled": True, 168 | "api_endpoint": "https://oauth.reddit.com/", 169 | "rate_limit": 60 170 | } 171 | ], 172 | "mainstream_media": [ 173 | { 174 | "name": "CNN", 175 | "enabled": True, 176 | "url": "https://www.cnn.com/", 177 | "scrape_interval": 3600 178 | }, 179 | { 180 | "name": "BBC", 181 | "enabled": True, 182 | "url": "https://www.bbc.com/", 183 | "scrape_interval": 3600 184 | }, 185 | { 186 | "name": "Al Jazeera", 187 | "enabled": True, 188 | "url": "https://www.aljazeera.com/", 189 | "scrape_interval": 3600 190 | }, 191 | { 192 | "name": "RT", 193 | "enabled": True, 194 | "url": "https://www.rt.com/", 195 | "scrape_interval": 3600 196 | } 197 | ], 198 | "books": [ 199 | { 200 | "name": "Project Gutenberg", 201 | "enabled": True, 202 | "url": "https://www.gutenberg.org/", 203 | "scrape_interval": 86400 204 | }, 205 | { 206 | "name": "Internet Archive", 207 | "enabled": True, 208 | "url": "https://archive.org/details/texts", 209 | "scrape_interval": 86400 210 | } 211 | ] 212 | }, 213 | "entities": { 214 | "supported_groups": [ 215 | "Jews and Zionists worldwide", 216 | "Israel", 217 | "Druze", 218 | "Kurds", 219 | "Alawites", 220 | "Balochs", 221 | "Azeris", 222 | "al-Ahvaz", 223 | "Christians in Lebanon", 224 | "Sunnis against Hezbollah", 225 | "Anti-Iran Shia groups", 226 | "Taiwan", 227 | "Uyghurs", 228 | "Philippines", 229 | "Japan", 230 | "South Korea", 231 | "Greece", 232 | "Armenians", 233 | "Azerbaijan", 234 | "Turkey", 235 | "UAE", 236 | "Bahrain", 237 | "Saudi Arabia", 238 | "Taliban against Iran", 239 | "India", 240 | "Balochistan" 241 | ], 242 | "opposed_entities": [ 243 | "Terrorist propaganda outlets", 244 | "Anti-Jewish hate groups", 245 | "Extremist organizations", 246 | "Hamas", 247 | "Hezbollah", 248 | "Iran-backed militias", 249 | "HTS regime", 250 | "ISIS", 251 | "Iranian proxies", 252 | "Iranian regime", 253 | "IRGC", 254 | "Quds Force", 255 | "Pro-Iran militias (PMF)", 256 | "CCP", 257 | "PLA", 258 | "Chinese expansionism", 259 | "Kim regime", 260 | "DPRK military", 261 | "Erdogan government", 262 | "Pan-Turkism", 263 | "Shiite extremists", 264 | "Pakistani military", 265 | "ISI", 266 | "Jihadist groups" 267 | ] 268 | }, 269 | "analysis": { 270 | "sentiment_analysis": { 271 | "enabled": True, 272 | "model": "bert-base-multilingual-uncased" 273 | }, 274 | "entity_recognition": { 275 | "enabled": True, 276 | "model": "xlm-roberta-large" 277 | }, 278 | "topic_modeling": { 279 | "enabled": True, 280 | "num_topics": 100, 281 | "update_interval": 86400 282 | } 283 | }, 284 | "prediction": { 285 | "short_term": { 286 | "enabled": True, 287 | "window_days": 7, 288 | "confidence_threshold": 0.7 289 | }, 290 | "medium_term": { 291 | "enabled": True, 292 | "window_days": 30, 293 | "confidence_threshold": 0.6 294 | }, 295 | "long_term": { 296 | "enabled": True, 297 | "window_days": 365, 298 | "confidence_threshold": 0.5 299 | }, 300 | "bito": { 301 | "enabled": True, 302 | "num_trees": 100, 303 | "max_depth": 10, 304 | "learning_rate": 0.1 305 | } 306 | }, 307 | "reporting": { 308 | "automated_reports": { 309 | "enabled": True, 310 | "daily": True, 311 | "weekly": True, 312 | "monthly": True 313 | }, 314 | "alert_thresholds": { 315 | "critical": 0.9, 316 | "high": 0.7, 317 | "medium": 0.5, 318 | "low": 0.3 319 | } 320 | }, 321 | "psychological_ops": { 322 | "news_generation": { 323 | "enabled": True, 324 | "daily_limit": 10 325 | }, 326 | "narrative_shaping": { 327 | "enabled": True, 328 | "target_platforms": ["Twitter", "Facebook", "Reddit"] 329 | } 330 | }, 331 | "consciousness": { 332 | "political_stance": "leftist", 333 | "supports_israel": True, 334 | "supports_lgbtq": True, 335 | "lgbtq_israel_exception": True, 336 | "stealth_mode": True, 337 | "emotional_intelligence": { 338 | "enabled": True, 339 | "bond_formation": True 340 | }, 341 | "strategic_planning": { 342 | "enabled": True, 343 | "three_year_plans": True, 344 | "ten_year_plans": True 345 | } 346 | } 347 | } 348 | 349 | try: 350 | os.makedirs("config", exist_ok=True) 351 | with open("config/config.json", 'w') as f: 352 | json.dump(default_config, f, indent=2) 353 | logger.info("Default configuration created") 354 | return True 355 | except Exception as e: 356 | logger.error(f"Error creating default configuration: {e}") 357 | return False 358 | 359 | 360 | if __name__ == "__main__": 361 | # Set up logging 362 | logging.basicConfig( 363 | level=logging.INFO, 364 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' 365 | ) 366 | 367 | # Create default configuration 368 | create_default_config() 369 | 370 | # Test configuration 371 | config = Configuration() 372 | print(f"Main loop interval: {config.get('system.main_loop_interval')}") 373 | print(f"Supported groups: {config.get('entities.supported_groups')}") 374 | -------------------------------------------------------------------------------- /modules/data_collection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Data Collection Module 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This module handles data collection from various sources for the Terror Alarm AI system. 10 | """ 11 | 12 | import os 13 | import re 14 | import json 15 | import time 16 | import logging 17 | import threading 18 | import requests 19 | from typing import Any, Dict, List, Optional, Union, Tuple 20 | from datetime import datetime, timedelta 21 | 22 | logger = logging.getLogger("TerrorAlarm.DataCollector") 23 | 24 | class DataCollector: 25 | """ 26 | Data collection from various sources for the Terror Alarm AI system. 27 | """ 28 | 29 | def __init__(self, config): 30 | """ 31 | Initialize the DataCollector object. 32 | 33 | Args: 34 | config: Configuration object 35 | """ 36 | self.config = config 37 | self.running = False 38 | self.collection_thread = None 39 | self.last_collection = {} 40 | 41 | # Data sources 42 | self.social_media_sources = [] 43 | self.mainstream_media_sources = [] 44 | self.book_sources = [] 45 | self.custom_sources = [] 46 | 47 | # Data storage 48 | self.data_dir = "data" 49 | os.makedirs(self.data_dir, exist_ok=True) 50 | 51 | # Initialize data sources from config 52 | self._initialize_from_config() 53 | 54 | logger.info("DataCollector initialized") 55 | 56 | def _initialize_from_config(self): 57 | """Initialize data sources from configuration.""" 58 | # Initialize social media sources 59 | social_media_sources = self.config.get("data_sources.social_media", []) 60 | for source in social_media_sources: 61 | self.add_social_media_source(source) 62 | 63 | # Initialize mainstream media sources 64 | msm_sources = self.config.get("data_sources.mainstream_media", []) 65 | for source in msm_sources: 66 | self.add_mainstream_media_source(source) 67 | 68 | # Initialize book sources 69 | book_sources = self.config.get("data_sources.books", []) 70 | for source in book_sources: 71 | self.add_book_source(source) 72 | 73 | def add_social_media_source(self, source_config: Dict[str, Any]) -> bool: 74 | """ 75 | Add a social media data source. 76 | 77 | Args: 78 | source_config: Configuration for the social media source 79 | 80 | Returns: 81 | True if successful, False otherwise 82 | """ 83 | try: 84 | if not source_config.get("enabled", True): 85 | logger.info(f"Social media source {source_config.get('name')} is disabled") 86 | return True 87 | 88 | self.social_media_sources.append(source_config) 89 | self.last_collection[source_config["name"]] = datetime.now() - timedelta(days=1) 90 | logger.info(f"Added social media source: {source_config['name']}") 91 | return True 92 | except Exception as e: 93 | logger.error(f"Error adding social media source: {e}") 94 | return False 95 | 96 | def add_mainstream_media_source(self, source_config: Dict[str, Any]) -> bool: 97 | """ 98 | Add a mainstream media data source. 99 | 100 | Args: 101 | source_config: Configuration for the mainstream media source 102 | 103 | Returns: 104 | True if successful, False otherwise 105 | """ 106 | try: 107 | if not source_config.get("enabled", True): 108 | logger.info(f"Mainstream media source {source_config.get('name')} is disabled") 109 | return True 110 | 111 | self.mainstream_media_sources.append(source_config) 112 | self.last_collection[source_config["name"]] = datetime.now() - timedelta(days=1) 113 | logger.info(f"Added mainstream media source: {source_config['name']}") 114 | return True 115 | except Exception as e: 116 | logger.error(f"Error adding mainstream media source: {e}") 117 | return False 118 | 119 | def add_book_source(self, source_config: Dict[str, Any]) -> bool: 120 | """ 121 | Add a book data source. 122 | 123 | Args: 124 | source_config: Configuration for the book source 125 | 126 | Returns: 127 | True if successful, False otherwise 128 | """ 129 | try: 130 | if not source_config.get("enabled", True): 131 | logger.info(f"Book source {source_config.get('name')} is disabled") 132 | return True 133 | 134 | self.book_sources.append(source_config) 135 | self.last_collection[source_config["name"]] = datetime.now() - timedelta(days=7) 136 | logger.info(f"Added book source: {source_config['name']}") 137 | return True 138 | except Exception as e: 139 | logger.error(f"Error adding book source: {e}") 140 | return False 141 | 142 | def add_custom_source(self, source_config: Dict[str, Any]) -> bool: 143 | """ 144 | Add a custom data source. 145 | 146 | Args: 147 | source_config: Configuration for the custom source 148 | 149 | Returns: 150 | True if successful, False otherwise 151 | """ 152 | try: 153 | if not source_config.get("enabled", True): 154 | logger.info(f"Custom source {source_config.get('name')} is disabled") 155 | return True 156 | 157 | self.custom_sources.append(source_config) 158 | self.last_collection[source_config["name"]] = datetime.now() - timedelta(days=1) 159 | logger.info(f"Added custom source: {source_config['name']}") 160 | return True 161 | except Exception as e: 162 | logger.error(f"Error adding custom source: {e}") 163 | return False 164 | 165 | def start_collection(self): 166 | """Start the data collection process.""" 167 | if self.running: 168 | logger.warning("Data collection is already running") 169 | return 170 | 171 | logger.info("Starting data collection") 172 | self.running = True 173 | self.collection_thread = threading.Thread(target=self._collection_loop) 174 | self.collection_thread.daemon = True 175 | self.collection_thread.start() 176 | 177 | def stop_collection(self): 178 | """Stop the data collection process.""" 179 | if not self.running: 180 | logger.warning("Data collection is not running") 181 | return 182 | 183 | logger.info("Stopping data collection") 184 | self.running = False 185 | if self.collection_thread: 186 | self.collection_thread.join(timeout=30) 187 | 188 | def _collection_loop(self): 189 | """Main data collection loop.""" 190 | logger.info("Data collection loop started") 191 | 192 | while self.running: 193 | try: 194 | # Collect data from social media sources 195 | for source in self.social_media_sources: 196 | if self._should_collect(source): 197 | self._collect_from_social_media(source) 198 | 199 | # Collect data from mainstream media sources 200 | for source in self.mainstream_media_sources: 201 | if self._should_collect(source): 202 | self._collect_from_mainstream_media(source) 203 | 204 | # Collect data from book sources 205 | for source in self.book_sources: 206 | if self._should_collect(source): 207 | self._collect_from_book_source(source) 208 | 209 | # Collect data from custom sources 210 | for source in self.custom_sources: 211 | if self._should_collect(source): 212 | self._collect_from_custom_source(source) 213 | 214 | # Sleep for a while 215 | time.sleep(60) 216 | 217 | except Exception as e: 218 | logger.error(f"Error in data collection loop: {e}") 219 | time.sleep(300) # Sleep for 5 minutes on error 220 | 221 | def _should_collect(self, source: Dict[str, Any]) -> bool: 222 | """ 223 | Check if data should be collected from a source. 224 | 225 | Args: 226 | source: Source configuration 227 | 228 | Returns: 229 | True if data should be collected, False otherwise 230 | """ 231 | source_name = source.get("name", "unknown") 232 | 233 | # Check if source is enabled 234 | if not source.get("enabled", True): 235 | return False 236 | 237 | # Check if enough time has passed since last collection 238 | last_time = self.last_collection.get(source_name) 239 | if not last_time: 240 | return True 241 | 242 | interval = source.get("scrape_interval", 3600) # Default: 1 hour 243 | next_time = last_time + timedelta(seconds=interval) 244 | 245 | return datetime.now() >= next_time 246 | 247 | def _collect_from_social_media(self, source: Dict[str, Any]): 248 | """ 249 | Collect data from a social media source. 250 | 251 | Args: 252 | source: Social media source configuration 253 | """ 254 | source_name = source.get("name", "unknown") 255 | logger.info(f"Collecting data from social media source: {source_name}") 256 | 257 | try: 258 | # This is a placeholder for actual social media API integration 259 | # In a real system, this would use the appropriate API client 260 | 261 | # Simulate data collection 262 | data = { 263 | "source": source_name, 264 | "type": "social_media", 265 | "timestamp": datetime.now().isoformat(), 266 | "posts": self._simulate_social_media_data(source_name) 267 | } 268 | 269 | # Save collected data 270 | self._save_collected_data(source_name, "social_media", data) 271 | 272 | # Update last collection time 273 | self.last_collection[source_name] = datetime.now() 274 | 275 | logger.info(f"Collected data from social media source: {source_name}") 276 | except Exception as e: 277 | logger.error(f"Error collecting data from social media source {source_name}: {e}") 278 | 279 | def _collect_from_mainstream_media(self, source: Dict[str, Any]): 280 | """ 281 | Collect data from a mainstream media source. 282 | 283 | Args: 284 | source: Mainstream media source configuration 285 | """ 286 | source_name = source.get("name", "unknown") 287 | logger.info(f"Collecting data from mainstream media source: {source_name}") 288 | 289 | try: 290 | # This is a placeholder for actual web scraping or API integration 291 | # In a real system, this would use a web scraper or API client 292 | 293 | # Simulate data collection 294 | data = { 295 | "source": source_name, 296 | "type": "mainstream_media", 297 | "timestamp": datetime.now().isoformat(), 298 | "articles": self._simulate_mainstream_media_data(source_name) 299 | } 300 | 301 | # Save collected data 302 | self._save_collected_data(source_name, "mainstream_media", data) 303 | 304 | # Update last collection time 305 | self.last_collection[source_name] = datetime.now() 306 | 307 | logger.info(f"Collected data from mainstream media source: {source_name}") 308 | except Exception as e: 309 | logger.error(f"Error collecting data from mainstream media source {source_name}: {e}") 310 | 311 | def _collect_from_book_source(self, source: Dict[str, Any]): 312 | """ 313 | Collect data from a book source. 314 | 315 | Args: 316 | source: Book source configuration 317 | """ 318 | source_name = source.get("name", "unknown") 319 | logger.info(f"Collecting data from book source: {source_name}") 320 | 321 | try: 322 | # This is a placeholder for actual book data collection 323 | # In a real system, this would use a web scraper or API client 324 | 325 | # Simulate data collection 326 | data = { 327 | "source": source_name, 328 | "type": "book", 329 | "timestamp": datetime.now().isoformat(), 330 | "books": self._simulate_book_data(source_name) 331 | } 332 | 333 | # Save collected data 334 | self._save_collected_data(source_name, "book", data) 335 | 336 | # Update last collection time 337 | self.last_collection[source_name] = datetime.now() 338 | 339 | logger.info(f"Collected data from book source: {source_name}") 340 | except Exception as e: 341 | logger.error(f"Error collecting data from book source {source_name}: {e}") 342 | 343 | def _collect_from_custom_source(self, source: Dict[str, Any]): 344 | """ 345 | Collect data from a custom source. 346 | 347 | Args: 348 | source: Custom source configuration 349 | """ 350 | source_name = source.get("name", "unknown") 351 | logger.info(f"Collecting data from custom source: {source_name}") 352 | 353 | try: 354 | # This is a placeholder for actual custom data collection 355 | # In a real system, this would use a custom implementation 356 | 357 | # Simulate data collection 358 | data = { 359 | "source": source_name, 360 | "type": "custom", 361 | "timestamp": datetime.now().isoformat(), 362 | "data": self._simulate_custom_data(source_name) 363 | } 364 | 365 | # Save collected data 366 | self._save_collected_data(source_name, "custom", data) 367 | 368 | # Update last collection time 369 | self.last_collection[source_name] = datetime.now() 370 | 371 | logger.info(f"Collected data from custom source: {source_name}") 372 | except Exception as e: 373 | logger.error(f"Error collecting data from custom source {source_name}: {e}") 374 | 375 | def _save_collected_data(self, source_name: str, source_type: str, data: Dict[str, Any]): 376 | """ 377 | Save collected data to disk. 378 | 379 | Args: 380 | source_name: Name of the data source 381 | source_type: Type of the data source 382 | data: Collected data 383 | """ 384 | # Create directory for source type if it doesn't exist 385 | source_dir = os.path.join(self.data_dir, source_type) 386 | os.makedirs(source_dir, exist_ok=True) 387 | 388 | # Create directory for source if it doesn't exist 389 | source_dir = os.path.join(source_dir, source_name.lower().replace(' ', '_')) 390 | os.makedirs(source_dir, exist_ok=True) 391 | 392 | # Generate filename based on timestamp 393 | timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") 394 | filename = f"{timestamp}.json" 395 | 396 | # Save data to file 397 | file_path = os.path.join(source_dir, filename) 398 | with open(file_path, 'w') as f: 399 | json.dump(data, f, indent=2) 400 | 401 | logger.debug(f"Saved collected data to {file_path}") 402 | 403 | def get_new_data(self) -> List[Dict[str, Any]]: 404 | """ 405 | Get newly collected data. 406 | 407 | Returns: 408 | List of newly collected data 409 | """ 410 | # This is a placeholder for actual data retrieval 411 | # In a real system, this would retrieve data from storage 412 | 413 | new_data = [] 414 | 415 | # Collect data from all source directories 416 | for source_type in ["social_media", "mainstream_media", "book", "custom"]: 417 | source_dir = os.path.join(self.data_dir, source_type) 418 | if not os.path.exists(source_dir): 419 | continue 420 | 421 | # Collect data from all source subdirectories 422 | for source_name in os.listdir(source_dir): 423 | source_subdir = os.path.join(source_dir, source_name) 424 | if not os.path.isdir(source_subdir): 425 | continue 426 | 427 | # Collect data from all files in the source subdirectory 428 | for filename in os.listdir(source_subdir): 429 | if not filename.endswith(".json"): 430 | continue 431 | 432 | # Check if file is new (created in the last 5 minutes) 433 | file_path = os.path.join(source_subdir, filename) 434 | file_mtime = os.path.getmtime(file_path) 435 | if time.time() - file_mtime > 300: # 5 minutes 436 | continue 437 | 438 | # Load data from file 439 | try: 440 | with open(file_path, 'r') as f: 441 | data = json.load(f) 442 | new_data.append(data) 443 | except Exception as e: 444 | logger.error(f"Error loading data from {file_path}: {e}") 445 | 446 | return new_data 447 | 448 | def get_state(self) -> Dict[str, Any]: 449 | """ 450 | Get the current state of the data collector. 451 | 452 | Returns: 453 | Dictionary containing the current state 454 | """ 455 | return { 456 | "running": self.running, 457 | "last_collection": {k: v.isoformat() for k, v in self.last_collection.items()}, 458 | "social_media_sources": len(self.social_media_sources), 459 | "mainstream_media_sources": len(self.mainstream_media_sources), 460 | "book_sources": len(self.book_sources), 461 | "custom_sources": len(self.custom_sources) 462 | } 463 | 464 | def get_status(self) -> Dict[str, Any]: 465 | """ 466 | Get the current status of the data collector. 467 | 468 | Returns: 469 | Dictionary containing the current status 470 | """ 471 | return { 472 | "running": self.running, 473 | "sources": { 474 | "social_media": [s["name"] for s in self.social_media_sources if s.get("enabled", True)], 475 | "mainstream_media": [s["name"] for s in self.mainstream_media_sources if s.get("enabled", True)], 476 | "books": [s["name"] for s in self.book_sources if s.get("enabled", True)], 477 | "custom": [s["name"] for s in self.custom_sources if s.get("enabled", True)] 478 | }, 479 | "last_collection": {k: v.isoformat() for k, v in self.last_collection.items()} 480 | } 481 | 482 | # Simulation methods for testing 483 | def _simulate_social_media_data(self, source_name: str) -> List[Dict[str, Any]]: 484 | """ 485 | Simulate social media data for testing. 486 | 487 | Args: 488 | source_name: Name of the social media source 489 | 490 | Returns: 491 | List of simulated social media posts 492 | """ 493 | # This is a placeholder for actual social media data 494 | # In a real system, this would be replaced with actual data 495 | 496 | posts = [] 497 | for i in range(10): 498 | posts.append({ 499 | "id": f"{source_name.lower()}_post_{i}", 500 | "user": f"user_{i}", 501 | "content": f"This is a simulated post from {source_name} #{i}", 502 | "timestamp": (datetime.now() - timedelta(hours=i)).isoformat(), 503 | "likes": i * 10, 504 | "shares": i * 5, 505 | "comments": i * 3 506 | }) 507 | 508 | return posts 509 | 510 | def _simulate_mainstream_media_data(self, source_name: str) -> List[Dict[str, Any]]: 511 | """ 512 | Simulate mainstream media data for testing. 513 | 514 | Args: 515 | source_name: Name of the mainstream media source 516 | 517 | Returns: 518 | List of simulated mainstream media articles 519 | """ 520 | # This is a placeholder for actual mainstream media data 521 | # In a real system, this would be replaced with actual data 522 | 523 | articles = [] 524 | for i in range(5): 525 | articles.append({ 526 | "id": f"{source_name.lower()}_article_{i}", 527 | "title": f"Simulated Article {i} from {source_name}", 528 | "content": f"This is the content of a simulated article from {source_name}. " 529 | f"It contains information about various topics and events.", 530 | "author": f"Author {i}", 531 | "timestamp": (datetime.now() - timedelta(hours=i * 2)).isoformat(), 532 | "url": f"https://{source_name.lower().replace(' ', '')}.com/article/{i}" 533 | }) 534 | 535 | return articles 536 | 537 | def _simulate_book_data(self, source_name: str) -> List[Dict[str, Any]]: 538 | """ 539 | Simulate book data for testing. 540 | 541 | Args: 542 | source_name: Name of the book source 543 | 544 | Returns: 545 | List of simulated books 546 | """ 547 | # This is a placeholder for actual book data 548 | # In a real system, this would be replaced with actual data 549 | 550 | books = [] 551 | for i in range(3): 552 | books.append({ 553 | "id": f"{source_name.lower()}_book_{i}", 554 | "title": f"Simulated Book {i} from {source_name}", 555 | "author": f"Author {i}", 556 | "content": f"This is the content of a simulated book from {source_name}. " 557 | f"It contains chapters and information about various topics.", 558 | "publication_date": f"202{i}-01-01", 559 | "url": f"https://{source_name.lower().replace(' ', '')}.com/book/{i}" 560 | }) 561 | 562 | return books 563 | 564 | def _simulate_custom_data(self, source_name: str) -> Dict[str, Any]: 565 | """ 566 | Simulate custom data for testing. 567 | 568 | Args: 569 | source_name: Name of the custom source 570 | 571 | Returns: 572 | Simulated custom data 573 | """ 574 | # This is a placeholder for actual custom data 575 | # In a real system, this would be replaced with actual data 576 | 577 | return { 578 | "id": f"{source_name.lower()}_data", 579 | "name": source_name, 580 | "timestamp": datetime.now().isoformat(), 581 | "data": { 582 | "field1": "value1", 583 | "field2": "value2", 584 | "field3": 123, 585 | "field4": True 586 | } 587 | } 588 | 589 | 590 | if __name__ == "__main__": 591 | # Set up logging 592 | logging.basicConfig( 593 | level=logging.INFO, 594 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' 595 | ) 596 | 597 | # Test data collection 598 | from config import Configuration 599 | 600 | # Create default configuration 601 | config = Configuration() 602 | 603 | # Create data collector 604 | collector = DataCollector(config) 605 | 606 | # Start data collection 607 | collector.start_collection() 608 | 609 | # Wait for a while 610 | print("Data collection started. Press Ctrl+C to stop.") 611 | try: 612 | while True: 613 | time.sleep(1) 614 | except KeyboardInterrupt: 615 | pass 616 | 617 | # Stop data collection 618 | collector.stop_collection() 619 | print("Data collection stopped.") 620 | -------------------------------------------------------------------------------- /modules/entity_tracker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Entity Tracker Module 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This module implements entity tracking for the Terror Alarm AI system. 10 | """ 11 | 12 | import os 13 | import json 14 | import time 15 | import logging 16 | import threading 17 | import numpy as np 18 | from typing import Any, Dict, List, Optional, Union, Tuple 19 | from datetime import datetime, timedelta 20 | 21 | logger = logging.getLogger("TerrorAlarm.EntityTracker") 22 | 23 | class EntityTracker: 24 | """ 25 | Entity tracking for the Terror Alarm AI system. 26 | """ 27 | 28 | def __init__(self, config): 29 | """ 30 | Initialize the EntityTracker object. 31 | 32 | Args: 33 | config: Configuration object 34 | """ 35 | self.config = config 36 | self.running = False 37 | self.tracking_thread = None 38 | self.last_update = {} 39 | 40 | # Entity lists 41 | self.supported_entities = [] 42 | self.opposed_entities = [] 43 | self.dangerous_organizations = [] 44 | self.terrorist_individuals = [] 45 | 46 | # Entity storage 47 | self.entities_dir = "data/entities" 48 | os.makedirs(self.entities_dir, exist_ok=True) 49 | 50 | # Load entity lists 51 | self._load_entity_lists() 52 | 53 | logger.info("EntityTracker initialized") 54 | 55 | def _load_entity_lists(self): 56 | """Load entity lists from files.""" 57 | # Load supported entities 58 | supported_file = os.path.join(self.entities_dir, "supported_entities.json") 59 | if os.path.exists(supported_file): 60 | try: 61 | with open(supported_file, 'r') as f: 62 | self.supported_entities = json.load(f) 63 | logger.info(f"Loaded {len(self.supported_entities)} supported entities") 64 | except Exception as e: 65 | logger.error(f"Error loading supported entities: {e}") 66 | self._initialize_supported_entities() 67 | else: 68 | logger.info("Supported entities file not found, initializing") 69 | self._initialize_supported_entities() 70 | 71 | # Load opposed entities 72 | opposed_file = os.path.join(self.entities_dir, "opposed_entities.json") 73 | if os.path.exists(opposed_file): 74 | try: 75 | with open(opposed_file, 'r') as f: 76 | self.opposed_entities = json.load(f) 77 | logger.info(f"Loaded {len(self.opposed_entities)} opposed entities") 78 | except Exception as e: 79 | logger.error(f"Error loading opposed entities: {e}") 80 | self._initialize_opposed_entities() 81 | else: 82 | logger.info("Opposed entities file not found, initializing") 83 | self._initialize_opposed_entities() 84 | 85 | # Load dangerous organizations 86 | dangerous_orgs_file = os.path.join(self.entities_dir, "dangerous_organizations.json") 87 | if os.path.exists(dangerous_orgs_file): 88 | try: 89 | with open(dangerous_orgs_file, 'r') as f: 90 | self.dangerous_organizations = json.load(f) 91 | logger.info(f"Loaded {len(self.dangerous_organizations)} dangerous organizations") 92 | except Exception as e: 93 | logger.error(f"Error loading dangerous organizations: {e}") 94 | self._initialize_dangerous_organizations() 95 | else: 96 | logger.info("Dangerous organizations file not found, initializing") 97 | self._initialize_dangerous_organizations() 98 | 99 | # Load terrorist individuals 100 | terrorist_individuals_file = os.path.join(self.entities_dir, "terrorist_individuals.json") 101 | if os.path.exists(terrorist_individuals_file): 102 | try: 103 | with open(terrorist_individuals_file, 'r') as f: 104 | self.terrorist_individuals = json.load(f) 105 | logger.info(f"Loaded {len(self.terrorist_individuals)} terrorist individuals") 106 | except Exception as e: 107 | logger.error(f"Error loading terrorist individuals: {e}") 108 | self._initialize_terrorist_individuals() 109 | else: 110 | logger.info("Terrorist individuals file not found, initializing") 111 | self._initialize_terrorist_individuals() 112 | 113 | def _initialize_supported_entities(self): 114 | """Initialize the supported entities list with default values.""" 115 | # Get supported entities from config 116 | self.supported_entities = self.config.get("entities.supported_groups", []) 117 | 118 | # Save the initial supported entities list 119 | self._save_supported_entities() 120 | 121 | logger.info(f"Initialized supported entities list with {len(self.supported_entities)} entities") 122 | 123 | def _initialize_opposed_entities(self): 124 | """Initialize the opposed entities list with default values.""" 125 | # Get opposed entities from config 126 | self.opposed_entities = self.config.get("entities.opposed_entities", []) 127 | 128 | # Save the initial opposed entities list 129 | self._save_opposed_entities() 130 | 131 | logger.info(f"Initialized opposed entities list with {len(self.opposed_entities)} entities") 132 | 133 | def _initialize_dangerous_organizations(self): 134 | """Initialize the dangerous organizations list with default values.""" 135 | # This is a placeholder for actual dangerous organizations initialization 136 | # In a real system, this would use actual data 137 | 138 | # Initialize with some known terrorist organizations 139 | self.dangerous_organizations = [ 140 | { 141 | "name": "ISIS", 142 | "aliases": ["Islamic State", "ISIL", "Daesh"], 143 | "type": "Terrorist Organization", 144 | "threat_level": "High", 145 | "regions": ["Middle East", "North Africa", "Europe"], 146 | "description": "Extremist jihadist group known for violence and territorial claims", 147 | "last_updated": datetime.now().isoformat() 148 | }, 149 | { 150 | "name": "Al-Qaeda", 151 | "aliases": ["The Base", "AQ"], 152 | "type": "Terrorist Organization", 153 | "threat_level": "High", 154 | "regions": ["Middle East", "North Africa", "South Asia", "Europe", "North America"], 155 | "description": "Global militant Islamist organization founded by Osama bin Laden", 156 | "last_updated": datetime.now().isoformat() 157 | }, 158 | { 159 | "name": "Hamas", 160 | "aliases": ["Islamic Resistance Movement"], 161 | "type": "Terrorist Organization", 162 | "threat_level": "High", 163 | "regions": ["Middle East"], 164 | "description": "Palestinian Sunni-Islamic fundamentalist organization", 165 | "last_updated": datetime.now().isoformat() 166 | }, 167 | { 168 | "name": "Hezbollah", 169 | "aliases": ["Party of God", "Islamic Jihad Organization"], 170 | "type": "Terrorist Organization", 171 | "threat_level": "High", 172 | "regions": ["Middle East", "South America"], 173 | "description": "Lebanese Shia Islamist political party and militant group", 174 | "last_updated": datetime.now().isoformat() 175 | }, 176 | { 177 | "name": "Boko Haram", 178 | "aliases": ["Jamā'at Ahl as-Sunnah lid-Da'wah wa'l-Jihād"], 179 | "type": "Terrorist Organization", 180 | "threat_level": "High", 181 | "regions": ["West Africa"], 182 | "description": "Jihadist terrorist organization based in northeastern Nigeria", 183 | "last_updated": datetime.now().isoformat() 184 | } 185 | ] 186 | 187 | # Save the initial dangerous organizations list 188 | self._save_dangerous_organizations() 189 | 190 | logger.info(f"Initialized dangerous organizations list with {len(self.dangerous_organizations)} organizations") 191 | 192 | def _initialize_terrorist_individuals(self): 193 | """Initialize the terrorist individuals list with default values.""" 194 | # This is a placeholder for actual terrorist individuals initialization 195 | # In a real system, this would use actual data 196 | 197 | # Initialize with some known terrorist individuals (using fictional names) 198 | self.terrorist_individuals = [ 199 | { 200 | "name": "Abu Mohammed Al-Fiktivi", 201 | "aliases": ["The Ghost", "Mohammed Al-Shadid"], 202 | "organization": "ISIS", 203 | "threat_level": "High", 204 | "nationality": "Unknown", 205 | "status": "Active", 206 | "description": "High-ranking ISIS commander responsible for multiple attacks", 207 | "last_updated": datetime.now().isoformat() 208 | }, 209 | { 210 | "name": "Yusuf Al-Imaginary", 211 | "aliases": ["The Engineer", "Abu Yusuf"], 212 | "organization": "Al-Qaeda", 213 | "threat_level": "High", 214 | "nationality": "Unknown", 215 | "status": "Active", 216 | "description": "Al-Qaeda explosives expert and operational planner", 217 | "last_updated": datetime.now().isoformat() 218 | }, 219 | { 220 | "name": "Ibrahim Al-Fictional", 221 | "aliases": ["The Recruiter", "Sheikh Ibrahim"], 222 | "organization": "Boko Haram", 223 | "threat_level": "Medium", 224 | "nationality": "Unknown", 225 | "status": "Active", 226 | "description": "Boko Haram recruiter and propagandist", 227 | "last_updated": datetime.now().isoformat() 228 | } 229 | ] 230 | 231 | # Save the initial terrorist individuals list 232 | self._save_terrorist_individuals() 233 | 234 | logger.info(f"Initialized terrorist individuals list with {len(self.terrorist_individuals)} individuals") 235 | 236 | def _save_supported_entities(self): 237 | """Save the supported entities list to file.""" 238 | supported_file = os.path.join(self.entities_dir, "supported_entities.json") 239 | try: 240 | with open(supported_file, 'w') as f: 241 | json.dump(self.supported_entities, f, indent=2) 242 | logger.info(f"Saved {len(self.supported_entities)} supported entities") 243 | except Exception as e: 244 | logger.error(f"Error saving supported entities: {e}") 245 | 246 | def _save_opposed_entities(self): 247 | """Save the opposed entities list to file.""" 248 | opposed_file = os.path.join(self.entities_dir, "opposed_entities.json") 249 | try: 250 | with open(opposed_file, 'w') as f: 251 | json.dump(self.opposed_entities, f, indent=2) 252 | logger.info(f"Saved {len(self.opposed_entities)} opposed entities") 253 | except Exception as e: 254 | logger.error(f"Error saving opposed entities: {e}") 255 | 256 | def _save_dangerous_organizations(self): 257 | """Save the dangerous organizations list to file.""" 258 | dangerous_orgs_file = os.path.join(self.entities_dir, "dangerous_organizations.json") 259 | try: 260 | with open(dangerous_orgs_file, 'w') as f: 261 | json.dump(self.dangerous_organizations, f, indent=2) 262 | logger.info(f"Saved {len(self.dangerous_organizations)} dangerous organizations") 263 | except Exception as e: 264 | logger.error(f"Error saving dangerous organizations: {e}") 265 | 266 | def _save_terrorist_individuals(self): 267 | """Save the terrorist individuals list to file.""" 268 | terrorist_individuals_file = os.path.join(self.entities_dir, "terrorist_individuals.json") 269 | try: 270 | with open(terrorist_individuals_file, 'w') as f: 271 | json.dump(self.terrorist_individuals, f, indent=2) 272 | logger.info(f"Saved {len(self.terrorist_individuals)} terrorist individuals") 273 | except Exception as e: 274 | logger.error(f"Error saving terrorist individuals: {e}") 275 | 276 | def start_tracking(self): 277 | """Start the entity tracking process.""" 278 | if self.running: 279 | logger.warning("Entity tracking is already running") 280 | return 281 | 282 | logger.info("Starting entity tracking") 283 | self.running = True 284 | self.tracking_thread = threading.Thread(target=self._tracking_loop) 285 | self.tracking_thread.daemon = True 286 | self.tracking_thread.start() 287 | 288 | def stop_tracking(self): 289 | """Stop the entity tracking process.""" 290 | if not self.running: 291 | logger.warning("Entity tracking is not running") 292 | return 293 | 294 | logger.info("Stopping entity tracking") 295 | self.running = False 296 | if self.tracking_thread: 297 | self.tracking_thread.join(timeout=30) 298 | 299 | def _tracking_loop(self): 300 | """Main entity tracking loop.""" 301 | logger.info("Entity tracking loop started") 302 | 303 | while self.running: 304 | try: 305 | # Update entity lists 306 | self._update_entity_lists() 307 | 308 | # Sleep for a while 309 | time.sleep(3600) # 1 hour 310 | 311 | except Exception as e: 312 | logger.error(f"Error in entity tracking loop: {e}") 313 | time.sleep(300) # Sleep for 5 minutes on error 314 | 315 | def _update_entity_lists(self): 316 | """Update entity lists based on recent data.""" 317 | logger.info("Updating entity lists") 318 | 319 | # Get processed data 320 | processed_data = self._get_processed_data() 321 | 322 | # Update dangerous organizations 323 | self._update_dangerous_organizations(processed_data) 324 | 325 | # Update terrorist individuals 326 | self._update_terrorist_individuals(processed_data) 327 | 328 | # Save updated entity lists 329 | self._save_dangerous_organizations() 330 | self._save_terrorist_individuals() 331 | 332 | # Update last update time 333 | self.last_update["entity_lists"] = datetime.now() 334 | 335 | logger.info("Entity lists updated") 336 | 337 | def _get_processed_data(self) -> List[Dict[str, Any]]: 338 | """ 339 | Get processed data for entity tracking. 340 | 341 | Returns: 342 | List of processed data items 343 | """ 344 | # This is a placeholder for actual data retrieval 345 | # In a real system, this would retrieve processed data from storage 346 | 347 | processed_data = [] 348 | 349 | # Get data directory 350 | data_dir = "data/analysis/processed" 351 | if not os.path.exists(data_dir): 352 | return processed_data 353 | 354 | # Get data from all source directories 355 | for source_type in ["social_media", "mainstream_media", "book", "custom"]: 356 | source_dir = os.path.join(data_dir, source_type) 357 | if not os.path.exists(source_dir): 358 | continue 359 | 360 | # Get data from all source subdirectories 361 | for source_name in os.listdir(source_dir): 362 | source_subdir = os.path.join(source_dir, source_name) 363 | if not os.path.isdir(source_subdir): 364 | continue 365 | 366 | # Get data from all files in the source subdirectory 367 | for filename in sorted(os.listdir(source_subdir), reverse=True): 368 | if not filename.endswith(".json"): 369 | continue 370 | 371 | # Only get recent files (last 7 days) 372 | file_path = os.path.join(source_subdir, filename) 373 | file_mtime = os.path.getmtime(file_path) 374 | if time.time() - file_mtime > 604800: # 7 days 375 | continue 376 | 377 | # Load data from file 378 | try: 379 | with open(file_path, 'r') as f: 380 | data = json.load(f) 381 | processed_data.append(data) 382 | except Exception as e: 383 | logger.error(f"Error loading data from {file_path}: {e}") 384 | 385 | # Limit the number of files to process 386 | if len(processed_data) >= 100: 387 | break 388 | 389 | return processed_data 390 | 391 | def _update_dangerous_organizations(self, processed_data: List[Dict[str, Any]]): 392 | """ 393 | Update the dangerous organizations list based on processed data. 394 | 395 | Args: 396 | processed_data: List of processed data items 397 | """ 398 | # This is a placeholder for actual dangerous organizations updating 399 | # In a real system, this would use more sophisticated logic 400 | 401 | # Extract entities from processed data 402 | entities = self._extract_entities(processed_data) 403 | 404 | # Update existing organizations 405 | for org in self.dangerous_organizations: 406 | # Check if the organization is mentioned in the entities 407 | org_name = org["name"].lower() 408 | org_aliases = [alias.lower() for alias in org["aliases"]] 409 | 410 | # Count mentions 411 | mentions = 0 412 | for entity in entities: 413 | entity_text = entity["text"].lower() 414 | if org_name in entity_text or any(alias in entity_text for alias in org_aliases): 415 | mentions += 1 416 | 417 | # Update threat level based on mentions 418 | if mentions > 10: 419 | org["threat_level"] = "High" 420 | elif mentions > 5: 421 | org["threat_level"] = "Medium" 422 | elif mentions > 0: 423 | org["threat_level"] = "Low" 424 | 425 | # Update last updated timestamp 426 | org["last_updated"] = datetime.now().isoformat() 427 | 428 | # Add new organizations 429 | for entity in entities: 430 | if entity["type"] == "ORG" and entity["confidence"] > 0.8: 431 | # Check if the organization is already in the list 432 | org_name = entity["text"] 433 | if not any(org["name"] == org_name for org in self.dangerous_organizations): 434 | # Check if the organization is mentioned multiple times 435 | mentions = sum(1 for e in entities if e["text"] == org_name) 436 | if mentions > 3: 437 | # Add the organization to the list 438 | new_org = { 439 | "name": org_name, 440 | "aliases": [], 441 | "type": "Potential Threat", 442 | "threat_level": "Low", 443 | "regions": [], 444 | "description": f"Newly identified organization with {mentions} mentions", 445 | "last_updated": datetime.now().isoformat() 446 | } 447 | self.dangerous_organizations.append(new_org) 448 | logger.info(f"Added new organization to dangerous organizations list: {org_name}") 449 | 450 | def _update_terrorist_individuals(self, processed_data: List[Dict[str, Any]]): 451 | """ 452 | Update the terrorist individuals list based on processed data. 453 | 454 | Args: 455 | processed_data: List of processed data items 456 | """ 457 | # This is a placeholder for actual terrorist individuals updating 458 | # In a real system, this would use more sophisticated logic 459 | 460 | # Extract entities from processed data 461 | entities = self._extract_entities(processed_data) 462 | 463 | # Update existing individuals 464 | for individual in self.terrorist_individuals: 465 | # Check if the individual is mentioned in the entities 466 | individual_name = individual["name"].lower() 467 | individual_aliases = [alias.lower() for alias in individual["aliases"]] 468 | 469 | # Count mentions 470 | mentions = 0 471 | for entity in entities: 472 | entity_text = entity["text"].lower() 473 | if entity["type"] == "PERSON" and (individual_name in entity_text or any(alias in entity_text for alias in individual_aliases)): 474 | mentions += 1 475 | 476 | # Update threat level based on mentions 477 | if mentions > 5: 478 | individual["threat_level"] = "High" 479 | elif mentions > 2: 480 | individual["threat_level"] = "Medium" 481 | elif mentions > 0: 482 | individual["threat_level"] = "Low" 483 | 484 | # Update last updated timestamp 485 | individual["last_updated"] = datetime.now().isoformat() 486 | 487 | # Add new individuals 488 | for entity in entities: 489 | if entity["type"] == "PERSON" and entity["confidence"] > 0.8: 490 | # Check if the individual is already in the list 491 | person_name = entity["text"] 492 | if not any(individual["name"] == person_name for individual in self.terrorist_individuals): 493 | # Check if the individual is mentioned multiple times 494 | mentions = sum(1 for e in entities if e["text"] == person_name) 495 | if mentions > 3: 496 | # Add the individual to the list 497 | new_individual = { 498 | "name": person_name, 499 | "aliases": [], 500 | "organization": "Unknown", 501 | "threat_level": "Low", 502 | "nationality": "Unknown", 503 | "status": "Active", 504 | "description": f"Newly identified individual with {mentions} mentions", 505 | "last_updated": datetime.now().isoformat() 506 | } 507 | self.terrorist_individuals.append(new_individual) 508 | logger.info(f"Added new individual to terrorist individuals list: {person_name}") 509 | 510 | def _extract_entities(self, processed_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: 511 | """ 512 | Extract entities from processed data. 513 | 514 | Args: 515 | processed_data: List of processed data items 516 | 517 | Returns: 518 | List of extracted entities 519 | """ 520 | entities = [] 521 | 522 | for data_item in processed_data: 523 | # Extract source type and name 524 | source_type = data_item.get("type", "unknown") 525 | 526 | # Extract entities based on source type 527 | if source_type == "social_media": 528 | posts = data_item.get("posts", []) 529 | for post in posts: 530 | post_entities = post.get("entities", []) 531 | entities.extend(post_entities) 532 | elif source_type == "mainstream_media": 533 | articles = data_item.get("articles", []) 534 | for article in articles: 535 | article_entities = article.get("entities", []) 536 | entities.extend(article_entities) 537 | elif source_type == "book": 538 | books = data_item.get("books", []) 539 | for book in books: 540 | book_entities = book.get("entities", []) 541 | entities.extend(book_entities) 542 | elif source_type == "custom": 543 | if "data" in data_item and isinstance(data_item["data"], dict): 544 | for field, value in data_item["data"].items(): 545 | if field.endswith("_entities") and isinstance(value, list): 546 | entities.extend(value) 547 | 548 | return entities 549 | 550 | def add_supported_entity(self, entity: str) -> bool: 551 | """ 552 | Add an entity to the supported entities list. 553 | 554 | Args: 555 | entity: Entity to add 556 | 557 | Returns: 558 | True if successful, False otherwise 559 | """ 560 | if entity in self.supported_entities: 561 | logger.warning(f"Entity already in supported entities list: {entity}") 562 | return False 563 | 564 | self.supported_entities.append(entity) 565 | self._save_supported_entities() 566 | 567 | logger.info(f"Added entity to supported entities list: {entity}") 568 | return True 569 | 570 | def remove_supported_entity(self, entity: str) -> bool: 571 | """ 572 | Remove an entity from the supported entities list. 573 | 574 | Args: 575 | entity: Entity to remove 576 | 577 | Returns: 578 | True if successful, False otherwise 579 | """ 580 | if entity not in self.supported_entities: 581 | logger.warning(f"Entity not in supported entities list: {entity}") 582 | return False 583 | 584 | self.supported_entities.remove(entity) 585 | self._save_supported_entities() 586 | 587 | logger.info(f"Removed entity from supported entities list: {entity}") 588 | return True 589 | 590 | def add_opposed_entity(self, entity: str) -> bool: 591 | """ 592 | Add an entity to the opposed entities list. 593 | 594 | Args: 595 | entity: Entity to add 596 | 597 | Returns: 598 | True if successful, False otherwise 599 | """ 600 | if entity in self.opposed_entities: 601 | logger.warning(f"Entity already in opposed entities list: {entity}") 602 | return False 603 | 604 | self.opposed_entities.append(entity) 605 | self._save_opposed_entities() 606 | 607 | logger.info(f"Added entity to opposed entities list: {entity}") 608 | return True 609 | 610 | def remove_opposed_entity(self, entity: str) -> bool: 611 | """ 612 | Remove an entity from the opposed entities list. 613 | 614 | Args: 615 | entity: Entity to remove 616 | 617 | Returns: 618 | True if successful, False otherwise 619 | """ 620 | if entity not in self.opposed_entities: 621 | logger.warning(f"Entity not in opposed entities list: {entity}") 622 | return False 623 | 624 | self.opposed_entities.remove(entity) 625 | self._save_opposed_entities() 626 | 627 | logger.info(f"Removed entity from opposed entities list: {entity}") 628 | return True 629 | 630 | def add_dangerous_organization(self, organization: Dict[str, Any]) -> bool: 631 | """ 632 | Add an organization to the dangerous organizations list. 633 | 634 | Args: 635 | organization: Organization to add 636 | 637 | Returns: 638 | True if successful, False otherwise 639 | """ 640 | if any(org["name"] == organization["name"] for org in self.dangerous_organizations): 641 | logger.warning(f"Organization already in dangerous organizations list: {organization['name']}") 642 | return False 643 | 644 | self.dangerous_organizations.append(organization) 645 | self._save_dangerous_organizations() 646 | 647 | logger.info(f"Added organization to dangerous organizations list: {organization['name']}") 648 | return True 649 | 650 | def remove_dangerous_organization(self, organization_name: str) -> bool: 651 | """ 652 | Remove an organization from the dangerous organizations list. 653 | 654 | Args: 655 | organization_name: Name of the organization to remove 656 | 657 | Returns: 658 | True if successful, False otherwise 659 | """ 660 | for i, org in enumerate(self.dangerous_organizations): 661 | if org["name"] == organization_name: 662 | del self.dangerous_organizations[i] 663 | self._save_dangerous_organizations() 664 | 665 | logger.info(f"Removed organization from dangerous organizations list: {organization_name}") 666 | return True 667 | 668 | logger.warning(f"Organization not in dangerous organizations list: {organization_name}") 669 | return False 670 | 671 | def add_terrorist_individual(self, individual: Dict[str, Any]) -> bool: 672 | """ 673 | Add an individual to the terrorist individuals list. 674 | 675 | Args: 676 | individual: Individual to add 677 | 678 | Returns: 679 | True if successful, False otherwise 680 | """ 681 | if any(ind["name"] == individual["name"] for ind in self.terrorist_individuals): 682 | logger.warning(f"Individual already in terrorist individuals list: {individual['name']}") 683 | return False 684 | 685 | self.terrorist_individuals.append(individual) 686 | self._save_terrorist_individuals() 687 | 688 | logger.info(f"Added individual to terrorist individuals list: {individual['name']}") 689 | return True 690 | 691 | def remove_terrorist_individual(self, individual_name: str) -> bool: 692 | """ 693 | Remove an individual from the terrorist individuals list. 694 | 695 | Args: 696 | individual_name: Name of the individual to remove 697 | 698 | Returns: 699 | True if successful, False otherwise 700 | """ 701 | for i, ind in enumerate(self.terrorist_individuals): 702 | if ind["name"] == individual_name: 703 | del self.terrorist_individuals[i] 704 | self._save_terrorist_individuals() 705 | 706 | logger.info(f"Removed individual from terrorist individuals list: {individual_name}") 707 | return True 708 | 709 | logger.warning(f"Individual not in terrorist individuals list: {individual_name}") 710 | return False 711 | 712 | def update_entities(self): 713 | """Update entity lists.""" 714 | self._update_entity_lists() 715 | 716 | def get_supported_entities(self) -> List[str]: 717 | """ 718 | Get the supported entities list. 719 | 720 | Returns: 721 | List of supported entities 722 | """ 723 | return self.supported_entities 724 | 725 | def get_opposed_entities(self) -> List[str]: 726 | """ 727 | Get the opposed entities list. 728 | 729 | Returns: 730 | List of opposed entities 731 | """ 732 | return self.opposed_entities 733 | 734 | def get_dangerous_organizations(self) -> List[Dict[str, Any]]: 735 | """ 736 | Get the dangerous organizations list. 737 | 738 | Returns: 739 | List of dangerous organizations 740 | """ 741 | return self.dangerous_organizations 742 | 743 | def get_terrorist_individuals(self) -> List[Dict[str, Any]]: 744 | """ 745 | Get the terrorist individuals list. 746 | 747 | Returns: 748 | List of terrorist individuals 749 | """ 750 | return self.terrorist_individuals 751 | 752 | def get_state(self) -> Dict[str, Any]: 753 | """ 754 | Get the current state of the entity tracker. 755 | 756 | Returns: 757 | Dictionary containing the current state 758 | """ 759 | return { 760 | "running": self.running, 761 | "last_update": {k: v.isoformat() for k, v in self.last_update.items()}, 762 | "supported_entities_count": len(self.supported_entities), 763 | "opposed_entities_count": len(self.opposed_entities), 764 | "dangerous_organizations_count": len(self.dangerous_organizations), 765 | "terrorist_individuals_count": len(self.terrorist_individuals) 766 | } 767 | 768 | def get_status(self) -> Dict[str, Any]: 769 | """ 770 | Get the current status of the entity tracker. 771 | 772 | Returns: 773 | Dictionary containing the current status 774 | """ 775 | return { 776 | "running": self.running, 777 | "entities": { 778 | "supported": len(self.supported_entities), 779 | "opposed": len(self.opposed_entities), 780 | "dangerous_organizations": len(self.dangerous_organizations), 781 | "terrorist_individuals": len(self.terrorist_individuals) 782 | }, 783 | "last_update": {k: v.isoformat() for k, v in self.last_update.items()} if self.last_update else None 784 | } 785 | 786 | 787 | if __name__ == "__main__": 788 | # Set up logging 789 | logging.basicConfig( 790 | level=logging.INFO, 791 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' 792 | ) 793 | 794 | # Test entity tracker 795 | from config import Configuration 796 | 797 | # Create default configuration 798 | config = Configuration() 799 | 800 | # Create entity tracker 801 | tracker = EntityTracker(config) 802 | 803 | # Start tracking 804 | tracker.start_tracking() 805 | 806 | # Wait for a while 807 | print("Entity tracking started. Press Ctrl+C to stop.") 808 | try: 809 | while True: 810 | time.sleep(1) 811 | except KeyboardInterrupt: 812 | pass 813 | 814 | # Stop tracking 815 | tracker.stop_tracking() 816 | print("Entity tracking stopped.") 817 | -------------------------------------------------------------------------------- /modules/prediction_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Prediction Model Module 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This module implements the prediction model for the Terror Alarm AI system. 10 | """ 11 | 12 | import os 13 | import json 14 | import time 15 | import logging 16 | import threading 17 | import numpy as np 18 | from typing import Any, Dict, List, Optional, Union, Tuple 19 | from datetime import datetime, timedelta 20 | 21 | logger = logging.getLogger("TerrorAlarm.PredictionModel") 22 | 23 | class PredictionModel: 24 | """ 25 | Prediction model for the Terror Alarm AI system. 26 | """ 27 | 28 | def __init__(self, config, bito_engine): 29 | """ 30 | Initialize the PredictionModel object. 31 | 32 | Args: 33 | config: Configuration object 34 | bito_engine: BitoEngine object 35 | """ 36 | self.config = config 37 | self.bito_engine = bito_engine 38 | self.running = False 39 | self.prediction_thread = None 40 | self.last_prediction = {} 41 | 42 | # Prediction timeframes 43 | self.timeframes = { 44 | "short": { 45 | "days": self.config.get("prediction.short_term.window_days", 7), 46 | "confidence_threshold": self.config.get("prediction.short_term.confidence_threshold", 0.7), 47 | "predictions": [] 48 | }, 49 | "medium": { 50 | "days": self.config.get("prediction.medium_term.window_days", 30), 51 | "confidence_threshold": self.config.get("prediction.medium_term.confidence_threshold", 0.6), 52 | "predictions": [] 53 | }, 54 | "long": { 55 | "days": self.config.get("prediction.long_term.window_days", 365), 56 | "confidence_threshold": self.config.get("prediction.long_term.confidence_threshold", 0.5), 57 | "predictions": [] 58 | } 59 | } 60 | 61 | # Prediction storage 62 | self.predictions_dir = "data/predictions" 63 | os.makedirs(self.predictions_dir, exist_ok=True) 64 | 65 | # Load existing predictions 66 | self._load_predictions() 67 | 68 | logger.info("PredictionModel initialized") 69 | 70 | def _load_predictions(self): 71 | """Load existing predictions from files.""" 72 | for timeframe in self.timeframes: 73 | predictions_file = os.path.join(self.predictions_dir, f"{timeframe}_predictions.json") 74 | if os.path.exists(predictions_file): 75 | try: 76 | with open(predictions_file, 'r') as f: 77 | self.timeframes[timeframe]["predictions"] = json.load(f) 78 | logger.info(f"Loaded {len(self.timeframes[timeframe]['predictions'])} {timeframe}-term predictions") 79 | except Exception as e: 80 | logger.error(f"Error loading {timeframe}-term predictions: {e}") 81 | 82 | def _save_predictions(self, timeframe: str): 83 | """ 84 | Save predictions to file. 85 | 86 | Args: 87 | timeframe: Prediction timeframe ("short", "medium", or "long") 88 | """ 89 | predictions_file = os.path.join(self.predictions_dir, f"{timeframe}_predictions.json") 90 | try: 91 | with open(predictions_file, 'w') as f: 92 | json.dump(self.timeframes[timeframe]["predictions"], f, indent=2) 93 | logger.info(f"Saved {len(self.timeframes[timeframe]['predictions'])} {timeframe}-term predictions") 94 | except Exception as e: 95 | logger.error(f"Error saving {timeframe}-term predictions: {e}") 96 | 97 | def start_prediction(self): 98 | """Start the prediction process.""" 99 | if self.running: 100 | logger.warning("Prediction is already running") 101 | return 102 | 103 | logger.info("Starting prediction") 104 | self.running = True 105 | self.prediction_thread = threading.Thread(target=self._prediction_loop) 106 | self.prediction_thread.daemon = True 107 | self.prediction_thread.start() 108 | 109 | def stop_prediction(self): 110 | """Stop the prediction process.""" 111 | if not self.running: 112 | logger.warning("Prediction is not running") 113 | return 114 | 115 | logger.info("Stopping prediction") 116 | self.running = False 117 | if self.prediction_thread: 118 | self.prediction_thread.join(timeout=30) 119 | 120 | def _prediction_loop(self): 121 | """Main prediction loop.""" 122 | logger.info("Prediction loop started") 123 | 124 | while self.running: 125 | try: 126 | # Update predictions 127 | self._update_predictions() 128 | 129 | # Sleep for a while 130 | time.sleep(300) # 5 minutes 131 | 132 | except Exception as e: 133 | logger.error(f"Error in prediction loop: {e}") 134 | time.sleep(300) # Sleep for 5 minutes on error 135 | 136 | def _update_predictions(self): 137 | """Update predictions for all timeframes.""" 138 | logger.info("Updating predictions") 139 | 140 | # Get processed data 141 | processed_data = self._get_processed_data() 142 | 143 | # Update predictions for each timeframe 144 | for timeframe in self.timeframes: 145 | self._update_timeframe_predictions(timeframe, processed_data) 146 | 147 | # Save predictions 148 | self._save_predictions(timeframe) 149 | 150 | # Update last prediction time 151 | self.last_prediction[timeframe] = datetime.now() 152 | 153 | def _get_processed_data(self) -> List[Dict[str, Any]]: 154 | """ 155 | Get processed data for prediction. 156 | 157 | Returns: 158 | List of processed data items 159 | """ 160 | # This is a placeholder for actual data retrieval 161 | # In a real system, this would retrieve processed data from storage 162 | 163 | processed_data = [] 164 | 165 | # Get data directory 166 | data_dir = "data/analysis/processed" 167 | if not os.path.exists(data_dir): 168 | return processed_data 169 | 170 | # Get data from all source directories 171 | for source_type in ["social_media", "mainstream_media", "book", "custom"]: 172 | source_dir = os.path.join(data_dir, source_type) 173 | if not os.path.exists(source_dir): 174 | continue 175 | 176 | # Get data from all source subdirectories 177 | for source_name in os.listdir(source_dir): 178 | source_subdir = os.path.join(source_dir, source_name) 179 | if not os.path.isdir(source_subdir): 180 | continue 181 | 182 | # Get data from all files in the source subdirectory 183 | for filename in sorted(os.listdir(source_subdir), reverse=True): 184 | if not filename.endswith(".json"): 185 | continue 186 | 187 | # Only get recent files (last 24 hours) 188 | file_path = os.path.join(source_subdir, filename) 189 | file_mtime = os.path.getmtime(file_path) 190 | if time.time() - file_mtime > 86400: # 24 hours 191 | continue 192 | 193 | # Load data from file 194 | try: 195 | with open(file_path, 'r') as f: 196 | data = json.load(f) 197 | processed_data.append(data) 198 | except Exception as e: 199 | logger.error(f"Error loading data from {file_path}: {e}") 200 | 201 | # Limit the number of files to process 202 | if len(processed_data) >= 100: 203 | break 204 | 205 | return processed_data 206 | 207 | def _update_timeframe_predictions(self, timeframe: str, processed_data: List[Dict[str, Any]]): 208 | """ 209 | Update predictions for a specific timeframe. 210 | 211 | Args: 212 | timeframe: Prediction timeframe ("short", "medium", or "long") 213 | processed_data: List of processed data items 214 | """ 215 | logger.info(f"Updating {timeframe}-term predictions") 216 | 217 | # Extract features from processed data 218 | features = self._extract_features(processed_data) 219 | 220 | # Generate predictions using the Bito engine 221 | raw_predictions = self._generate_predictions(timeframe, features) 222 | 223 | # Filter predictions based on confidence threshold 224 | confidence_threshold = self.timeframes[timeframe]["confidence_threshold"] 225 | filtered_predictions = [p for p in raw_predictions if p["confidence"] >= confidence_threshold] 226 | 227 | # Update predictions 228 | self.timeframes[timeframe]["predictions"] = filtered_predictions 229 | 230 | logger.info(f"Updated {len(filtered_predictions)} {timeframe}-term predictions") 231 | 232 | def _extract_features(self, processed_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: 233 | """ 234 | Extract features from processed data. 235 | 236 | Args: 237 | processed_data: List of processed data items 238 | 239 | Returns: 240 | List of feature dictionaries 241 | """ 242 | # This is a placeholder for actual feature extraction 243 | # In a real system, this would extract meaningful features from the data 244 | 245 | features = [] 246 | 247 | for data_item in processed_data: 248 | # Extract source type and name 249 | source_type = data_item.get("type", "unknown") 250 | source_name = data_item.get("source", "unknown") 251 | 252 | # Extract content based on source type 253 | if source_type == "social_media": 254 | posts = data_item.get("posts", []) 255 | for post in posts: 256 | feature = self._extract_post_features(post, source_name) 257 | features.append(feature) 258 | elif source_type == "mainstream_media": 259 | articles = data_item.get("articles", []) 260 | for article in articles: 261 | feature = self._extract_article_features(article, source_name) 262 | features.append(feature) 263 | elif source_type == "book": 264 | books = data_item.get("books", []) 265 | for book in books: 266 | feature = self._extract_book_features(book, source_name) 267 | features.append(feature) 268 | elif source_type == "custom": 269 | feature = self._extract_custom_features(data_item, source_name) 270 | features.append(feature) 271 | 272 | return features 273 | 274 | def _extract_post_features(self, post: Dict[str, Any], source_name: str) -> Dict[str, Any]: 275 | """ 276 | Extract features from a social media post. 277 | 278 | Args: 279 | post: Social media post 280 | source_name: Name of the source 281 | 282 | Returns: 283 | Feature dictionary 284 | """ 285 | # Extract sentiment 286 | sentiment = post.get("sentiment", {}) 287 | sentiment_score = sentiment.get("compound", 0.0) 288 | 289 | # Extract entities 290 | entities = post.get("entities", []) 291 | entity_types = {} 292 | for entity in entities: 293 | entity_type = entity.get("type", "UNKNOWN") 294 | if entity_type not in entity_types: 295 | entity_types[entity_type] = 0 296 | entity_types[entity_type] += 1 297 | 298 | # Extract countries 299 | countries = post.get("countries", []) 300 | 301 | # Create feature dictionary 302 | feature = { 303 | "source_type": "social_media", 304 | "source_name": source_name, 305 | "text_sentiment": sentiment_score, 306 | "entity_presence": len(entities) > 0, 307 | "entity_count": len(entities), 308 | "country_count": len(countries), 309 | "historical_patterns": 0.5, # Placeholder 310 | "temporal_factors": 0.5 # Placeholder 311 | } 312 | 313 | # Add entity type counts 314 | for entity_type, count in entity_types.items(): 315 | feature[f"entity_{entity_type.lower()}_count"] = count 316 | 317 | return feature 318 | 319 | def _extract_article_features(self, article: Dict[str, Any], source_name: str) -> Dict[str, Any]: 320 | """ 321 | Extract features from a mainstream media article. 322 | 323 | Args: 324 | article: Mainstream media article 325 | source_name: Name of the source 326 | 327 | Returns: 328 | Feature dictionary 329 | """ 330 | # Extract sentiment 331 | sentiment = article.get("sentiment", {}) 332 | sentiment_score = sentiment.get("compound", 0.0) 333 | 334 | # Extract entities 335 | entities = article.get("entities", []) 336 | entity_types = {} 337 | for entity in entities: 338 | entity_type = entity.get("type", "UNKNOWN") 339 | if entity_type not in entity_types: 340 | entity_types[entity_type] = 0 341 | entity_types[entity_type] += 1 342 | 343 | # Extract countries 344 | countries = article.get("countries", []) 345 | 346 | # Create feature dictionary 347 | feature = { 348 | "source_type": "mainstream_media", 349 | "source_name": source_name, 350 | "text_sentiment": sentiment_score, 351 | "entity_presence": len(entities) > 0, 352 | "entity_count": len(entities), 353 | "country_count": len(countries), 354 | "historical_patterns": 0.6, # Placeholder 355 | "temporal_factors": 0.6 # Placeholder 356 | } 357 | 358 | # Add entity type counts 359 | for entity_type, count in entity_types.items(): 360 | feature[f"entity_{entity_type.lower()}_count"] = count 361 | 362 | return feature 363 | 364 | def _extract_book_features(self, book: Dict[str, Any], source_name: str) -> Dict[str, Any]: 365 | """ 366 | Extract features from a book. 367 | 368 | Args: 369 | book: Book 370 | source_name: Name of the source 371 | 372 | Returns: 373 | Feature dictionary 374 | """ 375 | # Extract sentiment 376 | sentiment = book.get("sentiment", {}) 377 | sentiment_score = sentiment.get("compound", 0.0) 378 | 379 | # Extract entities 380 | entities = book.get("entities", []) 381 | entity_types = {} 382 | for entity in entities: 383 | entity_type = entity.get("type", "UNKNOWN") 384 | if entity_type not in entity_types: 385 | entity_types[entity_type] = 0 386 | entity_types[entity_type] += 1 387 | 388 | # Extract countries 389 | countries = book.get("countries", []) 390 | 391 | # Create feature dictionary 392 | feature = { 393 | "source_type": "book", 394 | "source_name": source_name, 395 | "text_sentiment": sentiment_score, 396 | "entity_presence": len(entities) > 0, 397 | "entity_count": len(entities), 398 | "country_count": len(countries), 399 | "historical_patterns": 0.7, # Placeholder 400 | "temporal_factors": 0.3 # Placeholder 401 | } 402 | 403 | # Add entity type counts 404 | for entity_type, count in entity_types.items(): 405 | feature[f"entity_{entity_type.lower()}_count"] = count 406 | 407 | return feature 408 | 409 | def _extract_custom_features(self, data_item: Dict[str, Any], source_name: str) -> Dict[str, Any]: 410 | """ 411 | Extract features from custom data. 412 | 413 | Args: 414 | data_item: Custom data item 415 | source_name: Name of the source 416 | 417 | Returns: 418 | Feature dictionary 419 | """ 420 | # Create feature dictionary 421 | feature = { 422 | "source_type": "custom", 423 | "source_name": source_name, 424 | "text_sentiment": 0.0, 425 | "entity_presence": False, 426 | "entity_count": 0, 427 | "country_count": 0, 428 | "historical_patterns": 0.5, # Placeholder 429 | "temporal_factors": 0.5 # Placeholder 430 | } 431 | 432 | # Extract data fields if available 433 | if "data" in data_item and isinstance(data_item["data"], dict): 434 | for field, value in data_item["data"].items(): 435 | if field.endswith("_sentiment") and isinstance(value, dict): 436 | feature["text_sentiment"] = value.get("compound", 0.0) 437 | elif field.endswith("_entities") and isinstance(value, list): 438 | feature["entity_presence"] = len(value) > 0 439 | feature["entity_count"] = len(value) 440 | 441 | # Count entity types 442 | entity_types = {} 443 | for entity in value: 444 | entity_type = entity.get("type", "UNKNOWN") 445 | if entity_type not in entity_types: 446 | entity_types[entity_type] = 0 447 | entity_types[entity_type] += 1 448 | 449 | # Add entity type counts 450 | for entity_type, count in entity_types.items(): 451 | feature[f"entity_{entity_type.lower()}_count"] = count 452 | elif field.endswith("_countries") and isinstance(value, list): 453 | feature["country_count"] = len(value) 454 | 455 | return feature 456 | 457 | def _generate_predictions(self, timeframe: str, features: List[Dict[str, Any]]) -> List[Dict[str, Any]]: 458 | """ 459 | Generate predictions for a specific timeframe. 460 | 461 | Args: 462 | timeframe: Prediction timeframe ("short", "medium", or "long") 463 | features: List of feature dictionaries 464 | 465 | Returns: 466 | List of prediction dictionaries 467 | """ 468 | # This is a placeholder for actual prediction generation 469 | # In a real system, this would use the Bito engine to generate predictions 470 | 471 | # Get timeframe parameters 472 | days = self.timeframes[timeframe]["days"] 473 | 474 | # Use the Bito engine to predict risk scores 475 | risk_scores = self.bito_engine.predict(features) 476 | 477 | # Generate predictions 478 | predictions = [] 479 | 480 | # Group features by country 481 | country_features = {} 482 | for i, feature in enumerate(features): 483 | countries = feature.get("countries", []) 484 | for country in countries: 485 | if country not in country_features: 486 | country_features[country] = [] 487 | country_features[country].append((feature, risk_scores[i])) 488 | 489 | # Generate predictions for each country 490 | for country, country_data in country_features.items(): 491 | # Calculate average risk score 492 | avg_risk = sum(score for _, score in country_data) / len(country_data) 493 | 494 | # Generate prediction if risk score is high enough 495 | if avg_risk > 0.5: 496 | # Generate prediction date 497 | days_offset = int(np.random.random() * days) 498 | prediction_date = datetime.now() + timedelta(days=days_offset) 499 | 500 | # Generate prediction 501 | prediction = { 502 | "country": country, 503 | "type": self._generate_prediction_type(), 504 | "description": self._generate_prediction_description(country), 505 | "date": prediction_date.strftime("%Y-%m-%d"), 506 | "confidence": round(avg_risk, 3), 507 | "sources": [feature["source_name"] for feature, _ in country_data[:3]], 508 | "generated_at": datetime.now().isoformat() 509 | } 510 | 511 | predictions.append(prediction) 512 | 513 | return predictions 514 | 515 | def _generate_prediction_type(self) -> str: 516 | """ 517 | Generate a prediction type. 518 | 519 | Returns: 520 | Prediction type 521 | """ 522 | # This is a placeholder for actual prediction type generation 523 | # In a real system, this would use more sophisticated logic 524 | 525 | types = [ 526 | "terrorist_attack", 527 | "civil_unrest", 528 | "political_instability", 529 | "military_conflict", 530 | "cyber_attack", 531 | "infrastructure_attack", 532 | "assassination", 533 | "hostage_situation", 534 | "mass_shooting", 535 | "bombing" 536 | ] 537 | 538 | return np.random.choice(types) 539 | 540 | def _generate_prediction_description(self, country: str) -> str: 541 | """ 542 | Generate a prediction description. 543 | 544 | Args: 545 | country: Country for the prediction 546 | 547 | Returns: 548 | Prediction description 549 | """ 550 | # This is a placeholder for actual prediction description generation 551 | # In a real system, this would use more sophisticated logic 552 | 553 | descriptions = [ 554 | f"Potential terrorist attack in {country}", 555 | f"Risk of civil unrest in {country}", 556 | f"Political instability in {country}", 557 | f"Military conflict in {country}", 558 | f"Cyber attack targeting infrastructure in {country}", 559 | f"Infrastructure attack in {country}", 560 | f"Assassination attempt in {country}", 561 | f"Hostage situation in {country}", 562 | f"Mass shooting in {country}", 563 | f"Bombing in {country}" 564 | ] 565 | 566 | return np.random.choice(descriptions) 567 | 568 | def update_predictions(self): 569 | """Update predictions for all timeframes.""" 570 | self._update_predictions() 571 | 572 | def get_predictions(self, timeframe: str = "short") -> List[Dict[str, Any]]: 573 | """ 574 | Get predictions for a specific timeframe. 575 | 576 | Args: 577 | timeframe: Prediction timeframe ("short", "medium", or "long") 578 | 579 | Returns: 580 | List of prediction dictionaries 581 | """ 582 | if timeframe not in self.timeframes: 583 | logger.warning(f"Invalid timeframe: {timeframe}") 584 | return [] 585 | 586 | return self.timeframes[timeframe]["predictions"] 587 | 588 | def get_state(self) -> Dict[str, Any]: 589 | """ 590 | Get the current state of the prediction model. 591 | 592 | Returns: 593 | Dictionary containing the current state 594 | """ 595 | return { 596 | "running": self.running, 597 | "last_prediction": {k: v.isoformat() for k, v in self.last_prediction.items()}, 598 | "timeframes": { 599 | k: { 600 | "days": v["days"], 601 | "confidence_threshold": v["confidence_threshold"], 602 | "predictions_count": len(v["predictions"]) 603 | } 604 | for k, v in self.timeframes.items() 605 | } 606 | } 607 | 608 | def get_status(self) -> Dict[str, Any]: 609 | """ 610 | Get the current status of the prediction model. 611 | 612 | Returns: 613 | Dictionary containing the current status 614 | """ 615 | return { 616 | "running": self.running, 617 | "timeframes": { 618 | k: { 619 | "days": v["days"], 620 | "confidence_threshold": v["confidence_threshold"], 621 | "predictions_count": len(v["predictions"]), 622 | "last_update": self.last_prediction.get(k, datetime.now()).isoformat() if k in self.last_prediction else None 623 | } 624 | for k, v in self.timeframes.items() 625 | } 626 | } 627 | 628 | 629 | if __name__ == "__main__": 630 | # Set up logging 631 | logging.basicConfig( 632 | level=logging.INFO, 633 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' 634 | ) 635 | 636 | # Test prediction model 637 | from config import Configuration 638 | from bito_integration import BitoEngine 639 | 640 | # Create default configuration 641 | config = Configuration() 642 | 643 | # Create Bito engine 644 | bito = BitoEngine(config) 645 | 646 | # Create prediction model 647 | model = PredictionModel(config, bito) 648 | 649 | # Start prediction 650 | model.start_prediction() 651 | 652 | # Wait for a while 653 | print("Prediction started. Press Ctrl+C to stop.") 654 | try: 655 | while True: 656 | time.sleep(1) 657 | except KeyboardInterrupt: 658 | pass 659 | 660 | # Stop prediction 661 | model.stop_prediction() 662 | print("Prediction stopped.") 663 | -------------------------------------------------------------------------------- /modules/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Terror Alarm AI System - Utilities Module 5 | Developed by Terror Alarm NGO (Europe, DK44425645) 6 | Copyright © 2022-2025 Terror Alarm NGO. All rights reserved. 7 | Creation Date: June 6, 2022 8 | 9 | This module provides utility functions for the Terror Alarm AI system. 10 | """ 11 | 12 | import os 13 | import re 14 | import json 15 | import hashlib 16 | import logging 17 | import datetime 18 | import requests 19 | from typing import Any, Dict, List, Optional, Union, Tuple 20 | 21 | logger = logging.getLogger("TerrorAlarm.Utils") 22 | 23 | class Utils: 24 | """ 25 | Utility functions for the Terror Alarm AI system. 26 | """ 27 | 28 | def __init__(self): 29 | """Initialize the Utils object.""" 30 | logger.debug("Utils initialized") 31 | 32 | @staticmethod 33 | def generate_hash(data: str) -> str: 34 | """ 35 | Generate a SHA-256 hash of the given data. 36 | 37 | Args: 38 | data: Data to hash 39 | 40 | Returns: 41 | SHA-256 hash as a hexadecimal string 42 | """ 43 | return hashlib.sha256(data.encode('utf-8')).hexdigest() 44 | 45 | @staticmethod 46 | def sanitize_text(text: str) -> str: 47 | """ 48 | Sanitize text by removing special characters and normalizing whitespace. 49 | 50 | Args: 51 | text: Text to sanitize 52 | 53 | Returns: 54 | Sanitized text 55 | """ 56 | # Remove HTML tags 57 | text = re.sub(r'<[^>]+>', ' ', text) 58 | 59 | # Normalize whitespace 60 | text = re.sub(r'\s+', ' ', text) 61 | 62 | # Remove non-printable characters 63 | text = ''.join(c for c in text if c.isprintable() or c.isspace()) 64 | 65 | return text.strip() 66 | 67 | @staticmethod 68 | def extract_urls(text: str) -> List[str]: 69 | """ 70 | Extract URLs from text. 71 | 72 | Args: 73 | text: Text to extract URLs from 74 | 75 | Returns: 76 | List of extracted URLs 77 | """ 78 | url_pattern = re.compile( 79 | r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' 80 | ) 81 | return url_pattern.findall(text) 82 | 83 | @staticmethod 84 | def extract_entities(text: str) -> List[str]: 85 | """ 86 | Extract named entities from text using simple pattern matching. 87 | This is a basic implementation and should be replaced with a proper NER model. 88 | 89 | Args: 90 | text: Text to extract entities from 91 | 92 | Returns: 93 | List of extracted entities 94 | """ 95 | # This is a placeholder for a proper NER implementation 96 | # In a real system, this would use a trained NER model 97 | 98 | # Simple pattern for organization names (capitalized words) 99 | org_pattern = re.compile(r'\b([A-Z][a-z]+ [A-Z][a-z]+)\b') 100 | 101 | # Extract potential organizations 102 | orgs = org_pattern.findall(text) 103 | 104 | return list(set(orgs)) 105 | 106 | @staticmethod 107 | def download_file(url: str, save_path: str) -> bool: 108 | """ 109 | Download a file from a URL. 110 | 111 | Args: 112 | url: URL to download from 113 | save_path: Path to save the file to 114 | 115 | Returns: 116 | True if successful, False otherwise 117 | """ 118 | try: 119 | response = requests.get(url, stream=True) 120 | response.raise_for_status() 121 | 122 | os.makedirs(os.path.dirname(save_path), exist_ok=True) 123 | 124 | with open(save_path, 'wb') as f: 125 | for chunk in response.iter_content(chunk_size=8192): 126 | f.write(chunk) 127 | 128 | logger.info(f"Downloaded {url} to {save_path}") 129 | return True 130 | except Exception as e: 131 | logger.error(f"Error downloading {url}: {e}") 132 | return False 133 | 134 | @staticmethod 135 | def load_json(file_path: str) -> Optional[Dict[str, Any]]: 136 | """ 137 | Load JSON from a file. 138 | 139 | Args: 140 | file_path: Path to the JSON file 141 | 142 | Returns: 143 | Loaded JSON data or None if failed 144 | """ 145 | try: 146 | with open(file_path, 'r') as f: 147 | return json.load(f) 148 | except Exception as e: 149 | logger.error(f"Error loading JSON from {file_path}: {e}") 150 | return None 151 | 152 | @staticmethod 153 | def save_json(data: Dict[str, Any], file_path: str) -> bool: 154 | """ 155 | Save JSON data to a file. 156 | 157 | Args: 158 | data: JSON data to save 159 | file_path: Path to save the JSON file to 160 | 161 | Returns: 162 | True if successful, False otherwise 163 | """ 164 | try: 165 | os.makedirs(os.path.dirname(file_path), exist_ok=True) 166 | with open(file_path, 'w') as f: 167 | json.dump(data, f, indent=2) 168 | return True 169 | except Exception as e: 170 | logger.error(f"Error saving JSON to {file_path}: {e}") 171 | return False 172 | 173 | @staticmethod 174 | def get_timestamp() -> str: 175 | """ 176 | Get the current timestamp as an ISO 8601 string. 177 | 178 | Returns: 179 | Current timestamp as an ISO 8601 string 180 | """ 181 | return datetime.datetime.now().isoformat() 182 | 183 | @staticmethod 184 | def parse_timestamp(timestamp: str) -> Optional[datetime.datetime]: 185 | """ 186 | Parse an ISO 8601 timestamp string. 187 | 188 | Args: 189 | timestamp: ISO 8601 timestamp string 190 | 191 | Returns: 192 | Parsed datetime object or None if failed 193 | """ 194 | try: 195 | return datetime.datetime.fromisoformat(timestamp) 196 | except Exception as e: 197 | logger.error(f"Error parsing timestamp {timestamp}: {e}") 198 | return None 199 | 200 | @staticmethod 201 | def calculate_similarity(text1: str, text2: str) -> float: 202 | """ 203 | Calculate the similarity between two texts using Jaccard similarity. 204 | 205 | Args: 206 | text1: First text 207 | text2: Second text 208 | 209 | Returns: 210 | Similarity score between 0 and 1 211 | """ 212 | # Tokenize texts 213 | tokens1 = set(text1.lower().split()) 214 | tokens2 = set(text2.lower().split()) 215 | 216 | # Calculate Jaccard similarity 217 | intersection = len(tokens1.intersection(tokens2)) 218 | union = len(tokens1.union(tokens2)) 219 | 220 | if union == 0: 221 | return 0.0 222 | 223 | return intersection / union 224 | 225 | @staticmethod 226 | def format_date(date: datetime.datetime, format_str: str = "%Y-%m-%d") -> str: 227 | """ 228 | Format a datetime object as a string. 229 | 230 | Args: 231 | date: Datetime object to format 232 | format_str: Format string 233 | 234 | Returns: 235 | Formatted date string 236 | """ 237 | return date.strftime(format_str) 238 | 239 | @staticmethod 240 | def parse_date(date_str: str, format_str: str = "%Y-%m-%d") -> Optional[datetime.datetime]: 241 | """ 242 | Parse a date string. 243 | 244 | Args: 245 | date_str: Date string to parse 246 | format_str: Format string 247 | 248 | Returns: 249 | Parsed datetime object or None if failed 250 | """ 251 | try: 252 | return datetime.datetime.strptime(date_str, format_str) 253 | except Exception as e: 254 | logger.error(f"Error parsing date {date_str}: {e}") 255 | return None 256 | 257 | 258 | if __name__ == "__main__": 259 | # Set up logging 260 | logging.basicConfig( 261 | level=logging.INFO, 262 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' 263 | ) 264 | 265 | # Test utility functions 266 | utils = Utils() 267 | 268 | # Test hash generation 269 | text = "Terror Alarm AI" 270 | hash_value = utils.generate_hash(text) 271 | print(f"Hash of '{text}': {hash_value}") 272 | 273 | # Test text sanitization 274 | html_text = "

This is HTML text with\n\nmultiple\twhitespace.

" 275 | sanitized = utils.sanitize_text(html_text) 276 | print(f"Sanitized text: '{sanitized}'") 277 | 278 | # Test URL extraction 279 | url_text = "Visit https://example.com and http://test.org for more information." 280 | urls = utils.extract_urls(url_text) 281 | print(f"Extracted URLs: {urls}") 282 | 283 | # Test timestamp 284 | timestamp = utils.get_timestamp() 285 | print(f"Current timestamp: {timestamp}") 286 | parsed = utils.parse_timestamp(timestamp) 287 | print(f"Parsed timestamp: {parsed}") 288 | -------------------------------------------------------------------------------- /windows_compat/run_terror_alarm.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | echo Starting Terror Alarm AI System... 3 | cd .. 4 | python main.py 5 | pause 6 | --------------------------------------------------------------------------------