├── utils └── README.md ├── deployments ├── services │ ├── redis │ │ └── .env │ ├── webapp │ │ ├── .env │ │ ├── .dockerignore │ │ ├── requirements.txt │ │ ├── Dockerfile │ │ └── webapp.ini │ ├── collector │ │ ├── .env │ │ ├── requirements.txt │ │ ├── Dockerfile │ │ └── collector.ini │ ├── nginx │ │ ├── telemetry.cert.pem │ │ ├── telemetry.key.pem │ │ ├── Dockerfile │ │ ├── nginx.conf │ │ └── sites.conf │ ├── processing │ │ ├── requirements.txt │ │ ├── run.sh │ │ ├── Dockerfile │ │ └── entrypoint.py │ ├── production.env │ └── testing.env ├── docker-compose.yaml └── docker-compose.prod.yaml ├── processing ├── processing │ ├── __init__.py │ └── main.py ├── tests │ └── blacklist.py └── process.py ├── .gitignore ├── telemetryui ├── migrations │ ├── README │ ├── script.py.mako │ ├── alembic.ini │ ├── versions │ │ └── ac1c5921bf2f_.py │ └── env.py ├── telemetryui │ ├── static │ │ ├── js │ │ │ ├── telemetryui.js │ │ │ ├── stats.js │ │ │ ├── builds.js │ │ │ ├── thermal.js │ │ │ ├── population.js │ │ │ └── mce.js │ │ ├── css │ │ │ └── style.css │ │ └── vendor │ │ │ ├── bootstrap │ │ │ └── css │ │ │ │ ├── bootstrap-reboot.min.css │ │ │ │ └── bootstrap-reboot.css │ │ │ └── jquery │ │ │ └── core.js │ ├── __init__.py │ ├── templates │ │ ├── records_list.html │ │ ├── builds.html │ │ ├── guilty_edit.html │ │ ├── guilty_reset.html │ │ ├── guilty_remove.html │ │ ├── thermal.html │ │ ├── guilty_hidden.html │ │ ├── crashes_list.html │ │ ├── guilty_add.html │ │ ├── base.html │ │ ├── population.html │ │ ├── stats.html │ │ ├── guilty_details.html │ │ ├── record_details.html │ │ ├── guilty_backtraces.html │ │ ├── guilty_edit_one.html │ │ ├── crashes_filter.html │ │ ├── mce.html │ │ ├── crashes.html │ │ └── records.html │ ├── config.py │ ├── cache.py │ ├── jinja_filters.py │ ├── utils.py │ └── forms.py └── run.py ├── AUTHORS ├── collector └── collector │ ├── log_requests.py │ ├── purge.py │ ├── __init__.py │ ├── tests │ ├── validation.py │ ├── api.py │ ├── purging.py │ ├── testcase.py │ └── headers.py │ ├── config.py │ ├── lib │ └── validation.py │ └── report_handler.py ├── README.md └── LICENSE /utils/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deployments/services/redis/.env: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deployments/services/webapp/.env: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /processing/processing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deployments/services/collector/.env: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deployments/services/nginx/telemetry.cert.pem: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deployments/services/nginx/telemetry.key.pem: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.log 3 | *.swp 4 | *.idea 5 | -------------------------------------------------------------------------------- /deployments/services/webapp/.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | -------------------------------------------------------------------------------- /telemetryui/migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /deployments/services/processing/requirements.txt: -------------------------------------------------------------------------------- 1 | luigi 2 | psycopg2-binary 3 | cxxfilt 4 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Archana Shinde 2 | Patrick McCarty 3 | Gabriel Briones Sayeg 4 | Alex Jaramillo 5 | -------------------------------------------------------------------------------- /deployments/services/production.env: -------------------------------------------------------------------------------- 1 | # Database access 2 | POSTGRES_HOSTNAME=db 3 | POSTGRES_PASSWORD= 4 | POSTGRES_USER=telemetry 5 | POSTGRES_DB=telemetry 6 | # Luigi log processing level 7 | PROCESSING_LOG_LEVEL=ERROR 8 | # Redis configuration 9 | REDIS_HOSTNAME=redis 10 | REDIS_PORT=6379 11 | REDIS_PASSWD= 12 | 13 | -------------------------------------------------------------------------------- /deployments/services/testing.env: -------------------------------------------------------------------------------- 1 | # Database access 2 | POSTGRES_HOSTNAME=db 3 | POSTGRES_PASSWORD=postgres 4 | POSTGRES_USER=telemetry 5 | POSTGRES_DB=telemetry 6 | PGDATA=/var/lib/postgresql/data/pgdata 7 | # URL prefix 8 | APPLICATION_ROOT=telemetryui 9 | # Redis configuration 10 | REDIS_HOSTNAME=redis 11 | REDIS_PORT=6379 12 | REDIS_PASSWD= 13 | # Luigi log level 14 | PROCESSING_LOG_LEVEL=DEBUG 15 | -------------------------------------------------------------------------------- /deployments/services/collector/requirements.txt: -------------------------------------------------------------------------------- 1 | alembic==1.1.0 2 | click==7.0 3 | Flask==2.3.2 4 | Flask-Migrate==2.5.2 5 | Flask-SQLAlchemy==2.4.0 6 | Flask-WTF==0.14.2 7 | itsdangerous==1.1.0 8 | Jinja2>=3.1 9 | Mako==1.2.2 10 | MarkupSafe==1.1.1 11 | psycopg2-binary==2.8.4 12 | python-dateutil==2.8.0 13 | python-editor==1.0.4 14 | redis==4.5.4 15 | six==1.12.0 16 | SQLAlchemy==1.3.5 17 | Werkzeug==3.0.6 18 | WTForms==2.2.1 19 | -------------------------------------------------------------------------------- /deployments/services/webapp/requirements.txt: -------------------------------------------------------------------------------- 1 | alembic==1.1.0 2 | click==7.0 3 | Flask==2.3.2 4 | Flask-Migrate==2.5.2 5 | Flask-SQLAlchemy==2.4.0 6 | Flask-WTF==0.14.2 7 | itsdangerous==1.1.0 8 | Jinja2>=3.1 9 | Mako==1.2.2 10 | MarkupSafe==1.1.1 11 | psycopg2-binary==2.8.4 12 | python-dateutil==2.8.0 13 | python-editor==1.0.4 14 | redis==4.5.4 15 | six==1.12.0 16 | SQLAlchemy==1.3.5 17 | Werkzeug==3.0.6 18 | WTForms==2.2.1 19 | cxxfilt 20 | -------------------------------------------------------------------------------- /deployments/services/processing/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (C) 2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | set -e 6 | 7 | LOG_LEVEL=${PROCESSING_LOG_LEVEL:-DEBUG} 8 | 9 | # Wait for database to be initialized 10 | until python3 entrypoint.py; do 11 | >&2 echo "Postgres is unavailable - sleeping" 12 | sleep 1 13 | done 14 | 15 | # Processing loop 16 | while true 17 | do 18 | sleep 2m 19 | python3 process.py ProcessCrashes --local-scheduler --log-level ${LOG_LEVEL} 20 | done 21 | -------------------------------------------------------------------------------- /deployments/services/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | FROM clearlinux:latest 4 | 5 | RUN swupd bundle-add nginx 6 | RUN useradd nginx 7 | COPY deployments/services/nginx/nginx.conf /etc/nginx/nginx.conf 8 | COPY deployments/services/nginx/sites.conf /etc/nginx/conf.d/sites.conf 9 | COPY deployments/services/nginx/*.pem /etc/nginx/ssl/ 10 | COPY telemetryui/telemetryui/static /var/www/static 11 | RUN chown -R nginx:nginx /var/www/static 12 | 13 | STOPSIGNAL SIGTERM 14 | 15 | CMD ["nginx", "-g", "daemon off;"] 16 | -------------------------------------------------------------------------------- /telemetryui/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /deployments/services/processing/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | FROM clearlinux:latest 4 | 5 | WORKDIR /srv/processing 6 | 7 | RUN swupd bundle-add python3-basic 8 | RUN mkdir -p /srv/processing 9 | COPY ./processing . 10 | COPY ./deployments/services/processing/. . 11 | COPY ./utils/shared_utils/crash.py ./processing/crash.py 12 | RUN pip3 install -r requirements.txt 13 | RUN groupadd -r appuser && useradd -r -g appuser appuser 14 | RUN chown -R appuser:appuser /srv/processing 15 | 16 | USER appuser 17 | 18 | CMD ["process.py", "ProcessCrashes", "--local-scheduler"] 19 | -------------------------------------------------------------------------------- /processing/tests/blacklist.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from process import GuilyBlacklist 3 | 4 | guilties = [('fa', 'mb'), ('fc', 'md')] 5 | 6 | 7 | class BlacklistTest(unittest.TestCase): 8 | 9 | def setUp(self): 10 | self.bl = GuilyBlacklist(guilties) 11 | 12 | def test_blacklisted_item(self): 13 | existing = guilties[1] 14 | self.assertEqual(self.bl.contains(existing), True) 15 | 16 | def test_non_blacklisted_item(self): 17 | non_existing = ('fx', 'mx') 18 | self.assertEqual(self.bl.contains(non_existing), False) 19 | 20 | 21 | if __name__ == '__main__': 22 | unittest.main() 23 | -------------------------------------------------------------------------------- /deployments/services/collector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM clearlinux:latest 2 | 3 | WORKDIR /var/www/collector 4 | 5 | RUN swupd bundle-add python3-basic uwsgi 6 | RUN mkdir -p /var/www/collector/log /var/www/collector/socket 7 | COPY ./deployments/services/collector/requirements.txt . 8 | COPY ./deployments/services/collector/collector.ini . 9 | COPY ./collector/collector /var/www/collector/collector 10 | COPY ./utils/shared_utils/model.py /var/www/collector/collector/ 11 | RUN pip3 install -r requirements.txt 12 | RUN groupadd -r webapp && useradd -r -g webapp webapp 13 | RUN chown -R webapp:webapp /var/www/collector 14 | 15 | USER webapp 16 | 17 | CMD ["uwsgi", "--ini", "/var/www/collector/collector.ini"] 18 | -------------------------------------------------------------------------------- /deployments/services/webapp/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | FROM clearlinux:latest 4 | 5 | WORKDIR /var/www/webapp 6 | 7 | RUN swupd bundle-add python3-basic uwsgi 8 | RUN mkdir -p /var/www/webapp/uwsgi-spool /var/www/webapp/log /var/www/webapp/socket 9 | COPY ./deployments/services/webapp/requirements.txt ./deployments/services/webapp/webapp.ini ./ 10 | COPY ./telemetryui/. /var/www/webapp 11 | COPY ./utils/shared_utils/crash.py ./utils/shared_utils/model.py /var/www/webapp/telemetryui/ 12 | RUN pip3 install -r requirements.txt 13 | RUN groupadd -r webapp && useradd -r -g webapp webapp 14 | RUN chown -R webapp:webapp /var/www/webapp 15 | 16 | USER webapp 17 | 18 | CMD ["uwsgi", "--ini", "/var/www/webapp/webapp.ini"] 19 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/js/telemetryui.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017-2020 Intel Corporation 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | (function(root, factory){ 7 | 8 | root.telemetryUI = factory(root); 9 | 10 | })(this, function(rootObj){ 11 | 12 | function newChart(ctx, type, data, options) { 13 | new Chart(ctx, { 14 | type: type, 15 | data: data, 16 | options: options 17 | }); 18 | } 19 | 20 | return { 21 | backgroundColors: ["#3e95cd", "#8e5ea2","#3cba9f","#e8c3b9","#c45850", 22 | "#ff8c00", "#483d8b", "#00bfff", "#1e90ff", "#008000", 23 | "#df42f4", "#c7f441", "#f47641", "#f44141", "#43f441"], 24 | newChart: newChart, 25 | }; 26 | 27 | }); 28 | -------------------------------------------------------------------------------- /telemetryui/run.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from telemetryui import app 18 | from telemetryui.model import db 19 | from flask_migrate import Migrate 20 | 21 | migrate = Migrate(app, db) 22 | 23 | 24 | # vi: ts=4 et sw=4 sts=4 25 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015-2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from flask import ( 5 | Flask) 6 | from . import config 7 | from .jinja_filters import ( 8 | timesince, 9 | local_datetime_since, 10 | basename, 11 | get_severity_label 12 | ) 13 | 14 | # App instance 15 | app = Flask(__name__, static_folder="static", static_url_path="/telemetryui/static") 16 | app.config.from_object(config.Config) 17 | # Load views 18 | from .views import views_bp 19 | # Add filters 20 | app.add_template_filter(timesince) 21 | app.add_template_filter(local_datetime_since) 22 | app.add_template_filter(basename) 23 | app.add_template_filter(get_severity_label) 24 | # Register routes 25 | app.register_blueprint(views_bp, url_prefix="/{}".format(app.config.get('APPLICATION_ROOT', ''))) 26 | 27 | # vi: ts=4 et sw=4 sts=4 28 | -------------------------------------------------------------------------------- /deployments/services/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | error_log /var/log/nginx/error.log; 4 | pid /run/nginx.pid; 5 | include /usr/share/nginx/modules/*.conf; 6 | 7 | events { 8 | worker_connections 1024; 9 | } 10 | 11 | http { 12 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 13 | '$status $body_bytes_sent "$http_referer" ' 14 | '"$http_user_agent" "$http_x_forwarded_for"'; 15 | 16 | access_log /var/log/nginx/access.log main; 17 | 18 | sendfile on; 19 | tcp_nopush on; 20 | tcp_nodelay on; 21 | keepalive_timeout 65; 22 | types_hash_max_size 2048; 23 | 24 | default_type text/html; 25 | server_tokens off; 26 | 27 | types { 28 | text/html html; 29 | text/css css; 30 | application/x-javascript js; 31 | } 32 | 33 | include /etc/nginx/conf.d/*.conf; 34 | } 35 | -------------------------------------------------------------------------------- /telemetryui/migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # template used to generate migration files 5 | # file_template = %%(rev)s_%%(slug)s 6 | 7 | # set to 'true' to run the environment during 8 | # the 'revision' command, regardless of autogenerate 9 | # revision_environment = false 10 | 11 | 12 | # Logging configuration 13 | [loggers] 14 | keys = root,sqlalchemy,alembic 15 | 16 | [handlers] 17 | keys = console 18 | 19 | [formatters] 20 | keys = generic 21 | 22 | [logger_root] 23 | level = WARN 24 | handlers = console 25 | qualname = 26 | 27 | [logger_sqlalchemy] 28 | level = WARN 29 | handlers = 30 | qualname = sqlalchemy.engine 31 | 32 | [logger_alembic] 33 | level = INFO 34 | handlers = 35 | qualname = alembic 36 | 37 | [handler_console] 38 | class = StreamHandler 39 | args = (sys.stderr,) 40 | level = NOTSET 41 | formatter = generic 42 | 43 | [formatter_generic] 44 | format = %(levelname)-5.5s [%(name)s] %(message)s 45 | datefmt = %H:%M:%S 46 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/records_list.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | {% for rec in records %} 6 | 7 | {%- if rec.external %} 8 | external 9 | {%- else %} 10 | internal 11 | {%- endif %} 12 |
{{ rec.machine_id }}
 13 | {{ rec.timestamp_server | int | timesince }} 14 | {{ (rec.severity | get_severity_label).1 }} 15 | {{ rec.classification|replace("org.clearlinux/", "") }} 16 | {{ rec.system_name }} 17 | {{ rec.payload }} 18 | 19 | 20 | 21 | 22 | {% endfor %} 23 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/config.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015-2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import logging 6 | 7 | class Config(object): 8 | db_host = os.environ.get('POSTGRES_HOSTNAME', 'db') 9 | db_user = os.environ.get('POSTGRES_USER', 'telemetry') 10 | db_passwd = os.environ.get('POSTGRES_PASSWORD','postgres') 11 | redis_passwd = os.environ.get('REDIS_PASSWD', '') 12 | redis_hostname = os.environ.get('REDIS_HOSTNAME', 'redis') 13 | DEBUG = False 14 | TESTING = False 15 | LOG_LEVEL = logging.ERROR 16 | SQLALCHEMY_DATABASE_URI = 'postgres://{user}:{passwd}@{host}/telemetry'.format(user=db_user, passwd=db_passwd, host=db_host) 17 | SQLALCHEMY_TRACK_MODIFICATIONS = True 18 | LOG_FILE = 'handler.log' 19 | WTF_CSRF_ENABLED = True 20 | SECRET_KEY = os.urandom(32) 21 | APPLICATION_ROOT = os.environ.get('APPLICATION_ROOT', '') 22 | RECORDS_PER_PAGE = 50 23 | REDIS_HOSTNAME = redis_hostname 24 | REDIS_PORT = 6379 25 | REDIS_PASSWD = redis_passwd 26 | 27 | # vi: ts=4 et sw=4 sts=4 28 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/builds.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | {% extends "base.html" %} 7 | 8 | {%- block page_title %} 9 | Build Stats - CLR Telemetry 10 | {%- endblock %} 11 | 12 | {% block content %} 13 | 14 |

Records per build

15 |
16 |
17 | 18 |
19 |
20 | 21 | 22 | 31 | 32 | 33 | {% block record_content %} 34 | {% endblock %} 35 | {% endblock %} 36 | 37 | {#- 38 | # vi: ft=jinja ts=8 et sw=4 sts=4 39 | #} 40 | -------------------------------------------------------------------------------- /collector/collector/log_requests.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from flask import request, current_app, app 18 | import logging 19 | 20 | 21 | @app.before_request 22 | def before_request(): 23 | headers = request.headers 24 | payload = request.data 25 | print('Before request') 26 | current_app.logger.info('\t'.join([ 27 | datetime.datetime.today().ctime(), 28 | request.remote_addr, 29 | request.method, 30 | request.url, 31 | str(request.data), 32 | ', '.join([': '.join(x) for x in request.headers])]) 33 | ) 34 | 35 | 36 | # vi: ts=4 et sw=4 sts=4 37 | -------------------------------------------------------------------------------- /deployments/services/processing/entrypoint.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import psycopg2 6 | from processing.main import ( 7 | get_latest_id, 8 | connect, 9 | PROCESSED_VIEW,) 10 | 11 | 12 | LAST_PROCESSED = """ 13 | CREATE OR REPLACE VIEW {} AS 14 | SELECT MAX(id) AS last_id FROM records WHERE processed = True 15 | """.format(PROCESSED_VIEW) 16 | 17 | 18 | class Conf(): 19 | db_name = os.environ['POSTGRES_DB'] 20 | db_host = os.environ['POSTGRES_HOSTNAME'] 21 | db_user = os.environ['POSTGRES_USER'] 22 | db_passwd = os.environ['POSTGRES_PASSWORD'] 23 | 24 | 25 | def check_processed_id(conn): 26 | cursor = conn.cursor() 27 | try: 28 | get_latest_id(cursor) 29 | except psycopg2.errors.UndefinedTable: 30 | cursor.execute(LAST_PROCESSED) 31 | finally: 32 | cursor.close() 33 | 34 | 35 | def main(): 36 | try: 37 | conn = connect(Conf()) 38 | check_processed_id(conn) 39 | conn.close() 40 | except psycopg2.OperationalError as op_error: 41 | print("could not connect to server: Connection refused") 42 | exit(1) 43 | 44 | if __name__ == '__main__': 45 | main() 46 | 47 | # vi: ts=4 et sw=4 sts=4 48 | -------------------------------------------------------------------------------- /collector/collector/purge.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from .model import Record 18 | from . import app 19 | 20 | try: 21 | from uwsgidecorators import cron 22 | 23 | PURGE_OLD_RECORDS = app.config.get("PURGE_OLD_RECORDS", True) 24 | 25 | # Runs cron job at 4:30 every day 26 | @cron(30, 4, -1, -1, -1, target='spooler') 27 | def purge_task(signum): 28 | if PURGE_OLD_RECORDS: 29 | app.logger.info("Running cron job for purging records") 30 | with app.app_context(): 31 | Record.delete_records() 32 | 33 | except ImportError: 34 | app.logger.info("Import error for uwsgidecorators") 35 | 36 | 37 | # vi: ts=4 et sw=4 sts=4 38 | -------------------------------------------------------------------------------- /collector/collector/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from flask import Flask 18 | from . import config 19 | from flask_sqlalchemy import SQLAlchemy 20 | from logging.handlers import RotatingFileHandler 21 | 22 | 23 | def configure_app(config_object, app): 24 | app.config.from_object(config_object) 25 | db = SQLAlchemy(app) 26 | 27 | app = Flask(__name__) 28 | db = SQLAlchemy() 29 | app.config.from_object(config.Config) 30 | 31 | from .model import * 32 | from . import report_handler 33 | 34 | handler = RotatingFileHandler(app.config['LOG_FILE'], maxBytes=10000, backupCount=1) 35 | handler.setLevel(app.config['LOG_LEVEL']) 36 | app.logger.addHandler(handler) 37 | 38 | 39 | # vi: ts=4 et sw=4 sts=4 40 | -------------------------------------------------------------------------------- /deployments/services/nginx/sites.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | #listen 443 ssl; 4 | server_name localhost; 5 | charset utf-8; 6 | client_max_body_size 75M; 7 | underscores_in_headers on; 8 | root /var/www/; 9 | 10 | #ssl_certificate /etc/nginx/ssl/telemetry.cert.pem; 11 | #ssl_certificate_key /etc/nginx/ssl/telemetry.key.pem; 12 | 13 | ssl_protocols TLSv1.2; 14 | ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDH-ECDSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:AES128-GCM-SHA256:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES256-CCM:AES128-CCM:AES128-SHA256:AES256-CCM:AES256-SHA256:DHE-RSA-AES128-CCM:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-CCM:DHE-RSA-AES256-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-DSS-AES128-SHA256:DHE-DSS-AES256-SHA256"; 15 | 16 | location /v2/collector { try_files $uri @collectorapp; } 17 | location @collectorapp { 18 | include uwsgi_params; 19 | uwsgi_pass unix:/var/run/uwsgi/collector.sock; 20 | error_log /var/log/nginx/collector_error.log; 21 | } 22 | 23 | location / { try_files $uri @uiapp; } 24 | location @uiapp { 25 | include uwsgi_params; 26 | uwsgi_pass unix:/var/run/uwsgi/webapp.sock; 27 | error_log /var/log/nginx/telemetryui_error.log; 28 | } 29 | 30 | location /static { 31 | try_files $uri $uri/ =404; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /processing/process.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import luigi 6 | import psycopg2 7 | from processing.main import ( 8 | connect, 9 | init_guilty_blacklist, 10 | get_latest_id, 11 | get_crashes, 12 | process_crashes, 13 | ) 14 | 15 | from processing import crash 16 | 17 | 18 | class GlobalParams(luigi.Config): 19 | 20 | db_name = os.environ['POSTGRES_DB'] 21 | db_host = os.environ['POSTGRES_HOSTNAME'] 22 | db_user = os.environ['POSTGRES_USER'] 23 | db_passwd = os.environ['POSTGRES_PASSWORD'] 24 | 25 | 26 | class ProcessCrashes(luigi.Task): 27 | """ 28 | process.py ProcessCrashes --local-scheduler 29 | """ 30 | is_complete = False 31 | 32 | def complete(self): 33 | return self.is_complete 34 | 35 | def run(self): 36 | db = connect(GlobalParams()) 37 | cur = db.cursor() 38 | crash.GUILTY_BLACKLIST = init_guilty_blacklist(cur) 39 | last_id = get_latest_id(cur) 40 | print("# Start processing after record id: {}".format(last_id)) 41 | crashes = get_crashes(cur, last_id) 42 | print("# Will process [{}] crashes".format(len(crashes))) 43 | process_crashes(cur, crashes) 44 | print("# Done processing\n") 45 | cur.close() 46 | db.close() 47 | self.is_complete = True 48 | 49 | 50 | if __name__ == "__main__": 51 | luigi.run() 52 | 53 | # vi: ts=4 et sw=4 sts=4 54 | -------------------------------------------------------------------------------- /deployments/services/collector/collector.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | #application's base folder 3 | chdir = /var/www/collector/ 4 | 5 | #python module to import 6 | app = collector 7 | module = %(app) 8 | 9 | #socket file's location 10 | socket = /var/www/collector/socket/%n.sock 11 | 12 | #permissions for the socket file 13 | chmod-socket = 666 14 | 15 | #the variable that holds a flask application inside the module imported at line #6 16 | callable = app 17 | 18 | processes = 4 19 | 20 | threads = 2 21 | 22 | #enable thread support.Need to figure out the optimal no of threads. 23 | enable-threads = true 24 | 25 | #location of log files 26 | logto = /var/www/collector/log/%n.log 27 | 28 | #maximum size of log file before rotation (100MB) 29 | log-maxsize = 104857600 30 | 31 | #backup log file (created after rotation) 32 | log-backupname = /var/www/collector/log/%n.log.bk 33 | 34 | #if request takes more than this parameter(in sec), request will be dropped 35 | harakiri = 60 36 | 37 | #respawn processes after serving 5000 requests 38 | max-requests = 5000 39 | 40 | #get verbose logs when a process gets stuck 41 | harakiri-verbose = true 42 | 43 | #http://uwsgi-docs.readthedocs.org/en/latest/Tracebacker.html#combining-the-tracebacker-with-harakiri 44 | #Traceback is automatically logged during harakiri phase. 45 | py-tracebacker=collectorsocket 46 | 47 | #enable stats. Use this for fine-tuning no of processes. 48 | #connect uwsgitop to the stats socket as: uwsgitop /tmp/collectorstats.socket 49 | stats=/tmp/collectorstats.socket 50 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_edit.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | {% block content %} 18 | 19 |

Crash guilty management

20 | 26 | 27 | {%- with messages = get_flashed_messages(with_categories=true) %} 28 | {%- if messages %} 29 | {%- for category, m in messages %} 30 | 31 | {%- endfor %} 32 | {%- endif %} 33 | {%- endwith %} 34 | 35 | {% endblock %} 36 | 37 | 38 | {#- 39 | # vi: ft=jinja ts=8 et sw=4 sts=4 40 | #} 41 | -------------------------------------------------------------------------------- /collector/collector/tests/validation.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import unittest 18 | from collector.tests.testcase import ( 19 | RecordTestCases, 20 | get_record_v3, 21 | kernel_version) 22 | 23 | 24 | class TestHandlerRecordValidation(RecordTestCases): 25 | """ 26 | Test record validation 27 | """ 28 | @staticmethod 29 | def get_version_records(): 30 | return get_record_v3() 31 | 32 | def test_post_kernel_version_validation_1(self): 33 | headers = get_record_v3() 34 | headers.update({kernel_version: '3.16.generic'}) 35 | response = self.client.post('/', headers=headers, data='test') 36 | self.assertTrue(response.status_code == 201, response.data.decode('utf-8')) 37 | 38 | 39 | if __name__ == '__main__' and __package__ is None: 40 | from os import sys, path 41 | sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) 42 | unittest.main() 43 | 44 | 45 | # vi: ts=4 et sw=4 sts=4 46 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_reset.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | {% block content %} 18 | 19 |

Crash guilty management

20 |

<< Back

21 | 22 |

Click the red button below to reset all guilties.

23 | 24 |
25 | {{ form.hidden_tag() }} 26 |
27 | {{ form.confirm(id="confirm_field", class="form-control") }} 28 | 29 |
30 |
31 | 32 | {%- with messages = get_flashed_messages(with_categories=true) %} 33 | {%- if messages %} 34 | {%- for category, m in messages %} 35 | 36 | {%- endfor %} 37 | {%- endif %} 38 | {%- endwith %} 39 | 40 | {% endblock %} 41 | 42 | 43 | {#- 44 | # vi: ft=jinja ts=8 et sw=4 sts=4 45 | #} 46 | -------------------------------------------------------------------------------- /deployments/services/webapp/webapp.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | #application's base folder 3 | chdir = /var/www/webapp/ 4 | 5 | #python module to import 6 | app = telemetryui 7 | module = %(app) 8 | 9 | #socket file's location 10 | socket = /var/www/webapp/socket/%n.sock 11 | 12 | #permissions for the socket file 13 | chmod-socket = 666 14 | 15 | #the variable that holds a flask application inside the module imported at line #6 16 | callable = app 17 | 18 | processes = 4 19 | 20 | threads = 2 21 | 22 | #enable thread support.Need to figure out the optimal no of threads. 23 | enable-threads = true 24 | 25 | #location of log files 26 | logto = /var/www/webapp/log/%n.log 27 | 28 | #maximum size of log file before rotation (100MB) 29 | log-maxsize = 104857600 30 | 31 | #backup log file (created after rotation) 32 | log-backupname = /var/www/webapp/log/%n.log.bk 33 | 34 | #if request takes more than this parameter(in sec), request will be dropped 35 | harakiri = 60 36 | 37 | #respawn processes after serving 5000 requests 38 | max-requests = 5000 39 | 40 | #get verbose logs when a process gets stuck 41 | harakiri-verbose = true 42 | 43 | #http://uwsgi-docs.readthedocs.org/en/latest/Tracebacker.html#combining-the-tracebacker-with-harakiri 44 | #Traceback is automatically logged during harakiri phase. 45 | py-tracebacker=telemetryuisocket 46 | 47 | #enable stats. Use this for fine-tuning no of processes. 48 | #connect uwsgitop to the stats socket as: uwsgitop /tmp/telemetryuistats.socket 49 | stats=/tmp/telemetryuistats.socket 50 | 51 | #for asynchronous processing 52 | spooler = %(chdir)/uwsgi-spool 53 | 54 | #default size is TOO LOW 55 | buffer-size = 8192 56 | -------------------------------------------------------------------------------- /collector/collector/tests/api.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import json 18 | import unittest 19 | from collector.tests.testcase import ( 20 | RecordTestCases, 21 | get_record) 22 | 23 | 24 | class TestHandler(RecordTestCases): 25 | 26 | def test_query_report(self): 27 | rec = get_record() 28 | response = self.client.post('/', headers=rec, data='test') 29 | self.assertTrue(response.status_code == 201) 30 | filters = { 31 | 'severity': 1, 32 | } 33 | response = self.client.get('/api/records', query_string=filters) 34 | resp_obj = json.loads(response.data.decode('utf-8')) 35 | self.assertEqual(len(resp_obj['records']), 1) 36 | filters1 = { 37 | 'build': '17780', 38 | } 39 | response = self.client.get('/api/records', query_string=filters1) 40 | resp_obj = json.loads(response.data.decode('utf-8')) 41 | self.assertEqual(len(resp_obj['records']), 0) 42 | 43 | 44 | if __name__ == '__main__': 45 | unittest.main() 46 | 47 | 48 | # vi: ts=4 et sw=4 sts=4 49 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/js/stats.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Intel Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | (function(root, factory){ 19 | 20 | factory(root); 21 | 22 | })(this.telemetryUI, function(rootObj){ 23 | 24 | 25 | function renderStats(ctx, title, labels, values){ 26 | 27 | var data = { 28 | labels: labels, 29 | datasets: [{ 30 | label: title, 31 | backgroundColor: rootObj.backgroundColors, 32 | data: values 33 | }] 34 | }; 35 | 36 | var options = { 37 | title: { 38 | display: true, 39 | fontSize: 18, 40 | text: title 41 | }, 42 | legend: { 43 | display: false 44 | } 45 | }; 46 | 47 | rootObj.newChart(ctx, "pie", data, options); 48 | } 49 | 50 | var renderStatsPlatform = renderStats; 51 | var renderStatsClass = renderStats; 52 | 53 | 54 | rootObj.renderStatsClass = renderStatsClass; 55 | rootObj.renderStatsPlatform = renderStatsPlatform; 56 | 57 | }); 58 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/cache.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015-2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import ast 5 | import redis 6 | from . import app 7 | 8 | REDIS_HOSTNAME = app.config.get('REDIS_HOSTNAME', 'localhost') 9 | REDIS_PORT = app.config.get('REDIS_PORT', 6379) 10 | REDIS_PASSWD = app.config.get('REDIS_PASSWD', None) 11 | 12 | 13 | def get_cached_data(varname, expiration, funct, *args, **kwargs): 14 | try: 15 | redis_client = redis.StrictRedis(decode_responses=True, 16 | host=REDIS_HOSTNAME, 17 | port=REDIS_PORT, 18 | password=REDIS_PASSWD,); 19 | # Try to get data from redis first 20 | ret = redis_client.get(varname) 21 | if ret is not None: 22 | # Convert to original type if successful 23 | ret = ast.literal_eval(ret) 24 | else: 25 | # If nothing was found, query the database 26 | ret = funct(*args, **kwargs) 27 | # Convert to string representation and cache via redis 28 | redis_client.set(varname, repr(ret), ex=expiration) 29 | except redis.exceptions.ConnectionError as e: 30 | print("%s Redis probably isn't running?" % str(e)) 31 | # If we can't connect to redis, just query directly 32 | ret = funct(*args, **kwargs) 33 | return ret 34 | 35 | 36 | def uncache_data(varname): 37 | try: 38 | redis_client = redis.StrictRedis(decode_responses=True); 39 | redis_client.delete(varname) 40 | except redis.exceptions.ConnectionError as e: 41 | print("%s Redis probably isn't running?" % str(e)) 42 | return 43 | 44 | # vi: ts=4 et sw=4 sts=4 45 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_remove.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | {% block content %} 18 | 19 |

Crash guilty management

20 |

<< Back

21 |
Remove guilty filters
22 | 23 | {%- if count != 0 %} 24 |
25 | {{ form.hidden_tag() }} 26 |
27 | {{ form.funcmod.label(class="control-label", for="guilty_field") }} 28 | {{ form.funcmod(id="guilty_field", class="form-control") }} 29 |
30 | 31 |
32 | {%- else %} 33 | 34 | {%- endif %} 35 | 36 | {%- with messages = get_flashed_messages(with_categories=true) %} 37 | {%- if messages %} 38 | {%- for category, m in messages %} 39 | 40 | {%- endfor %} 41 | {%- endif %} 42 | {%- endwith %} 43 | 44 | {% endblock %} 45 | 46 | 47 | {#- 48 | # vi: ft=jinja ts=8 et sw=4 sts=4 49 | #} 50 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/thermal.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | 18 | {%- block page_title %} 19 | Thermal Stats - CLR Telemetry 20 | {%- endblock %} 21 | 22 | {% block content %} 23 | 24 |

Thermal stats

25 | Thermal reports by week of the current year 26 | 27 | {%- with messages = get_flashed_messages() %} 28 | {%- if messages %} 29 | {%- for m in messages %} 30 |

{{ m }}

31 | {%- endfor %} 32 | {%- endif %} 33 | {%- endwith %} 34 | 35 |
36 | {%- if thermal_chart %} 37 |
38 | 39 |
40 | {%- endif %} 41 |
42 | 43 | 44 | 45 | 52 | 53 | 54 | {% endblock %} 55 | 56 | 57 | {#- 58 | # vi: ft=jinja ts=8 et sw=4 sts=4 59 | #} 60 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_hidden.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | {% block content %} 18 | 19 |

Crash guilty management

20 |

<< Back

21 |
Unhide guilties
22 | 23 | {%- if count != 0 %} 24 |
25 | {{ form.hidden_tag() }} 26 |
27 | {{ form.funcmod.label(class="control-label", for="hiddenguilty_field") }} 28 | {{ form.funcmod(id="hiddenguilty_field", class="form-control") }} 29 |
30 | 31 |
32 | {%- else %} 33 | 34 | {%- endif %} 35 | 36 | {%- with messages = get_flashed_messages(with_categories=true) %} 37 | {%- if messages %} 38 | {%- for category, m in messages %} 39 | 40 | {%- endfor %} 41 | {%- endif %} 42 | {%- endwith %} 43 | 44 | {% endblock %} 45 | 46 | 47 | {#- 48 | # vi: ft=jinja ts=8 et sw=4 sts=4 49 | #} 50 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/crashes_list.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | {%- set maxtotal = guilties[0].total %} 7 | {%- for g in guilties %} 8 | 9 | {{ loop.index + page_offset }} 10 | 11 | {{ g.guilty|truncate(30, True) }} 12 | 13 | 14 | 15 | 16 | 17 | {%- set width = g.total / maxtotal * 100 %} 18 | 19 | 20 | 21 | 22 | 23 | {%- for c in g.builds %} 24 | {%- for b in builds %} 25 | {%- if b[0] == c[0] and c[1] != "0" %} 26 | 27 | 28 | 29 | {%- set width = c[1] / b[1] * 100 %} 30 | 31 | 32 | 33 | 34 | 35 | {%- elif b[0] == c[0] %} 36 | 37 | {%- endif %} 38 | {%- endfor %} 39 | {%- endfor %} 40 | 41 |
42 |
{% if g.comment %}{{ g.comment|truncate(30, True) }}{% endif %}
43 |
44 | 45 | 46 | {%- endfor %} 47 | 48 | {#- 49 | # vi: ft=jinja ts=8 et sw=4 sts=4 50 | #} 51 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/jinja_filters.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | from time import ( 19 | time, 20 | strftime, 21 | localtime) 22 | 23 | 24 | def timesince(dt, default="just now"): 25 | now = time() 26 | diff = int(now - dt) 27 | 28 | months = diff // (86400 * 30) 29 | diff = diff % (86400 * 30) 30 | days = diff // 86400 31 | diff = diff % 86400 32 | hours = diff // 3600 33 | diff = diff % 3600 34 | minutes = diff // 60 35 | diff = diff % 60 36 | seconds = diff 37 | 38 | periods = ( 39 | (months, "month", "months"), 40 | (days, "day", "days"), 41 | (hours, "hour", "hours"), 42 | (minutes, "minute", "minutes"), 43 | (seconds, "second", "seconds"), 44 | ) 45 | 46 | for period, singular, plural in periods: 47 | if period: 48 | return "{} {}".format(period, singular if period == 1 else plural) 49 | 50 | return default 51 | 52 | 53 | def local_datetime_since(sec): 54 | return strftime("%a, %d %b %Y %H:%M:%S", localtime(sec)) 55 | 56 | 57 | def basename(path): 58 | return os.path.basename(path) 59 | 60 | 61 | def get_severity_label(severity): 62 | return { 63 | 2: ("info", "med",), 64 | 3: ("warning", "high",), 65 | 4: ("danger", "crit",) 66 | }.get(severity, ("default", "low",)) 67 | 68 | # vi: ts=4 et sw=4 sts=4 69 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_add.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | {% block content %} 18 | 19 |

Crash guilty management

20 |

<< Back

21 |
Add new guilty filters
22 | 23 |
24 |
25 | {{ form.hidden_tag() }} 26 |
27 | {{ form.funcmod.label(class="control-label", for="funcmod_field") }} 28 | {{ form.funcmod(id="funcmod_field", class="form-control") }} 29 |
30 |
31 | 32 |
33 |
34 |
35 | 36 |
37 |
38 |

Current filters:

39 |
    40 | {%- for f in filters %} 41 |
  • {{ f }}
    • 42 | {%- endfor %} 43 |
44 |
45 | {%- with messages = get_flashed_messages(with_categories=true) %} 46 | {%- if messages %} 47 | {%- for category, m in messages %} 48 | 49 | {%- endfor %} 50 | {%- endif %} 51 | {%- endwith %} 52 |
53 | 54 | {% endblock %} 55 | 56 | 57 | {#- 58 | # vi: ft=jinja ts=8 et sw=4 sts=4 59 | #} 60 | -------------------------------------------------------------------------------- /collector/collector/config.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2020 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import logging 19 | 20 | 21 | class Config(object): 22 | db_host = os.environ['POSTGRES_HOSTNAME'] 23 | db_user = os.environ['POSTGRES_USER'] 24 | db_passwd = os.environ['POSTGRES_PASSWORD'] 25 | DEBUG = False 26 | TESTING = False 27 | LOG_LEVEL = logging.ERROR 28 | SQLALCHEMY_DATABASE_URI = 'postgres://{user}:{passwd}@{host}/telemetry'.format(user=db_user, passwd=db_passwd, host=db_host) 29 | SQLALCHEMY_TRACK_MODIFICATIONS = True 30 | LOG_FILE = 'handler.log' 31 | 32 | # When PURGE_OLD_RECORDS == True then a purging system of old records will 33 | # be triggered daily. If this variable is not present, then no purging will be done. 34 | # If the purging system is enabled, then the following two variables must be set 35 | PURGE_OLD_RECORDS = True 36 | # The maximum retention time in days for records stored in the database 37 | # which do not match the filters in PURGE_FILTERED_RECORDS. 38 | # Use 0 to avoid deletion of all unfiltered records. 39 | MAX_DAYS_KEEP_UNFILTERED_RECORDS = 35 40 | # See config_example.py for details about PURGE_FILTERED_RECORDS 41 | PURGE_FILTERED_RECORDS = { 42 | "classification": { 43 | "org.clearlinux/hello/world": 1, 44 | } 45 | } 46 | 47 | # The Telemetry ID (TID) accepted by this `collector` app. The ID should be a 48 | # random UUID, generated with (for example) `uuidgen`. The default value 49 | # set here is used for records from the Clear Linux OS for Intel 50 | # Architecture. 51 | TELEMETRY_ID = "6907c830-eed9-4ce9-81ae-76daf8d88f0f" 52 | 53 | # vi: ts=4 et sw=4 sts=4 54 | -------------------------------------------------------------------------------- /deployments/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | db: 4 | image: postgres:latest 5 | networks: 6 | - backend 7 | env_file: 8 | - ./services/testing.env 9 | 10 | nginx: 11 | image: telemetry/proxy:1.0 12 | networks: 13 | - frontend 14 | volumes: 15 | - "telemetry-socket-volume:/var/run/uwsgi/" 16 | build: 17 | context: .. 18 | dockerfile: ./deployments/services/nginx/Dockerfile 19 | ports: 20 | - 80:80 21 | - 443:443 22 | 23 | redis: 24 | image: redis:latest 25 | networks: 26 | - backend 27 | env_file: 28 | - ./services/testing.env 29 | 30 | collector: 31 | image: telemetry/collector:1.0 32 | networks: 33 | - backend 34 | volumes: 35 | - "telemetry-socket-volume:/var/www/collector/socket" 36 | build: 37 | context: .. 38 | dockerfile: ./deployments/services/collector/Dockerfile 39 | env_file: 40 | - services/testing.env 41 | depends_on: ["db", "redis"] 42 | command: uwsgi --ini /var/www/collector/collector.ini 43 | 44 | webapp: 45 | image: telemetry/webapp:1.0 46 | networks: 47 | - backend 48 | volumes: 49 | - "telemetry-socket-volume:/var/www/webapp/socket" 50 | build: 51 | context: .. 52 | dockerfile: ./deployments/services/webapp/Dockerfile 53 | env_file: 54 | - ./services/testing.env 55 | depends_on: ["db", "redis"] 56 | command: uwsgi --ini /var/www/webapp/webapp.ini 57 | 58 | process: 59 | image: telemetry/process:1.0 60 | networks: 61 | - backend 62 | build: 63 | context: .. 64 | dockerfile: ./deployments/services/processing/Dockerfile 65 | env_file: 66 | - ./services/testing.env 67 | depends_on: ["db", "migrate"] 68 | command: /srv/processing/run.sh 69 | 70 | migrate: 71 | image: telemetry/webapp:1.0 72 | networks: 73 | - backend 74 | env_file: 75 | - ./services/testing.env 76 | environment: 77 | FLASK_APP: /var/www/webapp/run.py 78 | working_dir: /var/www/webapp 79 | depends_on: ["db"] 80 | restart: on-failure 81 | command: flask db upgrade 82 | 83 | volumes: 84 | telemetry-socket-volume: 85 | 86 | networks: 87 | frontend: 88 | driver: bridge 89 | backend: 90 | driver: overlay 91 | internal: true 92 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/js/builds.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Intel Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | (function(root, factory){ 19 | 20 | factory(root); 21 | 22 | })(this.telemetryUI, function(rootObj){ 23 | 24 | 25 | function renderBuilds(ctx, labels, values){ 26 | 27 | var data = { 28 | labels: labels, 29 | datasets:[ 30 | { 31 | label: "Records per Build", 32 | backgroundColor: rootObj.backgroundColors[0], 33 | data: values 34 | } 35 | ] 36 | }; 37 | 38 | var options = { 39 | legend: { display: false }, 40 | title: { 41 | display: true, 42 | text: "Records per Build", 43 | fontSize: 18 44 | }, 45 | scales: { 46 | yAxes: [{ 47 | scaleLabel: { 48 | display: true, 49 | labelString: "Records", 50 | fontSize: 15, 51 | fontStyle: "italic" 52 | }, 53 | ticks: { 54 | beginAtZero: true 55 | } 56 | }], 57 | xAxes: [{ 58 | scaleLabel: { 59 | display: true, 60 | labelString: "Builds", 61 | fontSize: 15, 62 | fontStyle: "italic", 63 | }, 64 | barThickness: 15 65 | }] 66 | } 67 | }; 68 | 69 | rootObj.newChart(ctx, "bar", data, options); 70 | } 71 | 72 | rootObj.renderBuilds = renderBuilds; 73 | 74 | }); 75 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/css/style.css: -------------------------------------------------------------------------------- 1 | .payload { 2 | overflow: hidden; 3 | text-overflow: ellipsis; 4 | white-space: nowrap; 5 | float:left; 6 | color:blue; 7 | } 8 | 9 | .machine_id { 10 | overflow: hidden; 11 | text-overflow: ellipsis; 12 | white-space: nowrap; 13 | max-width: 230px; 14 | } 15 | 16 | .ellipsize { 17 | overflow: hidden; 18 | text-overflow: ellipsis; 19 | white-space: nowrap; 20 | max-width: 200px; 21 | } 22 | 23 | #more_button { 24 | color: red; 25 | float: right; 26 | } 27 | 28 | #records_page_div{ 29 | margin-top: 30px; 30 | } 31 | 32 | ul#records_pagination { 33 | margin-top: 0px; 34 | margin-bottom: 0px; 35 | } 36 | 37 | #records_page_bar { 38 | display:inline; 39 | } 40 | 41 | #page_size_menu { 42 | vertical-align: inherit; 43 | } 44 | 45 | .diagonal_cell { 46 | color:red; 47 | background-color: #F0F0F0; 48 | font-weight: bold; 49 | } 50 | .dimmed_cell { 51 | background-color: lightgrey; 52 | font-weight: bold; 53 | } 54 | 55 | /* Bootstrap tooltip overrides */ 56 | .tooltip.left .tooltip-inner { 57 | background-color: #337AB7; 58 | } 59 | 60 | .tooltip.left .tooltip-arrow { 61 | border-left-color: #337AB7; 62 | } 63 | 64 | 65 | /* Stats tab special cases */ 66 | ul.legend-1 li, 67 | ul.legend-2 li { 68 | white-space: nowrap; 69 | overflow: hidden; 70 | cursor: default; 71 | text-overflow: ellipsis; 72 | } 73 | 74 | div.series-legend { 75 | display: inline-block; 76 | height: 10px; 77 | width: 30px; 78 | margin-right: 5px; 79 | background-color: #E0E0E0; 80 | } 81 | 82 | /* Records table */ 83 | table.telem-records { 84 | table-layout: fixed; 85 | } 86 | 87 | th.row-id { 88 | width: 10%; 89 | white-space: nowrap; 90 | overflow: hidden; 91 | text-overflow: ellipsis; 92 | } 93 | 94 | th.row-source { 95 | width: 5%; 96 | } 97 | 98 | th.row-machine-id { 99 | width: 17%; 100 | } 101 | 102 | th.row-when { 103 | width: 8%; 104 | } 105 | 106 | th.row-severity { 107 | width: 5%; 108 | } 109 | 110 | th.row-class { 111 | width: 15%; 112 | } 113 | 114 | th.row-os { 115 | width: 8%; 116 | } 117 | 118 | th.row-version { 119 | width: 5%; 120 | } 121 | 122 | th.row-payload { 123 | width: 22%; 124 | } 125 | 126 | th.row-button { 127 | width: 5%; 128 | } 129 | 130 | td.row-payload { 131 | color: blue; 132 | } 133 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/base.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | {%- block page_title %} 18 | CLR Telemetry 19 | {%- endblock %} 20 | 21 | 22 |
23 | 32 | 33 | {% block content %} {% endblock %} 34 |
35 | 45 | 46 | 47 | 48 | {#- 49 | # vi: ft=jinja ts=8 et sw=4 sts=4 50 | #} 51 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/js/thermal.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Intel Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | (function(root, factory){ 19 | 20 | factory(root); 21 | 22 | })(this.telemetryUI, function(rootObj){ 23 | 24 | 25 | // [["Thermal records", "Week: 30", "Week: 29", "Week: 28", "Week: 27", "Week: 26", "Week: 25"], ["org.clearlinux/mce/thermal", 155444, 28, 2734, 522, 54, 20]] 26 | 27 | function parseData(data) { 28 | 29 | var dataSets = {}; 30 | 31 | if (data.length > 0) { 32 | var weeks = data[0]; 33 | var values = data[1]; 34 | var dataSet = []; 35 | var labels = values.shift(1); 36 | weeks.shift(1); 37 | 38 | for(var x in values){ 39 | dataSet.push({ 40 | label: weeks[x], 41 | backgroundColor: rootObj.backgroundColors[x], 42 | data: [values[x]] 43 | }); 44 | } 45 | 46 | dataSets = { 47 | labels: [labels], 48 | datasets: dataSet 49 | }; 50 | } 51 | 52 | return dataSets; 53 | } 54 | 55 | function renderThermal(ctx, values){ 56 | 57 | var parsedData = parseData(values); 58 | 59 | var options = { 60 | legend: { 61 | display: true, 62 | position: "right" 63 | }, 64 | title: { 65 | display: false, 66 | }, 67 | scales: { 68 | xAxes: [{ 69 | barThickness: 30 70 | }], 71 | yAxes: [{ 72 | ticks: { 73 | beginAtZero: true 74 | } 75 | }] 76 | } 77 | }; 78 | 79 | rootObj.newChart(ctx, "bar", parsedData, options); 80 | } 81 | 82 | rootObj.renderThermal = renderThermal; 83 | 84 | }); 85 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/js/population.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Intel Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | (function(root, factory){ 19 | 20 | factory(root); 21 | 22 | })(this.telemetryUI, function(rootObj){ 23 | 24 | 25 | function renderChart(ctx, title, labels, a, b){ 26 | 27 | var data = { 28 | labels: labels, 29 | datasets:[ 30 | { 31 | label: "Internal", 32 | backgroundColor: rootObj.backgroundColors[0], 33 | data: a 34 | }, 35 | { 36 | label: "External", 37 | backgroundColor: rootObj.backgroundColors[1], 38 | data: b 39 | }, 40 | ] 41 | }; 42 | 43 | var options = { 44 | legend: { 45 | display: true, 46 | position: "right" 47 | }, 48 | title: { 49 | display: true, 50 | fontSize: 18, 51 | text: title 52 | }, 53 | responsive: true, 54 | scales: { 55 | xAxes: [{ 56 | stacked: true, 57 | scaleLabel: { 58 | display: true, 59 | labelString: "Builds", 60 | fontSize: 15, 61 | fontStyle: "italic" 62 | }, 63 | barThickness: 15 64 | }], 65 | yAxes: [{ 66 | stacked: true, 67 | scaleLabel: { 68 | display: true, 69 | labelString: "Machines", 70 | fontSize: 15, 71 | fontStyle: "italic" 72 | } 73 | }] 74 | } 75 | }; 76 | 77 | rootObj.newChart(ctx, "bar", data, options); 78 | } 79 | 80 | rootObj.renderMachinesPerBuild = renderChart; 81 | 82 | }); 83 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/population.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | 18 | {%- block page_title %} 19 | Population Stats - CLR Telemetry 20 | {%- endblock %} 21 | 22 | {% block content %} 23 |

Machines per build

24 |
25 | 26 |
27 | 28 |
29 | 30 |
31 | 32 |
33 | 34 |
35 | 36 |
37 | 38 |
39 | 40 |
41 | 42 |
43 | 44 |
45 | 46 |
47 | 48 | 49 | 50 | 67 | 68 | {% block record_content %} 69 | {% endblock %} 70 | {% endblock %} 71 | 72 | 73 | {#- 74 | # vi: ft=jinja ts=8 et sw=4 sts=4 75 | #} 76 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/stats.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | {% extends "base.html" %} 7 | {%- block page_title %} 8 | General Stats - CLR Telemetry 9 | {%- endblock %} 10 | 11 | {% block content %} 12 | 13 |

Statistics

14 | 15 |
16 |
17 | 18 |
19 |
20 |
Legend
21 |
    22 | {% for s in charts[0].record_stats %} 23 |
  • {{ s[0] }}
    • 24 | {% endfor %} 25 |
26 |
27 |
28 | 29 |
30 |
31 |
Legend
32 |
    33 | {% for s in charts[1].record_stats %} 34 |
  • {{ s[0] }}
    • 35 | {% endfor %} 36 |
37 |
38 |
39 | 40 | 41 | 69 | 70 | 71 | {% endblock %} 72 | 73 | 74 | {#- 75 | # vi: ft=jinja ts=8 et sw=4 sts=4 76 | #} 77 | -------------------------------------------------------------------------------- /telemetryui/migrations/versions/ac1c5921bf2f_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: ac1c5921bf2f 4 | Revises: 5 | Create Date: 2020-02-27 19:23:46.589251 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'ac1c5921bf2f' 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('guilty', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('function', sa.String(), nullable=True), 24 | sa.Column('module', sa.String(), nullable=True), 25 | sa.Column('comment', sa.String(), nullable=True), 26 | sa.Column('hide', sa.Boolean(), nullable=False, default=False), 27 | sa.PrimaryKeyConstraint('id'), 28 | sa.UniqueConstraint('function', 'module', name='unique_guilty_constraint') 29 | ) 30 | op.create_table('guilty_blacklisted', 31 | sa.Column('id', sa.Integer(), nullable=False), 32 | sa.Column('function', sa.String(), nullable=True), 33 | sa.Column('module', sa.String(), nullable=True), 34 | sa.PrimaryKeyConstraint('id') 35 | ) 36 | op.create_table('records', 37 | sa.Column('id', sa.Integer(), nullable=False), 38 | sa.Column('architecture', sa.Text(), nullable=True), 39 | sa.Column('bios_version', sa.Text(), nullable=True), 40 | sa.Column('board_name', sa.Text(), nullable=True), 41 | sa.Column('build', sa.Text(), nullable=False), 42 | sa.Column('classification', sa.Text(), nullable=False), 43 | sa.Column('cpu_model', sa.Text(), nullable=True), 44 | sa.Column('event_id', sa.Text(), nullable=True), 45 | sa.Column('external', sa.Boolean(), nullable=True), 46 | sa.Column('host_type', sa.Text(), nullable=True), 47 | sa.Column('kernel_version', sa.Text(), nullable=True), 48 | sa.Column('machine_id', sa.Text(), nullable=True), 49 | sa.Column('payload_version', sa.Integer(), nullable=True), 50 | sa.Column('record_version', sa.Integer(), nullable=True), 51 | sa.Column('severity', sa.Integer(), nullable=True), 52 | sa.Column('system_name', sa.Text(), nullable=True), 53 | sa.Column('timestamp_client', sa.Numeric(), nullable=True), 54 | sa.Column('timestamp_server', sa.Numeric(), nullable=False), 55 | sa.Column('payload', sa.Text(), nullable=False), 56 | sa.Column('processed', sa.Boolean(), nullable=True), 57 | sa.Column('guilty_id', sa.Integer(), nullable=True), 58 | sa.ForeignKeyConstraint(['guilty_id'], ['guilty.id'], ), 59 | sa.PrimaryKeyConstraint('id') 60 | ) 61 | # ### end Alembic commands ### 62 | 63 | 64 | def downgrade(): 65 | # ### commands auto generated by Alembic - please adjust! ### 66 | op.drop_table('records') 67 | op.drop_table('guilty_blacklisted') 68 | op.drop_table('guilty') 69 | # ### end Alembic commands ### 70 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_details.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | {% extends "base.html" %} 6 | 7 | {%- block page_title %} 8 | Guilty Details - CLR Telemetry 9 | {%- endblock %} 10 | 11 | {% block content %} 12 |

13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | {%- if query %} 32 | 33 | 34 | 35 | 36 | {% endif %} 37 | 38 |
Guilty Id{{ guilty_id }}
Function{{ func }}
Module{{ mod }}
Hidden?{{ hide }}
Backtracesview
39 | 40 | {%- if query %} 41 |

Machine ID mappings

42 | 43 |
44 | {%- for group in query|groupby('build')|sort(reverse=True) %} 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | {# The list is reverse numeric sorted by "total", so the first row has the maximum value. #} 56 | {%- set maxtotal = group.list[0].total %} 57 | {%- for row in group.list %} 58 | 59 | 60 | 61 | 62 | 63 | 64 | 72 | 73 | {%- endfor %} 74 | 75 |
Mappings for build {{ group.grouper }}
#Machine IDEvent Time SeriesBacktracesTotal
{{ loop.index }}{{ row.machine_id }}viewview 65 | 66 | {%- set width = row.total / maxtotal * 100 %} 67 | 68 | 69 | 70 | 71 |
76 | {%- endfor %} 77 |
78 | 79 | {%- else %} 80 | 81 | {%- endif %} 82 | 83 | 88 | 89 | {% endblock %} 90 | 91 | 92 | {#- 93 | # vi: ft=jinja ts=8 et sw=4 sts=4 94 | #} 95 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/record_details.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | {% block content %} 7 |

8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 |
Record Id{{ record.id }}
Machine Id{{ record.machine_id }}
Machine Type{{ record.host_type }}
Operating System{{ record.system_name }}
Kernel Version{{ record.kernel_version }}
Architecture{{ record.architecture }}
Board Name{{ record.board_name if record.board_name else 'N/A'}}
BIOS Version{{ record.bios_version if record.bios_version else 'N/A'}}
CPU Model{{ record.cpu_model if record.cpu_model else 'N/A'}}
Record Time{{ record.timestamp_client|local_datetime_since }}
Server Time{{ record.timestamp_server|local_datetime_since }}
Build{{ record.build }}
Severity{{ record.severity }}
Classification{{ record.classification }}
Guilty Id 69 | {%- if record.guilty_id %} 70 | {{ record.guilty_id }} 71 | {%- else %} 72 | N/A 73 | {%- endif %} 74 |
Payload Version{{ record.payload_version }}
External{{ record.external }}
86 | 
{{ record.payload }}
87 | 88 | {% endblock %} 89 | 90 | {#- 91 | # vi: ft=jinja ts=8 et sw=4 sts=4 92 | #} 93 | -------------------------------------------------------------------------------- /deployments/docker-compose.prod.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | db: 4 | image: postgres:latest 5 | networks: 6 | - backend 7 | restart: always 8 | volumes: 9 | - type: volume 10 | source: telemetry-data 11 | target: /var/lib/postgresql/data/pgdata 12 | env_file: 13 | - ./services/production.env 14 | environment: 15 | PGDATA: /var/lib/postgresql/data/pgdata 16 | POSTGRES_DB: telemetry 17 | 18 | nginx: 19 | image: telemetry/proxy:1.0 20 | networks: 21 | - frontend 22 | restart: always 23 | build: 24 | context: .. 25 | dockerfile: ./deployments/services/nginx/Dockerfile 26 | volumes: 27 | - "telemetry-socket-volume:/var/run/uwsgi/" 28 | - "telemetry-nginx-logs:/var/log/nginx" 29 | ports: 30 | - 80:80 31 | - 443:443 32 | 33 | redis: 34 | image: redis:latest 35 | networks: 36 | - backend 37 | restart: always 38 | env_file: 39 | - ./services/production.env 40 | 41 | collector: 42 | image: telemetry/collector:1.0 43 | networks: 44 | - backend 45 | volumes: 46 | - "telemetry-socket-volume:/var/www/collector/socket" 47 | - "telemetry-uwsgi-logs:/var/www/collector/log" 48 | restart: always 49 | build: 50 | context: .. 51 | dockerfile: ./deployments/services/collector/Dockerfile 52 | env_file: 53 | - services/production.env 54 | depends_on: ["db", "redis"] 55 | command: uwsgi --ini /var/www/collector/collector.ini 56 | 57 | webapp: 58 | image: telemetry/webapp:1.0 59 | networks: 60 | - backend 61 | volumes: 62 | - "telemetry-socket-volume:/var/www/webapp/socket" 63 | - "telemetry-uwsgi-logs:/var/www/webapp/log" 64 | restart: always 65 | build: 66 | context: .. 67 | dockerfile: ./deployments/services/webapp/Dockerfile 68 | env_file: 69 | - ./services/production.env 70 | depends_on: ["db", "redis"] 71 | command: uwsgi --ini /var/www/webapp/webapp.ini 72 | 73 | process: 74 | image: telemetry/process:1.0 75 | networks: 76 | - backend 77 | build: 78 | context: .. 79 | dockerfile: ./deployments/services/processing/Dockerfile 80 | env_file: 81 | - ./services/production.env 82 | depends_on: ["db", "migrate"] 83 | command: /srv/processing/run.sh 84 | 85 | migrate: 86 | image: telemetry/webapp:1.0 87 | networks: 88 | - backend 89 | env_file: 90 | - ./services/production.env 91 | environment: 92 | FLASK_APP: /var/www/webapp/run.py 93 | working_dir: /var/www/webapp 94 | depends_on: ["db"] 95 | restart: on-failure 96 | command: flask db upgrade 97 | 98 | volumes: 99 | telemetry-socket-volume: 100 | telemetry-uwsgi-logs: 101 | telemetry-nginx-logs: 102 | telemetry-data: 103 | 104 | networks: 105 | frontend: 106 | driver: bridge 107 | backend: 108 | driver: overlay 109 | internal: true 110 | -------------------------------------------------------------------------------- /telemetryui/migrations/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | from alembic import context 3 | from sqlalchemy import engine_from_config, pool 4 | from logging.config import fileConfig 5 | import logging 6 | 7 | # this is the Alembic Config object, which provides 8 | # access to the values within the .ini file in use. 9 | config = context.config 10 | 11 | # Interpret the config file for Python logging. 12 | # This line sets up loggers basically. 13 | fileConfig(config.config_file_name) 14 | logger = logging.getLogger('alembic.env') 15 | 16 | # add your model's MetaData object here 17 | # for 'autogenerate' support 18 | # from myapp import mymodel 19 | # target_metadata = mymodel.Base.metadata 20 | from flask import current_app 21 | config.set_main_option('sqlalchemy.url', 22 | current_app.config.get('SQLALCHEMY_DATABASE_URI')) 23 | target_metadata = current_app.extensions['migrate'].db.metadata 24 | 25 | # other values from the config, defined by the needs of env.py, 26 | # can be acquired: 27 | # my_important_option = config.get_main_option("my_important_option") 28 | # ... etc. 29 | 30 | 31 | def run_migrations_offline(): 32 | """Run migrations in 'offline' mode. 33 | 34 | This configures the context with just a URL 35 | and not an Engine, though an Engine is acceptable 36 | here as well. By skipping the Engine creation 37 | we don't even need a DBAPI to be available. 38 | 39 | Calls to context.execute() here emit the given string to the 40 | script output. 41 | 42 | """ 43 | url = config.get_main_option("sqlalchemy.url") 44 | context.configure(url=url) 45 | 46 | with context.begin_transaction(): 47 | context.run_migrations() 48 | 49 | 50 | def run_migrations_online(): 51 | """Run migrations in 'online' mode. 52 | 53 | In this scenario we need to create an Engine 54 | and associate a connection with the context. 55 | 56 | """ 57 | 58 | # this callback is used to prevent an auto-migration from being generated 59 | # when there are no changes to the schema 60 | # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html 61 | def process_revision_directives(context, revision, directives): 62 | if getattr(config.cmd_opts, 'autogenerate', False): 63 | script = directives[0] 64 | if script.upgrade_ops.is_empty(): 65 | directives[:] = [] 66 | logger.info('No changes in schema detected.') 67 | 68 | engine = engine_from_config(config.get_section(config.config_ini_section), 69 | prefix='sqlalchemy.', 70 | poolclass=pool.NullPool) 71 | 72 | connection = engine.connect() 73 | context.configure(connection=connection, 74 | target_metadata=target_metadata, 75 | process_revision_directives=process_revision_directives, 76 | **current_app.extensions['migrate'].configure_args) 77 | 78 | try: 79 | with context.begin_transaction(): 80 | context.run_migrations() 81 | finally: 82 | connection.close() 83 | 84 | if context.is_offline_mode(): 85 | run_migrations_offline() 86 | else: 87 | run_migrations_online() 88 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_backtraces.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | {% extends "base.html" %} 7 | 8 | {%- block page_title %} 9 | Backtraces - CLR Telemetry 10 | {%- endblock %} 11 | 12 | {% block content %} 13 |

Records with guilty {{ func }} - [{{ mod }}]

14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | {%- for b in backtraces %} 24 | 25 | 26 | 27 | 28 | 41 | 42 | {%- endfor %} 43 | 44 | 45 | 46 | 47 | 61 | 62 | 81 | {% endblock %} 82 | 83 | 84 | {#- 85 | # vi: ft=jinja ts=8 et sw=4 sts=4 86 | #} 87 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/guilty_edit_one.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | 18 | {%- block page_title %} 19 | Adjust guilties - CLR Telemetry 20 | {%- endblock %} 21 | 22 | {% block content %} 23 |

Update guilty blacklist based on record {{ record_id }}

24 | 25 | {%- if crash_info.backtrace %} 26 | 27 | 31 | 32 | 41 | 42 |
43 |

Frame List

44 | 45 | {%- with messages = get_flashed_messages(with_categories=true) %} 46 | {%- if messages %} 47 | {%- for category, m in messages %} 48 | 49 | {%- endfor %} 50 | {%- endif %} 51 | {%- endwith %} 52 | 53 |
54 | 55 | {{ form.hidden_tag() }} 56 |
57 | {{ form.frames() }} 58 |
59 | 60 |
61 | 62 | 63 |
64 | 65 | 66 |
67 | 68 | 76 | {%- else %} 77 | 78 | {%- endif %} {# crash_info.backtrace #} 79 | 80 | {% endblock %} 81 | 82 | 83 | {#- 84 | # vi: ft=jinja ts=8 et sw=4 sts=4 85 | #} 86 | -------------------------------------------------------------------------------- /processing/processing/main.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015-2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import json 5 | import psycopg2 6 | from . import crash 7 | from .crash import ( 8 | parse_crash, 9 | BACKTRACE_CLASSES as classifications, 10 | ) 11 | 12 | 13 | PROCESSED_VIEW = """last_processed""" 14 | 15 | CRASHES = """ 16 | SELECT id, payload, classification FROM records WHERE classification in {} AND id > {} ORDER BY id ASC 17 | """ 18 | 19 | INSERT_GUILTY = """ 20 | INSERT INTO {}.guilty (function, module, hide) VALUES (%s, %s, False) ON CONFLICT (function, module) DO NOTHING RETURNING id 21 | """ 22 | 23 | GUILTY_BLACKLIST = """ 24 | SELECT function, module FROM guilty_blacklisted 25 | """ 26 | 27 | LAST_ID = """ 28 | SELECT last_id FROM {} LIMIT 1 29 | """.format(PROCESSED_VIEW) 30 | 31 | UPDATE_PROCESSED_RECORD = """ 32 | UPDATE records SET processed = True WHERE id = %s 33 | """ 34 | 35 | CHECK_GUILTY = """ 36 | SELECT id FROM {}.guilty WHERE function = %s AND module = %s 37 | """ 38 | 39 | UPDATE_GUILTY = """ 40 | UPDATE records SET guilty_id = %s WHERE id = %s 41 | """ 42 | 43 | 44 | class GuilyBlacklist(object): 45 | 46 | def __init__(self, blacklisted): 47 | self.blacklisted = blacklisted 48 | 49 | def contains(self, item): 50 | return item in self.blacklisted 51 | 52 | 53 | def init_guilty_blacklist(cur): 54 | cur.execute(GUILTY_BLACKLIST) 55 | rows = cur.fetchall() 56 | bl = [(row[0], row[1]) for row in rows] 57 | return GuilyBlacklist(bl) 58 | 59 | 60 | def get_latest_id(cur): 61 | cur.execute(LAST_ID) 62 | row = cur.fetchone() 63 | if row[0] is None: 64 | return 0 65 | return row[0] 66 | 67 | 68 | def get_crashes(cur, last_id): 69 | classifications_str = "('{}')".format("', '".join(classifications)) 70 | cur.execute(CRASHES.format(classifications_str, last_id)) 71 | rows = cur.fetchall() 72 | return rows 73 | 74 | 75 | def process_crashes(cur, crashes, schema="public", debug=False): 76 | if len(crashes) == 0: 77 | return 78 | insert_guilty = INSERT_GUILTY.format(schema) 79 | for rid, backtrace, _ in crashes: 80 | if backtrace is None: 81 | print("## backtrace record {} is None".format(rid)) 82 | continue 83 | function, module = parse_crash(backtrace) 84 | if function is not None and module is not None: 85 | # Save new function and module 86 | cur.execute(insert_guilty, (function, module,)) 87 | # Row will have a value only when insertion happens 88 | row = cur.fetchone() 89 | if row is None: 90 | check_guilty = CHECK_GUILTY.format(schema) 91 | cur.execute(check_guilty, (function, module,)) 92 | row = cur.fetchone() 93 | gid = row[0] 94 | ### Update record table with guilty_id 95 | cur.execute(UPDATE_GUILTY, (gid, rid,)) 96 | else: 97 | print("## function or module is None for record id {}".format(rid)) 98 | 99 | ### Update latest processed record table with record_id 100 | cur.execute(UPDATE_PROCESSED_RECORD, (rid,)) 101 | print("## Updating last processed id to {}\n".format(rid)) 102 | 103 | 104 | def connect(conf): 105 | db = psycopg2.connect(host=conf.db_host, dbname=conf.db_name, 106 | user=conf.db_user, password=conf.db_passwd) 107 | db.autocommit = True 108 | return db 109 | 110 | # vi: ts=4 et sw=4 sts=4 111 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/vendor/bootstrap/css/bootstrap-reboot.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap Reboot v4.4.1 (https://getbootstrap.com/) 3 | * Copyright 2011-2019 The Bootstrap Authors 4 | * Copyright 2011-2019 Twitter, Inc. 5 | * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) 6 | * Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md) 7 | */*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus:not(:focus-visible){outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]){color:inherit;text-decoration:none}a:not([href]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important} 8 | /*# sourceMappingURL=bootstrap-reboot.min.css.map */ -------------------------------------------------------------------------------- /telemetryui/telemetryui/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015-2020 Intel Corporation 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import time 5 | from . import ( 6 | crash, 7 | app,) 8 | from .model import Record 9 | from flask import ( 10 | redirect, 11 | url_for, 12 | render_template) 13 | 14 | 15 | RECORDS_PER_PAGE = app.config.get('RECORDS_PER_PAGE', 25) 16 | 17 | 18 | def records_get(form, request, lastid=None): 19 | classification = request.args.get('classification') 20 | build = request.args.get('build') 21 | severity = request.args.get('severity') 22 | machine_id = request.args.get('machine_id') 23 | payload = request.args.get('payload') 24 | not_payload = request.args.get('not_payload') 25 | data_source = request.args.get('data_source') 26 | 27 | if classification is not None: 28 | form.classification.default = classification 29 | if build is not None: 30 | form.build.default = build 31 | if severity is not None: 32 | form.severity.default = severity 33 | if machine_id is not None: 34 | form.machine_id.default = machine_id 35 | if payload is not None: 36 | form.payload.default = payload 37 | if not_payload is not None: 38 | form.not_payload.default = not_payload 39 | if data_source is not None: 40 | form.data_source.default = data_source 41 | 42 | form.process() 43 | return Record.query_records(build, classification, severity, machine_id, 44 | payload=payload, not_payload=not_payload, 45 | data_source=data_source, from_id=lastid, 46 | limit=RECORDS_PER_PAGE) 47 | 48 | 49 | def records_post(form, request): 50 | 51 | if form.validate_on_submit() is False: 52 | out_records = Record.query.order_by(Record.id.desc()).limit(RECORDS_PER_PAGE).all() 53 | return render_template('records.html', records=out_records, form=form) 54 | else: 55 | classification = request.form.get('classification') 56 | system_name = request.form.get('system_name') 57 | build = request.form.get('build') 58 | severity = request.form.get('severity') 59 | machine_id = request.form.get('machine_id') 60 | payload = request.form.get('payload') 61 | not_payload = request.form.get('not_payload') 62 | from_date = request.form.get('from_date') 63 | to_date = request.form.get('to_date') 64 | data_source = request.form.get('data_source') 65 | 66 | redirect_args = { 67 | "system_name": system_name if system_name != "All" else None, 68 | "build": build if build != "All" else None, 69 | "severity": severity if severity != "All" else None, 70 | "classification": classification if classification != "All" else None, 71 | "machine_id": machine_id if machine_id != "" else None, 72 | "payload": payload if payload != "" else None, 73 | "not_payload": not_payload if not_payload != "" else None, 74 | "from_date": from_date if from_date != "" else None, 75 | "to_date": to_date if to_date != "" else None, 76 | "data_source": data_source if data_source != "All" else None, 77 | } 78 | dest = 'views_bp.records' 79 | 80 | if 'csv_attachment' in request.form: 81 | redirect_args['format_type'] = 'attach' 82 | redirect_args['timestamp'] = time.time() 83 | dest = 'views_bp.export_csv' 84 | 85 | url = url_for(dest, **redirect_args) 86 | return redirect(url) 87 | 88 | 89 | def explode_backtraces(classes=None, guilty_id=None, machine_id=None, build=None): 90 | crashes = [] 91 | backtraces = Record.get_crash_backtraces(classes, guilty_id, machine_id, build) 92 | for b in backtraces: 93 | crashes.append(crash.parse_backtrace(b)) 94 | return crashes 95 | 96 | 97 | # vi: ts=4 et sw=4 sts=4 98 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/forms.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from flask_wtf import FlaskForm 18 | from wtforms import ( 19 | StringField, 20 | SelectField, 21 | HiddenField, 22 | TextField, 23 | SelectMultipleField) 24 | from wtforms.fields.html5 import DateField 25 | from wtforms.validators import ( 26 | InputRequired, 27 | Length, 28 | Regexp, 29 | Optional) 30 | from wtforms.widgets import ( 31 | html_params, 32 | HTMLString) 33 | 34 | 35 | class RecordFilterForm(FlaskForm): 36 | system_name = SelectField('OS Name', coerce=str, validators=[Optional()]) 37 | build = SelectField('Version', coerce=str, validators=[Optional()]) 38 | classification = SelectField('Classification', coerce=str, validators=[InputRequired()]) 39 | severity = SelectField('Severity', coerce=str, validators=[InputRequired()]) 40 | machine_id = TextField('Machine ID') 41 | payload = TextField('Payload REGEX') 42 | not_payload = TextField('Payload NOT REGEX') 43 | data_source = SelectField('Source', coerce=str) 44 | from_date = DateField('From date', validators=[Optional()]) 45 | to_date = DateField('To date', validators=[Optional()]) 46 | 47 | 48 | class GuiltyDetailsForm(FlaskForm): 49 | comment = StringField('Comment (80 characters max)', validators=[Length(max=80)]) 50 | guilty_id = HiddenField('Guilty ID (autopopulated)', validators=[Regexp('[0-9]+')]) 51 | hidden = SelectField('Hide this guilty?', choices=[('no', 'no'), ('yes', 'yes')], default='no', validators=[InputRequired()]) 52 | 53 | 54 | class GuiltyAddForm(FlaskForm): 55 | funcmod = SelectField('Function [Module]', coerce=str, validators=[InputRequired()]) 56 | 57 | 58 | class GuiltyRemoveForm(FlaskForm): 59 | funcmod = SelectField('Function [Module]', coerce=str, validators=[InputRequired()]) 60 | 61 | 62 | class GuiltyHiddenForm(FlaskForm): 63 | funcmod = SelectField('Function [Module]', coerce=str, validators=[InputRequired()]) 64 | 65 | 66 | # TODO: eventually the 'confirm' field should be unhidden and populated with 67 | # the results of a prompt for the user's confirmation for carrying out the reset 68 | class GuiltyResetForm(FlaskForm): 69 | confirm = HiddenField('Confirm reset (autopopulated)', default='yes', validators=[Regexp('yes')]) 70 | 71 | 72 | # Custom widget derived from the example in the docs: 73 | # https://wtforms.readthedocs.io/en/latest/widgets.html 74 | def select_multi_checkbox(field, ul_class='', li_class='', **kwargs): 75 | kwargs.setdefault('type', 'checkbox') 76 | field_id = kwargs.pop('id', field.id) 77 | html = ['
    \n'.format(html_params(id=field_id, class_=ul_class))] 78 | for index, (value, label, checked) in enumerate(field.iter_choices()): 79 | choice_id = '{}{}'.format(field_id, index) 80 | options = dict(kwargs, name=field.name, value=value, id=choice_id) 81 | if checked: 82 | options['checked'] = 'checked' 83 | html.append('
  • \n'.format(html_params(class_=li_class))) 84 | html.append('\n'.format(html_params(**options))) 85 | html.append('\n
    • \n'.format(field_id, label)) 86 | html.append('
\n') 87 | return HTMLString(''.join(html)) 88 | 89 | 90 | # For modifying the guilty blacklist according to the backtrace frames selected 91 | class GuiltyEditOneForm(FlaskForm): 92 | frames = SelectMultipleField(widget=select_multi_checkbox, 93 | render_kw={'ul_class': 'list-group', 94 | 'li_class': 'list-group-item'}) 95 | 96 | 97 | # vi: ts=4 et sw=4 sts=4 98 | -------------------------------------------------------------------------------- /collector/collector/tests/purging.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import unittest 18 | import time 19 | from collector import db 20 | from flask import current_app 21 | from collector.model import ( 22 | app, 23 | Record, 24 | Classification, 25 | Build) 26 | 27 | from collector.tests.testcase import ( 28 | RecordTestCases, 29 | get_record,) 30 | 31 | 32 | def get_insert_params(days_old, severity, classification): 33 | record = get_record() 34 | record["classification"] = classification 35 | record["severity"] = severity 36 | db_class = Classification.query.filter_by(classification=record["classification"]).first() 37 | if db_class is None: 38 | db_class = Classification(record["classification"]) 39 | db_build = Build.query.filter_by(build=record["build"]).first() 40 | if db_build is None: 41 | db_build = Build(record["build"]) 42 | return [ 43 | record["machine_id"], 44 | record["host_type"], 45 | record["severity"], 46 | db_class, 47 | db_build, 48 | record["arch"], 49 | record["kernel_version"], 50 | record["record_format_version"], 51 | int(time.time()-3600*24*days_old), 52 | int(time.time()-3600*24*days_old), 53 | record["payload_format_version"], 54 | record["system_name"], 55 | record["board_name"], 56 | record["bios_version"], 57 | record["cpu_model"], 58 | "39cc109a1079df96376693ebc7a0f632", 59 | False, 60 | "Test" 61 | ] 62 | 63 | 64 | class TestPurging(RecordTestCases): 65 | """ Generic object for telemetry record tests """ 66 | 67 | def setUp(self): 68 | app.testing = True 69 | app.config.from_object('config_local.Testing') 70 | app.config["MAX_DAYS_KEEP_UNFILTERED_RECORDS"] = 5 71 | app.config["PURGE_FILTERED_RECORDS"] = { 72 | "severity": { 73 | 1: 1, 74 | 4: 0 75 | }, 76 | "classification": { 77 | "test/keep/one": 0, 78 | "test/discard/*": 1, 79 | } 80 | } 81 | app.debug = False 82 | self.app_context = app.app_context() 83 | self.app_context.push() 84 | db.init_app(current_app) 85 | db.create_all() 86 | self.client = app.test_client() 87 | 88 | def test_purge_delete(self): 89 | Record.create(*get_insert_params(2, 1, "test/discard/one")) 90 | Record.create(*get_insert_params(2, 2, "test/discard/two")) 91 | Record.create(*get_insert_params(2, 2, "test/discard/three")) 92 | Record.create(*get_insert_params(2, 4, "test/discard/two")) 93 | Record.create(*get_insert_params(2, 4, "test/discard/three")) 94 | Record.create(*get_insert_params(6, 2, "test/test/one")) 95 | Record.create(*get_insert_params(2, 1, "test/keep/one")) 96 | self.assertTrue(Record.query.count() == 7) 97 | Record.delete_records() 98 | self.assertTrue(Record.query.count() == 0) 99 | 100 | def test_purge_keep(self): 101 | Record.create(*get_insert_params(6, 2, "test/keep/one")) 102 | Record.create(*get_insert_params(3, 2, "test/test/one")) 103 | Record.create(*get_insert_params(6, 4, "test/test/one")) 104 | Record.create(*get_insert_params(2, 4, "test/discard/three")) 105 | Record.create(*get_insert_params(6, 2, "test/test/one")) 106 | Record.create(*get_insert_params(2, 1, "test/keep/one")) 107 | self.assertTrue(Record.query.count() == 6) 108 | Record.delete_records() 109 | self.assertTrue(len(Record.query.all()) == 3) 110 | 111 | 112 | if __name__ == '__main__' and __package__ is None: 113 | from os import sys, path 114 | sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) 115 | unittest.main() 116 | -------------------------------------------------------------------------------- /collector/collector/tests/testcase.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import unittest 18 | from collector import ( 19 | app, 20 | db, 21 | report_handler,) 22 | from flask import current_app 23 | 24 | classification = 'classification' 25 | severity = 'severity' 26 | kernel_version = 'kernel_version' 27 | record_version = 'record_format_version' 28 | machine_id = 'machine_id' 29 | host_type = 'host_type' 30 | arch = 'arch' 31 | build = 'build' 32 | timestamp = 'creation_timestamp' 33 | tid = 'X-Telemetry-Tid' 34 | board_name = 'Board-Name' 35 | cpu_model = 'Cpu-Model' 36 | bios_version = 'Bios-Version' 37 | system_name = 'System-Name' 38 | payload_version = 'Payload-Format-Version' 39 | event_id = 'Event-Id' 40 | 41 | 42 | REQUIRED_HEADERS_V1 = ( 43 | 'Arch', 44 | 'Build', 45 | 'Creation-Timestamp', 46 | 'Classification', 47 | 'Host-Type', 48 | 'Kernel-Version', 49 | 'Machine-Id', 50 | 'Severity', 51 | 'Record-Format-Version', 52 | ) 53 | 54 | 55 | def get_record_v1(): 56 | return { 57 | arch: 'x86_64', 58 | build: '550', 59 | timestamp: 1483232401, 60 | classification: 'a/b/c', 61 | host_type: 'LenovoT20', 62 | kernel_version: '3.16.4-123.generic', 63 | machine_id: '1234', 64 | severity: 2, 65 | record_version: 1, 66 | } 67 | 68 | 69 | def get_record_v2(): 70 | v2 = get_record_v1() 71 | v2.update({ 72 | record_version: 2, 73 | tid: '6907c830-eed9-4ce9-81ae-76daf8d88f0f', 74 | system_name: 'clear-linux-os', 75 | payload_version: 1 76 | }) 77 | return v2 78 | 79 | 80 | def get_record_v3(): 81 | v3 = get_record_v2() 82 | v3.update({ 83 | record_version: 3, 84 | board_name: 'D54250WYK|Intel Corporation', 85 | cpu_model: 'Intel(R) Core(TM) i5-4250U CPU @ 1.30GHz', 86 | bios_version: 'WYLPT10H.86A.0041.2015.0720.1108', 87 | 88 | }) 89 | return v3 90 | 91 | 92 | def get_record_v4(): 93 | v4 = get_record_v3() 94 | v4.update({ 95 | record_version: 4, 96 | event_id: '39cc109a1079df96376693ebc7a0f632', 97 | }) 98 | return v4 99 | 100 | 101 | def get_record(): 102 | return { 103 | "X-Telemetry-TID": "6907c830-eed9-4ce9-81ae-76daf8d88f0f", 104 | "record_format_version": "2", 105 | "severity": "1", 106 | "classification": "org.clearlinux/hello/world", 107 | "machine_id": "clr-linux-avj01", 108 | "creation_timestamp": "1505235249", 109 | "arch": "x86_64", 110 | "host_type": "blank|blank|blank", 111 | "kernel_version": "4.12.5-374.native", 112 | "system_name": "clear-linux-os", 113 | "build": "17700", 114 | "payload_format_version": "1", 115 | "board_name": "D54250WYK|Intel Corporation", 116 | "cpu_model": "Intel(R) Core(TM) i5-4250U CPU @ 1.30GHz", 117 | "bios_version": "WYLPT10H.86A.0041.2015.0720.1108" 118 | } 119 | 120 | 121 | class RecordTestCases(unittest.TestCase): 122 | """ Generic object for telemetry record tests """ 123 | 124 | def setUp(self): 125 | app.testing = True 126 | app.config.from_object('collector.config_local.Testing') 127 | app.debug = False 128 | self.app_context = app.app_context() 129 | self.app_context.push() 130 | db.init_app(current_app) 131 | db.create_all() 132 | self.client = app.test_client() 133 | 134 | def tearDown(self): 135 | db.session.remove() 136 | db.drop_all() 137 | self.app_context.pop() 138 | 139 | def missing_header(self, header, header_name): 140 | headers = self.get_version_records() 141 | del headers[header_name] 142 | response = self.client.post('/', headers=headers, data='test') 143 | self.assertTrue(response.status_code == 400) 144 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/crashes_filter.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | 18 | {%- block page_title %} 19 | Top Crashes ({{ build }}) - CLR Telemetry 20 | {%- endblock %} 21 | 22 | {% block content %} 23 |

Crash stats

24 |

Top Crashes ({{ build }})

25 | 26 |
27 | 28 |
#ProgramSignalDetails
{{ loop.index }}{{ b.program }}{{ b.signal }} 29 |
30 | 32 | Update blacklist 34 |
35 | 
36 |         {%- for f in b.backtrace %}
37 | #{{ loop.index0 }} {{ f[0] }} - [{{ f[1] }}]{% if f[2] %} - {{ f[2]|basename }}{% endif %}
38 |         {%- endfor %}
39 |
40 |
29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | {%- set maxtotal = guilties[0].total %} 38 | {%- for g in guilties %} 39 | 40 | 41 | 45 | 46 | 54 | 55 | 56 | {%- endfor %} 57 | 58 |
Top 10 crashes in the last 7 days
#GuiltyTotalComment
{{ loop.index }} 42 | {{ g.guilty }} 43 | 44 | 47 | 48 | {%- set width = g.total / maxtotal * 100 %} 49 | 50 | 51 | 52 | 53 | {{ g.comment }}
59 | 60 | 61 | 62 | 63 | 93 | 94 | 113 | 114 | {% endblock %} 115 | 116 | 117 | {#- 118 | # vi: ft=jinja ts=8 et sw=4 sts=4 119 | #} 120 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/js/mce.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Intel Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | (function(root, factory){ 19 | 20 | factory(root); 21 | 22 | })(this.telemetryUI, function(rootObj){ 23 | 24 | 25 | function parseMCEChartData(name, data){ 26 | 27 | var dataSet = []; 28 | var counter = 0 29 | var weekList = Object.keys(data).sort(function(a, b) {return a - b;}); 30 | for(var weekIndex in weekList){ 31 | var label = weekList[weekIndex]; 32 | dataSet.push({ 33 | label: "Week "+label, 34 | backgroundColor: rootObj.backgroundColors[counter%14], 35 | data: [data[label]] 36 | }); 37 | counter++; 38 | } 39 | 40 | return { 41 | labels: [name], 42 | datasets: dataSet 43 | } 44 | } 45 | 46 | 47 | function renderOneChart(ctx, data, max){ 48 | 49 | var options = { 50 | legend: { 51 | display: true, 52 | position: "right" 53 | }, 54 | title: { 55 | display: false 56 | }, 57 | scales: { 58 | xAxes: [{ 59 | stacked: false, 60 | barThickness: 30 61 | }] 62 | } 63 | }; 64 | 65 | rootObj.newChart(ctx, "bar", data, options); 66 | } 67 | 68 | 69 | 70 | function renderMCEClass(ctx, labels, values){ 71 | 72 | var data = { 73 | labels: labels, 74 | datasets: [{ 75 | label: "MCE reports by classification", 76 | backgroundColor: rootObj.backgroundColors, 77 | data: values 78 | }] 79 | }; 80 | 81 | var options = { 82 | title: { 83 | display: true, 84 | text: "MCE reports by classification", 85 | fontSize: 18 86 | }, 87 | legend: { 88 | position: 'right' 89 | } 90 | }; 91 | 92 | 93 | rootObj.newChart(ctx, "pie", data, options); 94 | } 95 | 96 | 97 | function renderMCEReports(ctx, labels, values){ 98 | 99 | var data = { 100 | labels: labels, 101 | datasets:[ 102 | { 103 | label: "MCE reports by build", 104 | data: values 105 | } 106 | ] 107 | }; 108 | 109 | var options = { 110 | legend: { display: false }, 111 | title: { 112 | display: true, 113 | text: "MCE reports by build", 114 | fontSize: 18 115 | }, 116 | scales: { 117 | yAxes: [{ 118 | scaleLabel: { 119 | display: true, 120 | labelString: "MCE count", 121 | fontSize: 15, 122 | fontStyle: "italic" 123 | }, 124 | ticks: { 125 | beginAtZero: true 126 | } 127 | }], 128 | xAxes: [{ 129 | scaleLabel: { 130 | display: true, 131 | labelString: "Builds", 132 | fontSize: 15, 133 | fontStyle: "italic" 134 | }, 135 | barThickness: 15 136 | }] 137 | } 138 | }; 139 | 140 | rootObj.newChart(ctx, "bar", data, options); 141 | } 142 | 143 | function renderMCEStatus(ctx, labels, values){ 144 | 145 | var data = { 146 | labels: labels, 147 | datasets:[ 148 | { 149 | label: "MCE counts by status", 150 | data: values 151 | } 152 | ] 153 | }; 154 | 155 | var options = { 156 | legend: { display: false }, 157 | title: { 158 | display: true, 159 | text: "MCE reports by status", 160 | fontSize: 18 161 | }, 162 | scales: { 163 | yAxes: [{ 164 | scaleLabel: { 165 | display: true, 166 | labelString: "MCE count", 167 | fontSize: 15, 168 | fontStyle: "italic" 169 | }, 170 | ticks: { 171 | beginAtZero: true 172 | } 173 | }], 174 | xAxes: [{ 175 | scaleLabel: { 176 | display: true, 177 | labelString: "Last 32 bits of status code", 178 | fontSize: 15, 179 | fontStyle: "italic" 180 | }, 181 | barThickness: 15 182 | }] 183 | } 184 | }; 185 | 186 | rootObj.newChart(ctx, "bar", data, options); 187 | } 188 | 189 | rootObj.renderMCEClass = renderMCEClass; 190 | rootObj.renderMCEReports = renderMCEReports; 191 | rootObj.renderMCEStatus = renderMCEStatus; 192 | rootObj.parseMCEChartData = parseMCEChartData; 193 | rootObj.renderOneChart = renderOneChart; 194 | }); 195 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/mce.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright 2016-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -#} 16 | {% extends "base.html" %} 17 | 18 | {%- block page_title %} 19 | MCE Stats - CLR Telemetry 20 | {%- endblock %} 21 | 22 | {% block content %} 23 | 24 |

MCE stats

25 | MCE reports by week of the current year 26 | 27 | {%- with messages = get_flashed_messages() %} 28 | {%- if messages %} 29 | {%- for m in messages %} 30 | 31 | {%- endfor %} 32 | {%- endif %} 33 | {%- endwith %} 34 | 35 |
36 | {%- for clas in fullmce %} 37 |
38 | 39 |
40 | {%- endfor %} 41 |
42 |

Top MCE

43 | 44 | 63 | 64 | 65 | 66 | {%- for b in builds %} 67 | 68 | {%- endfor %} 69 | 70 | 71 | 72 | {%- for row in top10 %} 73 | 74 | 75 | 78 | {%- for b in builds %} 79 | {%- if row.builds[b] %} 80 | 88 | {%- else %} 89 | 90 | {%- endif %} 91 | {%- endfor %} 92 | 93 | 94 | {%- endfor %} 95 | 96 |
Top MCE by machine id
#Machine ID{{ b }}Total
{{ loop.index }} 76 | {{ row.machine_id }} 77 | 81 | 82 | {%- set width = row.builds[b] / maxcnt * 100 %} 83 | 84 | 85 | 86 | 87 | {{ row.recordscnt }}
97 |
98 |
99 | 100 |
101 |
102 | 103 |
104 |
105 | 106 |
107 | 108 |
109 | 132 | 133 | 158 | 159 | 160 | {% endblock %} 161 | 162 | 163 | {#- 164 | # vi: ft=jinja ts=8 et sw=4 sts=4 165 | #} 166 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/crashes.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | {% extends "base.html" %} 7 | 8 | {%- block page_title %} 9 | Crash Stats - CLR Telemetry 10 | {%- endblock %} 11 | 12 | {% block content %} 13 | 14 |

Crash stats

15 |

Top Crashes

16 | 17 | {%- with messages = get_flashed_messages(with_categories=true) %} 18 | {%- if messages %} 19 | {%- for category, m in messages %} 20 | 21 | {%- endfor %} 22 | {%- endif %} 23 | {%- endwith %} 24 | 25 |
26 | 27 | {%- if guilties %} 28 | 29 | 30 | 31 | 32 | 33 | 34 | {%- for b in builds %} 35 | {%- set currentbuild = b[0] %} 36 | 37 | {%- endfor %} 38 | 39 | 40 | 41 | {% include "crashes_list.html" %} 42 | 43 |
Top crashes in the last 7 days, with frequency per build
#GuiltyTotal{{ b[0] }}Comment
44 | 45 | 46 |
47 | 51 |
52 | 53 | {%- else %} {# guilties #} 54 | 55 | {%- endif %} {# guilties #} 56 | 57 |
58 | 59 |
60 |
61 | 62 |
63 | 64 |
65 | 66 | 67 | 96 | 97 | 142 | 143 | {% endblock %} 144 | 145 | {#- 146 | # vi: ft=jinja ts=8 et sw=4 sts=4 147 | #} 148 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/vendor/bootstrap/css/bootstrap-reboot.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap Reboot v4.4.1 (https://getbootstrap.com/) 3 | * Copyright 2011-2019 The Bootstrap Authors 4 | * Copyright 2011-2019 Twitter, Inc. 5 | * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) 6 | * Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md) 7 | */ 8 | *, 9 | *::before, 10 | *::after { 11 | box-sizing: border-box; 12 | } 13 | 14 | html { 15 | font-family: sans-serif; 16 | line-height: 1.15; 17 | -webkit-text-size-adjust: 100%; 18 | -webkit-tap-highlight-color: rgba(0, 0, 0, 0); 19 | } 20 | 21 | article, aside, figcaption, figure, footer, header, hgroup, main, nav, section { 22 | display: block; 23 | } 24 | 25 | body { 26 | margin: 0; 27 | font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; 28 | font-size: 1rem; 29 | font-weight: 400; 30 | line-height: 1.5; 31 | color: #212529; 32 | text-align: left; 33 | background-color: #fff; 34 | } 35 | 36 | [tabindex="-1"]:focus:not(:focus-visible) { 37 | outline: 0 !important; 38 | } 39 | 40 | hr { 41 | box-sizing: content-box; 42 | height: 0; 43 | overflow: visible; 44 | } 45 | 46 | h1, h2, h3, h4, h5, h6 { 47 | margin-top: 0; 48 | margin-bottom: 0.5rem; 49 | } 50 | 51 | p { 52 | margin-top: 0; 53 | margin-bottom: 1rem; 54 | } 55 | 56 | abbr[title], 57 | abbr[data-original-title] { 58 | text-decoration: underline; 59 | -webkit-text-decoration: underline dotted; 60 | text-decoration: underline dotted; 61 | cursor: help; 62 | border-bottom: 0; 63 | -webkit-text-decoration-skip-ink: none; 64 | text-decoration-skip-ink: none; 65 | } 66 | 67 | address { 68 | margin-bottom: 1rem; 69 | font-style: normal; 70 | line-height: inherit; 71 | } 72 | 73 | ol, 74 | ul, 75 | dl { 76 | margin-top: 0; 77 | margin-bottom: 1rem; 78 | } 79 | 80 | ol ol, 81 | ul ul, 82 | ol ul, 83 | ul ol { 84 | margin-bottom: 0; 85 | } 86 | 87 | dt { 88 | font-weight: 700; 89 | } 90 | 91 | dd { 92 | margin-bottom: .5rem; 93 | margin-left: 0; 94 | } 95 | 96 | blockquote { 97 | margin: 0 0 1rem; 98 | } 99 | 100 | b, 101 | strong { 102 | font-weight: bolder; 103 | } 104 | 105 | small { 106 | font-size: 80%; 107 | } 108 | 109 | sub, 110 | sup { 111 | position: relative; 112 | font-size: 75%; 113 | line-height: 0; 114 | vertical-align: baseline; 115 | } 116 | 117 | sub { 118 | bottom: -.25em; 119 | } 120 | 121 | sup { 122 | top: -.5em; 123 | } 124 | 125 | a { 126 | color: #007bff; 127 | text-decoration: none; 128 | background-color: transparent; 129 | } 130 | 131 | a:hover { 132 | color: #0056b3; 133 | text-decoration: underline; 134 | } 135 | 136 | a:not([href]) { 137 | color: inherit; 138 | text-decoration: none; 139 | } 140 | 141 | a:not([href]):hover { 142 | color: inherit; 143 | text-decoration: none; 144 | } 145 | 146 | pre, 147 | code, 148 | kbd, 149 | samp { 150 | font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; 151 | font-size: 1em; 152 | } 153 | 154 | pre { 155 | margin-top: 0; 156 | margin-bottom: 1rem; 157 | overflow: auto; 158 | } 159 | 160 | figure { 161 | margin: 0 0 1rem; 162 | } 163 | 164 | img { 165 | vertical-align: middle; 166 | border-style: none; 167 | } 168 | 169 | svg { 170 | overflow: hidden; 171 | vertical-align: middle; 172 | } 173 | 174 | table { 175 | border-collapse: collapse; 176 | } 177 | 178 | caption { 179 | padding-top: 0.75rem; 180 | padding-bottom: 0.75rem; 181 | color: #6c757d; 182 | text-align: left; 183 | caption-side: bottom; 184 | } 185 | 186 | th { 187 | text-align: inherit; 188 | } 189 | 190 | label { 191 | display: inline-block; 192 | margin-bottom: 0.5rem; 193 | } 194 | 195 | button { 196 | border-radius: 0; 197 | } 198 | 199 | button:focus { 200 | outline: 1px dotted; 201 | outline: 5px auto -webkit-focus-ring-color; 202 | } 203 | 204 | input, 205 | button, 206 | select, 207 | optgroup, 208 | textarea { 209 | margin: 0; 210 | font-family: inherit; 211 | font-size: inherit; 212 | line-height: inherit; 213 | } 214 | 215 | button, 216 | input { 217 | overflow: visible; 218 | } 219 | 220 | button, 221 | select { 222 | text-transform: none; 223 | } 224 | 225 | select { 226 | word-wrap: normal; 227 | } 228 | 229 | button, 230 | [type="button"], 231 | [type="reset"], 232 | [type="submit"] { 233 | -webkit-appearance: button; 234 | } 235 | 236 | button:not(:disabled), 237 | [type="button"]:not(:disabled), 238 | [type="reset"]:not(:disabled), 239 | [type="submit"]:not(:disabled) { 240 | cursor: pointer; 241 | } 242 | 243 | button::-moz-focus-inner, 244 | [type="button"]::-moz-focus-inner, 245 | [type="reset"]::-moz-focus-inner, 246 | [type="submit"]::-moz-focus-inner { 247 | padding: 0; 248 | border-style: none; 249 | } 250 | 251 | input[type="radio"], 252 | input[type="checkbox"] { 253 | box-sizing: border-box; 254 | padding: 0; 255 | } 256 | 257 | input[type="date"], 258 | input[type="time"], 259 | input[type="datetime-local"], 260 | input[type="month"] { 261 | -webkit-appearance: listbox; 262 | } 263 | 264 | textarea { 265 | overflow: auto; 266 | resize: vertical; 267 | } 268 | 269 | fieldset { 270 | min-width: 0; 271 | padding: 0; 272 | margin: 0; 273 | border: 0; 274 | } 275 | 276 | legend { 277 | display: block; 278 | width: 100%; 279 | max-width: 100%; 280 | padding: 0; 281 | margin-bottom: .5rem; 282 | font-size: 1.5rem; 283 | line-height: inherit; 284 | color: inherit; 285 | white-space: normal; 286 | } 287 | 288 | progress { 289 | vertical-align: baseline; 290 | } 291 | 292 | [type="number"]::-webkit-inner-spin-button, 293 | [type="number"]::-webkit-outer-spin-button { 294 | height: auto; 295 | } 296 | 297 | [type="search"] { 298 | outline-offset: -2px; 299 | -webkit-appearance: none; 300 | } 301 | 302 | [type="search"]::-webkit-search-decoration { 303 | -webkit-appearance: none; 304 | } 305 | 306 | ::-webkit-file-upload-button { 307 | font: inherit; 308 | -webkit-appearance: button; 309 | } 310 | 311 | output { 312 | display: inline-block; 313 | } 314 | 315 | summary { 316 | display: list-item; 317 | cursor: pointer; 318 | } 319 | 320 | template { 321 | display: none; 322 | } 323 | 324 | [hidden] { 325 | display: none !important; 326 | } 327 | /*# sourceMappingURL=bootstrap-reboot.css.map */ -------------------------------------------------------------------------------- /collector/collector/tests/headers.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import unittest 18 | import json 19 | from collector.tests.testcase import ( 20 | RecordTestCases, 21 | classification, 22 | severity, 23 | kernel_version, 24 | record_version, 25 | machine_id, 26 | host_type, 27 | arch, 28 | build, 29 | timestamp, 30 | tid, 31 | board_name, 32 | cpu_model, 33 | bios_version, 34 | system_name, 35 | payload_version, 36 | event_id, 37 | get_record_v1, 38 | get_record_v2, 39 | get_record_v3, 40 | get_record_v4,) 41 | 42 | 43 | class TestHandlerRecordV2(RecordTestCases): 44 | """ 45 | Tests record creation for record version 2 and headers validation 46 | """ 47 | @staticmethod 48 | def get_version_records(): 49 | return get_record_v2() 50 | 51 | def test_record_created(self): 52 | headers = get_record_v2() 53 | data = "hello" 54 | response = self.client.post('/', headers=headers, data=data) 55 | self.assertTrue(response.status_code == 201, response.data.decode('utf-8')) 56 | json_resp = json.loads(response.data.decode('utf-8')) 57 | self.assertTrue(json_resp['classification'] == headers[classification]) 58 | self.assertTrue(str(json_resp['severity']) == str(headers[severity])) 59 | self.assertTrue(json_resp['kernel_version'] == headers[kernel_version]) 60 | self.assertTrue(str(json_resp['record_format_version']) == str(headers[record_version])) 61 | self.assertTrue(json_resp['machine_id'] == headers[machine_id]) 62 | self.assertTrue(json_resp['machine_type'] == headers[host_type]) 63 | self.assertTrue(json_resp['arch'] == headers[arch]) 64 | self.assertTrue(json_resp['build'] == headers[build]) 65 | self.assertTrue(json_resp['payload'] == data) 66 | 67 | def missing_header(self, header, header_name): 68 | headers = get_record_v2() 69 | del headers[header_name] 70 | response = self.client.post('/', headers=headers, data='test') 71 | self.assertTrue(response.status_code == 400, response.data.decode('utf-8')) 72 | 73 | def test_post_fail_missing_classifiction(self): 74 | self.missing_header('Classification', severity) 75 | 76 | def test_post_fail_missing_severity(self): 77 | self.missing_header('Severity', severity) 78 | 79 | def test_post_fail_missing_kernel_version(self): 80 | self.missing_header('Kernel-Version', kernel_version) 81 | 82 | def test_post_fail_missing_host_type(self): 83 | self.missing_header('Host-Type', host_type) 84 | 85 | def test_post_fail_missing_machine_id(self): 86 | self.missing_header('Machine-Id', machine_id) 87 | 88 | def test_post_fail_missing_arch(self): 89 | self.missing_header('Arch', arch) 90 | 91 | def test_post_fail_missing_build(self): 92 | self.missing_header('Build', build) 93 | 94 | def test_post_fail_missing_record_version(self): 95 | self.missing_header('Record-Format-Version', record_version) 96 | 97 | 98 | class TestHandlerRecordTransitionV2toV3(RecordTestCases): 99 | """ 100 | This test case tests a transition where the client is in record 101 | version 2 though is sending v3 headers, make sure the server 102 | understands the record as v2 and do not have problems creating it 103 | """ 104 | def test_record_created(self): 105 | headers = get_record_v3() 106 | data = "hello" 107 | response = self.client.post('/', headers=headers, data=data) 108 | self.assertTrue(response.status_code == 201, response.data.decode('utf-8')) 109 | json_resp = json.loads(response.data.decode('utf-8')) 110 | self.assertTrue(json_resp['classification'] == headers[classification]) 111 | self.assertTrue(int(json_resp['severity']) == int(headers[severity])) 112 | self.assertTrue(json_resp['kernel_version'] == headers[kernel_version]) 113 | self.assertTrue(int(json_resp['record_format_version']) == int(headers[record_version])) 114 | self.assertTrue(json_resp['machine_id'] == headers[machine_id]) 115 | self.assertTrue(json_resp['machine_type'] == headers[host_type]) 116 | self.assertTrue(json_resp['arch'] == headers[arch]) 117 | self.assertTrue(json_resp['build'] == headers[build]) 118 | 119 | 120 | class TestHandlerRecordV3(RecordTestCases): 121 | """ 122 | Test record v3 making sure to validate expected headers for v3 123 | if record version is properly set to 3. 124 | """ 125 | @staticmethod 126 | def get_version_records(): 127 | return get_record_v3() 128 | 129 | def test_post_record_v3_with_headers_v2(self): 130 | headers = get_record_v2() 131 | headers.update({record_version: 3, }) 132 | response = self.client.post('/', headers=headers, data='test') 133 | self.assertTrue(response.status_code == 400, response.data.decode('utf-8')) 134 | 135 | def test_post_record_v3(self): 136 | headers = get_record_v3() 137 | response = self.client.post('/', headers=headers, data='test') 138 | self.assertTrue(response.status_code == 201, response.data.decode('utf-8')) 139 | 140 | def test_post_missing_cpu_model(self): 141 | self.missing_header('Cpu-Model', cpu_model) 142 | 143 | def test_post_missing_board_name(self): 144 | self.missing_header('Board-Name', board_name) 145 | 146 | def test_post_missing_bios_version(self): 147 | self.missing_header('Bios-Version', bios_version) 148 | 149 | 150 | class TestHandlerRecordV4(RecordTestCases): 151 | """ 152 | Test record v4 153 | """ 154 | @staticmethod 155 | def get_version_records(): 156 | return get_record_v4() 157 | 158 | def test_post_record_v4(self): 159 | headers = get_record_v4() 160 | response = self.client.post('/', headers=headers, data='test') 161 | self.assertTrue(response.status_code == 201, response.data.decode('utf-8')) 162 | 163 | def test_post_missing_eid(self): 164 | self.missing_header('Event-Id', event_id) 165 | 166 | 167 | if __name__ == '__main__' and __package__ is None: 168 | from os import sys, path 169 | sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) 170 | unittest.main() 171 | 172 | 173 | # vi: ts=4 et sw=4 sts=4 174 | -------------------------------------------------------------------------------- /collector/collector/lib/validation.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2018 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import string 18 | from .. import app 19 | MAX_NUM_RECORDS = '1000' 20 | 21 | REQUIRED_HEADERS_V1 = ( 22 | 'Arch', 23 | 'Build', 24 | 'Creation-Timestamp', 25 | 'Classification', 26 | 'Host-Type', 27 | 'Kernel-Version', 28 | 'Machine-Id', 29 | 'Severity', 30 | 'Record-Format-Version', 31 | ) 32 | 33 | REQUIRED_HEADERS_V2 = REQUIRED_HEADERS_V1 + ( 34 | 'Payload-Format-Version', 35 | 'System-Name', 36 | 'X-Telemetry-Tid', 37 | ) 38 | 39 | REQUIRED_HEADERS_V3 = REQUIRED_HEADERS_V2 + ( 40 | 'Board-Name', 41 | 'Bios-Version', 42 | 'Cpu-Model', 43 | ) 44 | 45 | REQUIRED_HEADERS_V4 = REQUIRED_HEADERS_V3 + ( 46 | 'Event-Id', 47 | ) 48 | 49 | # see config.py for the meaning of this value 50 | TELEMETRY_ID = app.config.get("TELEMETRY_ID", "6907c830-eed9-4ce9-81ae-76daf8d88f0f") 51 | 52 | SEVERITY_VALUES = [1, 2, 3, 4] 53 | VALID_RECORD_FORMAT_VERSIONS = [1, 2, 3, 4] 54 | POSTGRES_INT_MAX = 2147483647 55 | MAXLEN_PRINTABLE = 200 56 | 57 | 58 | class InvalidUsage(Exception): 59 | status_code = 400 60 | 61 | def __init__(self, message, status_code=None, payload=None): 62 | Exception.__init__(self) 63 | self.message = message 64 | if status_code is not None: 65 | self.status_code = status_code 66 | self.payload = payload 67 | app.logger.error("InvalidUsage ({}): {}".format(self.status_code, self.message)) 68 | 69 | def to_dict(self): 70 | rv = dict(self.payload or ()) 71 | rv['message'] = self.message 72 | return rv 73 | 74 | def __str__(self): 75 | return self.message 76 | 77 | 78 | def validate_headers(headers, required_headers): 79 | """ Check for every single required header to be 80 | in the request """ 81 | req_headers = dict(headers) 82 | req_headers_keys = req_headers.keys() 83 | for header in required_headers: 84 | if header not in req_headers_keys: 85 | raise InvalidUsage("Record-Format-Version headers are invalid, {} missing".format(header), 400) 86 | return True 87 | 88 | 89 | def is_not_none(v): 90 | return v is not None 91 | 92 | 93 | def is_a_number(n): 94 | return str(n).isdigit() 95 | 96 | 97 | def value_is_printable(a_value): 98 | return all([x in string.printable for x in a_value]) 99 | 100 | 101 | def record_format_version_validation(record_format_version): 102 | return all([is_not_none(record_format_version), is_a_number(record_format_version)]) and int(record_format_version) in VALID_RECORD_FORMAT_VERSIONS 103 | 104 | 105 | def record_format_version_headers_validation(record_format_version, headers): 106 | # Validate required headers based on Record Version 107 | req_headrs = { 108 | '1': REQUIRED_HEADERS_V1, 109 | '2': REQUIRED_HEADERS_V2, 110 | '3': REQUIRED_HEADERS_V3, 111 | '4': REQUIRED_HEADERS_V4, 112 | } 113 | reqs = req_headrs.get(record_format_version, None) 114 | if reqs is None: 115 | return False 116 | return validate_headers(headers, reqs) 117 | 118 | 119 | def validation_tid_header(tid): 120 | return tid == TELEMETRY_ID 121 | 122 | 123 | def validate_severity(severity): 124 | return is_a_number(severity) and int(severity) in SEVERITY_VALUES 125 | 126 | 127 | def validate_classification(classification): 128 | return is_not_none(classification) and len(classification.split('/')) == 3 129 | 130 | 131 | def validate_machine_id(machine_id): 132 | return is_not_none(machine_id) and len(machine_id) <= 32 133 | 134 | 135 | def validate_timestamp(timestamp): 136 | return all([is_not_none(timestamp), is_a_number(timestamp)]) 137 | 138 | 139 | def validate_architecture(arch): 140 | return arch in ["armv7l", "armv6l", "aarch64", "amd64", "sparc64", "ppc64", "i686", "i386", "x86_64", "ppc"] 141 | 142 | 143 | def validate_host_type(host_type): 144 | return is_not_none(host_type) and len(host_type) < 250 145 | 146 | 147 | def validate_kernel_version(kernel_version): 148 | """ makes sure that the kernel version string has at least 2 numbers 149 | version, major and minor revision 150 | """ 151 | try: 152 | version, major_revision, _ = str(kernel_version).split('.', maxsplit=2) 153 | return all([is_a_number(x) for x in [version, major_revision]]) 154 | except ValueError as e: 155 | print(e) 156 | return False 157 | 158 | 159 | def validate_payload_format_version(payload_format_version): 160 | return int(payload_format_version) < POSTGRES_INT_MAX 161 | 162 | 163 | def validate_x_header(header_value): 164 | return is_not_none(header_value) and len(header_value) < MAXLEN_PRINTABLE and value_is_printable(header_value) 165 | 166 | 167 | def validate_created(created): 168 | return is_a_number(created) 169 | 170 | 171 | def validate_record_limit(limit): 172 | return is_a_number(limit) and limit <= MAX_NUM_RECORDS 173 | 174 | 175 | def validate_event_id(header_value): 176 | return len(header_value) == 32 and len([v for v in header_value if v in "0123456789abcdef"]) == 32 177 | 178 | 179 | def validate_header(name, value, expected=None): 180 | if expected is None: 181 | return { 182 | 'tid_header': validation_tid_header, 183 | 'record_format_version': record_format_version_validation, 184 | 'payload_format_version': validate_payload_format_version, 185 | 'severity': validate_severity, 186 | 'classification': validate_classification, 187 | 'machine_id': validate_machine_id, 188 | 'timestamp': validate_timestamp, 189 | 'architecture': validate_architecture, 190 | 'host_type': validate_host_type, 191 | 'kernel_version': validate_kernel_version, 192 | 'board_name': validate_x_header, 193 | 'cpu_model': validate_x_header, 194 | 'bios_version': validate_x_header, 195 | 'build': validate_x_header, 196 | 'event_id': validate_event_id, 197 | }.get(name, lambda x: False)(value) 198 | else: 199 | return value == expected 200 | 201 | 202 | def validate_query(name, value): 203 | return { 204 | 'id': is_a_number, 205 | 'ts_capture': is_a_number, 206 | 'severity': validate_severity, 207 | 'classification': validate_classification, 208 | 'build': validate_x_header, 209 | 'limit': validate_record_limit, 210 | 'machine_id': validate_machine_id, 211 | 'created_in_days': validate_created, 212 | 'created_in_sec': validate_created, 213 | }.get(name, lambda x: False)(value) 214 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/templates/records.html: -------------------------------------------------------------------------------- 1 | {#- 2 | # Copyright (C) 2015-2020 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | -#} 5 | 6 | {% extends "base.html" %} 7 | {% block content %} 8 |
9 |
10 | 11 | 12 |
13 |
14 | 15 |
16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | {% include "records_list.html" %} 31 | 32 |
SourceMachine IDAgeSeverityClassificationOSPayloadMore
33 | 34 | 35 |
36 | 40 |
41 | 42 | 43 | 57 | 58 | 59 | 73 | 74 | 75 | 125 | 126 | 213 | {% endblock %} 214 | 215 | {#- 216 | # vi: ft=jinja ts=8 et sw=4 sts=4 217 | #} 218 | -------------------------------------------------------------------------------- /telemetryui/telemetryui/static/vendor/jquery/core.js: -------------------------------------------------------------------------------- 1 | /* global Symbol */ 2 | // Defining this global in .eslintrc.json would create a danger of using the global 3 | // unguarded in another place, it seems safer to define global only for this module 4 | 5 | define( [ 6 | "./var/arr", 7 | "./var/document", 8 | "./var/getProto", 9 | "./var/slice", 10 | "./var/concat", 11 | "./var/push", 12 | "./var/indexOf", 13 | "./var/class2type", 14 | "./var/toString", 15 | "./var/hasOwn", 16 | "./var/fnToString", 17 | "./var/ObjectFunctionString", 18 | "./var/support", 19 | "./var/isFunction", 20 | "./var/isWindow", 21 | "./core/DOMEval", 22 | "./core/toType" 23 | ], function( arr, document, getProto, slice, concat, push, indexOf, 24 | class2type, toString, hasOwn, fnToString, ObjectFunctionString, 25 | support, isFunction, isWindow, DOMEval, toType ) { 26 | 27 | "use strict"; 28 | 29 | var 30 | version = "3.4.1", 31 | 32 | // Define a local copy of jQuery 33 | jQuery = function( selector, context ) { 34 | 35 | // The jQuery object is actually just the init constructor 'enhanced' 36 | // Need init if jQuery is called (just allow error to be thrown if not included) 37 | return new jQuery.fn.init( selector, context ); 38 | }, 39 | 40 | // Support: Android <=4.0 only 41 | // Make sure we trim BOM and NBSP 42 | rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g; 43 | 44 | jQuery.fn = jQuery.prototype = { 45 | 46 | // The current version of jQuery being used 47 | jquery: version, 48 | 49 | constructor: jQuery, 50 | 51 | // The default length of a jQuery object is 0 52 | length: 0, 53 | 54 | toArray: function() { 55 | return slice.call( this ); 56 | }, 57 | 58 | // Get the Nth element in the matched element set OR 59 | // Get the whole matched element set as a clean array 60 | get: function( num ) { 61 | 62 | // Return all the elements in a clean array 63 | if ( num == null ) { 64 | return slice.call( this ); 65 | } 66 | 67 | // Return just the one element from the set 68 | return num < 0 ? this[ num + this.length ] : this[ num ]; 69 | }, 70 | 71 | // Take an array of elements and push it onto the stack 72 | // (returning the new matched element set) 73 | pushStack: function( elems ) { 74 | 75 | // Build a new jQuery matched element set 76 | var ret = jQuery.merge( this.constructor(), elems ); 77 | 78 | // Add the old object onto the stack (as a reference) 79 | ret.prevObject = this; 80 | 81 | // Return the newly-formed element set 82 | return ret; 83 | }, 84 | 85 | // Execute a callback for every element in the matched set. 86 | each: function( callback ) { 87 | return jQuery.each( this, callback ); 88 | }, 89 | 90 | map: function( callback ) { 91 | return this.pushStack( jQuery.map( this, function( elem, i ) { 92 | return callback.call( elem, i, elem ); 93 | } ) ); 94 | }, 95 | 96 | slice: function() { 97 | return this.pushStack( slice.apply( this, arguments ) ); 98 | }, 99 | 100 | first: function() { 101 | return this.eq( 0 ); 102 | }, 103 | 104 | last: function() { 105 | return this.eq( -1 ); 106 | }, 107 | 108 | eq: function( i ) { 109 | var len = this.length, 110 | j = +i + ( i < 0 ? len : 0 ); 111 | return this.pushStack( j >= 0 && j < len ? [ this[ j ] ] : [] ); 112 | }, 113 | 114 | end: function() { 115 | return this.prevObject || this.constructor(); 116 | }, 117 | 118 | // For internal use only. 119 | // Behaves like an Array's method, not like a jQuery method. 120 | push: push, 121 | sort: arr.sort, 122 | splice: arr.splice 123 | }; 124 | 125 | jQuery.extend = jQuery.fn.extend = function() { 126 | var options, name, src, copy, copyIsArray, clone, 127 | target = arguments[ 0 ] || {}, 128 | i = 1, 129 | length = arguments.length, 130 | deep = false; 131 | 132 | // Handle a deep copy situation 133 | if ( typeof target === "boolean" ) { 134 | deep = target; 135 | 136 | // Skip the boolean and the target 137 | target = arguments[ i ] || {}; 138 | i++; 139 | } 140 | 141 | // Handle case when target is a string or something (possible in deep copy) 142 | if ( typeof target !== "object" && !isFunction( target ) ) { 143 | target = {}; 144 | } 145 | 146 | // Extend jQuery itself if only one argument is passed 147 | if ( i === length ) { 148 | target = this; 149 | i--; 150 | } 151 | 152 | for ( ; i < length; i++ ) { 153 | 154 | // Only deal with non-null/undefined values 155 | if ( ( options = arguments[ i ] ) != null ) { 156 | 157 | // Extend the base object 158 | for ( name in options ) { 159 | copy = options[ name ]; 160 | 161 | // Prevent Object.prototype pollution 162 | // Prevent never-ending loop 163 | if ( name === "__proto__" || target === copy ) { 164 | continue; 165 | } 166 | 167 | // Recurse if we're merging plain objects or arrays 168 | if ( deep && copy && ( jQuery.isPlainObject( copy ) || 169 | ( copyIsArray = Array.isArray( copy ) ) ) ) { 170 | src = target[ name ]; 171 | 172 | // Ensure proper type for the source value 173 | if ( copyIsArray && !Array.isArray( src ) ) { 174 | clone = []; 175 | } else if ( !copyIsArray && !jQuery.isPlainObject( src ) ) { 176 | clone = {}; 177 | } else { 178 | clone = src; 179 | } 180 | copyIsArray = false; 181 | 182 | // Never move original objects, clone them 183 | target[ name ] = jQuery.extend( deep, clone, copy ); 184 | 185 | // Don't bring in undefined values 186 | } else if ( copy !== undefined ) { 187 | target[ name ] = copy; 188 | } 189 | } 190 | } 191 | } 192 | 193 | // Return the modified object 194 | return target; 195 | }; 196 | 197 | jQuery.extend( { 198 | 199 | // Unique for each copy of jQuery on the page 200 | expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), 201 | 202 | // Assume jQuery is ready without the ready module 203 | isReady: true, 204 | 205 | error: function( msg ) { 206 | throw new Error( msg ); 207 | }, 208 | 209 | noop: function() {}, 210 | 211 | isPlainObject: function( obj ) { 212 | var proto, Ctor; 213 | 214 | // Detect obvious negatives 215 | // Use toString instead of jQuery.type to catch host objects 216 | if ( !obj || toString.call( obj ) !== "[object Object]" ) { 217 | return false; 218 | } 219 | 220 | proto = getProto( obj ); 221 | 222 | // Objects with no prototype (e.g., `Object.create( null )`) are plain 223 | if ( !proto ) { 224 | return true; 225 | } 226 | 227 | // Objects with prototype are plain iff they were constructed by a global Object function 228 | Ctor = hasOwn.call( proto, "constructor" ) && proto.constructor; 229 | return typeof Ctor === "function" && fnToString.call( Ctor ) === ObjectFunctionString; 230 | }, 231 | 232 | isEmptyObject: function( obj ) { 233 | var name; 234 | 235 | for ( name in obj ) { 236 | return false; 237 | } 238 | return true; 239 | }, 240 | 241 | // Evaluates a script in a global context 242 | globalEval: function( code, options ) { 243 | DOMEval( code, { nonce: options && options.nonce } ); 244 | }, 245 | 246 | each: function( obj, callback ) { 247 | var length, i = 0; 248 | 249 | if ( isArrayLike( obj ) ) { 250 | length = obj.length; 251 | for ( ; i < length; i++ ) { 252 | if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { 253 | break; 254 | } 255 | } 256 | } else { 257 | for ( i in obj ) { 258 | if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { 259 | break; 260 | } 261 | } 262 | } 263 | 264 | return obj; 265 | }, 266 | 267 | // Support: Android <=4.0 only 268 | trim: function( text ) { 269 | return text == null ? 270 | "" : 271 | ( text + "" ).replace( rtrim, "" ); 272 | }, 273 | 274 | // results is for internal usage only 275 | makeArray: function( arr, results ) { 276 | var ret = results || []; 277 | 278 | if ( arr != null ) { 279 | if ( isArrayLike( Object( arr ) ) ) { 280 | jQuery.merge( ret, 281 | typeof arr === "string" ? 282 | [ arr ] : arr 283 | ); 284 | } else { 285 | push.call( ret, arr ); 286 | } 287 | } 288 | 289 | return ret; 290 | }, 291 | 292 | inArray: function( elem, arr, i ) { 293 | return arr == null ? -1 : indexOf.call( arr, elem, i ); 294 | }, 295 | 296 | // Support: Android <=4.0 only, PhantomJS 1 only 297 | // push.apply(_, arraylike) throws on ancient WebKit 298 | merge: function( first, second ) { 299 | var len = +second.length, 300 | j = 0, 301 | i = first.length; 302 | 303 | for ( ; j < len; j++ ) { 304 | first[ i++ ] = second[ j ]; 305 | } 306 | 307 | first.length = i; 308 | 309 | return first; 310 | }, 311 | 312 | grep: function( elems, callback, invert ) { 313 | var callbackInverse, 314 | matches = [], 315 | i = 0, 316 | length = elems.length, 317 | callbackExpect = !invert; 318 | 319 | // Go through the array, only saving the items 320 | // that pass the validator function 321 | for ( ; i < length; i++ ) { 322 | callbackInverse = !callback( elems[ i ], i ); 323 | if ( callbackInverse !== callbackExpect ) { 324 | matches.push( elems[ i ] ); 325 | } 326 | } 327 | 328 | return matches; 329 | }, 330 | 331 | // arg is for internal usage only 332 | map: function( elems, callback, arg ) { 333 | var length, value, 334 | i = 0, 335 | ret = []; 336 | 337 | // Go through the array, translating each of the items to their new values 338 | if ( isArrayLike( elems ) ) { 339 | length = elems.length; 340 | for ( ; i < length; i++ ) { 341 | value = callback( elems[ i ], i, arg ); 342 | 343 | if ( value != null ) { 344 | ret.push( value ); 345 | } 346 | } 347 | 348 | // Go through every key on the object, 349 | } else { 350 | for ( i in elems ) { 351 | value = callback( elems[ i ], i, arg ); 352 | 353 | if ( value != null ) { 354 | ret.push( value ); 355 | } 356 | } 357 | } 358 | 359 | // Flatten any nested arrays 360 | return concat.apply( [], ret ); 361 | }, 362 | 363 | // A global GUID counter for objects 364 | guid: 1, 365 | 366 | // jQuery.support is not used in Core but other projects attach their 367 | // properties to it so it needs to exist. 368 | support: support 369 | } ); 370 | 371 | if ( typeof Symbol === "function" ) { 372 | jQuery.fn[ Symbol.iterator ] = arr[ Symbol.iterator ]; 373 | } 374 | 375 | // Populate the class2type map 376 | jQuery.each( "Boolean Number String Function Array Date RegExp Object Error Symbol".split( " " ), 377 | function( i, name ) { 378 | class2type[ "[object " + name + "]" ] = name.toLowerCase(); 379 | } ); 380 | 381 | function isArrayLike( obj ) { 382 | 383 | // Support: real iOS 8.2 only (not reproducible in simulator) 384 | // `in` check used to prevent JIT error (gh-2145) 385 | // hasOwn isn't used here due to false negatives 386 | // regarding Nodelist length in IE 387 | var length = !!obj && "length" in obj && obj.length, 388 | type = toType( obj ); 389 | 390 | if ( isFunction( obj ) || isWindow( obj ) ) { 391 | return false; 392 | } 393 | 394 | return type === "array" || length === 0 || 395 | typeof length === "number" && length > 0 && ( length - 1 ) in obj; 396 | } 397 | 398 | return jQuery; 399 | } ); 400 | -------------------------------------------------------------------------------- /collector/collector/report_handler.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015-2017 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import re 18 | import time 19 | import datetime 20 | import importlib 21 | from flask import request 22 | from flask import jsonify 23 | from flask import redirect 24 | from .lib.validation import ( 25 | validate_header, 26 | validate_query, 27 | MAX_NUM_RECORDS, 28 | record_format_version_headers_validation, 29 | InvalidUsage) 30 | from .model import Record 31 | from .purge import * 32 | 33 | tm_version_regex = re.compile("^[0-9]+\.[0-9]+$") 34 | client_id_regex = re.compile( 35 | "^[0-9]+[ \t]+[-/.:_+*a-zA-Z0-9]+[ \t]+[-/.:_+*a-zA-Z0-9]+$") 36 | ts_v3_regex = re.compile("^[0-9]+$") 37 | # FIXME: configurable limits 38 | max_payload_len_inline = 30 * 1024 # 30k 39 | max_payload_len = 300 * 1024 # 300k 40 | MAX_INTERVAL_SEC = 24 * 60 * 60 * 30 # 30 days in seconds 41 | 42 | 43 | @app.errorhandler(InvalidUsage) 44 | def handle_invalid_usage(error): 45 | response = jsonify(error.to_dict()) 46 | response.status_code = error.status_code 47 | return response 48 | 49 | 50 | @app.before_request 51 | def before_request(): 52 | headers = request.headers 53 | payload = request.data 54 | app.logger.info('\t'.join([ 55 | datetime.datetime.today().ctime(), 56 | request.method, 57 | request.url, 58 | str(request.data), 59 | ', '.join([': '.join(x) for x in request.headers])]) 60 | ) 61 | 62 | 63 | def clean_build_n_value(build): 64 | # It is common to see the build numbers quoted in os-release. We don't 65 | # need the quotes, so strip them from the semantic value. 66 | _build = build.replace('"', '').replace("'", "") 67 | return _build 68 | 69 | 70 | def validate_header_value(header_value, record_name, err_msg): 71 | try: 72 | if validate_header(record_name, header_value) is True: 73 | return header_value 74 | except Exception as e: 75 | err_msg = "Error parsing {}, {}".format(record_name, e) 76 | raise InvalidUsage(err_msg, 400) 77 | 78 | 79 | def validate_record_v3_headers(board_name, cpu_model, bios_version): 80 | validate_header_value(board_name, "board_name", "board name is invalid") 81 | validate_header_value(cpu_model, "cpu_model", "cpu model is invalid") 82 | validate_header_value(bios_version, "bios_version", "BIOS version is invalid") 83 | 84 | 85 | def collector_post_handler(): 86 | 87 | # The collector only accepts records with the configured TID value. 88 | # Make sure the TID in the collector config.py matches the TID 89 | # configured for telemetrics-client on the systems from which this 90 | # collector receives records. 91 | tid_header = request.headers.get("X-Telemetry-TID") 92 | validate_header_value(tid_header, "tid_header", "Telemetry ID mismatch") 93 | 94 | record_format_version = request.headers.get("Record-Format-Version") 95 | validate_header_value(record_format_version, "record_format_version", "Record-Format-Version is invalid") 96 | 97 | record_format_version_headers_validation(record_format_version, request.headers) 98 | 99 | severity = request.headers.get("Severity") 100 | validate_header_value(severity, "severity", "severity value is out of range") 101 | 102 | classification = request.headers.get("Classification") 103 | validate_header_value(classification, "classification", "Classification value is invalid") 104 | 105 | machine_id = request.headers.get("Machine-Id") 106 | validate_header_value(machine_id, "machine_id", "Machine id value is invalid") 107 | 108 | timestamp = request.headers.get("Creation-Timestamp") 109 | validate_header_value(timestamp, "timestamp", "timestamp is invalid") 110 | 111 | ts_capture = int(timestamp) 112 | ts_reception = time.time() 113 | 114 | architecture = request.headers.get("Arch") 115 | validate_header_value(architecture, "architecture", "architecture is invalid") 116 | 117 | host_type = request.headers.get("Host-Type") 118 | validate_header_value(host_type, "host_type", "host type is invalid") 119 | 120 | kernel_version = request.headers.get("Kernel-Version") 121 | validate_header_value(kernel_version, "kernel_version", "kernel version is invalid") 122 | 123 | # Record V3 format headers 124 | board_name = "N/A" 125 | cpu_model = "N/A" 126 | bios_version = "N/A" 127 | if record_format_version >= '3': 128 | board_name = request.headers.get("Board-Name") 129 | cpu_model = request.headers.get("Cpu-Model") 130 | bios_version = request.headers.get("Bios-Version") 131 | validate_record_v3_headers(board_name, cpu_model, bios_version) 132 | 133 | # Record V4 format headers 134 | event_id = "N/A" 135 | if record_format_version >= '4': 136 | event_id = request.headers.get("Event-Id") 137 | validate_header_value(event_id, "event_id", "Event id is invalid") 138 | 139 | os_name = request.headers.get('System-Name') 140 | os_name = os_name.replace('"', '').replace("'", "") 141 | build = request.headers.get('Build') 142 | build = clean_build_n_value(build) 143 | # The build number is stored as a string in the database, but if this 144 | # record is from a Clear Linux OS system, only accept an integer. 145 | # Otherwise, loosen the restriction to the characters listed for 146 | # VERSION_ID in os-release(5) in addition to the capital letters A-Z. 147 | if os_name == 'clear-linux-os': 148 | build_regex = re.compile(r"^[0-9]+$") 149 | if not build_regex.match(build): 150 | raise InvalidUsage("Clear Linux OS build version has invalid characters") 151 | else: 152 | build_regex = re.compile(r"^[-_a-zA-Z0-9.]+$") 153 | if not build_regex.match(build): 154 | raise InvalidUsage("Build version has invalid characters") 155 | 156 | payload_format_version = request.headers.get("Payload-Format-Version") 157 | validate_header_value(payload_format_version, "payload_format_version", 158 | "Payload format version outside of range supported") 159 | 160 | external = request.headers.get('X-CLR-External') 161 | if external and external == "true": 162 | external = True 163 | else: 164 | external = False 165 | 166 | try: 167 | # prefer UTF-8, if possible 168 | payload = request.data.decode('utf-8') 169 | except UnicodeError: 170 | # fallback to Latin-1, since it accepts all byte values 171 | payload = request.data.decode('latin-1') 172 | 173 | db_rec = Record.create(machine_id, host_type, severity, classification, build, architecture, kernel_version, 174 | record_format_version, ts_capture, ts_reception, payload_format_version, os_name, 175 | board_name, bios_version, cpu_model, event_id, external, payload) 176 | 177 | resp = jsonify(db_rec.to_dict()) 178 | resp.status_code = 201 179 | return resp 180 | 181 | 182 | def validate_query_value(query_value, query_name, err_msg): 183 | try: 184 | if validate_query(query_name, query_value) is True: 185 | return query_value 186 | except Exception as e: 187 | err_msg = "Error parsing {}, {}".format(query_name, e) 188 | raise InvalidUsage(err_msg, 400) 189 | 190 | 191 | def get_records_api_handler(): 192 | 193 | # Validate query parameters correctness 194 | severity = request.args.get("severity", None) 195 | if severity is not None: 196 | validate_query_value(severity, "severity", "Severity should be a numeric value") 197 | 198 | classification = request.args.get('classification', None) 199 | if classification is not None: 200 | validate_query_value(classification, "classification", "Classification value is invalid") 201 | 202 | build = request.args.get('build', None) 203 | if build is not None: 204 | build = clean_build_n_value(build) 205 | validate_query_value(build, "build", "Build value is invalid") 206 | 207 | machine_id = request.args.get('machine_id', None) 208 | if machine_id is not None: 209 | validate_query_value(machine_id, "machine_id", "Machine id value is invalid") 210 | 211 | created_in_days = request.args.get('created_in_days', None) 212 | if created_in_days is not None: 213 | validate_query_value(created_in_days, "created_in_days", "Created (in days) value is invalid") 214 | 215 | created_in_sec = request.args.get('created_in_sec', None) 216 | if created_in_sec is not None: 217 | validate_query_value(created_in_sec, "created_in_sec", "Created (in seconds) value is invalid") 218 | 219 | from_id = request.args.get('from_id', None) 220 | if from_id is not None: 221 | validate_query_value(from_id, "id", "Provided record id value is invalid") 222 | 223 | ts_capture = request.args.get('ts_capture', None) 224 | if ts_capture is not None: 225 | validate_query_value(ts_capture, "ts_capture", "Time stamp from record capture") 226 | 227 | limit = request.args.get('limit', MAX_NUM_RECORDS) 228 | if limit != MAX_NUM_RECORDS: 229 | validate_query_value(limit, "limit", "Record limit value is invalid") 230 | 231 | # Transform days interval to seconds 232 | if created_in_days is not None: 233 | created_in_days = int(created_in_days) 234 | interval_sec = 24 * 60 * 60 * created_in_days 235 | elif created_in_sec is not None: 236 | interval_sec = int(created_in_sec) 237 | else: 238 | interval_sec = MAX_INTERVAL_SEC 239 | 240 | if interval_sec is not None and interval_sec > MAX_INTERVAL_SEC: 241 | interval_sec = MAX_INTERVAL_SEC 242 | 243 | records = Record.query_records(build, classification, severity, machine_id, limit, interval_sec, 244 | from_id=from_id, ts_capture=ts_capture) 245 | record_list = [Record.to_dict(rec) for rec in records] 246 | 247 | return jsonify(records=record_list) 248 | 249 | # ########## Routes ########### 250 | 251 | 252 | @app.route("/", methods=['GET', 'POST']) 253 | @app.route("/v2/collector", methods=['GET', 'POST']) 254 | def handler(): 255 | if request.method == 'POST': 256 | return collector_post_handler() 257 | else: 258 | return redirect("/telemetryui", code=302) 259 | 260 | 261 | @app.route("/api/records", methods=['GET']) 262 | def records_api_handler(): 263 | """ 264 | query filters for simple query: 265 | classification 266 | severity 267 | build 268 | machine_id 269 | created_in_days - records created after given days 270 | created_in_sec - records created after given seconds 271 | 272 | TODO: Advanced query with pagination and following parameters? 273 | client_created_after - timestamp 274 | client_created_before - timestamp 275 | server_created_after - timestamp 276 | server_created_before - timestamp 277 | 278 | """ 279 | return get_records_api_handler() 280 | 281 | 282 | def verify_parser_module(parser_module): 283 | 284 | if getattr(parser_module, 'CLASSIFICATIONS', None) is None: 285 | raise PlugablePayloadParserException('Parser {} should have a CLASSIFICATIONS field') 286 | 287 | if getattr(parser_module, 'parse_payload', None) is None: 288 | raise PlugablePayloadParserException('Parser {} should have a parse_payload method') 289 | 290 | # vi: ts=4 et sw=4 sts=4 291 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## DISCONTINUATION OF PROJECT. 2 | 3 | This project will no longer be maintained by Intel. 4 | 5 | Intel will not provide or guarantee development of or support for this project, including but not limited to, maintenance, bug fixes, new releases or updates. Patches to this project are no longer accepted by Intel. If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the community, please create your own fork of the project. 6 | 7 | Contact: webadmin@linux.intel.com 8 | # telemetrics-backend 9 | 10 | ## Overview 11 | 12 | This project provides the server-side component of a complete telemetrics 13 | (telemetry + analytics) solution for Linux-based operating systems. The 14 | client-side component source repository lives at 15 | https://github.com/clearlinux/telemetrics-client. 16 | 17 | It consists of a Flask application `telemetryui`, 18 | that exposes several views to visualize the telemetry data. The `telemetryui` 19 | app also provides a REST API to perform queries on the data. 20 | 21 | The Flask apps have several dependencies listed [here](services/collector/requirements.txt) 22 | and [here](deployments/services/telemetryui/requirements.txt). The testing infrastructure 23 | is described by [this](deployments/docker-compose.yaml) docker-compose.yaml and production by 24 | [this](deployments/docker-compose.prod.yaml) docker-compose.prod.yaml 25 | 26 | ## Security considerations 27 | 28 | The telemetrics-backend was written with a particular deployment scenario in 29 | mind: internal LAN (e.g. a corporate network not exposed to the public internet). 30 | Also, no identity management, user authentication, or role-based access controls 31 | have yet been implemented for the applications. 32 | 33 | To control access to the applications, it is recommended that system 34 | administrators leverage web server authentication. 35 | 36 | To enable HTTPS connections replace the placeholders files [here](services/nginx/telemetry.cert.pem) 37 | and [here](services/nginx/telemetry.key.pem) with a certificate and private key 38 | for your server. In addition uncomment lines 3, 9, and 10 in the [sites](services/nginx/sites.conf). 39 | configuration file. 40 | 41 | ## Deployment 42 | 43 | The application is containerized to simplify the deployment. 44 | 45 | ``` 46 | # checkout latest tag source 47 | git clone --branch latest https://github.com/clearlinux/telemetrics-backend.git 48 | cd telemetrics-backend 49 | ``` 50 | 51 | For a deployment in production make sure to update the value for POSTGRES_PASSWORD otherwise 52 | the build step will fail. 53 | 54 | ``` 55 | # update services/production.env 56 | vi services/production.env 57 | 58 | # build production environment 59 | sudo -E docker-compose --file ./deployments/docker-compose.prod.yaml build --force-rm 60 | ``` 61 | 62 | Once the images are build successfully the environment can be started using: 63 | 64 | ``` 65 | # start environment on the background 66 | sudo -E docker-compose --file ./deployments/docker-compose.prod.yaml up --detach 67 | ``` 68 | 69 | ### Deploying as systemd service 70 | 71 | To deploy the environment as a systemd service use the following example: 72 | 73 | ``` 74 | [Unit] 75 | Description=Telemetry Backend 76 | Requires=docker.service 77 | After=docker.service 78 | 79 | [Service] 80 | Restart=always 81 | WorkingDirectory=/srv/telemetrics-backend 82 | ExecStart=/usr/local/bin/docker-compose --file deployments/docker-compose.prod.yaml up 83 | ExecStop=/usr/local/bin/docker-compose --file deployments/docker-compose.prod.yaml down -v 84 | 85 | [Install] 86 | WantedBy=multi-user.target 87 | ``` 88 | 89 | 90 | ## `telemetryui` views 91 | 92 | The `telemetryui` app is a web app that exposes several views to visualize the 93 | telemetry data and also provides a REST API to perform queries on record data. 94 | 95 | The current views are: 96 | 97 | * Records view - a paginated list of all records in the `telemetry` database that 98 | have been accepted by the `collector`. The records are presented in tabular 99 | format and the columns map to select fields from the `records` database table. 100 | At the top of the view, an HTML form can be used for "advanced searches", 101 | filtering the list of records to display. 102 | 103 | * Builds view - a basic column chart that displays how many records have been 104 | received for each OS build. Note that the view is optimized for Clear Linux 105 | OS, since the chart only displays data for records when their build numbers are 106 | integers. For example, records with non-integer build numbers, like "16.04" for 107 | Ubuntu, are not displayed in this view. 108 | 109 | * Stats view - two pie charts displaying the statistical breakdown of 110 | classifications and platforms for all records in the database. The 111 | "classification" field is used to identify the type of record sent by a 112 | specific client probe; classifications use the format DOMAIN/PROBE/REST, where 113 | DOMAIN is the vendor of the probe, PROBE is the probe name, and REST is a 114 | probe-defined field to classify what is contained in the payload. The 115 | "platform" field is a formatted string, 116 | `"sys_vendor|product_name|product_version"`, where the values are taken from 117 | files with those names in the `/sys/class/dmi/id/` directory; if any of these 118 | files are empty or contain only space characters, the client library 119 | substitutes "blank" for their value. 120 | 121 | * Crashes view - features a table displaying the top 10 crash reports from 122 | crash records received in the past week. It only consumes records from the 123 | telemetrics-client `crashprobe`, which extracts backtrace information from core 124 | files and creates/sends telemetry records containing this data. The crash 125 | reports are grouped by "guilties"; a guilty is a frame from a crash backtrace 126 | chosen as the best candidate for the cause of the crash. The logic for 127 | determining crash record guilty frames accepts user input; the user can 128 | identify which frames in a backtrace are never guilty. 129 | 130 | * MCE view - charts that display MCE (machine check exception) data from a 131 | patched version of `mcelog` that uses libtelemetry to create and send 132 | telemetry records. The mcelog patch is available from 133 | https://github.com/clearlinux-pkgs/mcelog. 134 | 135 | * Thermal view - similar to the MCE view, but it only displays a chart for MCE 136 | Thermal event records, also received from the patched `mcelog`. 137 | 138 | * Population view - contains column charts that display the number of unique 139 | systems that are running each version of an OS over a specific range of time. 140 | The "uniqueness" of a system is determined by its "machine ID" field, managed 141 | by the telemetrics-client daemon, which by default rotates the value every 3 142 | days. Thus, the analysis presented in this view is *fuzzy* due to the machine 143 | ID rotation. 144 | 145 | ## Custom `telemetryui` views 146 | 147 | To provided users with an extensible framework a concept of "plugin views" was 148 | implemented to add views without the need to make changes to the core of the 149 | application. To read more about [plugin view](/telemetryui/telemetryui/plugins/README.md) 150 | go to relevant documentation. 151 | 152 | ## Using the REST API 153 | 154 | A REST API for querying records is available at "/api/records". The API returns 155 | a JSON response that contains a list of records keyed on "records". 156 | 157 | Several parameters are available for filtering queries, similar to the filters 158 | available through the `telemetryui` Records view. 159 | 160 | * `classification`: The classification of the record. Right now this is 161 | restricted at 140 characters. If a classification with more that 140 162 | characters is supplied as a query parameter, an HTTP response 400 is sent back. 163 | * `severity`: The severity of the record. Restricted to integer value. 164 | * `machine_id`: The id of the machine where this record was generated on. 165 | Should be 32 characters in length. 166 | * `build`: The build on which the record was generated. Restricted to 256 167 | characters. 168 | * `created_in_days`: This should be an integer value. It causes the query to 169 | return records created after the last given days. Note: the server timestamp 170 | is used as a reference point. 171 | * `created_in_sec`: This should be an integer again. If returns the records 172 | created after the last given seconds. This is used only if the previous 173 | parameter is absent. Note: the server timestamp is used as a reference point. 174 | * `limit`: The maximum number of records to be returned. 175 | 176 | ### Example queries 177 | 178 | To query for records, simply make a GET call to the endpoint. 179 | 180 | * `GET /api/records` - Returns a maximum of 10000 most recent records in the 181 | backend database ordered by the record id (descending). 182 | * `GET /api/records?classification=org.clearlinux%2Fkernel%2Fwarning&severity=2&build=2980&created_in_sec=5&limit=100` - Returns at most 100 records with the classification 183 | "org.clearlinux/kernel/warning", severity 2, build 2980, and created in the 184 | last 5 seconds. As shown the query parameters need to be [URL encoded](https://en.wikipedia.org/wiki/Percent-encoding). 185 | 186 | ### Response object 187 | 188 | The response is a JSON object that contains a list of objects keyed on 189 | "records". This list is empty in case no records match the query parameters. 190 | Example response: 191 | 192 | ``` 193 | { 194 | "records": [ 195 | { 196 | "arch": "x86_64", 197 | "build": "2980", 198 | "classification": "org.clearlinux/hello/world", 199 | "kernel_version": "4.2.0-120", 200 | "machine_id": "66c196ce4222dd761470da5e7e35f6f1", 201 | "machine_type": "blank|blank|blank", 202 | "payload": "hello\n\n", 203 | "record_format_version": 2, 204 | "severity": 1, 205 | "ts_capture": "2015-09-30 00:39:35 UTC", 206 | "ts_reception": "2015-09-30 00:56:59 UTC" 207 | }, 208 | { 209 | "arch": "x86_64", 210 | "build": "2980", 211 | "classification": "org.clearlinux/hello/world", 212 | "kernel_version": "4.2.0-120", 213 | "machine_id": "66c196ce4222dd761470da5e7e35f6f1", 214 | "machine_type": "blank|blank|blank", 215 | "payload": "hello\n", 216 | "record_format_version": 2, 217 | "severity": 1, 218 | "ts_capture": "2015-09-30 00:36:22 UTC", 219 | "ts_reception": "2015-09-30 00:38:45 UTC" 220 | } 221 | ] 222 | } 223 | ``` 224 | 225 | ## Creating new database migrations 226 | 227 | Database migrations are managed using 228 | [Flask-Migrate](https://flask-migrate.readthedocs.io/en/latest/). Upon initial 229 | install of telemetrics-backend, the first migration will be applied, and any 230 | additional migrations in the `telemetryui/migrations/versions/` directory will 231 | be applied in sequence and upgrade the `telemetry` schema to the latest version. 232 | 233 | Any new migration from a new realease will be applied when the environment is 234 | started, this applies for both production and testing configurations. 235 | 236 | ## Development 237 | 238 | ``` 239 | # Build 240 | sudo -E docker-compose --file deployments/docker-compose.yaml build --force-rm 241 | 242 | # Launch 243 | sudo -E docker-compose --file deployments/docker-compose.yaml up 244 | ``` 245 | 246 | ## Software License 247 | 248 | The telemetrics-backend project is licensed under the Apache License, Version 249 | 2.0. The full license text is found in the LICENSE file, and individual source 250 | files contain the boilerplate notice described in the appendix of the LICENSE 251 | file. 252 | 253 | 254 | ## Security Disclosures 255 | 256 | To report a security issue or receive security advisories please follow procedures 257 | in this [link](https://01.org/security). 258 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | --------------------------------------------------------------------------------