├── repoxplorer ├── __init__.py ├── indexer │ ├── __init__.py │ └── git │ │ ├── __init__.py │ │ └── metadata_re.py ├── trends │ ├── __init__.py │ └── commits.py ├── controllers │ ├── __init__.py │ ├── status.py │ ├── metadata.py │ ├── renderers.py │ ├── root.py │ ├── search.py │ ├── tags.py │ ├── projects.py │ ├── histo.py │ ├── groups.py │ ├── commits.py │ ├── infos.py │ ├── users.py │ ├── utils.py │ └── tops.py ├── tests │ ├── __init__.py │ ├── config.py │ ├── test_tags.py │ ├── test_renderer.py │ ├── test_yamlbackend.py │ ├── test_trends.py │ ├── test_users.py │ └── gitshow.sample ├── model │ └── __init__.py ├── version.py ├── exceptions.py ├── app.py ├── public │ ├── css │ │ └── repoxplorer.css │ ├── projects.html │ ├── index.html │ ├── contributors.html │ ├── groups.html │ ├── home.html │ ├── contributor.html │ ├── group.html │ └── project.html ├── index │ ├── tags.py │ ├── __init__.py │ ├── yamlbackend.py │ └── users.py └── auth │ └── __init__.py ├── MANIFEST.in ├── playbooks ├── README.md ├── tox.yaml ├── deps-install.yaml ├── elastic6-install.yaml ├── elastic5-install.yaml └── elastic7-install.yaml ├── imgs ├── repoxplorer-cont.jpg ├── repoxplorer-plist.jpg └── repoxplorer-pstats.jpg ├── .gitreview ├── .zuul.yaml ├── bin ├── repoxplorer-git-credentials-helper ├── start-ui.sh ├── wipe_db.sh ├── repoxplorer-config-validate ├── bench │ └── fake-commit-gen.py ├── repoxplorer-github-organization ├── repoxplorer-indexer └── repoxplorer-fetch-web-assets ├── .gitignore ├── etc ├── groups.yaml ├── repoxplorer.service ├── repoxplorer-webui.service └── idents.yaml ├── requirements.txt ├── test-requirements.txt ├── docker ├── aoi-master │ ├── elasticsearch.repo │ ├── supervisord.conf │ ├── Dockerfile │ └── README └── aoi-last-stable │ ├── elasticsearch.repo │ ├── supervisord.conf │ ├── Dockerfile │ └── README ├── docker-compose.yaml ├── docker-compose-master.yaml ├── tox.ini ├── setup.py ├── config.py └── LICENSE /repoxplorer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /repoxplorer/indexer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /repoxplorer/trends/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /repoxplorer/controllers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /repoxplorer/indexer/git/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md config.py 2 | recursive-include repoxplorer/public * 3 | -------------------------------------------------------------------------------- /playbooks/README.md: -------------------------------------------------------------------------------- 1 | This directory contains only playbooks for repoxplorer testing in the CI. 2 | -------------------------------------------------------------------------------- /imgs/repoxplorer-cont.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/morucci/repoxplorer/HEAD/imgs/repoxplorer-cont.jpg -------------------------------------------------------------------------------- /imgs/repoxplorer-plist.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/morucci/repoxplorer/HEAD/imgs/repoxplorer-plist.jpg -------------------------------------------------------------------------------- /imgs/repoxplorer-pstats.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/morucci/repoxplorer/HEAD/imgs/repoxplorer-pstats.jpg -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=softwarefactory-project.io 3 | port=29418 4 | project=repoxplorer 5 | defaultbranch=master 6 | -------------------------------------------------------------------------------- /.zuul.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - project: 3 | check: 4 | jobs: 5 | - noop 6 | gate: 7 | jobs: 8 | - noop 9 | -------------------------------------------------------------------------------- /bin/repoxplorer-git-credentials-helper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | print("username=fakeusername") 4 | print("password=fakepassword") 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .tox 2 | *.pyc 3 | *.swp 4 | build 5 | dist 6 | *.egg-info 7 | .venv 8 | MANIFEST 9 | .coverage 10 | htmlcov 11 | docker/conf 12 | docker-data 13 | -------------------------------------------------------------------------------- /bin/start-ui.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | config=${1:-config.py} 6 | gunicorn_pecan --workers 3 --chdir / -b 0.0.0.0:51000 --name repoxplorer $config 7 | -------------------------------------------------------------------------------- /etc/groups.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | groups: {} 4 | 5 | #groups: 6 | # barbican-ptl: 7 | # description: Project team leaders of Barbican project 8 | # emails: {} 9 | -------------------------------------------------------------------------------- /bin/wipe_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -z "$1" ] && { 4 | echo "Please pass the index name as argument" 5 | exit 1 6 | } 7 | 8 | curl -XDELETE "http://localhost:9200/$1/" 9 | -------------------------------------------------------------------------------- /etc/repoxplorer.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=RepoXplorer indexer daemon 3 | 4 | [Service] 5 | ExecStart=/bin/repoxplorer-indexer --forever 6 | 7 | [Install] 8 | WantedBy=multi-user.target 9 | -------------------------------------------------------------------------------- /playbooks/tox.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: test-node 3 | tasks: 4 | - name: Run tox 5 | command: tox -e {{ tox_target }} 6 | args: 7 | chdir: "{{ zuul.project.src_dir }}/" 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML 2 | elasticsearch>7.0,<=7.10.1 3 | github3.py==1.0.0a4 4 | jsonschema 5 | pecan 6 | pycrypto 7 | pytz 8 | requests 9 | gunicorn 10 | pyjwt 11 | cryptography 12 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | pep8>=1.7 3 | nose 4 | mock 5 | coverage 6 | pecan 7 | WebOb>1.7.6 8 | elasticsearch>7.0,<=7.8 9 | PyYAML 10 | pycrypto 11 | jsonschema 12 | pytz 13 | pyjwt 14 | requests 15 | cryptography 16 | -------------------------------------------------------------------------------- /docker/aoi-master/elasticsearch.repo: -------------------------------------------------------------------------------- 1 | [elasticsearch-6.x] 2 | name=Elasticsearch repository for 6.x packages 3 | baseurl=https://artifacts.elastic.co/packages/6.x/yum 4 | gpgcheck=1 5 | gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch 6 | enabled=1 7 | autorefresh=1 8 | type=rpm-md 9 | -------------------------------------------------------------------------------- /docker/aoi-last-stable/elasticsearch.repo: -------------------------------------------------------------------------------- 1 | [elasticsearch-7.x] 2 | name=Elasticsearch repository for 7.x packages 3 | baseurl=https://artifacts.elastic.co/packages/7.x/yum 4 | gpgcheck=1 5 | gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch 6 | enabled=1 7 | autorefresh=1 8 | type=rpm-md 9 | -------------------------------------------------------------------------------- /etc/repoxplorer-webui.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=RepoXplorer web ui 3 | After=syslog.target 4 | 5 | [Service] 6 | ExecStart=/usr/bin/gunicorn_pecan --workers 10 --chdir / -b 0.0.0.0:51000 --name repoxplorer /etc/repoxplorer/config.py 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | -------------------------------------------------------------------------------- /etc/idents.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | #identities: 3 | # 0000-0000: 4 | # name: John Doe 5 | # default-email: john.doe@server.com 6 | # emails: 7 | # john.doe@server.com: 8 | # groups: 9 | # barbican-ptl: 10 | # jdoe@server.com: 11 | # groups: 12 | # barbican-ptl: 13 | 14 | identities: {} 15 | -------------------------------------------------------------------------------- /playbooks/deps-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: test-node 3 | tasks: 4 | - name: "Install some distro required packages" 5 | package: 6 | name: "{{ item }}" 7 | state: latest 8 | with_items: 9 | - git 10 | - gcc 11 | - python3-tox 12 | - python3-virtualenv 13 | - libffi-devel 14 | - openssl-devel 15 | - python3-devel 16 | become: true 17 | -------------------------------------------------------------------------------- /repoxplorer/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | from unittest import TestCase 3 | from pecan import set_config 4 | from pecan.testing import load_test_app 5 | 6 | __all__ = ['FunctionalTest'] 7 | 8 | 9 | class FunctionalTest(TestCase): 10 | 11 | def setUp(self): 12 | self.app = load_test_app(os.path.join( 13 | os.path.dirname(__file__), 14 | 'config.py' 15 | )) 16 | 17 | def tearDown(self): 18 | set_config({}, overwrite=True) 19 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Deploy last stable version of repoXplorer published on docker hub. 2 | 3 | version: '3' 4 | services: 5 | repoxplorer: 6 | image: morucci/repoxplorer 7 | container_name: repoxplorer 8 | ports: 9 | - 51000:51000 10 | volumes: 11 | - repoxplorer-data:/usr/local/share/repoxplorer 12 | - elasticsearch-data:/var/lib/elasticsearch 13 | - $PWD/docker-data/conf:/etc/repoxplorer/defs:z 14 | volumes: 15 | elasticsearch-data: 16 | repoxplorer-data: 17 | 18 | -------------------------------------------------------------------------------- /docker/aoi-master/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [program:elasticsearch] 5 | command=sudo -u elasticsearch /usr/share/elasticsearch/bin/elasticsearch 6 | autorestart=true 7 | 8 | [program:repoxplorer-indexor] 9 | command=repoxplorer-indexer --forever --config /etc/repoxplorer/config.py 10 | autorestart=true 11 | 12 | [program:repoxplorer-webui] 13 | command=gunicorn_pecan --worker-connections 10 --chdir / -b 0.0.0.0:51000 --name repoxplorer /etc/repoxplorer/config.py 14 | autorestart=true 15 | -------------------------------------------------------------------------------- /docker/aoi-last-stable/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [program:elasticsearch] 5 | command=sudo -u elasticsearch /usr/share/elasticsearch/bin/elasticsearch 6 | autorestart=true 7 | 8 | [program:repoxplorer-indexor] 9 | command=repoxplorer-indexer --forever --config /etc/repoxplorer/config.py 10 | autorestart=true 11 | 12 | [program:repoxplorer-webui] 13 | command=gunicorn_pecan --worker-connections 10 --chdir / -b 0.0.0.0:51000 --name repoxplorer /etc/repoxplorer/config.py 14 | autorestart=true 15 | -------------------------------------------------------------------------------- /repoxplorer/model/__init__.py: -------------------------------------------------------------------------------- 1 | from pecan import conf # noqa 2 | 3 | 4 | def init_model(): 5 | """ 6 | This is a stub method which is called at application startup time. 7 | 8 | If you need to bind to a parsed database configuration, set up tables or 9 | ORM classes, or perform any database initialization, this is the 10 | recommended place to do it. 11 | 12 | For more information working with databases, and some common recipes, 13 | see http://pecan.readthedocs.org/en/latest/databases.html 14 | """ 15 | pass 16 | -------------------------------------------------------------------------------- /docker-compose-master.yaml: -------------------------------------------------------------------------------- 1 | # Deploy last master version of repoXplorer published on docker hub. 2 | 3 | version: '3' 4 | services: 5 | repoxplorer-master: 6 | image: morucci/repoxplorer:master 7 | container_name: repoxplorer-master 8 | ports: 9 | - 51000:51000 10 | volumes: 11 | - repoxplorer-master-data:/usr/local/share/repoxplorer 12 | - elasticsearch-master-data:/var/lib/elasticsearch 13 | - $PWD/docker-data/conf:/etc/repoxplorer/defs:z 14 | volumes: 15 | elasticsearch-master-data: 16 | repoxplorer-master-data: 17 | 18 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36,py37,stats,pep8 3 | 4 | [testenv] 5 | setenv = 6 | PYTHONPATH = {envsitepackagesdir} 7 | PECAN_CONFIG = repoxplorer/tests/config.py 8 | deps = 9 | -rtest-requirements.txt 10 | commands = nosetests -v --with-coverage --cover-package=repoxplorer {posargs} 11 | 12 | [testenv:pep8] 13 | commands = flake8 repoxplorer 14 | 15 | [testenv:py36] 16 | basepython = python3.6 17 | 18 | [testenv:py37] 19 | basepython = python3.7 20 | 21 | [testenv:clean] 22 | commands = coverage erase 23 | 24 | [testenv:stats] 25 | commands = 26 | coverage report 27 | coverage html 28 | -------------------------------------------------------------------------------- /repoxplorer/version.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016-2017, Fabien Boucher 2 | # Copyright 2016-2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pkg_resources 17 | 18 | 19 | def get_version(): 20 | version = pkg_resources.get_distribution("repoxplorer").version 21 | 22 | return version 23 | -------------------------------------------------------------------------------- /repoxplorer/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019, Matthieu Huin 2 | # Copyright 2019, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | class UnauthorizedException(Exception): 18 | """Raise when the user is not allowed to perform an action. 19 | Typically trigger an abort(401) when caught.""" 20 | -------------------------------------------------------------------------------- /repoxplorer/tests/config.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | from repoxplorer.controllers.renderers import CSVRenderer 3 | 4 | # Server Specific Configurations 5 | server = { 6 | 'port': '8080', 7 | 'host': '0.0.0.0' 8 | } 9 | 10 | # Pecan Application Configurations 11 | app = { 12 | 'root': 'repoxplorer.controllers.root.RootController', 13 | 'modules': ['repoxplorer'], 14 | 'custom_renderers': {'csv': CSVRenderer}, 15 | 'static_root': '%(confdir)s/../../public', 16 | 'debug': True, 17 | 'errors': { 18 | '404': '/error/404', 19 | '__force_dict__': True 20 | } 21 | } 22 | 23 | projects_file_path = None 24 | git_store = None 25 | db_path = tempfile.mkdtemp() 26 | db_cache_path = tempfile.mkdtemp() 27 | db_default_file = None 28 | xorkey = None 29 | elasticsearch_host = 'localhost' 30 | elasticsearch_port = 9200 31 | elasticsearch_index = 'repoxplorertest' 32 | 33 | admin_token = '12345' 34 | -------------------------------------------------------------------------------- /playbooks/elastic6-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: test-node 3 | tasks: 4 | - name: "Install some distro required for Elastic" 5 | package: 6 | name: "{{ item }}" 7 | state: latest 8 | with_items: 9 | - which 10 | - java-1.8.0-openjdk 11 | become: true 12 | - name: Extract ElasticSearch 6.6.x archive 13 | unarchive: 14 | src: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.1.tar.gz 15 | dest: "{{ ansible_user_dir }}" 16 | remote_src: yes 17 | - name: Remove Xms/Xmx from default JVM options file 18 | lineinfile: 19 | path: "{{ ansible_user_dir }}/elasticsearch-6.6.1/config/jvm.options" 20 | state: absent 21 | regexp: '^-Xm.*$' 22 | - name: Start ElasticSearch 23 | command: "{{ ansible_user_dir }}/elasticsearch-6.6.1/bin/elasticsearch -d" 24 | environment: 25 | ES_JAVA_OPTS: "-Xms256m -Xmx1g" 26 | - name: Get processes list 27 | command: ps axf 28 | -------------------------------------------------------------------------------- /playbooks/elastic5-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: test-node 3 | tasks: 4 | - name: "Install some distro required for Elastic" 5 | package: 6 | name: "{{ item }}" 7 | state: latest 8 | with_items: 9 | - which 10 | - java-1.8.0-openjdk 11 | become: true 12 | - name: Extract ElasticSearch 5.5.x archive 13 | unarchive: 14 | src: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.15.tar.gz 15 | dest: "{{ ansible_user_dir }}" 16 | remote_src: yes 17 | - name: Remove Xms/Xmx from default JVM options file 18 | lineinfile: 19 | path: "{{ ansible_user_dir }}/elasticsearch-5.6.15/config/jvm.options" 20 | state: absent 21 | regexp: '^-Xm.*$' 22 | - name: Start ElasticSearch 23 | command: "{{ ansible_user_dir }}/elasticsearch-5.6.15/bin/elasticsearch -d" 24 | environment: 25 | ES_JAVA_OPTS: "-Xms256m -Xmx1g" 26 | - name: Get processes list 27 | command: ps axf 28 | -------------------------------------------------------------------------------- /playbooks/elastic7-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: test-node 3 | tasks: 4 | - name: "Install some distro required for Elastic" 5 | package: 6 | name: "{{ item }}" 7 | state: latest 8 | with_items: 9 | - which 10 | - java-1.8.0-openjdk 11 | become: true 12 | - name: Extract ElasticSearch 7.7.x archive 13 | unarchive: 14 | src: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.7.1-linux-x86_64.tar.gz 15 | dest: "{{ ansible_user_dir }}" 16 | remote_src: yes 17 | - name: Remove Xms/Xmx from default JVM options file 18 | lineinfile: 19 | path: "{{ ansible_user_dir }}/elasticsearch-7.7.1/config/jvm.options" 20 | state: absent 21 | regexp: '^-Xm.*$' 22 | - name: Start ElasticSearch 23 | command: "{{ ansible_user_dir }}/elasticsearch-7.7.1/bin/elasticsearch -d" 24 | environment: 25 | ES_JAVA_OPTS: "-Xms256m -Xmx1g" 26 | - name: Get processes list 27 | command: ps axf 28 | -------------------------------------------------------------------------------- /repoxplorer/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from pecan import make_app 17 | from pecan.middleware.static import StaticFileMiddleware 18 | from repoxplorer import model 19 | 20 | 21 | def setup_app(config): 22 | 23 | model.init_model() 24 | app_conf = dict(config.app) 25 | 26 | app = make_app( 27 | app_conf.pop('root'), 28 | logging=getattr(config, 'logging', {}), 29 | **app_conf 30 | ) 31 | return StaticFileMiddleware(app, app_conf.get('static_root')) 32 | -------------------------------------------------------------------------------- /docker/aoi-master/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM fedora:29 2 | LABEL maintainer="fabien.dot.boucher@gmail.com" 3 | 4 | ARG version=master 5 | 6 | RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch 7 | ADD elasticsearch.repo /etc/yum.repos.d/elasticsearch.repo 8 | 9 | RUN dnf -y install sudo elasticsearch java-1.8.0-openjdk libffi-devel \ 10 | openssl-devel python3-devel git gcc supervisor 11 | RUN dnf -y update 12 | RUN dnf clean all 13 | RUN rm -rf /var/cache/yum /var/cache/dnf 14 | 15 | RUN mkdir /etc/repoxplorer 16 | RUN git clone https://softwarefactory-project.io/r/repoxplorer 17 | RUN cd repoxplorer && git fetch origin ${version} 18 | RUN cd repoxplorer && git checkout FETCH_HEAD 19 | RUN cd repoxplorer && python3 -m pip install -r requirements.txt && python3 -m pip install . 20 | RUN cd repoxplorer && cp config.py /etc/repoxplorer/ 21 | RUN cd repoxplorer && python3 ./bin/repoxplorer-fetch-web-assets --config /etc/repoxplorer/config.py 22 | 23 | RUN mkdir /etc/repoxplorer/defs 24 | RUN sed -i "s|^db_path =.*|db_path = '/etc/repoxplorer/defs'|" /etc/repoxplorer/config.py 25 | 26 | ADD ./supervisord.conf /etc/supervisord.conf 27 | 28 | EXPOSE 51000 29 | 30 | CMD ["supervisord", "-n", "-c", "/etc/supervisord.conf"] 31 | -------------------------------------------------------------------------------- /docker/aoi-last-stable/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM fedora:32 2 | LABEL maintainer="fabien.dot.boucher@gmail.com" 3 | 4 | ARG version=1.6.1 5 | 6 | RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch 7 | ADD elasticsearch.repo /etc/yum.repos.d/elasticsearch.repo 8 | 9 | RUN dnf -y install sudo elasticsearch java-1.8.0-openjdk libffi-devel \ 10 | openssl-devel python3-devel git gcc supervisor 11 | RUN dnf -y update 12 | RUN dnf clean all 13 | RUN rm -rf /var/cache/yum /var/cache/dnf 14 | 15 | RUN mkdir /etc/repoxplorer 16 | RUN git clone https://softwarefactory-project.io/r/repoxplorer 17 | RUN cd repoxplorer && git fetch origin ${version} 18 | RUN cd repoxplorer && git checkout FETCH_HEAD 19 | RUN cd repoxplorer && python3 -m pip install -r requirements.txt && python3 -m pip install . 20 | RUN cd repoxplorer && cp config.py /etc/repoxplorer/ 21 | RUN cd repoxplorer && python3 ./bin/repoxplorer-fetch-web-assets --config /etc/repoxplorer/config.py 22 | # Remove keycloak until the support is fully fixed 23 | RUN rm /root/.local/repoxplorer/public/javascript/keycloak.min.js 24 | 25 | RUN mkdir /etc/repoxplorer/defs 26 | RUN sed -i "s|^db_path =.*|db_path = '/etc/repoxplorer/defs'|" /etc/repoxplorer/config.py 27 | 28 | ADD ./supervisord.conf /etc/supervisord.conf 29 | 30 | EXPOSE 51000 31 | 32 | CMD ["supervisord", "-n", "-c", "/etc/supervisord.conf"] 33 | -------------------------------------------------------------------------------- /docker/aoi-master/README: -------------------------------------------------------------------------------- 1 | # Container build 2 | docker build -t repoxplorer:master . 3 | 4 | mkdir conf 5 | chcon -Rt svirt_sandbox_file_t ./conf 6 | 7 | # Run the container 8 | docker run -dti -p 51000:51000 -v $(pwd)/conf:/etc/repoxplorer/defs:z repoxplorer-aoi:master 9 | 10 | # Access the UI 11 | firefox http://localhost:51000/index.html 12 | 13 | # You can set the projects/groups definition in conf/. 14 | # As an example set conf/projects.yaml with the yaml block below: 15 | 16 | project-templates: 17 | default: 18 | uri: https://github.com/openstack/%(name)s 19 | branches: 20 | - master 21 | gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s 22 | 23 | projects: 24 | Barbican: 25 | description: The Barbican project 26 | repos: 27 | barbican: 28 | template: default 29 | python-barbicanclient: 30 | template: default 31 | 32 | # You can also use the github helper directly from the container 33 | docker ps # get the 34 | docker exec -ti repoxplorer-github-organization --org git --skip-fork --output-path /etc/repoxplorer/defs/git 35 | 36 | # Check the indexer logs to see the progress of the indexation 37 | docker ps # get the 38 | docker exec -ti tail -f /root/.local/repoxplorer/repoxplorer-indexer.log 39 | 40 | # Using docker-compose-master.yaml 41 | docker-compose -f docker-compose-master.yaml up -d 42 | docker-compose -f docker-compose-master.yaml down (-v) 43 | -------------------------------------------------------------------------------- /docker/aoi-last-stable/README: -------------------------------------------------------------------------------- 1 | # Container build 2 | docker build -t repoxplorer:master . 3 | 4 | mkdir conf 5 | chcon -Rt svirt_sandbox_file_t ./conf 6 | 7 | # Run the container 8 | docker run -dti -p 51000:51000 -v $(pwd)/conf:/etc/repoxplorer/defs:z repoxplorer-aoi:master 9 | 10 | # Access the UI 11 | firefox http://localhost:51000/index.html 12 | 13 | # You can set the projects/groups definition in conf/. 14 | # As an example set conf/projects.yaml with the yaml block below: 15 | 16 | project-templates: 17 | default: 18 | uri: https://github.com/openstack/%(name)s 19 | branches: 20 | - master 21 | gitweb: https://github.com/openstack/%(name)s/commit/%%(sha)s 22 | 23 | projects: 24 | Barbican: 25 | description: The Barbican project 26 | repos: 27 | barbican: 28 | template: default 29 | python-barbicanclient: 30 | template: default 31 | 32 | # You can also use the github helper directly from the container 33 | docker ps # get the 34 | docker exec -ti repoxplorer-github-organization --org git --skip-fork --output-path /etc/repoxplorer/defs/git 35 | 36 | # Check the indexer logs to see the progress of the indexation 37 | docker ps # get the 38 | docker exec -ti tail -f /root/.local/repoxplorer/repoxplorer-indexer.log 39 | 40 | # Using docker-compose-master.yaml 41 | docker-compose -f docker-compose-master.yaml up -d 42 | docker-compose -f docker-compose-master.yaml down (-v) 43 | -------------------------------------------------------------------------------- /repoxplorer/public/css/repoxplorer.css: -------------------------------------------------------------------------------- 1 | .jumbotron { 2 | padding-top: 0.1%; 3 | padding-bottom: 0.1%; 4 | padding-left: 0.5%; 5 | } 6 | 7 | #groups-index { 8 | font-size: 150%; 9 | } 10 | 11 | #projects-filter-form { 12 | width: 100%; 13 | } 14 | 15 | #projects-filter-input { 16 | width: 80%; 17 | } 18 | 19 | .glyphicon-refresh-animate { 20 | -animation: spin .7s infinite linear; 21 | -ms-animation: spin .7s infinite linear; 22 | -webkit-animation: spinw .7s infinite linear; 23 | -moz-animation: spinm .7s infinite linear; 24 | } 25 | 26 | @keyframes spin { 27 | from { transform: scale(1) rotate(0deg);} 28 | to { transform: scale(1) rotate(360deg);} 29 | } 30 | 31 | @-webkit-keyframes spinw { 32 | from { -webkit-transform: rotate(0deg);} 33 | to { -webkit-transform: rotate(360deg);} 34 | } 35 | 36 | @-moz-keyframes spinm { 37 | from { -moz-transform: rotate(0deg);} 38 | to { -moz-transform: rotate(360deg);} 39 | } 40 | 41 | .row-flex { 42 | display: flex; 43 | flex-wrap: wrap; 44 | align-items: center; 45 | } 46 | 47 | .project-panel-logo { 48 | text-align: center; 49 | } 50 | 51 | .project-panel-logo img { 52 | height: 50px; 53 | width: 50px; 54 | } 55 | 56 | .project-panel-name h2 { 57 | margin-top: 10px; 58 | margin-bottom: 10px; 59 | } 60 | 61 | .project-panel-button { 62 | display: flex; 63 | flex-wrap: wrap; 64 | } 65 | 66 | .project-panel-detail { 67 | display: none; 68 | } 69 | 70 | .project-panel-detail h3 { 71 | background-color: #F2F2F2; 72 | } 73 | 74 | .blank-separator { 75 | margin-top: 15px; 76 | } 77 | -------------------------------------------------------------------------------- /repoxplorer/tests/test_tags.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from repoxplorer import index 4 | from repoxplorer.index.tags import Tags 5 | 6 | 7 | class TestTags(TestCase): 8 | 9 | @classmethod 10 | def setUpClass(cls): 11 | cls.con = index.Connector( 12 | index='repoxplorertest', 13 | index_suffix='tags') 14 | cls.t = Tags(cls.con) 15 | cls.tags = [ 16 | { 17 | 'sha': '3597334f2cb10772950c97ddf2f6cc17b184', 18 | 'date': 1410456010, 19 | 'name': 'tag1', 20 | 'repo': 'https://github.com/nakata/monkey.git:mon:master', 21 | }, 22 | { 23 | 'sha': '3597334f2cb10772950c97ddf2f6cc17b185', 24 | 'date': 1410456011, 25 | 'name': 'tag2', 26 | 'repo': 'https://github.com/nakata/monkey.git:mon:master', 27 | }, 28 | { 29 | 'sha': '3597334f2cb10772950c97ddf2f6cc17b186', 30 | 'date': 1410456012, 31 | 'name': 'tag3', 32 | 'repo': 'https://github.com/nakata/monkey.git:mon:stable-2', 33 | }, 34 | ] 35 | cls.t.add_tags(cls.tags) 36 | 37 | @classmethod 38 | def tearDownClass(cls): 39 | cls.con.ic.delete(index=cls.con.index) 40 | 41 | def test_get_tags(self): 42 | repos = ['https://github.com/nakata/monkey.git:mon:master'] 43 | ret = self.t.get_tags(repos=repos) 44 | tag_names = [d['_source']['name'] for d in ret] 45 | self.assertIn('tag1', tag_names) 46 | self.assertIn('tag2', tag_names) 47 | self.assertEqual(len(tag_names), 2) 48 | -------------------------------------------------------------------------------- /repoxplorer/tests/test_renderer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from unittest import TestCase 18 | 19 | from repoxplorer.controllers import renderers 20 | 21 | 22 | class TestCSVRenderer(TestCase): 23 | 24 | def setUp(self): 25 | self.csvrender = renderers.CSVRenderer(None, None) 26 | 27 | def test_rendering(self): 28 | data = {'f1': 'd1', 'f2': 2} 29 | output = self.csvrender.render(None, data) 30 | self.assertTrue( 31 | output == "f1,f2\r\nd1,2\r\n") 32 | 33 | data = {'f1': ['e1', 'e2'], 'f2': 2} 34 | output = self.csvrender.render(None, data) 35 | self.assertTrue( 36 | output == "f1,f2\r\ne1;e2,2\r\n") 37 | 38 | data = [ 39 | {'f1': 'd1', 'f2': 2}, 40 | {'f1': 'd2', 'f2': 3}, 41 | ] 42 | 43 | output = self.csvrender.render(None, data) 44 | self.assertTrue( 45 | output == "f1,f2\r\nd1,2\r\nd2,3\r\n") 46 | 47 | data = {'f1': {}} 48 | self.assertRaises(ValueError, self.csvrender.render, None, data) 49 | -------------------------------------------------------------------------------- /bin/repoxplorer-config-validate: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | 3 | # Copyright 2016,2017 Fabien Boucher 4 | # Copyright 2016,2017 Red Hat 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import imp 19 | import sys 20 | import argparse 21 | 22 | from pecan import configuration 23 | 24 | from repoxplorer.index import projects 25 | from repoxplorer.index import contributors 26 | 27 | parser = argparse.ArgumentParser(description='RepoXplorer config validator') 28 | parser.add_argument( 29 | '--config', required=True, 30 | help='Path to the repoXplorer configuration file') 31 | 32 | args = parser.parse_args() 33 | 34 | 35 | def validate(): 36 | projects_index = projects.Projects(vonly=True) 37 | issues = projects_index.validate() 38 | contributors_index = contributors.Contributors(vonly=True) 39 | issues.extend(contributors_index.validate()) 40 | for issue in issues: 41 | print(issue) 42 | return len(issues) 43 | 44 | 45 | if __name__ == "__main__": 46 | configuration.set_config(args.config) 47 | errs = validate() 48 | if errs: 49 | print("Corrupted configuration exit !") 50 | sys.exit(errs) 51 | else: 52 | print("Configuration OK.") 53 | -------------------------------------------------------------------------------- /repoxplorer/controllers/status.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import itertools 17 | 18 | from pecan import expose 19 | 20 | from pecan import conf 21 | from repoxplorer import version 22 | from repoxplorer.index.projects import Projects 23 | 24 | rx_version = version.get_version() 25 | index_custom_html = conf.get('index_custom_html', '') 26 | 27 | 28 | class StatusController(object): 29 | 30 | @expose('json') 31 | def version(self): 32 | return {'version': rx_version} 33 | 34 | def get_status(self): 35 | projects_index = Projects() 36 | projects = projects_index.get_projects(source=['name', 'refs']) 37 | num_projects = len(projects) 38 | num_repos = len(set([ 39 | ref['name'] for 40 | ref in itertools.chain( 41 | *[p['refs'] for p in list(projects.values())])])) 42 | return {'customtext': index_custom_html, 43 | 'projects': num_projects, 44 | 'repos': num_repos, 45 | 'users_endpoint': conf.get('users_endpoint', False), 46 | 'version': rx_version} 47 | 48 | @expose('json') 49 | def status(self): 50 | return self.get_status() 51 | -------------------------------------------------------------------------------- /repoxplorer/trends/commits.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import logging 17 | 18 | from repoxplorer.index.commits import Commits 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class CommitsAmountTrend(object): 24 | def __init__(self, connector=None): 25 | self.ic = Commits(connector) 26 | 27 | def get_trend(self, mails=[], repos=[], 28 | period_a=None, period_b=None, 29 | merge_commit=None): 30 | """ Return the amount diff and the percentil 31 | of amount evolution for perdiod a compared to 32 | period b 33 | """ 34 | assert isinstance(period_a, tuple) 35 | assert isinstance(period_b, tuple) 36 | c_amnt_a = self.ic.get_commits_amount(mails, repos, 37 | period_a[0], period_a[1], 38 | merge_commit) 39 | c_amnt_b = self.ic.get_commits_amount(mails, repos, 40 | period_b[0], period_b[1], 41 | merge_commit) 42 | diff = c_amnt_a - c_amnt_b 43 | trend = diff * 100 / (c_amnt_a or c_amnt_b) 44 | return diff, trend 45 | -------------------------------------------------------------------------------- /repoxplorer/controllers/metadata.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from pecan import expose 17 | 18 | from repoxplorer import index 19 | from repoxplorer.controllers import utils 20 | from repoxplorer.index.projects import Projects 21 | from repoxplorer.index.commits import Commits 22 | from repoxplorer.index.contributors import Contributors 23 | 24 | 25 | class MetadataController(object): 26 | 27 | @expose('json') 28 | def metadata(self, key=None, pid=None, tid=None, cid=None, gid=None, 29 | dfrom=None, dto=None, inc_merge_commit=None, 30 | inc_repos=None, exc_groups=None, inc_groups=None): 31 | 32 | c = Commits(index.Connector()) 33 | projects_index = Projects() 34 | idents = Contributors() 35 | 36 | query_kwargs = utils.resolv_filters( 37 | projects_index, idents, pid, tid, cid, gid, 38 | dfrom, dto, inc_repos, inc_merge_commit, None, exc_groups, 39 | inc_groups) 40 | del query_kwargs['metadata'] 41 | 42 | if not key: 43 | keys = c.get_metadata_keys(**query_kwargs) 44 | return keys 45 | else: 46 | vals = c.get_metadata_key_values(key, **query_kwargs) 47 | return vals 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os 17 | from setuptools import setup 18 | from setuptools import find_packages 19 | 20 | def read(fname): 21 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 22 | 23 | setup( 24 | name = "repoxplorer", 25 | version = "1.6.1", 26 | author = "Fabien Boucher", 27 | author_email = "fabien?dot.boucher@gmail.com", 28 | description = ("Git repositories metrics"), 29 | license = "ASL v 2.0", 30 | keywords = "git metrics statistics stats repo repositories elasticsearch", 31 | url = "https://github.com/morucci/repoxplorer", 32 | packages=find_packages(), 33 | include_package_data=True, 34 | zip_safe=False, 35 | long_description=read('README.md'), 36 | classifiers=[ 37 | "Development Status :: 5 - Production/Stable", 38 | "Intended Audience :: Information Technology", 39 | "License :: OSI Approved :: Apache Software License", 40 | "Operating System :: OS Independent", 41 | "Programming Language :: Python :: 3.6", 42 | "Programming Language :: Python :: 3.7", 43 | "Topic :: Software Development :: Version Control :: Git", 44 | ], 45 | scripts=['bin/repoxplorer-indexer', 46 | 'bin/repoxplorer-config-validate', 47 | 'bin/repoxplorer-fetch-web-assets', 48 | 'bin/repoxplorer-git-credentials-helper', 49 | 'bin/repoxplorer-github-organization'], 50 | ) 51 | -------------------------------------------------------------------------------- /repoxplorer/indexer/git/metadata_re.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019, Red Hat 2 | # Copyright 2019, Fabien Boucher 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import re 17 | 18 | METADATA_REs = { 19 | 'signed-of-by': re.compile('^[Ss]igned-[Oo]f(?:-[Bb]y)?:([^//].+)$'), 20 | 'reviewed-by': re.compile('^[Rr]evied(?:-[Bb]y)?:([^//].+)$'), 21 | 'tested-by': re.compile('^[Tt]ested(?:-[Bb]y)?:([^//].+)$'), 22 | 'rebased-by': re.compile('^[Rr]ebased(?:-[Bb]y)?:([^//].+)$'), 23 | 'reported-by': re.compile('^[Rr]eported(?:-[Bb]y)?:([^//].+)$'), 24 | 'co-authored-by': re.compile('^[Cc]o-[Aa]uthored(?:-[Bb]y)?:([^//].+)$'), 25 | 'helped-by': re.compile('^[Hh]elped(?:-[Bb]y)?:([^//].+)$'), 26 | 'acked-by': re.compile('^[Aa]cked(?:-[Bb]y)?:([^//].+)$'), 27 | 'suggested-by': re.compile('^[Ss]uggested(?:-[Bb]y)?:([^//].+)$'), 28 | 'noticed-by': re.compile('^[Nn]oticed(?:-[Bb]y)?:([^//].+)$'), 29 | 'mentored-by': re.compile('^[Mm]entored(?:-[Bb]y)?:([^//].+)$'), 30 | 'closes-bug': re.compile('^[Cc]loses?(?:-[Bb]ug)?:([^//].+)$'), 31 | 'fixes-bug': re.compile('^[Ff]ixe?s?(?:-[Bb]ug)?:([^//].+)$'), 32 | 'related-bug': re.compile('^[Rr]elated(?:-[Bb]ug)?:([^//].+)$'), 33 | 'depends-on': re.compile('^[Dd]epends(?:-[Oo]n)?:([^//].+)$'), 34 | 'resolves': re.compile('^[Rr]esolv(?:es)?:([^//].+)$'), 35 | 'issue': re.compile('^[Ii]ssue:([^//].+)$'), 36 | 'story': re.compile('^[Ss]tory:([^//].+)$'), 37 | 'task': re.compile('^[Tt]ask:([^//].+)$'), 38 | 'bug': re.compile('^[Bu]ug:([^//].+)$'), 39 | } 40 | -------------------------------------------------------------------------------- /repoxplorer/controllers/renderers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import csv 17 | from io import StringIO 18 | 19 | 20 | class CSVRenderer(object): 21 | def __init__(self, path, extra_vars): 22 | pass 23 | 24 | def render(self, template_path, namespace): 25 | buf = StringIO() 26 | # Assume namespace an array of dict 27 | if not isinstance(namespace, list): 28 | namespace = [namespace] 29 | for e in namespace: 30 | assert isinstance(e, dict) 31 | keys = list(namespace[0].keys()) 32 | keys.sort() 33 | w = csv.DictWriter(buf, fieldnames=keys) 34 | w.writeheader() 35 | for e in namespace: 36 | d = {} 37 | for k, v in e.items(): 38 | if not any([ 39 | isinstance(v, str), 40 | isinstance(v, list), 41 | isinstance(v, int), 42 | isinstance(v, float)]): 43 | raise ValueError( 44 | "'%s' (type: %s) is not supported for CSV output" % ( 45 | str(v), type(v))) 46 | if isinstance(v, str): 47 | d[k] = v 48 | elif isinstance(v, list): 49 | d[k] = ";".join(v) 50 | else: 51 | d[k] = str(v) 52 | w.writerow(d) 53 | buf.seek(0) 54 | return buf.read() 55 | -------------------------------------------------------------------------------- /repoxplorer/controllers/root.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016-2017, Fabien Boucher 2 | # Copyright 2016-2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from pecan import expose 18 | from pecan import request 19 | 20 | from repoxplorer.controllers import groups 21 | from repoxplorer.controllers import users 22 | from repoxplorer.controllers import histo 23 | from repoxplorer.controllers import infos 24 | from repoxplorer.controllers import tops 25 | from repoxplorer.controllers import search 26 | from repoxplorer.controllers import status 27 | from repoxplorer.controllers import projects 28 | from repoxplorer.controllers import metadata 29 | from repoxplorer.controllers import tags 30 | from repoxplorer.controllers import commits 31 | 32 | 33 | class V1Controller(object): 34 | 35 | infos = infos.InfosController() 36 | groups = groups.GroupsController() 37 | users = users.UsersController() 38 | histo = histo.HistoController() 39 | tops = tops.TopsController() 40 | search = search.SearchController() 41 | status = status.StatusController() 42 | projects = projects.ProjectsController() 43 | metadata = metadata.MetadataController() 44 | tags = tags.TagsController() 45 | commits = commits.CommitsController() 46 | 47 | 48 | class APIController(object): 49 | 50 | v1 = V1Controller() 51 | 52 | 53 | class ErrorController(object): 54 | 55 | @expose('json') 56 | def e404(self): 57 | message = str(request.environ.get('pecan.original_exception', '')) 58 | return dict(status=404, message=message) 59 | 60 | 61 | class RootController(object): 62 | 63 | api = APIController() 64 | error = ErrorController() 65 | -------------------------------------------------------------------------------- /repoxplorer/controllers/search.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hashlib 17 | from collections import OrderedDict 18 | 19 | from pecan import expose 20 | 21 | from pecan import conf 22 | from repoxplorer import index 23 | from repoxplorer.controllers import utils 24 | from repoxplorer.index.commits import Commits 25 | from repoxplorer.index.contributors import Contributors 26 | 27 | xorkey = conf.get('xorkey') or 'default' 28 | 29 | 30 | class SearchController(object): 31 | 32 | @expose('json') 33 | def search_authors(self, query=""): 34 | ret_limit = 100 35 | c = Commits(index.Connector()) 36 | ret = c.es.search( 37 | index=c.index, 38 | q=query, df="author_name", size=10000, 39 | default_operator="AND", 40 | _source_includes=["author_name", "author_email"]) 41 | ret = ret['hits']['hits'] 42 | if not len(ret): 43 | return {} 44 | idents = Contributors() 45 | authors = dict([(d['_source']['author_email'], 46 | d['_source']['author_name']) for d in ret]) 47 | result = {} 48 | _idents = idents.get_idents_by_emails(list(authors.keys())[:ret_limit]) 49 | for iid, ident in _idents.items(): 50 | email = ident['default-email'] 51 | name = ident['name'] or authors[email] 52 | result[utils.encrypt(xorkey, iid)] = { 53 | 'name': name, 54 | 'gravatar': hashlib.md5( 55 | email.encode(errors='ignore')).hexdigest()} 56 | result = OrderedDict( 57 | sorted(list(result.items()), key=lambda t: t[1]['name'])) 58 | return result 59 | -------------------------------------------------------------------------------- /repoxplorer/controllers/tags.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from pecan import expose 17 | 18 | from repoxplorer.controllers import utils 19 | from repoxplorer import index 20 | from repoxplorer.index.projects import Projects 21 | from repoxplorer.index.tags import Tags 22 | from repoxplorer.index.contributors import Contributors 23 | 24 | 25 | class TagsController(object): 26 | 27 | @expose('json') 28 | def tags(self, pid=None, tid=None, 29 | dfrom=None, dto=None, inc_repos=None): 30 | t = Tags(index.Connector(index_suffix='tags')) 31 | projects_index = Projects() 32 | idents = Contributors() 33 | 34 | query_kwargs = utils.resolv_filters( 35 | projects_index, idents, pid, tid, None, None, 36 | dfrom, dto, inc_repos, None, None, None, None) 37 | 38 | p_filter = [":".join(r.split(':')[:-1]) for r in query_kwargs['repos']] 39 | dfrom = query_kwargs['fromdate'] 40 | dto = query_kwargs['todate'] 41 | ret = [r['_source'] for r in t.get_tags(p_filter, dfrom, dto)] 42 | # TODO: if tid is given we can include user defined releases 43 | # for repo tagged with tid. 44 | if not pid: 45 | return ret 46 | # now append user defined releases 47 | ur = {} 48 | project = projects_index.get(pid, source=['refs', 'releases']) 49 | for release in project.get('releases', []): 50 | ur[release['name']] = release 51 | for ref in project['refs']: 52 | for release in ref.get('releases', []): 53 | ur[release['name']] = release 54 | for rel in ur.values(): 55 | ret.append(rel) 56 | return ret 57 | -------------------------------------------------------------------------------- /repoxplorer/tests/test_yamlbackend.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Red Hat 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os 17 | import shutil 18 | import tempfile 19 | from unittest import TestCase 20 | 21 | from repoxplorer.index.yamlbackend import YAMLBackend 22 | 23 | 24 | class TestYAMLBackend(TestCase): 25 | 26 | def setUp(self): 27 | pass 28 | 29 | def tearDown(self): 30 | if os.path.isdir(self.db): 31 | shutil.rmtree(self.db) 32 | 33 | def create_db(self, files): 34 | self.db = tempfile.mkdtemp() 35 | for filename, content in files.items(): 36 | open(os.path.join(self.db, filename), 'w+').write(content) 37 | 38 | def test_yamlbackend_load(self): 39 | f1 = """ 40 | --- 41 | key: value 42 | """ 43 | f2 = """ 44 | --- 45 | key2: value2 46 | """ 47 | files = {'f1.yaml': f1, 'f2.yaml': f2} 48 | self.create_db(files) 49 | backend = YAMLBackend(db_path=self.db) 50 | backend.load_db() 51 | default_data, data = backend.get_data() 52 | self.assertEqual(default_data, None) 53 | self.assertEqual(len(data), 2) 54 | 55 | def test_yamlbackend_load_with_default(self): 56 | f1 = """ 57 | --- 58 | key: value 59 | """ 60 | f2 = """ 61 | --- 62 | key2: value2 63 | """ 64 | files = {'default.yaml': f1, 'f2.yaml': f2} 65 | self.create_db(files) 66 | backend = YAMLBackend( 67 | db_path=self.db, 68 | db_default_file=os.path.join(self.db, 'default.yaml')) 69 | backend.load_db() 70 | default_data, data = backend.get_data() 71 | self.assertDictEqual(default_data, {'key': 'value'}) 72 | self.assertEqual(len(data), 1) 73 | self.assertDictEqual(data[0], {'key2': 'value2'}) 74 | -------------------------------------------------------------------------------- /repoxplorer/controllers/projects.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from collections import OrderedDict 17 | 18 | from pecan import abort 19 | from pecan import expose 20 | 21 | from repoxplorer import version 22 | from repoxplorer.index.projects import Projects 23 | 24 | rx_version = version.get_version() 25 | 26 | 27 | class ProjectsController(object): 28 | 29 | @expose('json') 30 | def projects(self, pid=None): 31 | projects_index = Projects() 32 | if pid: 33 | project = projects_index.get(pid) 34 | if not project: 35 | abort(404, detail="Project ID has not been found") 36 | return {pid: projects_index.get(pid)} 37 | else: 38 | projects = projects_index.get_projects( 39 | source=['name', 'description', 'logo', 'refs']) 40 | _projects = OrderedDict( 41 | sorted(list(projects.items()), key=lambda t: t[0])) 42 | return {'projects': _projects, 43 | 'tags': projects_index.get_tags()} 44 | 45 | @expose('json') 46 | def repos(self, pid=None, tid=None): 47 | projects_index = Projects() 48 | if not pid and not tid: 49 | abort(404, 50 | detail="A tag ID or project ID must be passed as parameter") 51 | if pid: 52 | project = projects_index.get(pid) 53 | else: 54 | if tid in projects_index.get_tags(): 55 | refs = projects_index.get_references_from_tags(tid) 56 | project = {'refs': refs} 57 | else: 58 | project = None 59 | if not project: 60 | abort(404, 61 | detail='Project ID or Tag ID has not been found') 62 | return project['refs'] 63 | -------------------------------------------------------------------------------- /repoxplorer/public/projects.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 |
43 |
44 |
45 |

Listing of projects referenced in the database

46 |
47 |
48 |
49 | 50 |
51 |
52 | 53 |
54 |
55 | 56 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /repoxplorer/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Welcome to RepoXplorer 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 |
43 |
44 |
45 |
46 |
47 |

Welcome on RepoXplorer

48 |
49 |

50 |
51 |

52 |

Browse projects, contributors, groups indexes

53 |

54 |
55 |

56 |
57 |
58 |
59 |
60 |
61 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /repoxplorer/public/contributors.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 | 43 |
44 | 45 |
46 |
47 |

Search commit's authors

48 |
49 |
50 |
51 | 52 |
53 |
54 |
55 |
56 |
57 | 58 | 59 |
60 |
61 |
62 |
63 |
64 | 65 |
66 |
67 |
68 | 69 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /repoxplorer/public/groups.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 | 43 |
44 |
45 |
46 |

Listing of groups and memberships referenced in the database

47 |
48 |
49 |
50 | 51 |
52 |
53 |

54 |
55 |
56 | 57 |
58 |
59 |
60 |
61 |

Groups list

62 |
63 |
64 |
65 |
66 |
67 |
68 | 69 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /repoxplorer/controllers/histo.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pecan import expose 16 | 17 | from repoxplorer import index 18 | from repoxplorer.controllers import utils 19 | from repoxplorer.index.commits import Commits 20 | from repoxplorer.index.projects import Projects 21 | from repoxplorer.index.contributors import Contributors 22 | 23 | 24 | class HistoController(object): 25 | 26 | @expose('json') 27 | def authors(self, pid=None, tid=None, cid=None, gid=None, 28 | dfrom=None, dto=None, inc_merge_commit=None, 29 | inc_repos=None, metadata=None, exc_groups=None, 30 | inc_groups=None): 31 | 32 | projects_index = Projects() 33 | idents = Contributors() 34 | 35 | query_kwargs = utils.resolv_filters( 36 | projects_index, idents, pid, tid, cid, gid, 37 | dfrom, dto, inc_repos, inc_merge_commit, 38 | metadata, exc_groups, inc_groups) 39 | 40 | c = Commits(index.Connector()) 41 | if not c.get_commits_amount(**query_kwargs): 42 | return [] 43 | ret = c.get_authors_histo(**query_kwargs)[1] 44 | for bucket in ret: 45 | _idents = idents.get_idents_by_emails(bucket['authors_email']) 46 | bucket['value'] = len(_idents) 47 | bucket['date'] = bucket['key_as_string'] 48 | del bucket['authors_email'] 49 | del bucket['doc_count'] 50 | del bucket['key_as_string'] 51 | del bucket['key'] 52 | 53 | return ret 54 | 55 | @expose('json') 56 | def commits(self, pid=None, tid=None, cid=None, gid=None, 57 | dfrom=None, dto=None, inc_merge_commit=None, 58 | inc_repos=None, metadata=None, exc_groups=None, 59 | inc_groups=None): 60 | 61 | projects_index = Projects() 62 | idents = Contributors() 63 | 64 | query_kwargs = utils.resolv_filters( 65 | projects_index, idents, pid, tid, cid, gid, 66 | dfrom, dto, inc_repos, inc_merge_commit, 67 | metadata, exc_groups, inc_groups) 68 | 69 | c = Commits(index.Connector()) 70 | if not c.get_commits_amount(**query_kwargs): 71 | return [] 72 | ret = c.get_commits_histo(**query_kwargs) 73 | ret = [{'date': d['key_as_string'], 74 | 'value': d['doc_count']} for d in ret[1]] 75 | return ret 76 | -------------------------------------------------------------------------------- /bin/bench/fake-commit-gen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2016, Fabien Boucher 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import string 18 | import random 19 | import hashlib 20 | 21 | from repoxplorer.index.commits import Commits 22 | from repoxplorer import index 23 | 24 | epoch_start = 1476633000 25 | 26 | 27 | def create_random_str(lenght=6): 28 | value = "".join([random.choice(string.ascii_lowercase) 29 | for _ in range(lenght)]) 30 | return value 31 | 32 | 33 | def gen_emails(amount): 34 | ret = [] 35 | for i in range(amount): 36 | email = "%s@%s.%s" % ( 37 | create_random_str(8), 38 | create_random_str(5), 39 | create_random_str(3), 40 | ) 41 | name = "%s %s" % ( 42 | create_random_str(8), 43 | create_random_str(8), 44 | ) 45 | ret.append((name, email)) 46 | return ret 47 | 48 | 49 | def gen_commit_msg(): 50 | return " ".join([create_random_str(random.randint(0, 10)) 51 | for _ in range(5)]) 52 | 53 | 54 | def gen_fake_commits(amount=10000): 55 | print("Start generation of %s fake commits" % amount) 56 | email_amount = amount * 0.03 57 | email_amount = int(email_amount) 58 | if not email_amount: 59 | email_amount = 1 60 | emails = gen_emails(email_amount) 61 | project = '%s:%s:%s' % ( 62 | 'https://github.com/openstack/test', 63 | 'test', 'master') 64 | ret = [] 65 | for i in range(amount): 66 | author_date = random.randint( 67 | epoch_start, epoch_start + 1000000) 68 | author = emails[random.randint(0, email_amount - 1)] 69 | committer = emails[random.randint(0, email_amount - 1)] 70 | c = {} 71 | c['sha'] = hashlib.sha256(create_random_str(10)).hexdigest() 72 | c['author_name'] = author[0] 73 | c['committer_name'] = committer[0] 74 | c['author_email'] = author[1] 75 | c['committer_email'] = committer[1] 76 | c['author_date'] = author_date 77 | c['committer_date'] = random.randint( 78 | author_date + 1, author_date + 10000) 79 | c['ttl'] = random.randint(0, 10000) 80 | c['commit_msg'] = gen_commit_msg() 81 | c['line_modifieds'] = random.randint(0, 10000) 82 | c['merge_commit'] = False 83 | c['projects'] = [project, ] 84 | ret.append(c) 85 | print("Generation of %s fake commits done." % amount) 86 | return ret 87 | 88 | 89 | if __name__ == '__main__': 90 | amount = 100000 91 | c = Commits(index.Connector()) 92 | c.add_commits(gen_fake_commits(amount)) 93 | print("Indexation done.") 94 | -------------------------------------------------------------------------------- /repoxplorer/tests/test_trends.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from repoxplorer import index 4 | from repoxplorer.index.commits import Commits 5 | from repoxplorer.trends.commits import CommitsAmountTrend 6 | 7 | 8 | class TestCommitsAmountTrend(TestCase): 9 | 10 | @classmethod 11 | def setUpClass(cls): 12 | cls.con = index.Connector(index='repoxplorertest') 13 | cls.c = Commits(cls.con) 14 | cls.t = CommitsAmountTrend(cls.con) 15 | cls.commits = [ 16 | { 17 | 'sha': '3597334f2cb10772950c97ddf2f6cc17b184', 18 | 'author_date': 1410456005, 19 | 'committer_date': 1410456010, 20 | 'ttl': 5, 21 | 'author_name': 'Nakata Daisuke', 22 | 'committer_name': 'Nakata Daisuke', 23 | 'author_email': 'n.suke@joker.org', 24 | 'committer_email': 'n.suke@joker.org', 25 | 'repos': [ 26 | 'https://github.com/nakata/monkey.git:monkey:master', ], 27 | 'line_modifieds': 10, 28 | 'merge_commit': False, 29 | 'commit_msg': 'Add init method', 30 | }, 31 | { 32 | 'sha': '3597334f2cb10772950c97ddf2f6cc17b185', 33 | 'author_date': 1410457005, 34 | 'committer_date': 1410457005, 35 | 'ttl': 0, 36 | 'author_name': 'Keiko Amura', 37 | 'committer_name': 'Keiko Amura', 38 | 'author_email': 'keiko.a@joker.org', 39 | 'committer_email': 'keiko.a@joker.org', 40 | 'repos': [ 41 | 'https://github.com/nakata/monkey.git:monkey:master', ], 42 | 'line_modifieds': 100, 43 | 'merge_commit': False, 44 | 'commit_msg': 'Merge "Fix sanity unittest"', 45 | }, 46 | { 47 | 'sha': '3597334f2cb10772950c97ddf2f6cc17b186', 48 | 'author_date': 1410458005, 49 | 'committer_date': 1410458005, 50 | 'ttl': 0, 51 | 'author_name': 'Jean Bon', 52 | 'committer_name': 'Jean Bon', 53 | 'author_email': 'jean.bon@joker.org', 54 | 'committer_email': 'jean.bon@joker.org', 55 | 'repos': [ 56 | 'https://github.com/nakata/monkey.git:monkey:master', ], 57 | 'line_modifieds': 200, 58 | 'merge_commit': False, 59 | 'commit_msg': 'Add request customer feature 19', 60 | }, 61 | ] 62 | cls.c.add_commits(cls.commits) 63 | 64 | @classmethod 65 | def tearDownClass(cls): 66 | cls.con.ic.delete(index=cls.con.index) 67 | 68 | def test_get_trend(self): 69 | ret = self.t.get_trend( 70 | repos=['https://github.com/nakata/monkey.git:monkey:master'], 71 | period_a=(1410457000, 1410459000), 72 | period_b=(1410456005, 1410456015)) 73 | self.assertEqual(ret[0], 1) 74 | self.assertEqual(ret[1], 50) 75 | 76 | ret = self.t.get_trend( 77 | repos=['https://github.com/nakata/monkey.git:monkey:master'], 78 | period_a=(1410457000, 1410459000), 79 | period_b=(1410455005, 1410455015)) 80 | self.assertEqual(ret[0], 2) 81 | self.assertEqual(ret[1], 100) 82 | 83 | ret = self.t.get_trend( 84 | repos=['https://github.com/nakata/monkey.git:monkey:master'], 85 | period_a=(1410455005, 1410455015), 86 | period_b=(1410457000, 1410459000)) 87 | self.assertEqual(ret[0], -2) 88 | self.assertEqual(ret[1], -100) 89 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import copy 4 | 5 | from repoxplorer.controllers.renderers import CSVRenderer 6 | 7 | runtimedir = os.path.join(os.path.expanduser('~'), '.local', 'repoxplorer') 8 | 9 | # RepoXplorer configuration file 10 | base_logging = { 11 | 'version': 1, 12 | 'root': {'level': 'DEBUG', 'handlers': ['normal']}, 13 | 'loggers': { 14 | 'indexerDaemon': { 15 | 'level': 'DEBUG', 16 | 'handlers': ['normal', 'console'], 17 | 'propagate': False, 18 | }, 19 | 'repoxplorer': { 20 | 'level': 'DEBUG', 21 | 'handlers': ['normal', 'console'], 22 | 'propagate': False, 23 | }, 24 | 'elasticsearch': { 25 | 'level': 'WARN', 26 | 'handlers': ['normal', 'console'], 27 | 'propagate': False, 28 | }, 29 | }, 30 | 'handlers': { 31 | 'console': { 32 | 'level': 'INFO', 33 | 'class': 'logging.StreamHandler', 34 | 'formatter': 'console' 35 | }, 36 | 'normal': { 37 | 'class': 'logging.handlers.TimedRotatingFileHandler', 38 | 'level': 'DEBUG', 39 | 'formatter': 'normal', 40 | 'filename': '', 41 | 'when': 'D', 42 | 'interval': 1, 43 | 'backupCount': 30, 44 | }, 45 | }, 46 | 'formatters': { 47 | 'console': {'format': ('%(levelname)-5.5s [%(name)s] %(message)s')}, 48 | 'normal': {'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' 49 | ' %(message)s')}, 50 | } 51 | } 52 | 53 | # Internal dev pecan server 54 | server = { 55 | 'port': '51000', 56 | 'host': '0.0.0.0' 57 | } 58 | 59 | # Pecan REST and rendering configuration 60 | app = { 61 | 'root': 'repoxplorer.controllers.root.RootController', 62 | 'modules': ['repoxplorer'], 63 | 'custom_renderers': {'csv': CSVRenderer}, 64 | 'static_root': '%s/public' % runtimedir, 65 | 'debug': False, 66 | 'errors': { 67 | 404: '/error/e404', 68 | '__force_dict__': True 69 | } 70 | } 71 | 72 | # Additional RepoXplorer configurations 73 | db_default_file = None 74 | db_path = runtimedir 75 | db_cache_path = db_path 76 | git_store = '%s/git_store' % runtimedir 77 | xorkey = None 78 | elasticsearch_host = 'localhost' 79 | elasticsearch_port = 9200 80 | elasticsearch_index = 'repoxplorer' 81 | elasticsearch_user = None 82 | elasticsearch_password = None 83 | indexer_loop_delay = 60 84 | indexer_skip_projects = [] 85 | index_custom_html = "" 86 | users_endpoint = False 87 | admin_token = 'admin_token' 88 | # Absolute path to a helper. If not set or None use the default provided helper 89 | git_credential_helper_path = None 90 | 91 | # Logging configuration for the wsgi app 92 | logging = copy.deepcopy(base_logging) 93 | logging['handlers']['normal']['filename'] = ( 94 | '%s/repoxplorer-api.log' % runtimedir) 95 | 96 | # Logging configuration for the indexer 97 | indexer_logging = copy.deepcopy(base_logging) 98 | indexer_logging['handlers']['normal']['filename'] = ( 99 | '%s/repoxplorer-indexer.log' % runtimedir) 100 | 101 | # Add this to activate OpenID Connect as your authentication method. 102 | oidc = { 103 | # issuer_url: used to fetch the issuer's configuration by appending 104 | # .well-known/openid-configuration to it 105 | 'issuer_url': 'https://path/to/idp', 106 | # Defaults to True, set to False for development 107 | 'verify_ssl': True, 108 | # This must be equal to the "aud" claim in the tokens sent back by your 109 | # OpenID Connect provider. 110 | 'audience': 'repoxplorer', 111 | } 112 | -------------------------------------------------------------------------------- /repoxplorer/index/tags.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Red Hat 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import logging 17 | 18 | from elasticsearch.helpers import bulk 19 | from elasticsearch.helpers import scan as scanner 20 | 21 | from repoxplorer.index import add_params 22 | from repoxplorer.index import clean_empty 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | PROPERTIES = { 27 | "sha": {"type": "keyword"}, 28 | "date": {"type": "date", "format": "epoch_second"}, 29 | "name": {"type": "keyword"}, 30 | "repo": {"type": "keyword"}, 31 | } 32 | 33 | 34 | class Tags(object): 35 | def __init__(self, connector=None): 36 | self.es = connector.es 37 | self.ic = connector.ic 38 | self.index = connector.index 39 | self.dbname = 'tags' 40 | self.mapping = { 41 | self.dbname: { 42 | "properties": PROPERTIES, 43 | } 44 | } 45 | if not self.ic.exists_type(index=self.index, 46 | doc_type=self.dbname): 47 | kwargs = add_params(self.es) 48 | self.ic.put_mapping(index=self.index, doc_type=self.dbname, 49 | body=self.mapping, **kwargs) 50 | 51 | def add_tags(self, source_it): 52 | def gen(it): 53 | for source in it: 54 | d = {} 55 | d['_index'] = self.index 56 | d['_type'] = self.dbname 57 | d['_op_type'] = 'index' 58 | d['_source'] = source 59 | yield d 60 | bulk(self.es, gen(source_it)) 61 | self.es.indices.refresh(index=self.index) 62 | 63 | def del_tags(self, id_list): 64 | def gen(it): 65 | for i in it: 66 | d = {} 67 | d['_index'] = self.index 68 | d['_type'] = self.dbname 69 | d['_op_type'] = 'delete' 70 | d['_id'] = i 71 | yield d 72 | bulk(self.es, gen(id_list)) 73 | self.es.indices.refresh(index=self.index) 74 | 75 | def get_tags(self, repos, fromdate=None, todate=None): 76 | 77 | qfilter = { 78 | "bool": { 79 | "must": [], 80 | "should": [], 81 | } 82 | } 83 | 84 | for repo in repos: 85 | should_repo_clause = { 86 | "bool": { 87 | "must": [] 88 | } 89 | } 90 | should_repo_clause["bool"]["must"].append( 91 | {"term": {"repo": repo}} 92 | ) 93 | qfilter["bool"]["should"].append(should_repo_clause) 94 | 95 | qfilter["bool"]["must"].append( 96 | { 97 | "range": { 98 | "date": { 99 | "gte": fromdate, 100 | "lt": todate, 101 | } 102 | } 103 | } 104 | ) 105 | 106 | body = { 107 | "query": { 108 | "bool": { 109 | "filter": qfilter 110 | } 111 | } 112 | } 113 | 114 | body = clean_empty(body) 115 | 116 | return [t for t in scanner(self.es, query=body, 117 | index=self.index)] 118 | -------------------------------------------------------------------------------- /repoxplorer/controllers/groups.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016-2017, Fabien Boucher 2 | # Copyright 2016-2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import hashlib 18 | 19 | from pecan import conf 20 | from pecan import expose 21 | 22 | from repoxplorer import index 23 | from repoxplorer.controllers import utils 24 | from repoxplorer.index.contributors import Contributors 25 | from repoxplorer.index.commits import Commits 26 | from repoxplorer.index.projects import Projects 27 | 28 | xorkey = conf.get('xorkey') or 'default' 29 | 30 | 31 | class GroupsController(object): 32 | 33 | @expose('json') 34 | def index(self, prefix=None, nameonly='false', withstats='false', 35 | pid=None, dfrom=None, dto=None, inc_merge_commit=None): 36 | ci = Commits(index.Connector()) 37 | contributors_index = Contributors() 38 | groups = contributors_index.get_groups() 39 | if withstats == 'true': 40 | projects_index = Projects() 41 | if nameonly == 'true': 42 | ret = dict([(k, None) for k in groups.keys()]) 43 | if prefix: 44 | ret = dict([(k, None) for k in ret.keys() if 45 | k.lower().startswith(prefix)]) 46 | return ret 47 | ret_groups = {} 48 | for group, data in groups.items(): 49 | if prefix and not group.lower().startswith(prefix.lower()): 50 | continue 51 | rg = {'members': {}, 52 | 'description': data.get('description', ''), 53 | 'domains': data.get('domains', [])} 54 | emails = list(data['emails'].keys()) 55 | members = contributors_index.get_idents_by_emails(emails) 56 | for id, member in members.items(): 57 | member['gravatar'] = hashlib.md5( 58 | member['default-email'].encode( 59 | errors='ignore')).hexdigest() 60 | # TODO(fbo): bounces should be a list of bounce 61 | # Let's deactivate that for now 62 | # member['bounces'] = bounces 63 | del member['emails'] 64 | if not member['name']: 65 | # Try to find it among commits 66 | suggested = ci.get_commits_author_name_by_emails( 67 | [member['default-email']]) 68 | name = suggested.get(member['default-email'], 69 | 'Unnamed') 70 | member['name'] = name 71 | del member['default-email'] 72 | rg['members'][utils.encrypt(xorkey, id)] = member 73 | 74 | if withstats == 'true': 75 | # Fetch the number of projects and repos contributed to 76 | query_kwargs = utils.resolv_filters( 77 | projects_index, contributors_index, pid, None, None, group, 78 | dfrom, dto, None, inc_merge_commit, None, None, None) 79 | 80 | repos = [r for r in ci.get_repos(**query_kwargs)[1] 81 | if not r.startswith('meta_ref: ')] 82 | projects = utils.get_projects_from_references( 83 | projects_index, repos) 84 | rg['repos_amount'] = len(repos) 85 | rg['projects_amount'] = len(projects) 86 | 87 | ret_groups[group] = rg 88 | 89 | return ret_groups 90 | -------------------------------------------------------------------------------- /repoxplorer/tests/test_users.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import copy 16 | from unittest import TestCase 17 | 18 | from repoxplorer import index 19 | from repoxplorer.index.users import Users 20 | 21 | 22 | class TestUsers(TestCase): 23 | 24 | @classmethod 25 | def setUpClass(cls): 26 | cls.con = index.Connector(index='repoxplorertest', 27 | index_suffix='users') 28 | cls.c = Users(cls.con) 29 | cls.user = { 30 | 'uid': '123', 31 | 'name': 'saboten', 32 | 'default-email': 'saboten@domain1', 33 | 'emails': [ 34 | {'email': 'saboten@domain1'}, 35 | {'email': 'saboten@domain2', 36 | 'groups': [ 37 | {'group': 'ugroup1', 38 | 'begin-date': '2016-01-01', 39 | 'end-date': '2016-01-09'}], 40 | }], 41 | 'last_cnx': 1410456005} 42 | cls.user2 = { 43 | 'uid': '124', 44 | 'name': 'ampanman', 45 | 'default-email': 'ampanman@domain1', 46 | 'emails': [ 47 | {'email': 'ampanman@domain1'}], 48 | 'last_cnx': 1410456006} 49 | 50 | @classmethod 51 | def tearDownClass(cls): 52 | cls.con.ic.delete(index=cls.con.index) 53 | 54 | def setUp(self): 55 | # Be sure users are deleted 56 | self.c.delete(self.user['uid']) 57 | self.c.delete(self.user2['uid']) 58 | 59 | def tearDown(self): 60 | # Be sure users are deleted 61 | self.c.delete(self.user['uid']) 62 | self.c.delete(self.user2['uid']) 63 | 64 | def test_user_crud(self): 65 | # Create and get a user 66 | self.c.create(self.user) 67 | ret = self.c.get(self.user['uid']) 68 | self.assertDictEqual(ret, self.user) 69 | self.c.create(self.user2) 70 | ret = self.c.get(self.user2['uid']) 71 | self.assertDictEqual(ret, self.user2) 72 | 73 | # Update and get a user 74 | u_user = copy.deepcopy(self.user) 75 | u_user['emails'] = [{'email': 'saboten@domain3', 76 | 'groups': [ 77 | {'group': 'ugroup2'} 78 | ]}] 79 | u_user['name'] = 'Cactus Saboten Junior' 80 | self.c.update(u_user) 81 | ret = self.c.get(self.user['uid']) 82 | self.assertDictEqual(ret, u_user) 83 | 84 | ret = self.c.get_idents_by_emails('saboten@domain3') 85 | self.assertDictEqual(ret[0], u_user) 86 | 87 | ret = self.c.get_idents_by_emails( 88 | ['saboten@domain3', 'ampanman@domain1']) 89 | self.assertEqual(len(ret), 2) 90 | self.assertIn('ampanman', [r['name'] for r in ret]) 91 | self.assertIn('Cactus Saboten Junior', [r['name'] for r in ret]) 92 | 93 | idents_list = self.c.get_idents_in_group('ugroup2') 94 | self.assertListEqual( 95 | idents_list, 96 | [{'uid': '123', 97 | 'default-email': 'saboten@domain1', 98 | 'name': 'Cactus Saboten Junior', 99 | 'emails': [{ 100 | 'groups': [{'group': 'ugroup2'}], 101 | 'email': 'saboten@domain3'}], 102 | 'last_cnx': 1410456005}]) 103 | 104 | # Delete and get a user 105 | self.c.delete(self.user['uid']) 106 | self.assertEqual( 107 | self.c.get(self.user['uid']), None) 108 | -------------------------------------------------------------------------------- /repoxplorer/controllers/commits.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # Copyright 2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hashlib 17 | 18 | from datetime import datetime 19 | 20 | from pecan import conf 21 | from pecan import expose 22 | 23 | from repoxplorer import index 24 | from repoxplorer.controllers import utils 25 | from repoxplorer.index.commits import Commits 26 | from repoxplorer.index.commits import PROPERTIES 27 | from repoxplorer.index.projects import Projects 28 | from repoxplorer.index.contributors import Contributors 29 | 30 | xorkey = conf.get('xorkey') or 'default' 31 | 32 | 33 | class CommitsController(object): 34 | 35 | @expose('json') 36 | def commits(self, pid=None, tid=None, cid=None, gid=None, 37 | start=0, limit=10, 38 | dfrom=None, dto=None, inc_merge_commit=None, 39 | inc_repos=None, metadata=None, exc_groups=None, 40 | inc_groups=None): 41 | 42 | c = Commits(index.Connector()) 43 | projects_index = Projects() 44 | idents = Contributors() 45 | 46 | query_kwargs = utils.resolv_filters( 47 | projects_index, idents, pid, tid, cid, gid, 48 | dfrom, dto, inc_repos, inc_merge_commit, 49 | metadata, exc_groups, inc_groups) 50 | query_kwargs.update( 51 | {'start': start, 'limit': limit}) 52 | 53 | resp = c.get_commits(**query_kwargs) 54 | 55 | for cmt in resp[2]: 56 | # Get extra metadata keys 57 | extra = set(cmt.keys()) - set(PROPERTIES.keys()) 58 | cmt['metadata'] = list(extra) 59 | cmt['repos'] = [r for r in cmt['repos'] 60 | if not r.startswith('meta_ref: ')] 61 | # Compute link to access commit diff based on the 62 | # URL template provided in projects.yaml 63 | cmt['gitwebs'] = [ 64 | projects_index.get_gitweb_link(r) % 65 | {'sha': cmt['sha']} for r in cmt['repos']] 66 | cmt['projects'] = utils.get_projects_from_references( 67 | projects_index, cmt['repos']) 68 | # Also remove the URI part 69 | cmt['repos'] = [":".join(p.split(':')[-2:]) for 70 | p in cmt['repos']] 71 | # Request the ident index to fetch author/committer name/email 72 | for elm in ('author', 'committer'): 73 | ident = list(idents.get_idents_by_emails( 74 | cmt['%s_email' % elm]).values())[0] 75 | cmt['%s_email' % elm] = ident['default-email'] 76 | if ident['name']: 77 | cmt['%s_name' % elm] = ident['name'] 78 | # Convert the TTL to something human readable 79 | cmt['ttl'] = str((datetime.fromtimestamp(cmt['ttl']) - 80 | datetime.fromtimestamp(0))) 81 | cmt['author_gravatar'] = \ 82 | hashlib.md5(cmt['author_email'].encode( 83 | errors='ignore')).hexdigest() 84 | cmt['committer_gravatar'] = \ 85 | hashlib.md5(cmt['committer_email'].encode( 86 | errors='ignore')).hexdigest() 87 | if len(cmt['commit_msg']) > 80: 88 | cmt['commit_msg'] = cmt['commit_msg'][0:76] + '...' 89 | # Add cid and ccid 90 | cmt['cid'] = utils.encrypt(xorkey, cmt['author_email']) 91 | cmt['ccid'] = utils.encrypt(xorkey, cmt['committer_email']) 92 | # Remove email details 93 | del cmt['author_email'] 94 | del cmt['committer_email'] 95 | return resp 96 | -------------------------------------------------------------------------------- /bin/repoxplorer-github-organization: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2016, 2017 Fabien Boucher 4 | # Copyright 2016, 2017 Red Hat 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import os 19 | import re 20 | import argparse 21 | import sys 22 | import yaml 23 | import github3 24 | 25 | parser = argparse.ArgumentParser( 26 | description='Read a Github organization and create' 27 | 'the repoXplorer description file') 28 | parser.add_argument( 29 | '--token', type=str, 30 | required=False, default=None, 31 | help='Specify an authentication token used to fetch private repositories') 32 | parser.add_argument( 33 | '--org', type=str, 34 | required=True, 35 | help='Specify the Github organization name') 36 | parser.add_argument( 37 | '--repo', type=str, 38 | required=False, 39 | help='Specify the repository name or a regular expression') 40 | parser.add_argument( 41 | '--mt-stars', type=int, 42 | required=False, 43 | help='Only repositories in top N by stargazers count') 44 | parser.add_argument( 45 | '--output-path', type=str, 46 | help='yaml file path to register organization repositories details') 47 | parser.add_argument( 48 | '--skip-fork', action='store_true', 49 | help='Do not consider forked repositories') 50 | parser.add_argument( 51 | '--all-branches', action='store_true', 52 | help='Include all branches in indexed repositories') 53 | 54 | args = parser.parse_args() 55 | 56 | if __name__ == "__main__": 57 | gh = github3.GitHub() 58 | if args.token: 59 | gh.login(token=args.token) 60 | org = gh.organization(args.org) 61 | if not org: 62 | print(( 63 | "Org %s not found, try to find single" 64 | " user's repos ..." % args.org)) 65 | if not args.token: 66 | repos = gh.repositories_by(args.org) 67 | else: 68 | repos = gh.repositories(type='owner') 69 | else: 70 | repos = org.repositories() 71 | templates = { 72 | args.org: { 73 | "branches": ["master", ], 74 | "uri": "http://github.com/%s/" % args.org + 75 | "%(name)s", 76 | "gitweb": "http://github.com/%s/" % args.org + 77 | "%(name)s/commit/%%(sha)s"} 78 | } 79 | projects = { 80 | args.org: { 81 | "repos": {}, 82 | "description": "The %s Github organization" % args.org, 83 | } 84 | } 85 | if args.mt_stars: 86 | stars = [] 87 | for r in repos: 88 | stars.append((r.name, int(r.stargazers_count))) 89 | stars_sorted = sorted(stars, key=lambda s: s[1], reverse=True) 90 | top = stars_sorted[:args.mt_stars] 91 | top = [t[0] for t in top] 92 | if args.repo: 93 | repo_re = re.compile(args.repo) 94 | for r in repos: 95 | if r.fork and args.skip_fork: 96 | continue 97 | if args.repo and not repo_re.match(r.name): 98 | continue 99 | if args.mt_stars: 100 | if r.name not in top: 101 | continue 102 | data = {r.name: {"template": args.org}} 103 | 104 | # Modified for addition of `--all-branches` option. 105 | # Revised to follow morruci's advice (set converted). 106 | branches = set([r.default_branch]) 107 | 108 | if args.all_branches: 109 | for branch in r.branches(): 110 | branches.add(branch.name) 111 | 112 | data[r.name]['branches'] = list(branches) 113 | 114 | projects[args.org]["repos"].update(data) 115 | print("Found %s" % r.name) 116 | 117 | struct = {'projects': projects, 118 | 'project-templates': templates} 119 | 120 | path = '%s.yaml' % args.org 121 | if args.output_path: 122 | path = os.path.expanduser(args.output_path) 123 | if not (path.endswith('.yaml') or path.endswith('.yml')): 124 | path += '.yaml' 125 | 126 | with open(path, 'w') as fd: 127 | fd.write(yaml.safe_dump(struct, 128 | default_flow_style=False)) 129 | print() 130 | print(("%s source repositories details" 131 | " has been written to %s" % (args.org, path))) 132 | 133 | print("Please edit the yaml file if needed (like adding additional" 134 | " branches to index, defines custom releases, ...)") 135 | 136 | sys.exit(0) 137 | -------------------------------------------------------------------------------- /repoxplorer/index/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import yaml 17 | import time 18 | import pytz 19 | import datetime 20 | 21 | from pecan import conf 22 | 23 | from Crypto.Hash import SHA 24 | 25 | from elasticsearch import client 26 | from jsonschema import validate as schema_validate 27 | 28 | from repoxplorer.index.yamlbackend import YAMLBackend 29 | 30 | 31 | def date2epoch(date): 32 | d = datetime.datetime.strptime(date, "%Y-%m-%d") 33 | d = d.replace(tzinfo=pytz.utc) 34 | epoch = (d - datetime.datetime(1970, 1, 1, 35 | tzinfo=pytz.utc)).total_seconds() 36 | return int(epoch) 37 | 38 | 39 | def get_elasticsearch_version(es): 40 | version = es.info()['version']['number'] 41 | return int(version.split('.')[0]) 42 | 43 | 44 | def add_params(es): 45 | if get_elasticsearch_version(es) >= 7: 46 | return {'include_type_name': 'true'} 47 | else: 48 | return {} 49 | 50 | 51 | # From https://stackoverflow.com/a/27974027/1966658 52 | def clean_empty(d): 53 | if not isinstance(d, (dict, list)): 54 | return d 55 | if isinstance(d, list): 56 | return [v for v in (clean_empty(v) for v in d) if v] 57 | return {k: v for k, v in ((k, clean_empty(v)) for k, v in d.items()) if ( 58 | v or v == False)} # noqa: E712 59 | 60 | 61 | class Connector(object): 62 | def __init__(self, host=None, port=None, index=None, index_suffix=None): 63 | self.host = (host or 64 | getattr(conf, 'elasticsearch_host', None) or 65 | 'localhost') 66 | self.port = (port or 67 | getattr(conf, 'elasticsearch_port', None) or 68 | 9200) 69 | self.index = (index or 70 | getattr(conf, 'elasticsearch_index', None) or 71 | 'repoxplorer') 72 | if index_suffix: 73 | self.index += "-%s" % index_suffix 74 | if (getattr(conf, 'elasticsearch_user', None) and 75 | getattr(conf, 'elasticsearch_password', None)): 76 | self.http_auth = "%s:%s" % ( 77 | getattr(conf, 'elasticsearch_user', None), 78 | getattr(conf, 'elasticsearch_password', None)) 79 | # NOTE(dpawlik) Opendistro is using self signed certs, 80 | # so verify_certs is set to False. 81 | self.es = client.Elasticsearch( 82 | [{"host": self.host, 83 | "port": self.port, 84 | "http_auth": self.http_auth, 85 | "use_ssl": True, 86 | "verify_certs": False, 87 | "ssl_show_warn": True}], timeout=60) 88 | else: 89 | self.es = client.Elasticsearch( 90 | [{"host": self.host, "port": self.port}], 91 | timeout=60) 92 | self.ic = client.IndicesClient(self.es) 93 | if not self.ic.exists(index=self.index): 94 | self.ic.create(index=self.index) 95 | # Give some time to have the index fully created 96 | time.sleep(1) 97 | 98 | 99 | class YAMLDefinition(object): 100 | def __init__(self, db_path=None, db_default_file=None, 101 | db_cache_path=None): 102 | db_cache_path = db_cache_path or conf.get('db_cache_path') or db_path 103 | self.yback = YAMLBackend( 104 | db_path or conf.get('db_path'), 105 | db_default_file=db_default_file or conf.get('db_default_file'), 106 | db_cache_path=db_cache_path) 107 | self.yback.load_db() 108 | self.hashes_str = SHA.new( 109 | "".join(self.yback.hashes).encode(errors='ignore')).hexdigest() 110 | self.default_data, self.data = self.yback.get_data() 111 | self._merge() 112 | 113 | def _check_basic(self, key, schema, identifier): 114 | """ Verify schema and no data duplicated 115 | """ 116 | issues = [] 117 | ids = set() 118 | for d in self.data: 119 | data = d.get(key, {}) 120 | try: 121 | schema_validate({key: data}, 122 | yaml.load(schema)) 123 | except Exception as e: 124 | issues.append(e.message) 125 | duplicated = set(data.keys()) & ids 126 | if duplicated: 127 | issues.append("%s IDs [%s,] are duplicated" % ( 128 | identifier, ",".join(duplicated))) 129 | ids.update(set(data.keys())) 130 | return ids, issues 131 | -------------------------------------------------------------------------------- /repoxplorer/public/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | My home 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 | 43 |
44 | 45 |
46 |
47 |
48 | 49 | 52 | 53 |
54 | 55 |
56 | 57 |
58 |
59 |

60 | User settings 61 | 62 | 63 | 64 |

65 |
66 | 67 |
68 |
69 |
70 | 71 |
72 | 73 |
74 |
75 |
76 | 77 |
78 | 79 |
80 |
81 |
82 | 83 |
84 | 85 |
86 |
87 |
88 |
89 |
90 | 91 |
92 |
93 |
94 |
95 |
96 | 97 |
98 |
99 |

100 | Request account deletion 101 |

102 |
103 | 104 |
105 |
106 |

107 | Click on the button below to delete your user account. This user account and associated emails will be delete from the database. 108 |

109 | 110 |
111 |
112 |
113 | 114 |
115 | 116 |
117 | 118 | 119 |
120 | 121 | 122 | 123 | 126 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /repoxplorer/controllers/infos.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from datetime import timedelta 16 | 17 | import re 18 | import hashlib 19 | 20 | from pecan import abort 21 | from pecan import expose 22 | from pecan import conf 23 | 24 | from repoxplorer import index 25 | from repoxplorer.controllers import utils 26 | from repoxplorer.index.commits import Commits 27 | from repoxplorer.index.projects import Projects 28 | from repoxplorer.index.contributors import Contributors 29 | 30 | xorkey = conf.get('xorkey') or 'default' 31 | 32 | 33 | class InfosController(object): 34 | 35 | def get_generic_infos( 36 | self, projects_index, commits_index, idents, pid, query_kwargs): 37 | infos = {} 38 | infos['commits_amount'] = commits_index.get_commits_amount( 39 | **query_kwargs) 40 | if not infos['commits_amount']: 41 | infos['line_modifieds_amount'] = 0 42 | return infos 43 | authors = commits_index.get_authors(**query_kwargs)[1] 44 | infos['authors_amount'] = len(utils.authors_sanitize(idents, authors)) 45 | first, last, duration = commits_index.get_commits_time_delta( 46 | **query_kwargs) 47 | infos['first'] = first 48 | infos['last'] = last 49 | infos['duration'] = duration 50 | ttl_average = commits_index.get_ttl_stats(**query_kwargs)[1]['avg'] 51 | infos['ttl_average'] = \ 52 | timedelta(seconds=int(ttl_average)) - timedelta(seconds=0) 53 | infos['ttl_average'] = int(infos['ttl_average'].total_seconds()) 54 | 55 | infos['line_modifieds_amount'] = int( 56 | commits_index.get_line_modifieds_stats(**query_kwargs)[1]['sum']) 57 | 58 | repos = [r for r in commits_index.get_repos(**query_kwargs)[1] 59 | if not r.startswith('meta_ref: ')] 60 | if pid: 61 | projects = (pid,) 62 | else: 63 | projects = utils.get_projects_from_references( 64 | projects_index, repos) 65 | infos['repos_amount'] = len(repos) 66 | infos['projects_amount'] = len(projects) 67 | return infos 68 | 69 | @expose('json') 70 | @expose('csv:', content_type='text/csv') 71 | def infos(self, pid=None, tid=None, cid=None, gid=None, 72 | dfrom=None, dto=None, inc_merge_commit=None, 73 | inc_repos=None, metadata=None, exc_groups=None, 74 | inc_groups=None): 75 | 76 | c = Commits(index.Connector()) 77 | projects_index = Projects() 78 | idents = Contributors() 79 | 80 | query_kwargs = utils.resolv_filters( 81 | projects_index, idents, pid, tid, cid, gid, 82 | dfrom, dto, inc_repos, inc_merge_commit, 83 | metadata, exc_groups, inc_groups) 84 | 85 | return self.get_generic_infos( 86 | projects_index, c, idents, pid, query_kwargs) 87 | 88 | @expose('json') 89 | @expose('csv:', content_type='text/csv') 90 | def contributor(self, cid=None): 91 | if not cid: 92 | abort(404, 93 | detail="No contributor specified") 94 | 95 | c = Commits(index.Connector()) 96 | 97 | if cid.endswith(','): 98 | mails = [e.lstrip(',') for e in re.findall('[^@]+@[^@,]+', cid)] 99 | name = c.get_commits_author_name_by_emails(mails[:1])[mails[0]] 100 | return { 101 | 'name': name, 102 | 'mails_amount': len(mails), 103 | 'gravatar': hashlib.md5( 104 | mails[0].encode(errors='ignore')).hexdigest() 105 | } 106 | 107 | try: 108 | cid = utils.decrypt(xorkey, cid) 109 | except Exception: 110 | abort(404, 111 | detail="The cid is incorrectly formated") 112 | 113 | idents = Contributors() 114 | _, ident = idents.get_ident_by_id(cid) 115 | if not ident: 116 | # No ident has been declared for that contributor 117 | ident = list(idents.get_idents_by_emails(cid).values())[0] 118 | mails = ident['emails'] 119 | name = ident['name'] 120 | if not name: 121 | raw_names = c.get_commits_author_name_by_emails([cid]) 122 | if cid not in raw_names: 123 | # TODO: get_commits_author_name_by_emails must 124 | # support look by committer email too 125 | name = 'Unnamed' 126 | else: 127 | name = raw_names[cid] 128 | 129 | infos = {} 130 | infos['name'] = name 131 | infos['mails_amount'] = len(mails) 132 | infos['gravatar'] = hashlib.md5( 133 | ident['default-email'].encode(errors='ignore')).hexdigest() 134 | return infos 135 | -------------------------------------------------------------------------------- /repoxplorer/index/yamlbackend.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2017 Red Hat, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import os 17 | import yaml 18 | import pickle 19 | import logging 20 | 21 | from Crypto.Hash import SHA 22 | 23 | from pecan import conf 24 | 25 | 26 | logger = logging.getLogger(__name__) 27 | 28 | 29 | class NoDatesSafeLoader(yaml.SafeLoader): 30 | @classmethod 31 | def remove_implicit_resolver(cls, tag_to_remove): 32 | """ 33 | We want to load datetimes as strings, not dates, because we 34 | go on to serialise as json which doesn't have the advanced types 35 | of yaml, and leads to incompatibilities down the track. 36 | """ 37 | if 'yaml_implicit_resolvers' not in cls.__dict__: 38 | cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy() 39 | 40 | for first_letter, mappings in cls.yaml_implicit_resolvers.items(): 41 | cls.yaml_implicit_resolvers[first_letter] = [ 42 | (tag, regexp) 43 | for tag, regexp in mappings 44 | if tag != tag_to_remove] 45 | 46 | 47 | NoDatesSafeLoader.remove_implicit_resolver('tag:yaml.org,2002:timestamp') 48 | 49 | 50 | class YAMLDBException(Exception): 51 | pass 52 | 53 | 54 | class YAMLBackend(object): 55 | def __init__(self, db_path, db_default_file=None, db_cache_path=None): 56 | """ Class to read YAML files from a DB path. 57 | db_default_file: is the path to a trusted file usually 58 | computed from an already verified data source. 59 | db_path: directory where data can be read. This is 60 | supposed to be user provided data to be verified 61 | by the caller and could overwrite data from the 62 | default_file. 63 | db_cache_path: directory to store cache files 64 | """ 65 | self.db_path = db_path or conf.get('db_path') 66 | self.db_default_file = db_default_file 67 | self.db_cache_path = ( 68 | db_cache_path or conf.get('db_cache_path') or self.db_path) 69 | 70 | self.default_data = None 71 | self.data = [] 72 | # List of hashes 73 | self.hashes = [] 74 | 75 | def load_db(self): 76 | def check_ext(f): 77 | return f.endswith('.yaml') or f.endswith('.yml') 78 | 79 | def load(path): 80 | data = None 81 | logger.debug("Check cache for %s ..." % path) 82 | basename = os.path.basename(path) 83 | if not os.path.isdir(self.db_path): 84 | os.makedirs(self.db_path) 85 | if self.db_cache_path and not os.path.isdir(self.db_cache_path): 86 | os.makedirs(self.db_cache_path) 87 | cached_hash_path = os.path.join( 88 | self.db_cache_path, basename + '.hash') 89 | cached_data_path = os.path.join( 90 | self.db_cache_path, basename + '.cached') 91 | hash = SHA.new(open(path).read().encode( 92 | errors='ignore')).hexdigest() 93 | if (os.path.isfile(cached_hash_path) and 94 | os.path.isfile(cached_data_path)): 95 | cached_hash = pickle.load(open(cached_hash_path, 'rb')) 96 | if cached_hash == hash: 97 | logger.debug("Reading %s from cache ..." % path) 98 | data = pickle.load(open(cached_data_path, 'rb')) 99 | self.hashes.append(hash) 100 | if not data: 101 | logger.debug("Reading %s from file ..." % path) 102 | try: 103 | data = yaml.load( 104 | open(path, 'rb'), Loader=NoDatesSafeLoader) 105 | except Exception as e: 106 | raise YAMLDBException( 107 | "YAML format corrupted in file %s (%s)" % (path, e)) 108 | pickle.dump( 109 | data, open(cached_data_path, 'wb'), 110 | pickle.HIGHEST_PROTOCOL) 111 | pickle.dump( 112 | hash, open(cached_hash_path, 'wb'), 113 | pickle.HIGHEST_PROTOCOL) 114 | self.hashes.append(hash) 115 | return data 116 | 117 | if self.db_default_file: 118 | self.default_data = load(self.db_default_file) 119 | 120 | yamlfiles = [f for f in os.listdir(self.db_path) if check_ext(f)] 121 | yamlfiles.sort() 122 | for f in yamlfiles: 123 | path = os.path.join(self.db_path, f) 124 | if path == self.db_default_file: 125 | continue 126 | self.data.append(load(path)) 127 | 128 | def get_data(self): 129 | """ Return the full raw data structure. 130 | """ 131 | return self.default_data, self.data 132 | -------------------------------------------------------------------------------- /repoxplorer/tests/gitshow.sample: -------------------------------------------------------------------------------- 1 | commit 1ef6088bb6678b78993672ffdec93c7c99a0405d 2 | tree 4a09e8fa7bc448ae2dba5bda4af45b92ebd86bbd 3 | parent 0e58c2fd54a50362138849a20bced510480dac8d 4 | author Author A 1493424649 -0400 5 | committer Author A 1493425136 -0400 6 | gpgsig -----BEGIN PGP SIGNATURE----- 7 | 8 | iQIzBAABCgAdFiEEItfJ0pcfUYWn/MzM/RKg8hTJ4XcFAlkD2/IACgkQ/RKg8hTJ 9 | 4XcpbRAA2NhbpeBEHsIcYMr2clvOie3AYFt1VaU7UDPuivQpx3SPsTqutt4gqG8p 10 | 91v951uaXqdAbBcRH4xSktOA4aYs+eNhKABs8Iw7khlN4IV4GwnUmeD40UsJedri 11 | iUaxSjv7S+q5mhCgJeYvP3Ln7Vhm0ojO/fQKAnXqaAPQoI04yehdr2++NYXcsWuT 12 | lcVaUi3PMz0i70pAx+U7ltr2te3jwvb4vzYv4Phuf7WEPx8DJFN+8jee/KO0kz+j 13 | xE49sRcCm5BYla5HSLnB3C8Dee2BYTPDWYwwXQrWWBzdlJX8S0SHyKkbF7U4y80N 14 | xDrNXFu4TUjOLpTJNoEjqZsO6QpBg6QUtebXqa3L+uPNqNtHqxWrGl/odUq4d2CF 15 | URRfJv3a0FH9aRu+05wzdJPariyf1okNn/tpjjpgLw8lxxOqGJNPQtA8IDBfbWRb 16 | 4igwZEu0ONrHa5PlglKIDTCMlzt3KL6xksR5tgiQyEJ6y5fIacMpliiAzljySB+K 17 | Mur9A8Mwo5w/4qaQzHmAleqGRD1U4in9rspJAP3nygh+yfg7AC3ysBR2GcVQJsPY 18 | vU5sNZcTJLu1tQiD37g8vtVgMU1eUTNRFOfG+mqKDUltTBMdBk/j677XnOqq8VjE 19 | Sxcsca2dfRNwRyfPF/XGdEQF2nXxKeKdV1WBJlK3iRqdlgTVis8= 20 | =vS+Q 21 | -----END PGP SIGNATURE----- 22 | 23 | Make playbook and task in topic singular 24 | 25 | The topic names need to be consistent between event types to ensure 26 | people can properly use wildcards to filter the messages on just the 27 | things they care about. However playbook and task were inconsistent in 28 | topics from between different methods in the callback plugin. This 29 | commit fixes that by making sure we always use 'playbook' and 'task', 30 | not 'playbooks' or 'tasks'. 31 | 32 | Change-Id: I3e6240560ad562e8f41f7e314ef7a4b0b1178e32 33 | 34 | 5 5 modules/openstack_project/files/puppetmaster/mqtt.py 35 | 36 | commit 0e58c2fd54a50362138849a20bced510480dac8d 37 | tree 01ee7e6331e400a12468b9651bf7083a9892d3bd 38 | parent 7c9af152dd5e99215c6f7e721be3331a2b00f43f 39 | parent 2bf2adb27175c68891712e4641b1874dfbe2c63a 40 | author Jenkins 1493423272 +0000 41 | committer Gerrit Code Review 1493423272 +0000 42 | 43 | Merge "Cast the playbook uuid as a string" 44 | 45 | 1 1 modules/openstack_project/files/puppetmaster/mqtt.py 46 | 47 | commit fb7d2712a907f8f01b817889e88abaf0dad6a109 48 | tree d052472f2a252ab0ec74df38889f37f70a15c074 49 | parent ad24b90f0891bdb7891bf8f13493472cf3885a9c 50 | parent 123bdfbb16a00e8e3abea9d84c72cdbe580e5046 51 | author Jenkins 1493413511 +0000 52 | committer Gerrit Code Review 1493413511 +0000 53 | 54 | Merge "Add subunit gearman worker mqtt info to firehose docs" 55 | 56 | 10 10 doc/source/firehose.rst 57 | 35 0 doc/source/firehose_schema.rst 58 | 59 | commit d9fda5b81f6c8d64fda2ca2c08246492e800292f 60 | tree 6540793b0a0348c00e8f1bd8fc52f024db95bea3 61 | parent 8cb34d026e9c290b83c52301d82b2011406fc7d8 62 | author Author A 1491593068 -0400 63 | committer Author B 1493244209 +0000 64 | 65 | Add firehose schema docs 66 | 67 | This commit starts a new page for documenting the firehose schema. Right 68 | now it only includes the schema for gerrit events, but it will be 69 | expanded in future patches to include the other services reporting. 70 | Hopefully this will serve as a better guide for people to actually being 71 | able to consume events from firehose. 72 | 73 | Change-Id: I2157f702c87f32055ba2fad842a05e31539bc857 74 | 75 | 1 0 doc/source/firehose.rst 76 | 59 0 doc/source/firehose_schema.rst 77 | 1 0 doc/source/systems.rst 78 | 3 0 doc/source/{arch.rst => components.rst} 79 | 80 | commit 8cb34d026e9c290b83c52301d82b2011406fc7d8 81 | tree 06ca044790deff0e862ea1748f8ac5d6bd62bb9c 82 | author Author C 1493240029 -0700 83 | committer Author C 1493240029 -0700 84 | 85 | Fix use of _ that should be - in mqtt-ca_certs 86 | 87 | This typo in the subunit2sql worker config was preventing it from 88 | running properly. Fix it. 89 | 90 | Change-Id: I4155bdd80523b73fdc69f45d6120e8eec986dda7 91 | 92 | 1 1 modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb 93 | 94 | commit 88364beba125cc8e6e314885db1c909b3d526340 95 | tree 7216d778a004570112903d36fdf9895288e1aad3 96 | parent 920bd292ea4cd3157a62c5deaaae0ea56740afa2 97 | author Author C 1493240029 -0700 98 | committer Author C 1493240029 -0700 99 | HG:extra histedit_source:c73fdac7f898c02211ccadfc1b24a1e55e54e6c9%2C383797830f6c21649c53cb112e29ea34c4021804 100 | HG:extra source:83ee8628972f4de30e4f10c2c75f3d8dec4eca03 101 | 102 | Add type declarations for Windows API calls as found in jaraco.windows 3.6.1. Fixes #758. 103 | 104 | 19 0 paramiko/_winapi.py 105 | 2 0 sites/www/changelog.rst 106 | 107 | commit f5d7eb5b623b625062cf0d3d8d552ee0ea9000dd 108 | tree c2f0679439901d9e167be239c93b4a9211f08415 109 | parent 5bca9f6aaf14129d5582e1b4ad6adc11253ebe4e 110 | author Author C 1493240029 -0700 111 | committer Author C 1493240029 -0700 112 | 113 | windows linefeed was breaking /usr/bin/env from executing correctly :/s/ 114 | //g 115 | 116 | 1 1 SickBeard.py 117 | 118 | commit 8e1cc08e799a83ace198ee7a3c6f9169635e7f46 119 | tree b66305787d63c93c778b41ca49ff12ff30d856b7 120 | parent ae272bb6f3a23b3f393ee4f6fd133cc407d2090d 121 | parent be272bb6f3a23b3f393ee4f6fd133cc407d2090d 122 | author Author C 1493240029 -0700 123 | committer Author C 1493240029 -0700 124 | 125 | Merge pull request #13155 from coolljt0725/fix_validate_tag_name 126 | 127 | Fix validate tag name. Fix #13149 128 | 129 | commit 1c939e7487986f1ada02f1414f6101b7cd696824 130 | tree 3acbcc73d05349df63e7b750322835c3a7bf3b2d 131 | parent 31d980a72fbc620a31e44929f831263b1ecab5e1 132 | author mysql-builder@oracle.com <> 1352117713 +0530 133 | committer mysql-builder@oracle.com <> 1352117713 +0530 134 | -------------------------------------------------------------------------------- /repoxplorer/index/users.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import logging 17 | 18 | from elasticsearch.exceptions import NotFoundError 19 | 20 | from repoxplorer.index import add_params 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | class Users(object): 26 | 27 | PROPERTIES = { 28 | "uid": {"type": "keyword"}, 29 | "name": {"type": "keyword"}, 30 | "default-email": {"type": "keyword"}, 31 | "last_cnx": {"type": "date", "format": "epoch_second"}, 32 | "emails": { 33 | "type": "nested", 34 | "properties": { 35 | "email": {"type": "keyword"}, 36 | "groups": { 37 | "type": "nested", 38 | "properties": { 39 | "group": { 40 | "type": "keyword"}, 41 | "begin-date": { 42 | "type": "keyword"}, 43 | "end-date": { 44 | "type": "keyword"} 45 | } 46 | } 47 | } 48 | } 49 | } 50 | 51 | def __init__(self, connector=None): 52 | self.es = connector.es 53 | self.ic = connector.ic 54 | self.index = connector.index 55 | self.dbname = 'users' 56 | self.mapping = { 57 | self.dbname: { 58 | "properties": self.PROPERTIES, 59 | } 60 | } 61 | if not self.ic.exists_type(index=self.index, 62 | doc_type=self.dbname): 63 | kwargs = add_params(self.es) 64 | self.ic.put_mapping(index=self.index, doc_type=self.dbname, 65 | body=self.mapping, **kwargs) 66 | 67 | def create(self, user): 68 | self.es.create(self.index, doc_type=self.dbname, 69 | id=user['uid'], 70 | body=user) 71 | self.es.indices.refresh(index=self.index) 72 | 73 | def update(self, user): 74 | self.es.update(self.index, 75 | doc_type=self.dbname, 76 | id=user['uid'], 77 | body={"doc": user}) 78 | self.es.indices.refresh(index=self.index) 79 | 80 | def get(self, uid, silent=True): 81 | try: 82 | res = self.es.get(index=self.index, 83 | doc_type=self.dbname, 84 | id=uid) 85 | return res['_source'] 86 | except Exception as e: 87 | if silent: 88 | return None 89 | logger.error('Unable to get user (%s). %s' % (uid, e)) 90 | 91 | def get_ident_by_id(self, id): 92 | return self.get(id) 93 | 94 | def get_idents_by_emails(self, emails): 95 | if not isinstance(emails, list): 96 | emails = (emails,) 97 | params = {'index': self.index} 98 | body = { 99 | "query": {"bool": { 100 | "filter": {"bool": {"must": { 101 | "nested": { 102 | "path": "emails", 103 | "query": { 104 | "bool": { 105 | "should": [ 106 | {"match": {"emails.email": email}} for 107 | email in emails 108 | ] 109 | } 110 | } 111 | } 112 | }}} 113 | }} 114 | } 115 | params['body'] = body 116 | # TODO(fbo): Improve by doing it by bulk instead 117 | params['size'] = 10000 118 | ret = self.es.search(**params)['hits']['hits'] 119 | ret = [r['_source'] for r in ret] 120 | return ret 121 | 122 | def get_idents_in_group(self, group): 123 | params = {'index': self.index} 124 | body = { 125 | "query": {"bool": { 126 | "filter": {"bool": {"must": { 127 | "nested": { 128 | "path": "emails", 129 | "query": { 130 | "bool": {"must": { 131 | "nested": { 132 | "path": "emails.groups", 133 | "query": { 134 | "bool": {"must": { 135 | "match": { 136 | "emails.groups.group": group} 137 | } 138 | } 139 | } 140 | } 141 | } 142 | } 143 | } 144 | } 145 | }}} 146 | }} 147 | } 148 | params['body'] = body 149 | # TODO(fbo): Improve by doing it by bulk instead 150 | params['size'] = 10000 151 | ret = self.es.search(**params)['hits']['hits'] 152 | return [r['_source'] for r in ret] 153 | 154 | def delete(self, uid): 155 | try: 156 | self.es.delete(self.index, doc_type=self.dbname, id=uid) 157 | self.es.indices.refresh(index=self.index) 158 | except NotFoundError: 159 | pass 160 | -------------------------------------------------------------------------------- /repoxplorer/controllers/users.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016-2017, Fabien Boucher 2 | # Copyright 2016-2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from pecan import conf 18 | from pecan import abort 19 | from pecan import expose 20 | from pecan import request 21 | from pecan import response 22 | from pecan.rest import RestController 23 | 24 | from repoxplorer import index 25 | from repoxplorer.exceptions import UnauthorizedException 26 | from repoxplorer.index import users 27 | from repoxplorer.controllers import utils 28 | 29 | if conf.get('users_endpoint', False) and conf.get('oidc', False): 30 | from repoxplorer.auth import OpenIDConnectEngine as AuthEngine 31 | else: 32 | from repoxplorer.auth import CAuthEngine as AuthEngine 33 | 34 | 35 | AUTH_ENGINE = AuthEngine() 36 | xorkey = conf.get('xorkey') or 'default' 37 | 38 | 39 | class UsersController(RestController): 40 | 41 | auth = AUTH_ENGINE 42 | 43 | def abort_if_not_active(self): 44 | if not self.auth.is_configured(): 45 | abort(403) 46 | 47 | def _authorize(self, uid=None): 48 | self.abort_if_not_active() 49 | # Shortcircuit the authorization for testing purpose 50 | # return 51 | try: 52 | self.auth.authorize(request, uid) 53 | except UnauthorizedException as e: 54 | abort(401, str(e)) 55 | except Exception as e: 56 | abort(500, "Unexpected error: %s" % e) 57 | self.auth.provision_user(request) 58 | 59 | def _validate(self, data): 60 | mandatory_keys = ( 61 | 'uid', 'name', 'default-email', 'emails') 62 | email_keys = ( 63 | ('email', True), 64 | ('groups', False)) 65 | group_keys = ( 66 | ('group', True), 67 | ('begin-date', False), 68 | ('end-date', False)) 69 | # All keys must be provided 70 | if set(data.keys()) != set(mandatory_keys): 71 | # Mandatory keys are missing 72 | return False 73 | if not isinstance(data['emails'], list): 74 | # Wrong data type for email 75 | return False 76 | if len(data['name']) >= 100: 77 | return False 78 | mekeys = set([mk[0] for mk in email_keys if mk[1]]) 79 | mgkeys = set([mk[0] for mk in group_keys if mk[1]]) 80 | if data['emails']: 81 | for email in data['emails']: 82 | if not mekeys.issubset(set(email.keys())): 83 | # Mandatory keys are missing 84 | return False 85 | if not set(email.keys()).issubset( 86 | set([k[0] for k in email_keys])): 87 | # Found extra keys 88 | return False 89 | if 'groups' in email.keys(): 90 | for group in email['groups']: 91 | if not mgkeys.issubset(set(group.keys())): 92 | # Mandatory keys are missing 93 | return False 94 | if not set(group.keys()).issubset( 95 | set([k[0] for k in group_keys])): 96 | # Found extra keys 97 | return False 98 | return True 99 | 100 | def _modify_protected_fields(self, prev, new): 101 | if new['uid'] != prev['uid']: 102 | return True 103 | if new['default-email'] != prev['default-email']: 104 | return True 105 | # Adding or removing emails is forbidden 106 | prev_emails = set([e['email'] for e in prev['emails']]) 107 | new_emails = set([e['email'] for e in new['emails']]) 108 | if (not new_emails.issubset(prev_emails) or 109 | not prev_emails.issubset(new_emails)): 110 | return True 111 | return False 112 | 113 | # curl -H 'Remote-User: admin' -H 'Admin-Token: abc' \ 114 | # "http://localhost:51000/api/v1/users/fabien" 115 | @expose('json') 116 | def get(self, uid): 117 | self._authorize(uid) 118 | _users = users.Users( 119 | index.Connector(index_suffix='users')) 120 | u = _users.get(uid) 121 | if not u: 122 | abort(404) 123 | u['cid'] = utils.encrypt(xorkey, u['default-email']) 124 | return u 125 | 126 | @expose('json') 127 | def delete(self, uid): 128 | self._authorize(uid) 129 | _users = users.Users( 130 | index.Connector(index_suffix='users')) 131 | u = _users.get(uid) 132 | if not u: 133 | abort(404) 134 | _users.delete(uid) 135 | 136 | # curl -X PUT -H 'Remote-User: admin' -H 'Admin-Token: abc' \ 137 | # -H "Content-Type: application/json" --data \ 138 | # '{"uid":"fabien","name":"Fabien Boucher","default-email": \ 139 | # "fboucher@redhat.com","emails": [{"email": "fboucher@redhat.com"}]}' \ 140 | # "http://localhost:51000/api/v1/users/fabien" 141 | @expose('json') 142 | def put(self, uid): 143 | # We don't pass uid to authorize, then only admin logged with 144 | # admin token will be authorized 145 | self._authorize() 146 | _users = users.Users( 147 | index.Connector(index_suffix='users')) 148 | u = _users.get(uid) 149 | if u: 150 | abort(409) 151 | infos = request.json if request.content_length else {} 152 | if not self._validate(infos): 153 | abort(400) 154 | # Need to check infos content 155 | infos['uid'] = uid 156 | _users.create(infos) 157 | response.status = 201 158 | 159 | # curl -X POST -H 'Remote-User: admin' -H 'Admin-Token: abc' \ 160 | # -H "Content-Type: application/json" --data \ 161 | # '{"uid":"fabien","name":"Fabien Boucher","default-email": \ 162 | # "fboucher@redhat.com","emails": [{"email": "fboucher@redhat.com"}, \ 163 | # {"email": "fabien.boucher@enovance.com"}]}' \ 164 | # "http://localhost:51000/api/v1/users/fabien" 165 | @expose('json') 166 | def post(self, uid): 167 | requester = self._authorize(uid) 168 | _users = users.Users( 169 | index.Connector(index_suffix='users')) 170 | u = _users.get(uid) 171 | if not u: 172 | abort(404) 173 | infos = request.json if request.content_length else {} 174 | infos['uid'] = uid 175 | # Can be provided by mistake, just remove it 176 | if 'cid' in infos: 177 | del infos['cid'] 178 | if not self._validate(infos): 179 | abort(400) 180 | if requester != 'admin': 181 | # User is not allowed to modify some raw_fields 182 | # like adding or removing emails ... 183 | if self._modify_protected_fields(u, infos): 184 | abort(403) 185 | _users.update(infos) 186 | -------------------------------------------------------------------------------- /repoxplorer/auth/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019, Matthieu Huin 2 | # Copyright 2019, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Various authentication engines supported by RepoXplorer.""" 17 | 18 | import base64 19 | import json 20 | import jwt 21 | from urllib.parse import urljoin 22 | import requests 23 | 24 | from pecan import conf 25 | 26 | from repoxplorer.exceptions import UnauthorizedException 27 | from repoxplorer import index 28 | from repoxplorer.index import users 29 | 30 | 31 | class BaseAuthEngine(object): 32 | """The base auth engine class.""" 33 | 34 | def is_configured(self) -> bool: 35 | """Activate the users REST endpoint if authentication is configured.""" 36 | return False 37 | 38 | def authorize(self, request, uid=None) -> str: 39 | """Make sure the authenticated user is allowed an action.""" 40 | raise UnauthorizedException("Not implemented") 41 | 42 | def provision_user(self, request) -> None: 43 | """If needed, the user can be provisioned based on the user info passed 44 | by the Identity Provider.""" 45 | return 46 | 47 | 48 | class CAuthEngine(BaseAuthEngine): 49 | """Cauth relies on Apache + mod_auth_authtkt to set a Remote-User header. 50 | User provisioning is done out of the band by Cauth itself, calling the 51 | PUT endpoint on the users API.""" 52 | 53 | def is_configured(self): 54 | return conf.get('users_endpoint', False) 55 | 56 | def authorize(self, request, uid=None): 57 | """Make sure the request is authorized. 58 | Returns the authorized user's uid or raises if unauthorized.""" 59 | if not request.remote_user: 60 | request.remote_user = request.headers.get('Remote-User') 61 | if not request.remote_user: 62 | request.remote_user = request.headers.get('X-Remote-User') 63 | if request.remote_user == '(null)': 64 | if request.headers.get('Authorization'): 65 | auth_header = request.headers.get('Authorization').split()[1] 66 | request.remote_user = base64.b64decode( 67 | auth_header).split(':')[0] 68 | if (request.remote_user == "admin" and 69 | request.headers.get('Admin-Token')): 70 | sent_admin_token = request.headers.get('Admin-Token') 71 | # If remote-user is admin and an admin-token is passed 72 | # authorized if the token is correct 73 | if sent_admin_token == conf.get('admin_token'): 74 | return 'admin' 75 | else: 76 | # If uid targeted by the request is the same 77 | # as the requester then authorize 78 | if uid and uid == request.remote_user: 79 | return uid 80 | if uid and uid != request.remote_user: 81 | raise UnauthorizedException("Admin action only") 82 | raise UnauthorizedException("unauthorized") 83 | 84 | 85 | class OpenIDConnectEngine(BaseAuthEngine): 86 | """Expects a Bearer token sent through the 'Authorization' header. 87 | The token is verified against a JWK, pulled from the well-known 88 | configuration of the OIDC provider. 89 | 90 | The claims will be used to provision users if authorization is 91 | successful.""" 92 | 93 | config = conf.get('oidc', {}) 94 | 95 | def is_configured(self): 96 | return self.config.get('issuer_url', False) 97 | 98 | def _get_issuer_info(self): 99 | issuer_url = self.config.get('issuer_url') 100 | verify_ssl = self.config.get('verify_ssl', True) 101 | issuer_info = requests.get( 102 | urljoin(issuer_url, '.well-known/openid-configuration'), 103 | verify=verify_ssl) 104 | if issuer_info.status_code > 399: 105 | raise UnauthorizedException( 106 | "Cannot fetch OpenID provider's configuration") 107 | return issuer_info.json() 108 | 109 | def _get_signing_key(self, jwks_uri, key_id): 110 | verify_ssl = self.config.get('verify_ssl', True) 111 | certs = requests.get(jwks_uri, verify=verify_ssl) 112 | if certs.status_code > 399: 113 | raise UnauthorizedException("Cannot fetch JWKS") 114 | for k in certs.json()['keys']: 115 | if k['kid'] == key_id: 116 | return (jwt.algorithms.RSAAlgorithm.from_jwk(json.dumps(k)), 117 | k['alg']) 118 | raise UnauthorizedException("Key %s not found" % key_id) 119 | 120 | def _get_raw_token(self, request): 121 | if request.headers.get('Authorization', None) is None: 122 | raise UnauthorizedException('Missing "Authorization" header') 123 | auth_header = request.headers.get('Authorization', None) 124 | if not auth_header.lower().startswith('bearer '): 125 | raise UnauthorizedException('Invalid "Authorization" header') 126 | token = auth_header[len('bearer '):] 127 | return token 128 | 129 | def authorize(self, request, uid=None): 130 | token = self._get_raw_token(request) 131 | issuer_info = self._get_issuer_info() 132 | unverified_headers = jwt.get_unverified_header(token) 133 | key_id = unverified_headers.get('kid', None) 134 | if key_id is None: 135 | raise UnauthorizedException("Missing key id in token") 136 | jwks_uri = issuer_info.get('jwks_uri') 137 | if jwks_uri is None: 138 | raise UnauthorizedException("Missing JWKS URI in config") 139 | key, algo = self._get_signing_key(jwks_uri, key_id) 140 | try: 141 | claims = jwt.decode(token, key, algorithms=algo, 142 | issuer=issuer_info['issuer'], 143 | audience=self.config['audience']) 144 | except Exception as e: 145 | raise UnauthorizedException('Invalid access token: %s' % e) 146 | if claims['preferred_username'] == self.config.get('admin_username', 147 | 'admin'): 148 | return 'admin' 149 | if uid and uid == claims['preferred_username']: 150 | return uid 151 | if uid and uid != claims['preferred_username']: 152 | raise UnauthorizedException("Only the admin ") 153 | raise UnauthorizedException('unauthorized') 154 | 155 | def provision_user(self, request): 156 | raw_token = self._get_raw_token(request) 157 | # verified before so it's totally okay 158 | claims = jwt.decode(raw_token, verify=False) 159 | # TODO assuming the presence of claims, but a specific scope might be 160 | # needed. 161 | # These are expected to be standard though, see 162 | # https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims 163 | email = claims['email'] 164 | uid = claims['preferred_username'] 165 | name = claims['name'] 166 | _users = users.Users(index.Connector(index_suffix='users')) 167 | u = _users.get(uid) 168 | infos = {'uid': uid, 169 | 'name': name, 170 | 'default-email': email, 171 | 'emails': [{'email': email}]} 172 | if u: 173 | _users.update(infos) 174 | else: 175 | _users.create(infos) 176 | -------------------------------------------------------------------------------- /bin/repoxplorer-indexer: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2016, Fabien Boucher 4 | # Copyright 2016, Red Hat 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import imp 19 | import sys 20 | import time 21 | import logging 22 | import argparse 23 | import logging.config 24 | 25 | from pecan import configuration 26 | 27 | from repoxplorer.indexer.git import indexer 28 | from repoxplorer.index import projects 29 | 30 | logger = logging.getLogger('indexerDaemon') 31 | 32 | parser = argparse.ArgumentParser(description='RepoXplorer indexer') 33 | parser.add_argument( 34 | '--forever', action='store_true', default=False, 35 | help='Make the indexer run forever') 36 | parser.add_argument( 37 | '--extract-workers', type=int, default=0, 38 | help='Specify the amount of worker processes for ' 39 | 'extracting commits information (default = auto)') 40 | parser.add_argument( 41 | '--config', required=True, 42 | help='Path to the repoXplorer configuration file') 43 | parser.add_argument( 44 | '--project', type=str, default=None, 45 | help='Specify the project to index') 46 | parser.add_argument( 47 | '--logfile', type=str, default=None, 48 | help='Override the logging file from config') 49 | parser.add_argument( 50 | '--loglevel', type=str, default=None, 51 | help='Override the logging level from config') 52 | parser.add_argument( 53 | '--clean-orphan', action='store_true', default=False, 54 | help="Clean refs and tags not known from the project's configuration") 55 | parser.add_argument( 56 | '--refresh-projects-index', action='store_true', default=False, 57 | help="Refresh projects index into the Elastic database") 58 | 59 | args = parser.parse_args() 60 | 61 | 62 | def refresh_projects_index(): 63 | logger.info("Start loading projects index configuration into EL") 64 | projects.Projects(dump_yaml_in_index=True) 65 | 66 | 67 | def clean(conf): 68 | logger.info("Start cleaning no longer referenced refs and tags") 69 | projects_index = projects.Projects() 70 | rc = indexer.RefsCleaner(projects_index) 71 | refs_to_clean = rc.find_refs_to_clean() 72 | rc.clean(refs_to_clean) 73 | 74 | 75 | def process(conf): 76 | projects_index = projects.Projects() 77 | prjs = projects_index.get_projects(source=['name', 'refs', 'meta-ref']) 78 | for project in prjs.values(): 79 | pid = project['name'] 80 | if args.project and args.project != pid: 81 | continue 82 | if not hasattr(conf, 'indexer_skip_projects'): 83 | conf.indexer_skip_projects = [] 84 | if not args.project and pid in conf.indexer_skip_projects: 85 | continue 86 | logger.info("Start indexing project %s" % pid) 87 | meta_ref = None 88 | if project.get('meta-ref') is True: 89 | meta_ref = pid 90 | for ref in project['refs']: 91 | r = indexer.RepoIndexer(ref['name'], 92 | ref['uri'], 93 | parsers=ref['parsers'], 94 | meta_ref=meta_ref) 95 | try: 96 | r.git_init() 97 | except Exception as e: 98 | logger.warning("Unable to init the repository %s: %s" % ( 99 | r.base_id, e)) 100 | continue 101 | try: 102 | r.get_refs() 103 | except Exception as e: 104 | logger.warning("Unable to access the repository %s: %s" % ( 105 | r.base_id, e)) 106 | continue 107 | r.get_heads() 108 | if ref.get('index-tags') is True: 109 | r.get_tags() 110 | if not [head for head in r.heads if 111 | head[1].endswith(ref['branch'])]: 112 | logger.warning( 113 | "Repository %s does not have the " 114 | "requested branch %s" % (r.base_id, ref['branch'])) 115 | continue 116 | r.set_branch(ref['branch']) 117 | if r.is_branch_fully_indexed(): 118 | logger.info("Repository branch fully indexed %s" % ( 119 | r.ref_id)) 120 | continue 121 | logger.info("Start indexing repository branch %s" % r.ref_id) 122 | try: 123 | r.git_fetch_branch() 124 | except Exception as e: 125 | logger.warning("Unable to fetch repository " 126 | "branch %s: %s" % (r.ref_id, e)) 127 | continue 128 | try: 129 | r.git_get_commit_obj() 130 | r.get_current_commits_indexed() 131 | r.compute_to_index_to_delete() 132 | r.index(args.extract_workers) 133 | except Exception as e: 134 | logger.warning("Unable to index repository " 135 | "branch %s: %s" % (r.ref_id, e)) 136 | logger.exception("Exception is:") 137 | continue 138 | try: 139 | if ref.get('index-tags') is True: 140 | r.index_tags() 141 | else: 142 | # Make sure to wipe tags for this repo if index-tags flag 143 | # is False. 144 | rc = indexer.RefsCleaner( 145 | projects_index, config=args.config) 146 | tags = rc.t.get_tags([r.base_id]) 147 | ids = [t['_id'] for t in tags] 148 | if ids: 149 | logger.info( 150 | "Found %s tags for %s but index-tags is False. " 151 | "Wipe tags ..." % (len(ids), r.base_id)) 152 | rc.t.del_tags(ids) 153 | except Exception as e: 154 | logger.warning("Unable to index repository tags " 155 | "%s: %s" % (r.base_id, e)) 156 | continue 157 | 158 | 159 | if __name__ == "__main__": 160 | conf = imp.load_source('config', args.config) 161 | configuration.set_config(args.config) 162 | if args.logfile: 163 | conf.indexer_logging['handlers']['normal']['filename'] = args.logfile 164 | if args.loglevel: 165 | conf.indexer_logging['handlers']['normal']['level'] = args.loglevel 166 | conf.indexer_logging['handlers']['console']['level'] = args.loglevel 167 | logging.config.dictConfig(conf.indexer_logging) 168 | if args.refresh_projects_index: 169 | refresh_projects_index() 170 | sys.exit() 171 | if args.clean_orphan: 172 | try: 173 | refresh_projects_index() 174 | clean(conf) 175 | sys.exit(0) 176 | except Exception: 177 | logger.exception("Unexcepted error occured") 178 | sys.exit(-1) 179 | if args.forever: 180 | while True: 181 | try: 182 | refresh_projects_index() 183 | process(conf) 184 | clean(conf) 185 | except Exception: 186 | logger.exception("Unexcepted error occured") 187 | if args.forever: 188 | logger.info( 189 | "Waiting the loop delay (%s/s)" % conf.indexer_loop_delay) 190 | time.sleep(conf.indexer_loop_delay) 191 | else: 192 | try: 193 | refresh_projects_index() 194 | process(conf) 195 | clean(conf) 196 | except Exception: 197 | logger.exception("Unexcepted error occured") 198 | sys.exit(-1) 199 | -------------------------------------------------------------------------------- /repoxplorer/public/contributor.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 |
43 | 44 |
45 |
46 |
47 | 48 |
49 | 50 | 55 | 56 |
57 | 58 |
59 |
60 |
61 |

62 | Infos (based on selected filters) 63 | 64 | 65 | 66 |

67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 | 87 |
88 |
89 |
90 |

Filters

91 |
92 |
93 |
94 |
95 |
96 | 115 | 116 | 117 |
118 |
119 | 120 | 121 |
122 |
123 | 124 |
125 |
126 | 127 |
128 |
129 |
130 | 131 | 133 |
134 |
135 |
136 |
137 |
138 |
139 | 140 |
141 | 142 | 149 | 150 |
151 |
152 |
153 |
154 |

Commits history

155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 | 164 |
165 | 166 |
167 |
168 |
169 |

170 | Top projects by commit 171 |

172 |
173 |
174 |
175 |
176 |
177 |
178 | 179 |
180 |
181 |
182 |

183 | Top projects by lines changed 184 |

185 |
186 |
187 |
188 |
189 |
190 |
191 | 192 |
193 | 194 |
195 |
196 |
197 |
198 |

Contributor commits (Limited to 100 pages)

199 |
200 |
201 |
202 |
203 |
204 | 206 |
207 |
208 |
209 |
210 | 211 |
212 | 213 | 216 | 217 | 218 | -------------------------------------------------------------------------------- /bin/repoxplorer-fetch-web-assets: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | 3 | # Copyright 2017, Fabien Boucher 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import sys 19 | import imp 20 | import glob 21 | import shutil 22 | import tarfile 23 | import zipfile 24 | import tempfile 25 | import argparse 26 | import requests 27 | 28 | from pkg_resources import resource_string 29 | 30 | archives = { 31 | # No release. Master of 03/05/2017 32 | "simplePagination": "https://codeload.github.com/flaviusmatis/simplePagination.js/tar.gz/07c37285fafc7dbabce0392f730037ac012cc3ea", # noqa 33 | "moment.js": "https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.17.1/moment.min.js", # noqa 34 | "d3.js": "https://cdnjs.cloudflare.com/ajax/libs/d3/3.1.6/d3.min.js", # noqa 35 | "jquery-ui": "https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip", # noqa 36 | "jquery": "https://cdnjs.cloudflare.com/ajax/libs/jquery/1.12.4/jquery.min.js", # noqa 37 | "dimple.js": "https://cdnjs.cloudflare.com/ajax/libs/dimple/2.2.0/dimple.latest.min.js", # noqa 38 | "bootstrap": "https://github.com/twbs/bootstrap/releases/download/v3.3.7/bootstrap-3.3.7-dist.zip", # noqa 39 | "keycloak": "https://downloads.jboss.org/keycloak/9.0.0/adapters/keycloak-oidc/keycloak-js-adapter-dist-9.0.0.zip", # noqa 40 | } 41 | 42 | 43 | def download_file(url, targetdir): 44 | local_filename = url.split('/')[-1] 45 | local_filename = os.path.join(targetdir, local_filename) 46 | r = requests.get(url, stream=True) 47 | with open(local_filename, 'wb') as f: 48 | for chunk in r.iter_content(chunk_size=1024): 49 | if chunk: 50 | f.write(chunk) 51 | return local_filename 52 | 53 | 54 | def install_archive(name, path, basepath): 55 | if name == "moment.js": 56 | shutil.move(path, 57 | os.path.join(basepath, 'javascript', 'moment.min.js')) 58 | elif name == "simplePagination": 59 | tar = tarfile.open(path) 60 | tar.extractall(os.path.dirname(path)) 61 | tar.close() 62 | shutil.move( 63 | os.path.join( 64 | os.path.dirname(path), 65 | "simplePagination.js-07c37285fafc7dbabce0392f730037ac012cc3ea", 66 | "jquery.simplePagination.js"), 67 | os.path.join(basepath, 'javascript', 'jquery.simplePagination.js') 68 | ) 69 | shutil.move( 70 | os.path.join( 71 | os.path.dirname(path), 72 | "simplePagination.js-07c37285fafc7dbabce0392f730037ac012cc3ea", 73 | "simplePagination.css"), 74 | os.path.join(basepath, 'css', 'simplePagination.css') 75 | ) 76 | shutil.rmtree(os.path.join( 77 | os.path.dirname(path), 78 | "simplePagination.js-07c37285fafc7dbabce0392f730037ac012cc3ea")) 79 | os.unlink(path) 80 | elif name == "d3.js": 81 | shutil.move(path, 82 | os.path.join(basepath, 'javascript', 'd3.min.js')) 83 | elif name == "jquery": 84 | shutil.move(path, 85 | os.path.join(basepath, 'javascript', 'jquery.min.js')) 86 | elif name == "dimple.js": 87 | shutil.move(path, 88 | os.path.join(basepath, 'javascript', 'dimple.min.js')) 89 | elif name == "jquery-ui": 90 | zi = zipfile.ZipFile(path) 91 | zi.extractall(os.path.dirname(path)) 92 | zi.close() 93 | shutil.move( 94 | os.path.join(os.path.dirname(path), 95 | "jquery-ui-1.12.1", 96 | "jquery-ui.min.js"), 97 | os.path.join(basepath, 'javascript', 'jquery-ui.min.js') 98 | ) 99 | shutil.move( 100 | os.path.join(os.path.dirname(path), 101 | "jquery-ui-1.12.1", 102 | "jquery-ui.min.css"), 103 | os.path.join(basepath, 'css', 'jquery-ui.min.css') 104 | ) 105 | for f in glob.glob(os.path.join(os.path.dirname(path), 106 | "jquery-ui-1.12.1", "images", "*")): 107 | shutil.move(f, os.path.join(basepath, "css", 108 | "images", os.path.basename(f))) 109 | shutil.rmtree(os.path.join(os.path.dirname(path), "jquery-ui-1.12.1")) 110 | os.unlink(path) 111 | elif name == "bootstrap": 112 | zi = zipfile.ZipFile(path) 113 | zi.extractall(os.path.dirname(path)) 114 | zi.close() 115 | shutil.move( 116 | os.path.join(os.path.dirname(path), 117 | "bootstrap-3.3.7-dist", "js", 118 | "bootstrap.min.js"), 119 | os.path.join(basepath, 'javascript', 'bootstrap.min.js') 120 | ) 121 | shutil.move( 122 | os.path.join(os.path.dirname(path), 123 | "bootstrap-3.3.7-dist", "css", 124 | "bootstrap.min.css"), 125 | os.path.join(basepath, 'css', 'bootstrap.min.css') 126 | ) 127 | for f in glob.glob(os.path.join(os.path.dirname(path), 128 | "bootstrap-3.3.7-dist", "fonts", "*")): 129 | shutil.move(f, os.path.join(basepath, 130 | 'fonts', os.path.basename(f))) 131 | shutil.rmtree( 132 | os.path.join(os.path.dirname(path), "bootstrap-3.3.7-dist")) 133 | os.unlink(path) 134 | elif name == "keycloak": 135 | zi = zipfile.ZipFile(path) 136 | zi.extractall(os.path.dirname(path)) 137 | zi.close() 138 | shutil.move( 139 | os.path.join(os.path.dirname(path), 140 | "keycloak-js-adapter-dist-9.0.0", 141 | "keycloak.min.js"), 142 | os.path.join(basepath, 'javascript', 'keycloak.min.js') 143 | ) 144 | shutil.rmtree(os.path.join(os.path.dirname(path), 145 | "keycloak-js-adapter-dist-9.0.0")) 146 | os.unlink(path) 147 | else: 148 | print("Install for %s not supported. skip." % name) 149 | 150 | 151 | if __name__ == "__main__": 152 | parser = argparse.ArgumentParser( 153 | description='Fetch and install web assets for repoXplorer.') 154 | parser.add_argument( 155 | '--config', required=True, 156 | help='Path to the repoXplorer configuration file') 157 | 158 | args = parser.parse_args() 159 | conf = imp.load_source('config', args.config) 160 | static_root = conf.app['static_root'] 161 | 162 | td = tempfile.mkdtemp() 163 | print("Assets will be installed in %s ..." % static_root) 164 | if not os.path.isdir(static_root): 165 | os.makedirs(static_root) 166 | print("Fetch/Deflate assets in a temp directory %s ..." % td) 167 | for d in ('css', 'css/images', 'javascript', 'fonts'): 168 | os.mkdir(os.path.join(td, d)) 169 | for name, url in list(archives.items()): 170 | print("Fetch and Install %s ..." % name) 171 | path = download_file(url, td) 172 | install_archive(name, path, td) 173 | 174 | # Copy repoxplorer assets 175 | for path in ( 176 | ('public/javascript/repoxplorer.js', 'javascript/repoxplorer.js'), 177 | ('public/css/repoxplorer.css', 'css/repoxplorer.css'), 178 | ('public/index.html', 'index.html'), 179 | ('public/contributor.html', 'contributor.html'), 180 | ('public/contributors.html', 'contributors.html'), 181 | ('public/group.html', 'group.html'), 182 | ('public/groups.html', 'groups.html'), 183 | ('public/project.html', 'project.html'), 184 | ('public/home.html', 'home.html'), 185 | ('public/projects.html', 'projects.html')): 186 | data = resource_string('repoxplorer', path[0]) 187 | open(os.path.join(td, path[1]), 'wb').write(data) 188 | print("Copied %s" % os.path.join(td, path[1])) 189 | 190 | shutil.rmtree(static_root) 191 | shutil.move(td, static_root) 192 | os.chmod(static_root, 0o755) 193 | print("Assets moved to %s" % static_root) 194 | -------------------------------------------------------------------------------- /repoxplorer/controllers/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016-2017, Fabien Boucher 2 | # Copyright 2016-2017, Red Hat 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import re 17 | import base64 18 | from datetime import datetime 19 | 20 | from Crypto.Cipher import XOR 21 | 22 | from pecan import conf 23 | from pecan import abort 24 | 25 | 26 | xorkey = conf.get('xorkey') or 'default' 27 | 28 | 29 | def encrypt(key, plaintext): 30 | cipher = XOR.new(key) 31 | data = cipher.encrypt(plaintext) 32 | data = base64.b64encode(data) 33 | return data.decode(errors='ignore').replace('=', '-') 34 | 35 | 36 | def decrypt(key, ciphertext): 37 | cipher = XOR.new(key) 38 | ret = cipher.decrypt(base64.b64decode(ciphertext.replace('-', '='))) 39 | return ret.decode(errors='ignore') 40 | 41 | 42 | def authors_sanitize(idents, authors): 43 | sanitized = {} 44 | _idents = idents.get_idents_by_emails(list(authors.keys())) 45 | for iid, ident in _idents.items(): 46 | sanitized[ident['default-email']] = [0, ident['name'], iid] 47 | for ident_email in ident['emails']: 48 | if ident_email in authors: 49 | sanitized[ident['default-email']][0] += authors[ident_email] 50 | return sanitized 51 | 52 | 53 | def get_projects_from_references(pi, references): 54 | projects = pi.get_projects_from_references(references) 55 | return list(projects) 56 | 57 | 58 | def get_references_filter(project, inc_references=None): 59 | r_filter = {} 60 | can_use_meta_ref = True 61 | if inc_references: 62 | # The use of the meta ref is possible if we want stats of the complete 63 | # project 64 | can_use_meta_ref = False 65 | if "refs" in project: 66 | for r in project['refs']: 67 | if inc_references: 68 | if not "%(name)s:%(branch)s" % r in inc_references: 69 | continue 70 | r_filter[r['fullrid']] = r.get('paths') 71 | if r.get('paths'): 72 | # There is a path restriction then we cannot use the meta ref 73 | can_use_meta_ref = False 74 | if can_use_meta_ref and project.get('meta-ref', False): 75 | r_filter = {'meta_ref: %s' % project['name']: None} 76 | return r_filter 77 | 78 | 79 | def get_mail_filter(idents, cid=None, gid=None, group=None): 80 | if cid: 81 | _, ident = idents.get_ident_by_id(cid) 82 | if not ident: 83 | # No ident has been declared for that contributor 84 | ident = list(idents.get_idents_by_emails(cid).values())[0] 85 | return ident['emails'] 86 | elif gid: 87 | if not group: 88 | _, group = idents.get_group_by_id(gid) 89 | return group['emails'] 90 | else: 91 | return {} 92 | 93 | 94 | def filters_validation(projects_index, idents, pid=None, tid=None, 95 | cid=None, gid=None, dfrom=None, dto=None, 96 | inc_merge_commit=None, inc_repos=None, metadata=None, 97 | exc_groups=None, inc_groups=None): 98 | 99 | if pid and tid: 100 | abort(400, detail="pid and tid are exclusive") 101 | if cid and gid: 102 | abort(400, detail="cid and gid are exclusive") 103 | 104 | if exc_groups and inc_groups: 105 | abort(400, detail="exc_groups and inc_groups are exclusive") 106 | 107 | if (exc_groups or inc_groups) and not (pid or tid): 108 | abort(400, detail=( 109 | 'exc_groups or inc_groups can only be used' 110 | 'with pid or tid parameters')) 111 | 112 | if exc_groups and len(exc_groups.split(',')) > 1: 113 | abort(400, detail="as of now exc_groups supports only one group") 114 | if inc_groups and len(inc_groups.split(',')) > 1: 115 | abort(400, detail="as of now inc_groups supports only one group") 116 | 117 | if pid: 118 | if not projects_index.exists(pid): 119 | abort(404, 120 | detail="The project has not been found") 121 | if tid: 122 | if tid not in projects_index.get_tags(): 123 | abort(404, 124 | detail="The tag has not been found") 125 | if cid: 126 | # A cid can contain a comma separated list of emails 127 | if not cid.endswith(','): 128 | try: 129 | cid = decrypt(xorkey, cid) 130 | except Exception: 131 | abort(404, 132 | detail="The cid is incorrectly formated") 133 | 134 | if gid: 135 | _, group = idents.get_group_by_id(gid) 136 | if not group: 137 | abort(404, 138 | detail="The group has not been found") 139 | 140 | try: 141 | if dfrom: 142 | datetime.strptime(dfrom, "%Y-%m-%d") 143 | if dto: 144 | datetime.strptime(dto, "%Y-%m-%d") 145 | except Exception: 146 | abort(400, 147 | detail="Date format is expected to be 'Y-m-d'") 148 | 149 | 150 | def resolv_filters(projects_index, idents, pid, 151 | tid, cid, gid, dfrom, dto, inc_repos, 152 | inc_merge_commit, metadata, exc_groups, 153 | inc_groups): 154 | 155 | filters_validation( 156 | projects_index, idents, pid=pid, tid=tid, cid=cid, gid=gid, 157 | dfrom=dfrom, dto=dto, inc_merge_commit=inc_merge_commit, 158 | inc_repos=inc_repos, metadata=metadata, exc_groups=exc_groups, 159 | inc_groups=inc_groups) 160 | 161 | mails = {} 162 | domains = [] 163 | 164 | if pid: 165 | project = projects_index.get( 166 | pid, source=['name', 'meta-ref', 'refs', 'bots-group']) 167 | p_filter = get_references_filter(project, inc_repos) 168 | elif tid: 169 | refs = projects_index.get_references_from_tags(tid) 170 | project = {'refs': refs, 'name': tid} 171 | p_filter = get_references_filter(project, inc_repos) 172 | else: 173 | p_filter = [] 174 | 175 | blacklisted_mails = [] 176 | if pid: 177 | bots_group = project.get('bots-group') 178 | if bots_group: 179 | _, group = idents.get_group_by_id(bots_group) 180 | blacklisted_mails = group['emails'] 181 | 182 | _metadata = [] 183 | if not metadata: 184 | metadata = "" 185 | metadata_splitted = metadata.split(',') 186 | for meta in metadata_splitted: 187 | try: 188 | key, value = meta.split(':') 189 | if value == '*': 190 | value = None 191 | except ValueError: 192 | continue 193 | _metadata.append((key, value)) 194 | 195 | mails_neg = False 196 | 197 | if exc_groups and (pid or tid): 198 | mails_to_exclude = {} 199 | domains_to_exclude = [] 200 | mails_neg = True 201 | groups_splitted = exc_groups.split(',') 202 | for _gid in groups_splitted: 203 | _, group = idents.get_group_by_id(_gid) 204 | mails_to_exclude.update(group['emails']) 205 | domains_to_exclude.extend(group.get('domains', [])) 206 | 207 | if inc_groups and (pid or tid): 208 | groups_splitted = inc_groups.split(',') 209 | for _gid in groups_splitted: 210 | _, group = idents.get_group_by_id(_gid) 211 | mails.update(group['emails']) 212 | domains.extend(group.get('domains', [])) 213 | 214 | if cid or gid: 215 | if cid: 216 | if cid.endswith(','): 217 | mails = [e.lstrip(',') for 218 | e in re.findall('[^@]+@[^@,]+', cid)] 219 | else: 220 | cid = decrypt(xorkey, cid) 221 | mails = get_mail_filter(idents, cid=cid) 222 | if gid: 223 | _, group = idents.get_group_by_id(gid) 224 | mails = get_mail_filter(idents, gid=gid, group=group) 225 | domains.extend(group.get('domains', [])) 226 | 227 | if dfrom: 228 | dfrom = datetime.strptime(dfrom, "%Y-%m-%d").strftime('%s') 229 | 230 | if dto: 231 | dto = str( 232 | int(datetime.strptime(dto, "%Y-%m-%d").strftime('%s')) + 24 * 3600) 233 | 234 | if inc_merge_commit == 'on': 235 | # The None value will return all whatever 236 | # the commit is a merge one or not 237 | inc_merge_commit = None 238 | else: 239 | inc_merge_commit = False 240 | 241 | if mails_neg: 242 | mails = mails_to_exclude 243 | domains = domains_to_exclude 244 | 245 | query_kwargs = { 246 | 'repos': p_filter, 247 | 'fromdate': dfrom, 248 | 'mails': mails, 249 | 'domains': domains, 250 | 'mails_neg': mails_neg, 251 | 'todate': dto, 252 | 'merge_commit': inc_merge_commit, 253 | 'metadata': _metadata, 254 | 'blacklisted_mails': blacklisted_mails, 255 | } 256 | return query_kwargs 257 | -------------------------------------------------------------------------------- /repoxplorer/public/group.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 | 43 |
44 | 45 |
46 |
47 |
48 | 49 |
50 | 51 | 56 | 57 |
58 | 59 |
60 |
61 |
62 |

63 | Infos (based on selected filters) 64 | 65 | 66 | 67 |

68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 | 85 |
86 |
87 |
88 |

Filters

89 |
90 |
91 |
92 |
93 |
94 | 113 | 114 | 115 |
116 |
117 | 118 | 119 |
120 |
121 | 122 |
123 |
124 | 125 |
126 |
127 |
128 | 129 | 131 |
132 |
133 |
134 |
135 |
136 |
137 | 138 |
139 | 140 | 147 | 148 |
149 | 150 |
151 |
152 |
153 |

154 | Top authors by commit 155 |

156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 | 165 |
166 |
167 |
168 |

169 | Top authors by lines changed 170 |

171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 | 180 |
181 | 182 |
183 | 184 |
185 |
186 |
187 |

188 | Top projects by commit 189 |

190 |
191 |
192 |
193 |
194 |
195 |
196 | 197 |
198 |
199 |
200 |

201 | Top projects by lines changed 202 |

203 |
204 |
205 |
206 |
207 |
208 |
209 | 210 |
211 | 212 | 213 |
214 |
215 |
216 |
217 |

Commits history

218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 | 227 |
228 |
229 |
230 |
231 |

Contributors history

232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 | 241 | 242 |
243 |
244 |
245 |
246 |

Group commits (Limited to 100 pages)

247 |
248 |
249 |
250 |
251 |
252 | 254 |
255 |
256 |
257 |
258 | 259 |
260 | 261 | 264 | 265 | 266 | 267 | 268 | -------------------------------------------------------------------------------- /repoxplorer/public/project.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 41 |
42 |
43 | 44 |
45 |
46 |
47 | 48 |
49 | 50 | 55 | 56 |
57 | 58 |
59 |
60 |
61 |

62 | Infos (based on selected filters) 63 | 64 | 65 | 66 |

67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 | 84 |
85 |
86 |
87 |

Filters

88 |
89 |
90 |
91 |
92 |
93 | 110 | 111 | 112 |
113 |
114 | 115 | 116 |
117 |
118 | 119 |
120 |
121 |
122 | 123 | 124 |
125 |
126 | 127 | 129 | 130 | 132 |

133 | 134 |

135 | 136 |

137 |
138 |
139 |
140 | 141 | 142 | 143 | 144 |
145 |
146 |
147 |
148 |
149 |
150 | 151 |
152 | 153 | 160 | 161 |
162 | 163 |
164 |
165 |
166 |

167 | Top authors by commit 168 |

169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 | 178 |
179 |
180 |
181 |

182 | Top authors by lines changed 183 |

184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 | 193 |
194 |
195 |
196 |

197 | Top new authors by commit 198 |

199 |
200 |
201 |
202 |
203 | 204 | 212 |
213 |
214 |
215 |
216 | 217 | 218 |
219 |
220 |
221 |

222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 | 231 |
232 | 233 |
234 | 235 |
236 |
237 |
238 |
239 |

Commits history

240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 | 248 |
249 |
250 |
251 |

Contributors history

252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 | 261 |
262 | 263 |
264 |
265 |
266 |
267 |

Project commits (Limited to 100 pages)

268 |
269 |
270 |
271 |
272 |
273 | 275 |
276 |
277 |
278 |
279 | 280 |
281 |
282 | 285 | 286 | 287 | -------------------------------------------------------------------------------- /repoxplorer/controllers/tops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Fabien Boucher 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import copy 16 | import hashlib 17 | 18 | from pecan import abort 19 | from pecan import conf 20 | from pecan import expose 21 | 22 | from repoxplorer import index 23 | from repoxplorer.controllers import utils 24 | from repoxplorer.index.commits import Commits 25 | from repoxplorer.index.projects import Projects 26 | from repoxplorer.index.contributors import Contributors 27 | 28 | xorkey = conf.get('xorkey') or 'default' 29 | 30 | 31 | class TopAuthorsController(object): 32 | 33 | def resolv_name(self, commits, authors): 34 | name_to_requests = [] 35 | for v in authors: 36 | if not v['name']: 37 | name_to_requests.append(v['email']) 38 | if name_to_requests: 39 | raw_names = commits.get_commits_author_name_by_emails( 40 | name_to_requests) 41 | for v in authors: 42 | v['name'] = v['name'] or raw_names[v['email']] 43 | del v['email'] 44 | 45 | def top_authors_sanitize( 46 | self, idents, authors, commits, top, 47 | resolv_name=True, clean_email=True): 48 | sanitized = utils.authors_sanitize(idents, authors) 49 | top_authors_s = [] 50 | for email, v in sanitized.items(): 51 | top_authors_s.append( 52 | {'cid': utils.encrypt(xorkey, v[2]), 53 | 'email': email, 54 | 'gravatar': hashlib.md5( 55 | email.encode(errors='ignore')).hexdigest(), 56 | 'amount': int(v[0]), 57 | 'name': v[1]}) 58 | top_authors_s_sorted = sorted(top_authors_s, 59 | key=lambda k: k['amount'], 60 | reverse=True) 61 | if top is None: 62 | top = 10 63 | else: 64 | top = int(top) 65 | # If top set to a negative value all results will be returned 66 | if top >= 0: 67 | top_authors_s_sorted = top_authors_s_sorted[:int(top)] 68 | 69 | if resolv_name: 70 | self.resolv_name(commits, top_authors_s_sorted) 71 | 72 | return top_authors_s_sorted 73 | 74 | def gbycommits( 75 | self, c, idents, query_kwargs, top, 76 | resolv_name=True, clean_email=True): 77 | authors = c.get_authors(**query_kwargs)[1] 78 | top_authors = self.top_authors_sanitize( 79 | idents, authors, c, top, resolv_name, clean_email) 80 | 81 | return top_authors 82 | 83 | def gbylchanged(self, c, idents, query_kwargs, top): 84 | top_authors_modified = c.get_top_authors_by_lines(**query_kwargs)[1] 85 | 86 | top_authors_modified = self.top_authors_sanitize( 87 | idents, top_authors_modified, c, top) 88 | 89 | return top_authors_modified 90 | 91 | @expose('json') 92 | @expose('csv:', content_type='text/csv') 93 | def bylchanged(self, pid=None, tid=None, cid=None, gid=None, 94 | dfrom=None, dto=None, inc_merge_commit=None, 95 | inc_repos=None, metadata=None, exc_groups=None, 96 | limit=None, inc_groups=None): 97 | 98 | c = Commits(index.Connector()) 99 | projects_index = Projects() 100 | idents = Contributors() 101 | 102 | query_kwargs = utils.resolv_filters( 103 | projects_index, idents, pid, tid, cid, gid, 104 | dfrom, dto, inc_repos, inc_merge_commit, metadata, 105 | exc_groups, inc_groups) 106 | 107 | return self.gbylchanged(c, idents, query_kwargs, limit) 108 | 109 | @expose('json') 110 | @expose('csv:', content_type='text/csv') 111 | def bycommits(self, pid=None, tid=None, cid=None, gid=None, 112 | dfrom=None, dto=None, inc_merge_commit=None, 113 | inc_repos=None, metadata=None, exc_groups=None, 114 | limit=None, inc_groups=None): 115 | 116 | c = Commits(index.Connector()) 117 | projects_index = Projects() 118 | idents = Contributors() 119 | 120 | query_kwargs = utils.resolv_filters( 121 | projects_index, idents, pid, tid, cid, gid, 122 | dfrom, dto, inc_repos, inc_merge_commit, metadata, 123 | exc_groups, inc_groups) 124 | 125 | return self.gbycommits(c, idents, query_kwargs, limit) 126 | 127 | @expose('json') 128 | @expose('csv:', content_type='text/csv') 129 | def diff(self, pid=None, tid=None, cid=None, gid=None, 130 | dfrom=None, dto=None, dfromref=None, dtoref=None, 131 | inc_merge_commit=None, inc_repos=None, metadata=None, 132 | exc_groups=None, limit=None, inc_groups=None): 133 | 134 | if not dfrom or not dto: 135 | abort(404, 136 | detail="Must specify dfrom and dto dates for the new " 137 | "contributors") 138 | 139 | if not dfromref or not dtoref: 140 | abort(404, 141 | detail="Must specify dfromref and dtoref dates for the " 142 | "reference period to compute new contributors") 143 | 144 | # Get contributors for the new period 145 | c = Commits(index.Connector()) 146 | projects_index = Projects() 147 | idents = Contributors() 148 | 149 | query_kwargs = utils.resolv_filters( 150 | projects_index, idents, pid, tid, cid, gid, 151 | dfrom, dto, inc_repos, inc_merge_commit, metadata, 152 | exc_groups, inc_groups) 153 | 154 | authors_new = self.gbycommits( 155 | c, idents, query_kwargs, top=-1, 156 | resolv_name=False, clean_email=False) 157 | 158 | # Now get contributors for the old reference period 159 | query_kwargs = utils.resolv_filters( 160 | projects_index, idents, pid, tid, cid, gid, 161 | dfromref, dtoref, inc_repos, inc_merge_commit, metadata, 162 | exc_groups, inc_groups) 163 | 164 | authors_old = self.gbycommits( 165 | c, idents, query_kwargs, top=-1, 166 | resolv_name=False, clean_email=False) 167 | 168 | # And compute the difference 169 | cids_new = set([auth['cid'] for auth in authors_new]) - \ 170 | set([auth['cid'] for auth in authors_old]) 171 | authors_diff = [author for author in authors_new 172 | if author['cid'] in cids_new] 173 | if limit is None: 174 | limit = 10 175 | else: 176 | limit = int(limit) 177 | # If limit set to a negative value all results will be returned 178 | if limit >= 0: 179 | authors_diff = authors_diff[:limit] 180 | 181 | self.resolv_name(c, authors_diff) 182 | 183 | return authors_diff 184 | 185 | 186 | class TopProjectsController(object): 187 | 188 | def gby(self, ci, pi, query_kwargs, 189 | inc_repos_detail, f1, f2, limit): 190 | repos = f1(**query_kwargs)[1] 191 | if inc_repos_detail: 192 | repos_contributed = [ 193 | (p, ca) for p, ca in repos.items() 194 | if not p.startswith('meta_ref: ')] 195 | else: 196 | repos_contributed = [] 197 | projects = utils.get_projects_from_references( 198 | pi, [r for r in repos.keys() 199 | if not r.startswith('meta_ref: ')]) 200 | for pname in projects: 201 | project = pi.get(pname, source=['name', 'meta-ref', 'refs']) 202 | p_filter = utils.get_references_filter(project) 203 | _query_kwargs = copy.deepcopy(query_kwargs) 204 | _query_kwargs['repos'] = p_filter 205 | ca = int(f2(**_query_kwargs) or 0) 206 | if ca: 207 | repos_contributed.append((pname, ca)) 208 | 209 | sorted_rc = sorted(repos_contributed, 210 | key=lambda i: i[1], 211 | reverse=True) 212 | ret = [] 213 | for item in sorted_rc: 214 | ret.append({"amount": int(item[1])}) 215 | if inc_repos_detail: 216 | ret[-1]["projects"] = utils.get_projects_from_references( 217 | pi, [item[0]]) 218 | ret[-1]["name"] = ":".join(item[0].split(':')[-2:]) 219 | 220 | if limit is None: 221 | limit = 10 222 | else: 223 | limit = int(limit) 224 | # If limit set to a negative value all results will be returned 225 | if limit >= 0: 226 | ret = ret[:limit] 227 | return ret 228 | 229 | def gbycommits(self, ci, pi, query_kwargs, inc_repos_detail, limit): 230 | ret = self.gby(ci, pi, query_kwargs, inc_repos_detail, 231 | ci.get_repos, ci.get_commits_amount, limit) 232 | return ret 233 | 234 | def gbylchanged(self, ci, pi, query_kwargs, inc_repos_detail, limit): 235 | 236 | def f2(**kwargs): 237 | return ci.get_line_modifieds_stats(**kwargs)[1]['sum'] 238 | 239 | ret = self.gby(ci, pi, query_kwargs, inc_repos_detail, 240 | ci.get_top_repos_by_lines, f2, limit) 241 | return ret 242 | 243 | @expose('json') 244 | @expose('csv:', content_type='text/csv') 245 | def bylchanged(self, pid=None, tid=None, cid=None, gid=None, 246 | dfrom=None, dto=None, inc_merge_commit=None, 247 | inc_repos=None, metadata=None, exc_groups=None, 248 | inc_repos_detail=None, inc_groups=None, limit=None): 249 | 250 | c = Commits(index.Connector()) 251 | projects_index = Projects() 252 | idents = Contributors() 253 | 254 | query_kwargs = utils.resolv_filters( 255 | projects_index, idents, pid, tid, cid, gid, 256 | dfrom, dto, inc_repos, inc_merge_commit, metadata, 257 | exc_groups, inc_groups) 258 | 259 | return self.gbylchanged(c, projects_index, query_kwargs, 260 | inc_repos_detail, limit) 261 | 262 | @expose('json') 263 | @expose('csv:', content_type='text/csv') 264 | def bycommits(self, pid=None, tid=None, cid=None, gid=None, 265 | dfrom=None, dto=None, inc_merge_commit=None, 266 | inc_repos=None, metadata=None, exc_groups=None, 267 | inc_repos_detail=None, inc_groups=None, limit=None): 268 | 269 | c = Commits(index.Connector()) 270 | projects_index = Projects() 271 | idents = Contributors() 272 | 273 | query_kwargs = utils.resolv_filters( 274 | projects_index, idents, pid, tid, cid, gid, 275 | dfrom, dto, inc_repos, inc_merge_commit, metadata, 276 | exc_groups, inc_groups) 277 | 278 | return self.gbycommits(c, projects_index, query_kwargs, 279 | inc_repos_detail, limit) 280 | 281 | 282 | class TopsController(object): 283 | 284 | authors = TopAuthorsController() 285 | projects = TopProjectsController() 286 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2016, Fabien Boucher 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------