├── .dockerignore ├── .flake8 ├── .github └── workflows │ ├── app-release.yml │ ├── appimage.yml │ ├── dockertests.yml │ ├── pythonapp.yml │ └── shiftleft-analysis.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── app-build.sh ├── appimage-builder-arm64.yml ├── appimage-builder.yml ├── appimage-reqs.sh ├── azure-pipelines.yml ├── builder.Dockerfile ├── builder.sh ├── builder_aarch64.Dockerfile ├── building_env.sh ├── ci ├── Dockerfile-csharp ├── Dockerfile-dynamic-lang ├── Dockerfile-java └── Dockerfile-oss ├── contrib ├── google-cloud │ ├── README.md │ ├── dashboard1.png │ ├── dashboard2.png │ └── store-api │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── __init__.py │ │ ├── app.py │ │ ├── cloudbuild.yaml │ │ ├── config.py │ │ └── requirements.txt └── seccomp.json ├── dev-build.sh ├── docs ├── README.md ├── TRICKS.md ├── azure-devops.md ├── azure-devops.png ├── azure-pipelines.yml.sample ├── build-breaker.png ├── circleci-artifacts.png ├── circleci-sample.png ├── circleci.md ├── findsecbugs-report.html ├── integration.md ├── pmd-report.html ├── pre-commit.sh └── sarif-online-viewer.png ├── dynamic-lang.sh ├── lib ├── __init__.py ├── aggregate.py ├── analysis.py ├── builder.py ├── cis.py ├── config.py ├── constants.py ├── context.py ├── convert.py ├── csv_parser.py ├── cwe.py ├── data │ ├── CREDITS │ ├── cis-aws.yaml │ ├── cis-k8s.yaml │ ├── cwe_research.csv │ └── cwe_software.csv ├── executor.py ├── inspect.py ├── integration │ ├── __init__.py │ ├── bitbucket.py │ ├── github.py │ ├── gitlab.py │ └── provider.py ├── issue.py ├── logger.py ├── pyt │ ├── LICENSE │ ├── README.md │ ├── __init__.py │ ├── analysis │ │ ├── __init__.py │ │ ├── constraint_table.py │ │ ├── definition_chains.py │ │ ├── fixed_point.py │ │ ├── lattice.py │ │ └── reaching_definitions_taint.py │ ├── cfg │ │ ├── __init__.py │ │ ├── alias_helper.py │ │ ├── expr_visitor.py │ │ ├── expr_visitor_helper.py │ │ ├── make_cfg.py │ │ ├── stmt_visitor.py │ │ └── stmt_visitor_helper.py │ ├── cfg_analyzer.py │ ├── core │ │ ├── __init__.py │ │ ├── ast_helper.py │ │ ├── astcheck.py │ │ ├── astsearch.py │ │ ├── module_definitions.py │ │ ├── node_types.py │ │ ├── project_handler.py │ │ └── transformer.py │ ├── formatters │ │ ├── __init__.py │ │ └── json.py │ ├── helper_visitors │ │ ├── __init__.py │ │ ├── call_visitor.py │ │ ├── label_visitor.py │ │ ├── right_hand_side_visitor.py │ │ └── vars_visitor.py │ ├── vulnerabilities │ │ ├── __init__.py │ │ ├── insights.py │ │ ├── rules.py │ │ ├── trigger_definitions_parser.py │ │ ├── vulnerabilities.py │ │ └── vulnerability_helper.py │ ├── vulnerability_definitions │ │ ├── all_sources_sinks.pyt │ │ ├── blackbox_mapping.json │ │ ├── taint.config │ │ ├── test_positions.pyt │ │ └── test_triggers.pyt │ └── web_frameworks │ │ ├── __init__.py │ │ ├── framework_adaptor.py │ │ └── framework_helper.py ├── remediate.py ├── telemetry.py ├── utils.py └── xml_parser.py ├── requirements-dev.txt ├── requirements.txt ├── scan ├── starlark └── scan.star ├── test ├── __init__.py ├── conftest.py ├── data │ ├── .sastscan.baseline │ ├── .sastscanrc │ ├── audit-php.json │ ├── bandit-report.json │ ├── bandit-report.sarif │ ├── bash-report.sarif │ ├── checkov-report.json │ ├── dep_check-report.json │ ├── depscan-report-java.json │ ├── depscan-report-nodejs.json │ ├── findsecbugs-report.sarif │ ├── findsecbugs-report.xml │ ├── gosec-report.sarif │ ├── inspect-nodejs.json │ ├── inspect-report.json │ ├── issue-259.php │ ├── njs2.json │ ├── njsscan-report.json │ ├── nodejsscan-report.json │ ├── nodejsscan-report.sarif │ ├── pmd-report.csv │ ├── pmd-report.sarif │ ├── retire-report.json │ ├── source-go-ignore.json │ ├── source-kt-report.xml │ ├── source-php-report.json │ ├── source-ruby.json │ ├── staticcheck-ignore-report.json │ ├── staticcheck-report.json │ ├── staticcheck-report.sarif │ ├── taint-php-report.json │ ├── taint-php-report2.json │ └── taint-python-report.json ├── integration │ ├── test_bitbucket.py │ ├── test_github.py │ ├── test_gitlab.py │ └── test_provider.py ├── pyt │ ├── test_ast.py │ ├── test_cfg.py │ └── test_insights.py ├── test_aggregate.py ├── test_analysis.py ├── test_cis.py ├── test_config.py ├── test_context.py ├── test_convert.py ├── test_csv.py ├── test_cwe.py ├── test_inspect.py ├── test_issue.py ├── test_utils.py └── test_xml.py ├── tools_config ├── credscan-config.toml ├── detekt-config.yml ├── io.shiftleft.scan.appdata.xml ├── phpstan.neon.dist ├── rules-pmd.xml ├── scan.png ├── scan.svg └── spotbugs │ ├── exclude.xml │ └── include.xml ├── ubuntu_build.sh ├── ubuntu_build_arm64.sh └── ubuntu_build_x86_64.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | docs 3 | test 4 | .* 5 | test* 6 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503 3 | max-line-length = 79 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | -------------------------------------------------------------------------------- /.github/workflows/app-release.yml: -------------------------------------------------------------------------------- 1 | name: Release Scan AppImage 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-22.04 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Run AppImage Build Script 14 | run: | 15 | sudo chmod +x ubuntu_build_x86_64.sh 16 | sudo chmod +x ubuntu_build.sh 17 | sudo ./ubuntu_build_x86_64.sh 18 | - name: Create Release 19 | id: create_release 20 | uses: actions/create-release@v1 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | with: 24 | tag_name: ${{ github.ref }} 25 | release_name: Release ${{ github.ref }} 26 | draft: false 27 | prerelease: false 28 | - name: Upload Release Asset 29 | id: upload-release-asset 30 | uses: actions/upload-release-asset@v1 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | with: 34 | upload_url: ${{ steps.create_release.outputs.upload_url }} 35 | asset_path: scan-latest-x86_64.AppImage 36 | asset_name: scan 37 | asset_content_type: application/octet-stream 38 | -------------------------------------------------------------------------------- /.github/workflows/appimage.yml: -------------------------------------------------------------------------------- 1 | name: Scan AppImage 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-22.04 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Run AppImage Build Script 11 | run: | 12 | sudo chmod +x ubuntu_build_x86_64.sh 13 | sudo chmod +x ubuntu_build.sh 14 | sudo ./ubuntu_build_x86_64.sh 15 | - uses: actions/upload-artifact@v2 16 | with: 17 | name: AppImage 18 | path: './scan*.AppImage*' 19 | -------------------------------------------------------------------------------- /.github/workflows/dockertests.yml: -------------------------------------------------------------------------------- 1 | name: docker tests 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-22.04 8 | strategy: 9 | matrix: 10 | os: [ubuntu-22.04, macos-latest, windows-latest] 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Test container images 14 | run: | 15 | docker pull shiftleft/scan-slim:latest 16 | docker pull shiftleft/scan:latest 17 | docker save -o scanslim.tar shiftleft/scan-slim:latest 18 | docker save -o scan.tar shiftleft/scan:latest 19 | docker run --rm -e "WORKSPACE=${PWD}" -v $PWD:/app shiftleft/scan:docker scan --src /app/scanslim.tar -o /app/reports --type docker 20 | docker run --rm -e "WORKSPACE=${PWD}" -e "FETCH_LICENSE=true" -e "ENABLE_OSS_RISK=true" -v $PWD:/app shiftleft/scan:docker scan --src /app/scan.tar -o /app/reports --type docker 21 | env: 22 | PYTHONPATH: "." 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | - uses: actions/upload-artifact@v1 25 | with: 26 | name: reports 27 | path: reports 28 | -------------------------------------------------------------------------------- /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | name: Python matrix CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-22.04 8 | strategy: 9 | matrix: 10 | os: [ubuntu-latest, macos-latest, windows-latest] 11 | python-version: [3.10.12] 12 | steps: 13 | - uses: actions/checkout@v1 14 | - name: Set up Python 15 | uses: actions/setup-python@v1 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Install dependencies 19 | run: | 20 | python3 -m pip install --upgrade pip 21 | pip install -r requirements-dev.txt 22 | - name: Lint with flake8 23 | run: | 24 | # stop the build if there are Python syntax errors or undefined names 25 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 26 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 27 | flake8 . --count --exit-zero --statistics 28 | - name: Test with pytest 29 | run: | 30 | pytest --cov=. test 31 | -------------------------------------------------------------------------------- /.github/workflows/shiftleft-analysis.yml: -------------------------------------------------------------------------------- 1 | # This workflow integrates Scan with GitHub's code scanning feature 2 | # Scan is a free open-source security tool for modern DevOps teams 3 | # Visit https://appthreat.com/en/latest/integrations/github-actions/ for help 4 | name: Scan 5 | 6 | # This section configures the trigger for the workflow. Feel free to customize depending on your convention 7 | on: 8 | pull_request: 9 | 10 | jobs: 11 | Scan-Build: 12 | runs-on: ubuntu-22.04 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Cache multiple paths 16 | uses: actions/cache@v2 17 | with: 18 | path: | 19 | ${{ github.workspace }}/db 20 | key: ${{ runner.os }}-${{ hashFiles('requirements*.txt') }} 21 | - name: Perform Scan 22 | uses: ShiftLeftSecurity/scan-action@master 23 | env: 24 | VDB_HOME: ${{ github.workspace }}/db 25 | WORKSPACE: "" 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | with: 28 | output: reports 29 | - name: Upload report 30 | uses: github/codeql-action/upload-sarif@v1 31 | with: 32 | sarif_file: reports 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_STORE 2 | .idea/ 3 | .gitleaks_bin/ 4 | repos/ 5 | .vscode/ 6 | .tox/ 7 | .tool-versions 8 | venv/ 9 | venv38/ 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | pip-wheel-metadata/ 32 | share/python-wheels/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | MANIFEST 37 | 38 | # PyInstaller 39 | # Usually these files are written by a python script from a template 40 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 41 | *.manifest 42 | *.spec 43 | 44 | # Installer logs 45 | pip-log.txt 46 | pip-delete-this-directory.txt 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .nox/ 52 | .coverage 53 | .coverage.* 54 | .cache 55 | nosetests.xml 56 | coverage.xml 57 | *.cover 58 | *.py,cover 59 | .hypothesis/ 60 | .pytest_cache/ 61 | 62 | # Translations 63 | *.mo 64 | *.pot 65 | 66 | # Django stuff: 67 | *.log 68 | local_settings.py 69 | db.sqlite3 70 | db.sqlite3-journal 71 | 72 | # Flask stuff: 73 | instance/ 74 | .webassets-cache 75 | 76 | # Scrapy stuff: 77 | .scrapy 78 | 79 | # Sphinx documentation 80 | docs/_build/ 81 | 82 | # PyBuilder 83 | target/ 84 | 85 | # Jupyter Notebook 86 | .ipynb_checkpoints 87 | 88 | # IPython 89 | profile_default/ 90 | ipython_config.py 91 | 92 | # pyenv 93 | .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 103 | __pypackages__/ 104 | 105 | # Celery stuff 106 | celerybeat-schedule 107 | celerybeat.pid 108 | 109 | # SageMath parsed files 110 | *.sage.py 111 | 112 | # Environments 113 | .env 114 | .venv 115 | env/ 116 | venv/ 117 | ENV/ 118 | env.bak/ 119 | venv.bak/ 120 | 121 | # Spyder project settings 122 | .spyderproject 123 | .spyproject 124 | 125 | # Rope project settings 126 | .ropeproject 127 | 128 | # mkdocs documentation 129 | /site 130 | 131 | # mypy 132 | .mypy_cache/ 133 | .dmypy.json 134 | dmypy.json 135 | 136 | # Pyre type checker 137 | .pyre/ 138 | .coverage 139 | reports/ 140 | AppDir/ 141 | appimage-builder-cache/ 142 | *.AppImage 143 | *.AppImage.zsync 144 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy as scan-base 2 | 3 | # This section of the multi-stage build is used to build the final Scan image 4 | # We install all building dependencies, run the process of getting and compiling additional tools and then 5 | # use an also ubuntu version in the next stage to only get the obtained tools. 6 | ARG CLI_VERSION 7 | ARG BUILD_DATE 8 | ARG ARCH 9 | 10 | ENV GOPATH=/opt/app-root/go \ 11 | GO_VERSION=1.21 \ 12 | PATH=${PATH}:${GOPATH}/bin:/usr/local/go/bin: 13 | 14 | LABEL maintainer="qwiet.ai" \ 15 | org.label-schema.schema-version="1.0" \ 16 | org.label-schema.vendor="qwiet.ai" \ 17 | org.label-schema.name="scan-base" \ 18 | org.label-schema.version=$CLI_VERSION \ 19 | org.label-schema.license="GPL-3.0-or-later" \ 20 | org.label-schema.description="Base image containing multiple programming languages" \ 21 | org.label-schema.url="https://qwiet.ai" \ 22 | org.label-schema.usage="https://github.com/ShiftLeftSecurity/sast-scan" \ 23 | org.label-schema.build-date=$BUILD_DATE \ 24 | org.label-schema.vcs-url="https://github.com/ShiftLeftSecurity/sast-scan.git" \ 25 | org.label-schema.docker.cmd="docker run --rm -it --name scan-base shiftleft/scan-base /bin/bash" 26 | 27 | RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/00-docker 28 | RUN echo 'APT::Install-Recommends "0";' >> /etc/apt/apt.conf.d/00-docker 29 | 30 | COPY appimage-reqs.sh / 31 | COPY building_env.sh / 32 | COPY dynamic-lang.sh / 33 | COPY requirements.txt / 34 | COPY scan /usr/local/src/ 35 | COPY lib /usr/local/src/lib 36 | COPY tools_config/ /usr/local/src/ 37 | 38 | USER root 39 | # this ensures there will be no pre-deletion of app folder. 40 | ENV KEEP_BUILD_ARTIFACTS=true 41 | ENV ARCH=$ARCH 42 | ENV DEBIAN_FRONTEND=noninteractive 43 | 44 | # Dependencies for scan, many of these are only necessary to compile/initialize the tools 45 | RUN apt-get update && apt-get install -y python3 python3-dev \ 46 | python3-pip python3-setuptools patchelf \ 47 | php php-curl php-zip php-bcmath php-json \ 48 | php-pear php-mbstring php-dev php-xml wget curl git unzip 49 | 50 | # Use the same script as we would use locally, for consistency 51 | RUN /appimage-reqs.sh / 52 | 53 | # We remove packages that are going to increase the size of our /usr folder. 54 | RUN apt-get remove -y apache2 python3-dev \ 55 | python3-pip python3-setuptools patchelf desktop-file-utils \ 56 | libgdk-pixbuf2.0-dev wget curl unzip gcc g++ make && apt-get autoremove -y && apt-get clean -y 57 | 58 | FROM ubuntu:jammy as sast-scan-tools 59 | 60 | LABEL maintainer="qwiet.ai" \ 61 | org.label-schema.schema-version="1.0" \ 62 | org.label-schema.vendor="qwiet.ai" \ 63 | org.label-schema.name="sast-scan" \ 64 | org.label-schema.version=$CLI_VERSION \ 65 | org.label-schema.license="Apache-2.0" \ 66 | org.label-schema.description="Container with various opensource static analysis security testing tools (shellcheck, gosec, tfsec, gitleaks, ...) for multiple programming languages" \ 67 | org.label-schema.url="https://qwiet.ai" \ 68 | org.label-schema.usage="https://github.com/ShiftLeftSecurity/sast-scan" \ 69 | org.label-schema.build-date=$BUILD_DATE \ 70 | org.label-schema.vcs-url="https://github.com/ShiftLeftSecurity/sast-scan.git" \ 71 | org.label-schema.docker.cmd="docker run --rm -it --name sast-scan shiftleft/sast-scan" 72 | 73 | # Beware, versions should be kept in sync with appimage-reqs.sh 74 | ENV APP_SRC_DIR=/usr/local/src \ 75 | DEPSCAN_CMD="/usr/local/bin/depscan" \ 76 | MVN_CMD="/usr/bin/mvn" \ 77 | PMD_CMD="/opt/pmd-bin/bin/run.sh pmd" \ 78 | PMD_JAVA_OPTS="--enable-preview" \ 79 | SB_VERSION=4.7.3 \ 80 | PMD_VERSION=6.55.0 \ 81 | SPOTBUGS_HOME=/opt/spotbugs \ 82 | JAVA_HOME=/usr/lib/jvm/jre-11-openjdk \ 83 | SCAN_JAVA_HOME=/usr/lib/jvm/jre-11-openjdk \ 84 | SCAN_JAVA_11_HOME=/usr/lib/jvm/jre-11-openjdk \ 85 | SCAN_JAVA_8_HOME=/usr/lib/jvm/jre-1.8.0 \ 86 | GRADLE_VERSION=7.2 \ 87 | GRADLE_HOME=/opt/gradle \ 88 | GRADLE_CMD=gradle \ 89 | PYTHONUNBUFFERED=1 \ 90 | DOTNET_CLI_TELEMETRY_OPTOUT=1 \ 91 | SHIFTLEFT_HOME=/opt/sl-cli \ 92 | GO111MODULE=auto \ 93 | GOARCH=amd64 \ 94 | GOOS=linux \ 95 | CGO_ENABLED=0 \ 96 | NVD_EXCLUDE_TYPES="o,h" \ 97 | PATH=/usr/local/src/:${PATH}:/opt/gradle/bin:/opt/apache-maven/bin:/usr/local/go/bin:/opt/sl-cli:/opt/phpsast/vendor/bin: 98 | 99 | # We only get what we need from the previous stage 100 | COPY --from=scan-base /opt /opt 101 | COPY --from=scan-base /usr /usr 102 | 103 | WORKDIR /app 104 | 105 | CMD [ "python3", "/usr/local/src/scan" ] 106 | -------------------------------------------------------------------------------- /app-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | rm -rf AppDir appimage-builder-cache 4 | rm *.AppImage* 5 | mkdir -p appimage-builder-cache 6 | wget https://github.com/AppImage/AppImageKit/releases/download/12/runtime-x86_64 -O appimage-builder-cache/runtime-x86_64 7 | UPDATE_INFO="gh-releases-zsync|ShiftLeftSecurity|sast-scan|latest|*x86_64.AppImage.zsync" appimage-builder --recipe appimage-builder.yml --skip-test 8 | rm -rf AppDir appimage-builder-cache 9 | chmod +x *.AppImage 10 | -------------------------------------------------------------------------------- /appimage-builder-arm64.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | script: 3 | - chmod +x ./appimage-reqs.sh && ./appimage-reqs.sh $(pwd)/AppDir 4 | - cp $(pwd)/AppDir/usr/share/icons/Adwaita/scalable/apps/utilities-terminal-symbolic.svg $(pwd)/AppDir/usr/share/icons/utilities-terminal.svg 5 | - cp $(pwd)/AppDir/usr/share/icons/Adwaita/scalable/apps/utilities-terminal-symbolic.svg $(pwd)/AppDir/usr/share/icons/Adwaita/scalable/apps/utilities-terminal.svg 6 | 7 | AppDir: 8 | path: ./AppDir 9 | app_info: 10 | id: io.shiftleft.scan 11 | name: scan 12 | icon: utilities-terminal 13 | version: latest 14 | # Set the python executable as entry point 15 | exec: usr/bin/python3 16 | # Set the application main script path as argument. Use '$@' to forward CLI parameters 17 | exec_args: "$APPDIR/usr/src/scan $@" 18 | 19 | apt: 20 | arch: aarch64 21 | sources: 22 | - sourceline: 'deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe multiverse' 23 | key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920D1991BC93C' 24 | - sourceline: 'deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe multiverse' 25 | key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3B4FE6ACC0B21F32' 26 | - sourceline: 'deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse' 27 | key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920D1991BC93C' 28 | include: 29 | - python3.8 30 | - python3-pkg-resources 31 | - git 32 | - php 33 | - php-json 34 | - php-pear 35 | - php-mbstring 36 | - tar 37 | files: 38 | exclude: 39 | - usr/share/man 40 | - usr/share/doc/*/README.* 41 | - usr/share/doc/*/changelog.* 42 | - usr/share/doc/*/NEWS.* 43 | - usr/share/doc/*/TODO.* 44 | 45 | runtime: 46 | env: 47 | PATH: '/bin:/usr/bin:${APPDIR}/usr/bin:${APPDIR}/usr/bin/nodejs/bin:${PATH}:${APPDIR}/opt/phpsast/vendor/bin:${APPDIR}/usr/local/lib/node_modules/.bin:' 48 | PYTHONHOME: '${APPDIR}/usr' 49 | PYTHONPATH: '${APPDIR}/usr/lib/python3.10/site-packages:$APPDIR/usr/lib/python3.10:$APPDIR/usr/lib/python3.10/lib-dynload' 50 | SSL_CERT_FILE: '${APPDIR}/usr/lib/python3.10/site-packages/certifi/cacert.pem' 51 | PYTHONUNBUFFERED: '1' 52 | APP_SRC_DIR: '${APPDIR}/usr/src' 53 | TOOLS_CONFIG_DIR: '${APPDIR}/usr/src/tools_config' 54 | DEPSCAN_CMD: '${APPDIR}/usr/bin/depscan' 55 | PMD_CMD: '${APPDIR}/opt/pmd-bin/bin/run.sh pmd' 56 | SPOTBUGS_HOME: '${APPDIR}/opt/spotbugs' 57 | DETEKT_JAR: '${APPDIR}/usr/bin/detekt-cli.jar' 58 | 59 | test: 60 | fedora: 61 | image: appimagecrafters/tests-env:fedora-30 62 | command: ./AppRun --help 63 | use_host_x: true 64 | debian: 65 | image: appimagecrafters/tests-env:debian-stable 66 | command: ./AppRun --help 67 | use_host_x: true 68 | arch: 69 | image: appimagecrafters/tests-env:archlinux-latest 70 | command: ./AppRun --help 71 | use_host_x: true 72 | centos: 73 | image: appimagecrafters/tests-env:centos-7 74 | command: ./AppRun --help 75 | use_host_x: true 76 | ubuntu: 77 | image: appimagecrafters/tests-env:ubuntu-xenial 78 | command: ./AppRun --help 79 | use_host_x: true 80 | 81 | AppImage: 82 | update-information: !ENV ${UPDATE_INFO} 83 | sign-key: None 84 | arch: aarch64 85 | -------------------------------------------------------------------------------- /appimage-builder.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | script: 3 | - chmod +x ./appimage-reqs.sh && APPIMAGE=true ./appimage-reqs.sh $(pwd)/AppDir 4 | - cp -r /usr/share/icons/Adwaita AppDir/usr/share/icons/ 5 | - cp $(pwd)/AppDir/usr/share/icons/Adwaita/scalable/apps/utilities-terminal-symbolic.svg $(pwd)/AppDir/usr/share/icons/utilities-terminal.svg 6 | - cp $(pwd)/AppDir/usr/share/icons/Adwaita/scalable/apps/utilities-terminal-symbolic.svg $(pwd)/AppDir/usr/share/icons/Adwaita/scalable/apps/utilities-terminal.svg 7 | 8 | AppDir: 9 | path: ./AppDir 10 | app_info: 11 | id: io.shiftleft.scan 12 | name: scan 13 | icon: utilities-terminal 14 | version: latest 15 | # Set the python executable as entry point 16 | exec: usr/bin/python3 17 | # Set the application main script path as argument. Use '$@' to forward CLI parameters 18 | exec_args: "$APPDIR/usr/src/scan $@" 19 | 20 | apt: 21 | arch: amd64 22 | sources: 23 | - sourceline: 'deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricted universe multiverse' 24 | key_url: 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920D1991BC93C' 25 | - sourceline: deb [arch=amd64] https://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe multiverse 26 | key_url: 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920D1991BC93C' 27 | - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse 28 | key_url: 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920D1991BC93C' 29 | include: 30 | - python3 31 | - python3-pkg-resources 32 | - python3-yaml 33 | - python3-requests 34 | - python3-urllib3 35 | - libyaml-0-2 36 | - git 37 | - php 38 | - php-json 39 | - php-pear 40 | - php-mbstring 41 | - tar 42 | files: 43 | exclude: 44 | - usr/share/man 45 | - usr/share/doc/*/README.* 46 | - usr/share/doc/*/changelog.* 47 | - usr/share/doc/*/NEWS.* 48 | - usr/share/doc/*/TODO.* 49 | 50 | runtime: 51 | env: 52 | PATH: '${APPDIR}/usr/bin:${APPDIR}/usr/bin/nodejs/bin:${PATH}:${APPDIR}/opt/phpsast/vendor/bin:${APPDIR}/usr/local/lib/node_modules/.bin:' 53 | PYTHONHOME: '${APPDIR}/usr' 54 | PYTHONPATH: '${APPDIR}/usr/lib/python3.10/site-packages:$APPDIR/usr/lib/python3.10:$APPDIR/usr/lib/python3.10/lib-dynload' 55 | SSL_CERT_FILE: '${APPDIR}/usr/lib/python3.10/site-packages/certifi/cacert.pem' 56 | PYTHONUNBUFFERED: '1' 57 | APP_SRC_DIR: '${APPDIR}/usr/src' 58 | TOOLS_CONFIG_DIR: '${APPDIR}/usr/src/tools_config' 59 | DEPSCAN_CMD: '${APPDIR}/usr/bin/depscan' 60 | PMD_CMD: '${APPDIR}/opt/pmd-bin/bin/run.sh pmd' 61 | SPOTBUGS_HOME: '${APPDIR}/opt/spotbugs' 62 | DETEKT_JAR: '${APPDIR}/usr/bin/detekt-cli.jar' 63 | 64 | test: 65 | fedora: 66 | image: appimagecrafters/tests-env:fedora-30 67 | command: ./AppRun --help 68 | use_host_x: true 69 | debian: 70 | image: appimagecrafters/tests-env:debian-stable 71 | command: ./AppRun --help 72 | use_host_x: true 73 | arch: 74 | image: appimagecrafters/tests-env:archlinux-latest 75 | command: ./AppRun --help 76 | use_host_x: true 77 | centos: 78 | image: appimagecrafters/tests-env:centos-7 79 | command: ./AppRun --help 80 | use_host_x: true 81 | ubuntu: 82 | image: appimagecrafters/tests-env:ubuntu-xenial 83 | command: ./AppRun --help 84 | use_host_x: true 85 | 86 | AppImage: 87 | update-information: !ENV ${UPDATE_INFO} 88 | sign-key: None 89 | arch: x86_64 90 | -------------------------------------------------------------------------------- /appimage-reqs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # This script is used by the image generated by builder.Dockerfile to create the AppImage for sast-scan 5 | # if you want to use this standalone, ensure ARCH is set to the right architecture, currently it has only 6 | # been tested on x86_64 and arm64 7 | # It is also used by the Dockerfile generating the scan image to satisfy all dependencies. 8 | 9 | ## App Versions 10 | source building_env.sh 11 | 12 | ## First parameter is the path to the AppDir where all the building happens, you can use whatever path you want 13 | ## but it needs to be the same as the one in appimage-builder.yml if you are using it too. 14 | APPDIR=$1 15 | echo "AppDir is ${APPDIR}" 16 | 17 | 18 | ## Remove any previous build 19 | if [ -z "$KEEP_BUILD_ARTIFACTS" ]; then 20 | rm -rf "${APPDIR}" 21 | mkdir -p "${APPDIR}" 22 | else 23 | echo "Keeping build artifacts from previous build" 24 | fi 25 | 26 | ## Make usr and icons dirs 27 | mkdir -p "${APPDIR}"/usr/src 28 | mkdir -p "${APPDIR}"/usr/local/lib/"${LIBARCH}"-linux-gnu 29 | mkdir -p "${APPDIR}"/usr/share/{metainfo,icons} 30 | 31 | ## Ensure the required folders exist. 32 | USR_BIN_PATH=${APPDIR}/usr/bin/ 33 | OPTDIR=${APPDIR}/opt 34 | mkdir -p "$USR_BIN_PATH" 35 | mkdir -p "$OPTDIR" 36 | 37 | ## Ensure our binaries to be downloaded are in the path. 38 | export PATH=$PATH:${USR_BIN_PATH}:${USR_BIN_PATH}/nodejs/bin 39 | 40 | echo "$PWD" 41 | 42 | ## Get all the packages we have in the dynamic lang version of this script 43 | source dynamic-lang.sh 44 | 45 | ## Download and install gosec (https://github.com/securego/gosec) 46 | GOSEC_TAR="gosec_${GOSEC_VERSION}_linux_${ARCH_ALT_NAME}.tar.gz" 47 | echo "Downloading ${GOSEC_TAR}" 48 | curl -LO "https://github.com/securego/gosec/releases/download/v${GOSEC_VERSION}/${GOSEC_TAR}" 49 | tar -C "${USR_BIN_PATH}" -xzvf "${GOSEC_TAR}" 50 | chmod +x "${USR_BIN_PATH}"gosec 51 | mayberm "${GOSEC_TAR}" 52 | 53 | ## Download and install staticcheck (https://github.com/dominikh/go-tools) 54 | STCHECK_TAR="staticcheck_linux_${ARCH_ALT_NAME}.tar.gz" 55 | echo "Downloading ${STCHECK_TAR}" 56 | curl -LO "https://github.com/dominikh/go-tools/releases/download/${SC_VERSION}/${STCHECK_TAR}" 57 | tar -C /tmp -xzvf "${STCHECK_TAR}" 58 | chmod +x /tmp/staticcheck/staticcheck 59 | cp /tmp/staticcheck/staticcheck "${USR_BIN_PATH}"staticcheck 60 | mayberm "${STCHECK_TAR}" 61 | 62 | 63 | ## Download and install pmd (https://github.com/pmd/pmd) 64 | PMD_ZIP=pmd-bin-${PMD_VERSION}.zip 65 | if [ ! -f "${PMD_ZIP}" ]; then 66 | echo "Downloading ${PMD_ZIP}" 67 | wget "https://github.com/pmd/pmd/releases/download/pmd_releases%2F${PMD_VERSION}/${PMD_ZIP}" 68 | fi 69 | if [ ! -d "${OPTDIR}"/pmd-bin ]; then 70 | echo "Installing ${PMD_ZIP}" 71 | unzip -q pmd-bin-${PMD_VERSION}.zip -d "${OPTDIR}"/ 72 | mv -f "${OPTDIR}"/pmd-bin-${PMD_VERSION} "${OPTDIR}"/pmd-bin 73 | mayberm ${PMD_ZIP} 74 | else 75 | echo "PMD already installed" 76 | fi 77 | 78 | 79 | ## Download and install detekt (https://github.com/detekt/detekt) 80 | curl -L "https://github.com/detekt/detekt/releases/download/v${DETEKT_VERSION}/detekt-cli-${DETEKT_VERSION}-all.jar" -o "${USR_BIN_PATH}detekt-cli.jar" 81 | 82 | # SpotBugs --------------------------------------------------------------- 83 | ## Download and install spotbugs (https://github.com/spotbugs/spotbugs) 84 | SPOTBUGS_TGZ="spotbugs-${SB_VERSION}.tgz" 85 | SPOTBUGS_OPTDIR="${OPTDIR}/spotbugs-${SB_VERSION}" 86 | if [ ! -d "${OPTDIR}"/spotbugs ]; then 87 | echo "Downloading ${SPOTBUGS_TGZ}" 88 | curl -LO "https://github.com/spotbugs/spotbugs/releases/download/${SB_VERSION}/${SPOTBUGS_TGZ}" 89 | tar -C "${OPTDIR}" -xzvf spotbugs-${SB_VERSION}.tgz 90 | rm ${SPOTBUGS_TGZ} 91 | 92 | ## Download and install findsecbugs plugin for spotbugs (https://find-sec-bugs.github.io/) 93 | curl -LO "https://repo1.maven.org/maven2/com/h3xstream/findsecbugs/findsecbugs-plugin/${FSB_VERSION}/findsecbugs-plugin-${FSB_VERSION}.jar" 94 | mv -f findsecbugs-plugin-${FSB_VERSION}.jar "${SPOTBUGS_OPTDIR}"/plugin/findsecbugs-plugin.jar 95 | 96 | ## Download and install sb-contrib plugin for spotbugs (https://github.com/mebigfatguy/fb-contrib) 97 | curl -LO "https://repo1.maven.org/maven2/com/mebigfatguy/sb-contrib/sb-contrib/${SB_CONTRIB_VERSION}/sb-contrib-${SB_CONTRIB_VERSION}.jar" 98 | mv -f sb-contrib-${SB_CONTRIB_VERSION}.jar "${SPOTBUGS_OPTDIR}"/plugin/sb-contrib.jar 99 | 100 | mv -f "${SPOTBUGS_OPTDIR}" "${OPTDIR}"/spotbugs 101 | else 102 | echo "SpotBugs already installed" 103 | fi 104 | 105 | # End SpotBugs ----------------------------------------------------------- 106 | 107 | ## install composer 108 | if [ ! -f composer-setup.php ]; then 109 | echo "Downloading composer" 110 | php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" 111 | fi 112 | php composer-setup.php 113 | mv -f composer.phar "${USR_BIN_PATH}"composer 114 | mayberm composer-setup.php 115 | 116 | # Install application dependencies 117 | npm install --no-audit --progress=false --omit=dev --production --no-save --prefix "${APPDIR}"/usr/local/lib yarn @cyclonedx/cdxgen @microsoft/rush 118 | mkdir -p "${APPDIR}"/opt/phpsast 119 | pushd "${APPDIR}"/opt/phpsast 120 | composer init --name shiftleft/scan --description scan --quiet 121 | composer require --quiet --no-cache -n --no-ansi --dev vimeo/psalm:^5.15 122 | popd 123 | # I suspect at the time of writing this, the behavoir of --prefix + --root is as described in this issue: 124 | # https://github.com/pypa/pip/issues/7829#issuecomment-596330888 125 | # TL;DR the resulting path prefix will be ${APPDIR}/usr which is consistent with what we want in all cases, if this is 126 | # invoked within a docker build APPDIR will be simply / 127 | python3 -m pip install -v --prefix=/usr --root="${APPDIR}" -r "${PWD}"/requirements.txt --no-warn-script-location --force-reinstall 128 | composer require --quiet --no-cache --dev phpstan/phpstan 129 | 130 | ## Copy the python application code into the AppDir if APPIMAGE is set 131 | if [ -n "${APPIMAGE}" ]; then 132 | cp -r scan lib tools_config "${APPDIR}"/usr/src 133 | cp tools_config/scan.png "${APPDIR}"/usr/share/icons/ 134 | cp tools_config/io.shiftleft.scan.appdata.xml "${APPDIR}"/usr/share/metainfo/ 135 | fi 136 | -------------------------------------------------------------------------------- /builder.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | # This dockerfile builds a dockerfile that can be used as an env to build the AppImage, beware,it is slow due to IO 4 | ENV DEBIAN_FRONTEND=noninteractive 5 | RUN apt-get update -y && apt-get install -y python3 python3-dev \ 6 | python3-pip python3-setuptools patchelf desktop-file-utils \ 7 | libgdk-pixbuf2.0-dev php php-curl php-zip php-bcmath php-json \ 8 | php-pear php-mbstring php-dev wget curl git unzip \ 9 | adwaita-icon-theme libfuse2 squashfs-tools zsync 10 | 11 | RUN wget https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage && chmod +x appimagetool-x86_64.AppImage && ./appimagetool-x86_64.AppImage --appimage-extract && ln -s /squashfs-root/AppRun /usr/local/bin/appimagetool 12 | RUN wget https://github.com/AppImage/AppImageKit/releases/download/continuous/runtime-x86_64 -O /usr/local/bin/runtime-x86_64 && chmod +x /usr/local/bin/runtime-x86_64 13 | RUN pip3 install git+https://github.com/perrito666/appimage-builder.git 14 | 15 | ENV UPDATE_INFO=gh-releases-zsync|ShiftLeftSecurity|sast-scan|latest|*x86_64.AppImage.zsync 16 | 17 | WORKDIR /build 18 | 19 | ENV PATH="/build/AppDir/usr/bin:/build/AppDir/usr/bin/nodejs/bin:${PATH}" 20 | ENV ARCH=x86_64 21 | 22 | CMD mkdir -p appimage-builder-cache && ln -fs /usr/local/bin/runtime-x86_64 appimage-builder-cache && appimage-builder --recipe appimage-builder.yml --skip-test 23 | -------------------------------------------------------------------------------- /builder.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker run -v `pwd`:/build shiftleft/sast-scan-builder 4 | -------------------------------------------------------------------------------- /builder_aarch64.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | # This dockerfile builds a dockerfile that can be used as an env to build the AppImage, beware,it is slow due to IO 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | RUN apt-get update -y && apt-get install -y python3.8 python3.8-dev \ 7 | python3-pip python3-setuptools patchelf desktop-file-utils \ 8 | libgdk-pixbuf2.0-dev php php-curl php-zip php-bcmath php-json \ 9 | php-pear php-mbstring php-dev wget curl git unzip \ 10 | adwaita-icon-theme libfuse2 squashfs-tools zsync 11 | 12 | RUN wget https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-aarch64.AppImage && chmod +x appimagetool-aarch64.AppImage && ./appimagetool-aarch64.AppImage --appimage-extract && ln -s /squashfs-root/AppRun /usr/local/bin/appimagetool 13 | RUN wget https://github.com/AppImage/AppImageKit/releases/download/continuous/runtime-aarch64 -O /usr/local/bin/runtime-aarch64 && chmod +x /usr/local/bin/runtime-aarch64 14 | RUN pip3 install git+https://github.com/perrito666/appimage-builder.git 15 | RUN chmod +x /usr/local/bin/runtime-aarch64 16 | 17 | ENV UPDATE_INFO=gh-releases-zsync|ShiftLeftSecurity|sast-scan|latest|*aarch64.AppImage.zsync 18 | 19 | WORKDIR /build 20 | 21 | ENV PATH="/build/AppDir/usr/bin:/build/AppDir/usr/bin/nodejs/bin:${PATH}" 22 | ENV ARCH=arm64 23 | 24 | CMD mkdir -p appimage-builder-cache && ln -fs /usr/local/bin/runtime-aarch64 appimage-builder-cache && appimage-builder --recipe appimage-builder.yml --skip-test 25 | -------------------------------------------------------------------------------- /building_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | export GOSEC_VERSION=2.17.0 3 | export TFSEC_VERSION=1.28.1 4 | export KUBESEC_VERSION=2.13.0 5 | export KUBE_SCORE_VERSION=1.17.0 6 | export DETEKT_VERSION=1.23.1 7 | export GITLEAKS_VERSION=8.17.0 8 | export SC_VERSION=2023.1.5 # 0.4.5, staticcheck actually uses date versions now 9 | export PMD_VERSION=6.55.0 10 | export FSB_VERSION=1.12.0 11 | export SB_CONTRIB_VERSION=7.4.7 12 | export SB_VERSION=4.7.3 13 | export NODE_VERSION=18.17.1 14 | 15 | 16 | ## Fail if ARCH is not set 17 | if [ -z "$ARCH" ]; then 18 | echo "ARCH is not set, please set it to the architecture you want to build for" 19 | exit 1 20 | fi 21 | 22 | 23 | # Normalize in case of docker invocation using TARGETARCH 24 | if [ "$ARCH" = "amd64" ]; then # docker uses this but lets normalize 25 | ARCH="x86_64" 26 | fi 27 | if [ "$ARCH" = "arm64" ]; then # docker uses this but lets normalize 28 | ARCH="aarch64" 29 | fi 30 | 31 | # Account for non conventional Arch names in downloadables 32 | if [ "$ARCH" = "x86_64" ]; then 33 | NODE_ARCH="x64" 34 | else 35 | NODE_ARCH="$ARCH" 36 | fi 37 | if [ "$ARCH" = "x86_64" ]; then 38 | ARCH_ALT_NAME="amd64" 39 | else 40 | ARCH_ALT_NAME="$ARCH" 41 | fi 42 | if [ "$ARCH" = "aarch64" ]; then 43 | LIBARCH="arm64" 44 | else 45 | LIBARCH="$ARCH" 46 | fi 47 | 48 | export NODE_ARCH 49 | export ARCH_ALT_NAME 50 | export LIBARCH 51 | 52 | 53 | ## mayberm deletes the passed file only if KEEP_BUILD_ARTIFACTS variable is not set 54 | mayberm() { 55 | if [ -z "$KEEP_BUILD_ARTIFACTS" ]; then 56 | rm "$1" 57 | fi 58 | } 59 | -------------------------------------------------------------------------------- /ci/Dockerfile-csharp: -------------------------------------------------------------------------------- 1 | FROM shiftleft/scan-base as builder 2 | 3 | ARG CLI_VERSION 4 | ARG BUILD_DATE 5 | 6 | ENV GOSEC_VERSION=2.14.0 \ 7 | TFSEC_VERSION=0.63.1 \ 8 | KUBESEC_VERSION=2.11.4 \ 9 | KUBE_SCORE_VERSION=1.13.0 \ 10 | SHELLCHECK_VERSION=0.7.2 \ 11 | DETEKT_VERSION=1.22.0 \ 12 | GITLEAKS_VERSION=7.6.1 \ 13 | SC_VERSION=0.3.3 \ 14 | JQ_VERSION=1.6 \ 15 | GOPATH=/opt/app-root/go \ 16 | SHIFTLEFT_HOME=/opt/sl-cli \ 17 | PATH=${PATH}:${GOPATH}/bin: 18 | 19 | USER root 20 | 21 | RUN mkdir -p /usr/local/bin/shiftleft \ 22 | && curl -LO "https://github.com/securego/gosec/releases/download/v${GOSEC_VERSION}/gosec_${GOSEC_VERSION}_linux_amd64.tar.gz" \ 23 | && tar -C /usr/local/bin/shiftleft/ -xvf gosec_${GOSEC_VERSION}_linux_amd64.tar.gz \ 24 | && chmod +x /usr/local/bin/shiftleft/gosec \ 25 | && rm gosec_${GOSEC_VERSION}_linux_amd64.tar.gz 26 | RUN curl -LO "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \ 27 | && tar -C /tmp/ -xvf shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz \ 28 | && cp /tmp/shellcheck-v${SHELLCHECK_VERSION}/shellcheck /usr/local/bin/shiftleft/shellcheck \ 29 | && chmod +x /usr/local/bin/shiftleft/shellcheck \ 30 | && curl -LO "https://github.com/dominikh/go-tools/releases/download/v${SC_VERSION}/staticcheck_linux_amd64.tar.gz" \ 31 | && tar -C /tmp -xvf staticcheck_linux_amd64.tar.gz \ 32 | && chmod +x /tmp/staticcheck/staticcheck \ 33 | && cp /tmp/staticcheck/staticcheck /usr/local/bin/shiftleft/staticcheck \ 34 | && rm staticcheck_linux_amd64.tar.gz 35 | RUN curl -L "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks-linux-amd64" -o "/usr/local/bin/shiftleft/gitleaks" \ 36 | && chmod +x /usr/local/bin/shiftleft/gitleaks \ 37 | && curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec-linux-amd64" -o "/usr/local/bin/shiftleft/tfsec" \ 38 | && chmod +x /usr/local/bin/shiftleft/tfsec \ 39 | && rm shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz 40 | RUN curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/kube-score_${KUBE_SCORE_VERSION}_linux_amd64" -o "/usr/local/bin/shiftleft/kube-score" \ 41 | && chmod +x /usr/local/bin/shiftleft/kube-score \ 42 | && curl -L "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64" -o "/usr/local/bin/shiftleft/jq" \ 43 | && chmod +x /usr/local/bin/shiftleft/jq 44 | RUN curl -L "https://github.com/detekt/detekt/releases/download/v${DETEKT_VERSION}/detekt-cli-${DETEKT_VERSION}-all.jar" -o "/usr/local/bin/shiftleft/detekt-cli.jar" \ 45 | && curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/kubesec_linux_amd64.tar.gz" \ 46 | && tar -C /usr/local/bin/shiftleft/ -xvf kubesec_linux_amd64.tar.gz \ 47 | && rm kubesec_linux_amd64.tar.gz \ 48 | && curl "https://cdn.shiftleft.io/download/sl" > /usr/local/bin/shiftleft/sl \ 49 | && chmod a+rx /usr/local/bin/shiftleft/sl \ 50 | && mkdir -p /opt/sl-cli \ 51 | && /usr/local/bin/shiftleft/sl update csharp2cpg 52 | 53 | FROM shiftleft/scan-base-csharp as sast-scan-tools 54 | 55 | LABEL maintainer="ShiftLeftSecurity" \ 56 | org.label-schema.schema-version="1.0" \ 57 | org.label-schema.vendor="shiftleft" \ 58 | org.label-schema.name="sast-scan" \ 59 | org.label-schema.version=$CLI_VERSION \ 60 | org.label-schema.license="Apache-2.0" \ 61 | org.label-schema.description="Container with various opensource static analysis security testing tools (shellcheck, gosec, tfsec, gitleaks, ...) for multiple programming languages" \ 62 | org.label-schema.url="https://www.shiftleft.io" \ 63 | org.label-schema.usage="https://github.com/ShiftLeftSecurity/sast-scan" \ 64 | org.label-schema.build-date=$BUILD_DATE \ 65 | org.label-schema.vcs-url="https://github.com/ShiftLeftSecurity/sast-scan.git" \ 66 | org.label-schema.docker.cmd="docker run --rm -it --name sast-scan shiftleft/scan-csharp" 67 | 68 | ENV APP_SRC_DIR=/usr/local/src \ 69 | DEPSCAN_CMD="/usr/local/bin/depscan" \ 70 | MVN_CMD="/usr/bin/mvn" \ 71 | PMD_CMD="/opt/pmd-bin/bin/run.sh pmd" \ 72 | PYTHONUNBUFFERED=1 \ 73 | DOTNET_CLI_TELEMETRY_OPTOUT=1 \ 74 | SHIFTLEFT_HOME=/opt/sl-cli \ 75 | GO111MODULE=auto \ 76 | GOARCH=amd64 \ 77 | GOOS=linux \ 78 | CGO_ENABLED=0 \ 79 | NVD_EXCLUDE_TYPES="o,h" \ 80 | PATH=/usr/local/src/:${PATH}:/usr/local/go/bin:/opt/sl-cli: 81 | 82 | COPY --from=builder /usr/local/bin/shiftleft /usr/local/bin 83 | COPY --from=builder /opt/sl-cli /opt/sl-cli 84 | COPY tools_config/ /usr/local/src/ 85 | COPY requirements.txt /usr/local/src/ 86 | 87 | USER root 88 | 89 | RUN python3 -m pip install --upgrade pip \ 90 | && pip3 install --no-cache-dir wheel \ 91 | && pip3 install --no-cache-dir -r /usr/local/src/requirements.txt \ 92 | && npm install --no-audit --progress=false --only=production -g @cyclonedx/cdxgen @microsoft/rush --unsafe-perm \ 93 | && microdnf clean all 94 | 95 | WORKDIR /app 96 | 97 | COPY scan /usr/local/src/ 98 | COPY lib /usr/local/src/lib 99 | 100 | CMD [ "python3", "/usr/local/src/scan" ] 101 | -------------------------------------------------------------------------------- /ci/Dockerfile-dynamic-lang: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy as builder 2 | 3 | ARG CLI_VERSION 4 | ARG BUILD_DATE 5 | ARG TARGETARCH 6 | 7 | # For now these are kept here but ensure they are in sync with building_env.sh 8 | ENV TFSEC_VERSION=1.28.1 \ 9 | GITLEAKS_VERSION=8.17.0 \ 10 | KUBESEC_VERSION=2.13.0 \ 11 | KUBE_SCORE_VERSION=1.17.0 \ 12 | PATH=/usr/local/src/:${PATH}:/usr/local/bin/shiftleft/:/usr/local/bin/shiftleft/nodejs/bin \ 13 | ARCH=$TARGETARCH 14 | 15 | USER root 16 | 17 | RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/00-docker 18 | RUN echo 'APT::Install-Recommends "0";' >> /etc/apt/apt.conf.d/00-docker 19 | 20 | # Dependencies to install other tools, node is downloaded as the apt version is too old 21 | RUN apt-get update && apt-get install -y gcc git python3 python3-dev \ 22 | python3-pip python3-setuptools curl 23 | COPY dynamic-lang.sh / 24 | COPY building_env.sh / 25 | RUN chmod +x /dynamic-lang.sh && /dynamic-lang.sh /usr/local/bin/shiftleft/ 26 | COPY requirements.txt /usr/local/src/ 27 | COPY scan /usr/local/src/ 28 | COPY lib /usr/local/src/lib 29 | COPY tools_config/ /usr/local/src/ 30 | 31 | RUN python3 -m pip install --upgrade pip 32 | RUN python3 -m pip install --no-cache-dir -r /usr/local/src/requirements.txt 33 | RUN /usr/local/bin/shiftleft/nodejs/bin/npm install --no-audit --progress=false --only=production -g @cyclonedx/cdxgen @microsoft/rush --unsafe-perm 34 | RUN apt-get remove -y gcc python3-dev curl && apt-get autoremove -y && apt-get clean -y 35 | 36 | FROM ubuntu:jammy as sast-scan-tools 37 | 38 | LABEL maintainer="ShiftLeftSecurity" \ 39 | org.label-schema.schema-version="1.0" \ 40 | org.label-schema.vendor="shiftleft" \ 41 | org.label-schema.name="sast-scan" \ 42 | org.label-schema.version=$CLI_VERSION \ 43 | org.label-schema.license="Apache-2.0" \ 44 | org.label-schema.description="Container with various opensource static analysis security testing tools (shellcheck, gosec, tfsec, gitleaks, ...) for multiple programming languages" \ 45 | org.label-schema.url="https://www.qwiet.ai" \ 46 | org.label-schema.usage="https://github.com/ShiftLeftSecurity/sast-scan" \ 47 | org.label-schema.build-date=$BUILD_DATE \ 48 | org.label-schema.vcs-url="https://github.com/ShiftLeftSecurity/sast-scan.git" \ 49 | org.label-schema.docker.cmd="docker run --rm -it --name sast-scan shiftleft/scan-slim" 50 | 51 | ENV APP_SRC_DIR=/usr/local/src \ 52 | DEPSCAN_CMD="/usr/local/bin/depscan" \ 53 | PMD_CMD="" \ 54 | PYTHONUNBUFFERED=1 \ 55 | NVD_EXCLUDE_TYPES="o,h" \ 56 | GIT_PYTHON_GIT_EXECUTABLE=/usr/bin/git \ 57 | PATH=/usr/local/src/:/usr/local/bin/shiftleft/:/usr/local/bin/shiftleft/nodejs/bin:/usr/bin:${PATH}: 58 | 59 | COPY --from=builder /usr /usr 60 | 61 | WORKDIR /app 62 | 63 | CMD [ "python3", "/usr/local/src/scan" ] 64 | -------------------------------------------------------------------------------- /contrib/google-cloud/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | Store AppThreat scan reports and create Application Security dashboards with Google Cloud. 4 | 5 | # Dashboard Screenshots 6 | 7 | ![Dashboard 1](./dashboard1.png) 8 | 9 | ![Dashboard 2](./dashboard2.png) 10 | -------------------------------------------------------------------------------- /contrib/google-cloud/dashboard1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/contrib/google-cloud/dashboard1.png -------------------------------------------------------------------------------- /contrib/google-cloud/dashboard2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/contrib/google-cloud/dashboard2.png -------------------------------------------------------------------------------- /contrib/google-cloud/store-api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | LABEL maintainer="ShiftLeftSecurity" \ 4 | org.label-schema.schema-version="1.0" \ 5 | org.label-schema.vendor="shiftleft" \ 6 | org.label-schema.name="store-api" \ 7 | org.label-schema.version="1.0.0" \ 8 | org.label-schema.license="UNLICENSED" \ 9 | org.label-schema.description="API to store and retrieve AppThreat reports" \ 10 | org.label-schema.url="https://www.shiftleft.io" \ 11 | org.label-schema.usage="https://github.com/ShiftLeftSecurity/sast-scan" \ 12 | org.label-schema.build-date=$BUILD_DATE \ 13 | org.label-schema.vcs-url="https://github.com/ShiftLeftSecurity/sast-scan.git" \ 14 | org.label-schema.docker.cmd="docker run --rm -it --name store-api shiftleft/store-api" 15 | 16 | USER root 17 | WORKDIR /app 18 | 19 | COPY requirements.txt /app/ 20 | RUN pip install --no-cache-dir -r requirements.txt 21 | 22 | ENV QUART_ENV=production \ 23 | PYTHONDONTWRITEBYTECODE=1 \ 24 | PYTHONUNBUFFERED=1 25 | 26 | COPY . /app/ 27 | CMD [ "python", "app.py" ] 28 | -------------------------------------------------------------------------------- /contrib/google-cloud/store-api/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This is a lightweight api service to store and retrieve AppThreat scan reports for Google Cloud. Google Cloud Run is used to run a lightweight quart microservice and Firestore is used as the backend. Data from firebase could be exported to Google BigQuery and visualized using Datastudio. 4 | 5 | ## Usage 6 | 7 | ``` 8 | export GOOGLE_APPLICATION_CREDENTIALS=path to json 9 | python3 app.py 10 | ``` 11 | 12 | ## API Endpoints 13 | 14 | ### Health check 15 | 16 | ```bash 17 | curl http://0.0.0.0:8080/healthcheck 18 | ``` 19 | 20 | ### Post summary data 21 | 22 | ```bash 23 | curl --header "Content-Type: application/json" -d '{"id": "foo", "data": "bar"}' http://0.0.0.0:8080/summary 24 | ``` 25 | 26 | ### Get summary data 27 | 28 | ```bash 29 | curl http://0.0.0.0:8080/summary?id=foo 30 | ``` 31 | 32 | ### Upload scan report 33 | 34 | You can upload either the individual SARIF files or the special aggregate file called scan-full-report.json. AppThreat produce scan-full-report.json which is a jsonlines file containing a single SARIF report in each line. 35 | 36 | ```bash 37 | curl -F 'file=@/home/guest/CodeAnalysisLogs/scan-full-report.json' http://0.0.0.0:8080/scans 38 | ``` 39 | 40 | ```bash 41 | curl -F 'file=@/home/guest/CodeAnalysisLogs/source-java-report.sarif' http://0.0.0.0:8080/scans 42 | ``` 43 | 44 | ### Retrieve scans 45 | 46 | By scan id (SARIF -> runs -> automationDetails.guid) 47 | 48 | ```bash 49 | curl http://0.0.0.0:8080/scans?id=c3983af9-7dc0-4a6e-8109-726ee127530d 50 | ``` 51 | 52 | ```bash 53 | curl "http://0.0.0.0:8080/scans?repositoryUri=https://github.com/AppThreat/WebGoat&branch=develop" 54 | ``` 55 | 56 | Create any composite index as required by firestore. Some suggested `Fields indexed` for the composite indexes are: 57 | 58 | - branch Ascending repositoryUri Ascending created_at Descending 59 | - repositoryUri Ascending created_at Descending 60 | 61 | Delete scans 62 | 63 | ```bash 64 | curl -X DELETE "http://0.0.0.0:8080/scans?repositoryUri=https://github.com/AppThreat/WebGoat&branch=develop&revisionId=210dbaf5f0f49a79cb1adf9760c36658c819ff7d" 65 | ``` 66 | 67 | ## Integration with Datastudio 68 | 69 | ```bash 70 | gcloud firestore export gs://at_scans --collection-ids=at_scans 71 | ``` 72 | 73 | Copy the outputUriPrefix from the above command and use it below. 74 | 75 | ```bash 76 | bq --location=US load \ 77 | --source_format=DATASTORE_BACKUP \ 78 | --replace \ 79 | at_scans_analysis.all_apps \ 80 | gs://at_scans//all_namespaces/kind_at_scans/all_namespaces_kind_at_scans.export_metadata 81 | ``` 82 | 83 | Eg: 84 | 85 | ``` 86 | gs://at_scans/2021-07-21T13:22:42_29189/all_namespaces/kind_at_scans/all_namespaces_kind_at_scans.export_metadata 87 | ``` 88 | -------------------------------------------------------------------------------- /contrib/google-cloud/store-api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/contrib/google-cloud/store-api/__init__.py -------------------------------------------------------------------------------- /contrib/google-cloud/store-api/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'docker:latest' 3 | args: [ 'build', '-t', 'gcr.io/${PROJECT_ID}/${_NAME_SPACE}/${_SERVICE_NAME}:${_TAG}', '.'] 4 | - name: 'docker:latest' 5 | args: ['push', 'gcr.io/${PROJECT_ID}/${_NAME_SPACE}/${_SERVICE_NAME}:${_TAG}'] 6 | - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:slim' 7 | args: 8 | - 'gcloud' 9 | - 'run' 10 | - 'deploy' 11 | - '${_SERVICE_NAME}' 12 | - '--image' 13 | - 'gcr.io/${PROJECT_ID}/${_NAME_SPACE}/${_SERVICE_NAME}:${_TAG}' 14 | - '--region' 15 | - '${_REGION}' 16 | - '--memory=1Gi' 17 | - '--platform' 18 | - 'managed' 19 | - '--allow-unauthenticated' 20 | 21 | substitutions: 22 | _NAME_SPACE: appthreat 23 | _TAG: 1.0.0 24 | _SERVICE_NAME: store-api 25 | _REGION: us-west1 26 | 27 | images: 28 | - gcr.io/${PROJECT_ID}/${_NAME_SPACE}/${_SERVICE_NAME}:${_TAG} 29 | -------------------------------------------------------------------------------- /contrib/google-cloud/store-api/config.py: -------------------------------------------------------------------------------- 1 | SUMMARY_COLLECTION = "at_summary" 2 | SCANS_COLLECTION = "at_scans" 3 | ALLOWED_EXTENSIONS = ["json", "jsonl", "sarif"] 4 | MAX_CONTENT_LENGTH = 10 * 1000 * 1000 # 10 Mb 5 | SCAN_FULL_REPORT = "scan-full-report.json" 6 | -------------------------------------------------------------------------------- /contrib/google-cloud/store-api/requirements.txt: -------------------------------------------------------------------------------- 1 | quart 2 | hypercorn 3 | google-cloud-firestore 4 | -------------------------------------------------------------------------------- /dev-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | DOCKER_CMD=docker 3 | if command -v podman >/dev/null 2>&1; then 4 | DOCKER_CMD=podman 5 | fi 6 | isort **/*.py 7 | python3 -m black lib 8 | python3 -m black scan 9 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 10 | flake8 . --count --exit-zero --statistics 11 | 12 | $DOCKER_CMD build -t shiftleft/sast-scan -t shiftleft/scan -f Dockerfile . 13 | $DOCKER_CMD build -t shiftleft/scan-java -f ci/Dockerfile-java . 14 | $DOCKER_CMD build -t shiftleft/scan-slim -f ci/Dockerfile-dynamic-lang . 15 | $DOCKER_CMD build -t shiftleft/scan-csharp -f ci/Dockerfile-csharp . 16 | $DOCKER_CMD build -t shiftleft/scan-oss -f ci/Dockerfile-oss . 17 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | Visit [https://appthreat.com](https://appthreat.com) to access the latest Scan documentation. 4 | 5 | -------------------------------------------------------------------------------- /docs/TRICKS.md: -------------------------------------------------------------------------------- 1 | # Tips and tricks 2 | 3 | This page captures advanced customisation and tweaks supported by sast-scan. 4 | 5 | ## Automatic build 6 | 7 | Scan can attempt to build certain project types such as Java, go, node.js, rust and csharp using the bundled runtimes. To enable auto build simply pass `--build` argument or set the environment variable `SCAN_AUTO_BUILD` to a non-empty value. 8 | 9 | ## Workspace path prefix 10 | 11 | sast-scan tool is typically invoked using the docker container image with volume mounts. Due to this behaviour, the source path the tools would see would be different to the source path in the developer laptop or in the CI environment. 12 | 13 | To override the prefix, simply pass the environment variable `WORKSPACE` with the path that should get prefixed in the reports. 14 | 15 | ```bash 16 | export WORKSPACE="/home/shiftleft/src" 17 | 18 | # To specify url 19 | export WORKSPACE="https://github.com/ShiftLeftSecurity/cdxgen/blob/master" 20 | ``` 21 | 22 | If your organization use `Azure Repos` for hosting git repositories then the above approach would not work because of the way url gets constructed. You can construct the url for Azure Repos as follows: 23 | 24 | ```bash 25 | export WORKSPACE="$(Build.Repository.Uri)?_a=contents&version=GB$(Build.SourceBranchName)&path=" 26 | ``` 27 | 28 | However, note that because of the way `Build.SourceBranchName` is [computed](https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml) this variable may not work if the branch contains slashes in them such as `feature/foo/bar`. In such cases, the branch name has to be derived based on the variable `Build.SourceBranch` by removing the `/refs/heads` or `/refs/pull/` prefixes. 29 | 30 | Let us know if you find a better way to support direct linking for Azure Repos. 31 | 32 | ## Config file 33 | 34 | sast-scan can load configurations automatically from `.sastscanrc` in the repo root directory. This file is a json file containing the keys from [config.py](lib/config.py). 35 | 36 | Below is an example. 37 | 38 | ```json 39 | { 40 | "scan_type": "java,credscan,bash", 41 | "scan_tools_args_map": { 42 | "credscan": [ 43 | "gitleaks", 44 | "--branch=master", 45 | "--repo-path=%(src)s", 46 | "--redact", 47 | "--report=%(report_fname_prefix)s.json", 48 | "--format=json" 49 | ] 50 | } 51 | } 52 | ``` 53 | 54 | With a local config you can override the scan type and even configure the command line args for the tools as shown. 55 | 56 | ## Use CI build reference as runGuid 57 | 58 | By setting the environment variable `SCAN_ID` you can re-use the CI build reference as the run guid for the reports. This is useful to reverse lookup the pipeline result based on the sast-scan result. 59 | -------------------------------------------------------------------------------- /docs/azure-devops.md: -------------------------------------------------------------------------------- 1 | # Integration with Azure DevOps Pipelines 2 | 3 | sast-scan has good integration with Azure Pipelines. This repo contains an [example for a yaml pipeline](https://github.com/ShiftLeftSecurity/WebGoat/blob/develop/azure-pipelines.yml) that invokes sast-scan as a build step. The step is reproduced below for convenience. 4 | 5 | ```yaml 6 | - script: | 7 | docker run -e "WORKSPACE=https://github.com/ShiftLeftSecurity/WebGoat/blob/$(Build.SourceVersion)" \ 8 | -v $(Build.SourcesDirectory):/app \ 9 | -v $(Build.ArtifactStagingDirectory):/reports \ 10 | shiftleft/sast-scan scan --src /app \ 11 | --out_dir /reports/CodeAnalysisLogs 12 | displayName: "Perform ShiftLeft Scan" 13 | continueOnError: "true" 14 | ``` 15 | 16 | ## Suggested DevSecOps workflow 17 | 18 | This section is mostly common for all dev and CI environments. 19 | 20 | ### pre-commit hook 21 | 22 | Use the example pre-commit script provided under `docs/pre-commit.sh` to enable automatic sast-scan prior to commits. 23 | 24 | ```bash 25 | cp docs/pre-commit.sh /.git/hooks/pre-commit 26 | ``` 27 | 28 | This pre-commit hook performs both credentials and sast-scan. Any identified credential will be displayed in plain-text to enable remediation. sast-scan reports would be stored under `reports` directory which could be added to .gitignore to prevent unwanted commits of the reports. 29 | 30 | ### Credentials scanning 31 | 32 | Include `credscan` along with the type parameter as shown to enable credentials scanning for the branch on the CI. This feature is powered by [gitleaks](https://github.com/zricethezav/gitleaks). Please note that identified secrets are automatically REDACTED in the CI environments to prevent leakage. 33 | 34 | ### Viewing sast-scan reports 35 | 36 | The following extension called [ShiftLeft Scan Reports]() must be installed and enabled by the administrator. 37 | 38 | The yaml pipeline should include the below steps to enable the analysis. 39 | 40 | ```yaml 41 | - task: PublishBuildArtifacts@1 42 | displayName: "Publish analysis logs" 43 | inputs: 44 | PathtoPublish: "$(Build.ArtifactStagingDirectory)/CodeAnalysisLogs" 45 | ArtifactName: "CodeAnalysisLogs" 46 | publishLocation: "Container" 47 | ``` 48 | -------------------------------------------------------------------------------- /docs/azure-devops.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/docs/azure-devops.png -------------------------------------------------------------------------------- /docs/azure-pipelines.yml.sample: -------------------------------------------------------------------------------- 1 | # Use jobs to seperate SAST scans from build and deployments 2 | jobs: 3 | - job: SastScans 4 | displayName: "Run SAST scan" 5 | pool: 6 | vmImage: "Ubuntu 16.04" 7 | steps: 8 | # Pull the sast-scan image from the container registry. Feel free to cache it locally within acr, ecr or gcr to 9 | # improve performance and security 10 | - script: docker pull shiftleft/sast-scan 11 | # This step assumes that python source code are inside a directory called python 12 | - script: | 13 | docker run -e "WORKSPACE=https://github.com/ShiftLeftSecurity/WebGoat/blob/$(Build.SourceVersion)" -v $(Build.SourcesDirectory)/python:/app shiftleft/sast-scan scan --src /app --type python --out_dir /app/reports 14 | displayName: "Python scan" 15 | # This step assumes that node.js source code are inside a directory called javascript 16 | - script: | 17 | docker run -e "WORKSPACE=https://github.com/ShiftLeftSecurity/WebGoat/blob/$(Build.SourceVersion)" -v $(Build.SourcesDirectory)/javascript:/app shiftleft/sast-scan scan --src /app --type nodejs --out_dir /app/reports 18 | displayName: "Node.js scan" 19 | # Bring together all the .sarif files to a directory called CodeAnalysisLogs 20 | - task: CopyFiles@2 21 | displayName: "Copy analysis logs" 22 | inputs: 23 | SourceFolder: "$(Build.SourcesDirectory)" 24 | Contents: "**/*.sarif" 25 | TargetFolder: "$(Build.ArtifactStagingDirectory)/CodeAnalysisLogs" 26 | flattenFolders: true 27 | # To integrate with the SARIF Azure DevOps Extension it is necessary to publish the CodeAnalysisLogs folder 28 | # as an artifact with the same name 29 | - task: PublishBuildArtifacts@1 30 | displayName: "Publish analysis logs" 31 | inputs: 32 | PathtoPublish: "$(Build.ArtifactStagingDirectory)/CodeAnalysisLogs" 33 | ArtifactName: "CodeAnalysisLogs" 34 | publishLocation: "Container" 35 | -------------------------------------------------------------------------------- /docs/build-breaker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/docs/build-breaker.png -------------------------------------------------------------------------------- /docs/circleci-artifacts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/docs/circleci-artifacts.png -------------------------------------------------------------------------------- /docs/circleci-sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/docs/circleci-sample.png -------------------------------------------------------------------------------- /docs/circleci.md: -------------------------------------------------------------------------------- 1 | # Integration with CircleCI 2 | 3 | sast-scan has decent integration with CircleCI. This repo contains an [example for a yaml pipeline](https://github.com/ShiftLeftSecurity/WebGoat/blob/develop/.circleci/config.yml) that invokes sast-scan using docker run. 4 | 5 | Note: 6 | 7 | - CircleCI does not support docker volume mounts. A suggested [workaround](https://circleci.com/docs/2.0/building-docker-images/#mounting-folders) is to use `docker create` along with `docker cp` to copy the repo with compiled code and then perform sast-scan against that. 8 | 9 | ## Usage 10 | 11 | The config is reproduced here for convenience. 12 | 13 | ```yaml 14 | steps: 15 | - checkout 16 | - setup_remote_docker 17 | - run: 18 | name: Compile and prepare volume 19 | command: | 20 | # Compile the code 21 | mvn compile 22 | # Create a dummy container with the application data 23 | docker create -v /app --name appcon alpine:3.4 /bin/true 24 | # Copy the repo to the temporary volume 25 | docker cp $PWD appcon:/app 26 | - run: 27 | name: Perform sast-scan 28 | command: | 29 | set +e 30 | # start an application container using this volume 31 | docker run --name sastscan -e "WORKSPACE=${CIRCLE_REPOSITORY_URL}" --volumes-from appcon shiftleft/sast-scan scan --src /app --out_dir /app/reports 32 | # Copy the reports 33 | docker cp sastscan:/app/reports reports 34 | - store_artifacts: 35 | path: reports 36 | destination: sast-scan-reports 37 | ``` 38 | 39 | In the above snippet, almost all the steps are necessary. 40 | 41 | - setup_remote_docker - This is required for performing docker create, cp commands etc 42 | - mvn compile - Certain tools expect both the source code and its compiled form. Hence it is necessary to compile and build the application before performing sast-scan 43 | - `docker create and cp` - This is the workaround for the lack of volume mounts in CircleCI. If you are not a fan of this, then ask CircleCI to implement the mount feature 😒 44 | - `set +e` - This prevents the docker run command from breaking the build so that the reports can be stored and viewed later on. Alternatively, if you indeed want the build to break then remove this line. 45 | - `docker cp` - This is how the produced reports are copied back to the build container 46 | - `store_artifacts` - Stores reports as an artifact for later use 47 | 48 | ## Screenshots 49 | 50 | ![CircleCI Sample](circleci-sample.png) 51 | 52 | ![CircleCI Build Artifacts](circleci-artifacts.png) 53 | -------------------------------------------------------------------------------- /docs/integration.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This document describes the output SARIF format emitted by `sast-scan` tool for integration purposes. 4 | 5 | ## SARIF specification 6 | 7 | sast-tool implements version 2.1.0 specification which can be found [here](https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Toc16012479). Every release of sast-tool is carefully tested to remain compliant and produce valid SARIF files. The [online validator](https://sarifweb.azurewebsites.net/Validation) can be used to validate the [sample files](test/data/bandit-report.sarif) attached with this repo. 8 | 9 | ## SARIF components 10 | 11 | ### sarifLog 12 | 13 | - version: 2.1.0 14 | - \$schema: https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json 15 | - inlineExternalProperties: 16 | - guid - UUID representing each report from the tool 17 | - runGuid - UUID representing an invocation of sast-scan which can produce multiple reports. This can be specified by setting the environment variable `SCAN_ID` 18 | - runs: Array with a single run object representing a single run of a tool. This might however change in the future to represent tools that perform multiple scans per invocation. 19 | 20 | ### run 21 | 22 | - tool: 23 | - driver: This section would describe the tool used to perform the scan along with the rules applied. Eg: A scan for python would lead to the below section 24 | 25 | ```json 26 | "tool": { 27 | "driver": { 28 | "name": "Security audit for python", 29 | "rules": [ 30 | { 31 | "id": "B322", 32 | "name": "blacklist", 33 | "helpUri": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b322-input" 34 | } 35 | ... 36 | ] 37 | } 38 | } 39 | ``` 40 | 41 | - conversion: This section would contain information on how sast-scan utilized the underlying tool to perform the report conversion to SARIF format. 42 | - invocations: This section would contain useful information such as: 43 | - endTimeUtc: Scan end time 44 | - workingDirectory: Working directory used for the scan 45 | - properties: 46 | - metrics: This section contains the scan summary such as total issues found as well as the number of critical, high, medium and low issues 47 | 48 | ```json 49 | "properties": { 50 | "metrics": { 51 | "total": 35, 52 | "critical": 0, 53 | "high": 5, 54 | "medium": 30, 55 | "low": 0 56 | } 57 | } 58 | ``` 59 | 60 | - results: An array of result object representing the findings 61 | 62 | ### result 63 | 64 | - message: Detailed message from the tool representing the finding 65 | - level: string representing the type of finding - can be error, warning, note 66 | - locations: An array of information representing the source code, line numbers, filename (`artifactLocation`) along with the code snippet highlighting the issue. artifactLocation would start with either https:// or file:// protocol depending on the `WORKSPACE` environment variable used 67 | - properties: 68 | - issue_confidence: UPPER case flag indicating the confidence level of the tool for the particular result. Valid values are: HIGH, MEDIUM, LOW 69 | - issue_severity: UPPER case flag indicating the severity level of the particular result. Valid values are: HIGH, MEDIUM, LOW 70 | - ruleId: ID of the rule used. This will be the present in the list of rules mentioned in the tool section 71 | - ruleIndex: Index of the rule in the tool section for faster lookups 72 | 73 | Example of a result is shown below: 74 | 75 | ```json 76 | { 77 | "message": { 78 | "text": "The input method in Python 2 will read from standard input, evaluate and run the resulting string as python source code. This is similar, though in many ways worse, then using eval. On Python 2, use raw_input instead, input is safe in Python 3." 79 | }, 80 | "level": "error", 81 | "locations": [ 82 | { 83 | "physicalLocation": { 84 | "region": { 85 | "snippet": { 86 | "text": " response = input('Enter the hash that follows ' + lastkey + ': ')\n" 87 | }, 88 | "startLine": 24 89 | }, 90 | "artifactLocation": { 91 | "uri": "file:///Users/guest/work/shiftleft/vulpy/utils/skey.py" 92 | }, 93 | "contextRegion": { 94 | "snippet": { 95 | "text": " while True:\n response = input('Enter the hash that follows ' + lastkey + ': ')\n result = hashlib.new(ALGORITHM, response.encode()).hexdigest()\n" 96 | }, 97 | "endLine": 25, 98 | "startLine": 23 99 | } 100 | } 101 | } 102 | ], 103 | "properties": { 104 | "issue_confidence": "HIGH", 105 | "issue_severity": "HIGH" 106 | }, 107 | "hostedViewerUri": "https://sarifviewer.azurewebsites.net", 108 | "ruleId": "B322", 109 | "ruleIndex": 0 110 | } 111 | ``` 112 | -------------------------------------------------------------------------------- /docs/pre-commit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # An example pre-commit hook to perform sast-scan on the repo. 4 | # Copy this file to .git/hooks/pre-commit 5 | echo ' 6 | 7 | █████╗ ██████╗ ██████╗ ████████╗██╗ ██╗██████╗ ███████╗ █████╗ ████████╗ 8 | ██╔══██╗██╔══██╗██╔══██╗╚══██╔══╝██║ ██║██╔══██╗██╔════╝██╔══██╗╚══██╔══╝ 9 | ███████║██████╔╝██████╔╝ ██║ ███████║██████╔╝█████╗ ███████║ ██║ 10 | ██╔══██║██╔═══╝ ██╔═══╝ ██║ ██╔══██║██╔══██╗██╔══╝ ██╔══██║ ██║ 11 | ██║ ██║██║ ██║ ██║ ██║ ██║██║ ██║███████╗██║ ██║ ██║ 12 | ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ ╚═╝ 13 | 14 | ' 15 | docker_state=$(docker info >/dev/null 2>&1) 16 | if [[ $? -ne 0 ]]; then 17 | echo "Docker does not seem to be running, please start the service or run the desktop application" 18 | exit 1 19 | fi 20 | docker pull shiftleft/sast-scan >/dev/null 2>&1 21 | 22 | # Scan credentials using gitleaks 23 | docker run --rm --tmpfs /tmp -e "WORKSPACE=${PWD}" -v $PWD:/app shiftleft/sast-scan gitleaks --uncommitted --repo-path=/app --pretty 24 | 25 | if [ $? == 1 ]; then 26 | echo "Remove the credentials identified by the scan" 27 | exit 1 28 | fi 29 | 30 | # Perform automatic scan 31 | echo "Performing SAST scan on the repo" 32 | docker run --rm --tmpfs /tmp -e "WORKSPACE=${PWD}" -v $PWD:/app shiftleft/sast-scan scan --src /app --out_dir /app/reports 33 | -------------------------------------------------------------------------------- /docs/sarif-online-viewer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/docs/sarif-online-viewer.png -------------------------------------------------------------------------------- /dynamic-lang.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Dynamic lang script installs a subset of the whole tools required by scan application, this is the set used by 4 | # scan-slim image. 5 | # This can be invoked standalone passing the path to the folder where the binaries will be installed, or it can be 6 | # sourced by another script, in which case the USR_BIN_PATH variable will be set to the path where the binaries go. 7 | 8 | # IF USR_BIN_PATH is not set, set it to the default, this means we are being called standalone. 9 | if [ -z "$USR_BIN_PATH" ]; then 10 | source building_env.sh 11 | # typically "/usr/local/bin/shiftleft/" 12 | USR_BIN_PATH=$1 13 | fi 14 | export USR_BIN_PATH 15 | 16 | mkdir -p ${USR_BIN_PATH} 17 | 18 | 19 | ## Download and install gitleaks (https://github.com/zricethezav/gitleaks) 20 | GLEAKS_FOLDER="gitleaks_${GITLEAKS_VERSION}_linux_${NODE_ARCH}" 21 | GLEAKS_TAR="${GLEAKS_FOLDER}.tar.gz" 22 | echo "Downloading ${GLEAKS_TAR}" 23 | curl -LO "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/${GLEAKS_TAR}" 24 | mkdir -p /tmp/"${GLEAKS_FOLDER}" 25 | tar -C /tmp/"${GLEAKS_FOLDER}" -xzvf "${GLEAKS_TAR}" 26 | cp /tmp/"${GLEAKS_FOLDER}"/gitleaks "${USR_BIN_PATH}"gitleaks 27 | chmod +x "${USR_BIN_PATH}"gitleaks 28 | 29 | ## Download and install kube-score (https://github.com/zegl/kube-score) 30 | K8SCORE_TAR="kube-score_${KUBE_SCORE_VERSION}_linux_${ARCH}" 31 | echo "Downloading ${K8SCORE_TAR}" 32 | curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/${K8SCORE_TAR}" -o "${USR_BIN_PATH}kube-score" 33 | chmod +x "${USR_BIN_PATH}"kube-score 34 | 35 | ## Download and install tfsec (https://github.com/aquasecurity/tfsec) 36 | TFSEC_TAR="tfsec-linux-${ARCH}" 37 | echo "Downloading ${TFSEC_TAR}" 38 | curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${TFSEC_TAR}" -o "${USR_BIN_PATH}tfsec" 39 | chmod +x "${USR_BIN_PATH}"tfsec 40 | 41 | ## Download and install kubesec (https://github.com/controlplaneio/kubesec) 42 | K8SSEC_TAR="kubesec_linux_${ARCH_ALT_NAME}.tar.gz" 43 | if [ ! -f "${K8SSEC_TAR}" ]; then 44 | echo "Downloading ${K8SSEC_TAR}" 45 | curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/${K8SSEC_TAR}" 46 | fi 47 | echo "Installing ${K8SSEC_TAR}" 48 | tar -C "${USR_BIN_PATH}" -xzvf "${K8SSEC_TAR}" 49 | mayberm "${K8SSEC_TAR}" 50 | 51 | ## Download and install nodeJS (https://nodejs.org) 52 | NODE_TAR=node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz 53 | # if file not there, download it 54 | if [ ! -f "${NODE_TAR}" ]; then 55 | echo "Downloading ${NODE_TAR}" 56 | curl -LO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TAR}" 57 | fi 58 | if [ ! -d "${USR_BIN_PATH}"nodejs/node-v${NODE_VERSION}-linux-"${NODE_ARCH}" ]; then 59 | echo "Installing ${NODE_TAR}" 60 | tar -C "${USR_BIN_PATH}" -xzf "${NODE_TAR}" 61 | mv -f "${USR_BIN_PATH}"node-v${NODE_VERSION}-linux-"${NODE_ARCH}" "${USR_BIN_PATH}"nodejs 62 | chmod +x "${USR_BIN_PATH}"nodejs/bin/node 63 | chmod +x "${USR_BIN_PATH}"nodejs/bin/npm 64 | mayberm "${NODE_TAR}" 65 | else 66 | echo "NodeJS already installed" 67 | fi 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/lib/__init__.py -------------------------------------------------------------------------------- /lib/aggregate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | import uuid 5 | from datetime import datetime 6 | 7 | import sarif_om as om 8 | from jschema_to_python.to_json import to_json 9 | 10 | import lib.config as config 11 | 12 | 13 | def jsonl_aggregate(run_data_list, out_file_name): 14 | """Produce aggregated report in jsonl format 15 | 16 | :param run_data_list: List of run data after parsing the sarif files 17 | :param out_file_name: Output filename 18 | """ 19 | if not run_data_list or not out_file_name: 20 | return 21 | with open(out_file_name, "w") as outfile: 22 | for data in run_data_list: 23 | json.dump(data, outfile) 24 | outfile.write("\n") 25 | 26 | 27 | def sarif_aggregate(run_data_list, out_sarif_name): 28 | """Produce aggregated sarif data (Unused) 29 | 30 | :param run_data_list: 31 | :param out_sarif_name: 32 | :return: 33 | """ 34 | log_uuid = str(uuid.uuid4()) 35 | run_uuid = config.get("run_uuid") 36 | log = om.SarifLog( 37 | schema_uri="https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", 38 | version="2.1.0", 39 | inline_external_properties=[ 40 | om.ExternalProperties(guid=log_uuid, run_guid=run_uuid) 41 | ], 42 | runs=run_data_list, 43 | ) 44 | serialized_log = to_json(log) 45 | with open(out_sarif_name, "w") as outfile: 46 | outfile.write(serialized_log) 47 | 48 | 49 | def store_baseline(baseline_fingerprints, baseline_file): 50 | """Produce baseline file 51 | 52 | :param baseline_fingerprints: Fingerprints to store 53 | :param baseline_file: Baseline filename 54 | """ 55 | with open(baseline_file, "w") as outfile: 56 | json.dump( 57 | { 58 | "baseline_fingerprints": baseline_fingerprints, 59 | "created_at": str(datetime.now()), 60 | }, 61 | outfile, 62 | indent=2, 63 | ) 64 | -------------------------------------------------------------------------------- /lib/cis.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import yaml 4 | 5 | cis_k8s_file = Path(__file__).parent / "data" / "cis-k8s.yaml" 6 | cis_aws_file = Path(__file__).parent / "data" / "cis-aws.yaml" 7 | 8 | cis_rules_dict = {} 9 | if not cis_rules_dict: 10 | for cf in [cis_k8s_file, cis_aws_file]: 11 | with open(cf) as fp: 12 | raw_data = fp.read().split("---") 13 | for tmp_data in raw_data: 14 | cdata = yaml.safe_load(tmp_data) 15 | if not cdata: 16 | continue 17 | for group in cdata.get("groups", []): 18 | for check in group.get("checks"): 19 | for rule in check.get("scan_rule_ids", []): 20 | cis_rules_dict[rule.upper()] = check 21 | 22 | 23 | def get_cis_rules(): 24 | """ 25 | Return data about all CIS data for Kubernetes 26 | :return: dict containing all CIS data 27 | """ 28 | return cis_rules_dict 29 | 30 | 31 | def get_rule(rule_id): 32 | return cis_rules_dict.get(str(rule_id).upper()) 33 | -------------------------------------------------------------------------------- /lib/constants.py: -------------------------------------------------------------------------------- 1 | RANKING = ["UNDEFINED", "LOW", "MEDIUM", "HIGH", "CRITICAL"] 2 | CONFIDENCE_DEFAULT = "MEDIUM" 3 | SEVERITY_DEFAULT = "LOW" 4 | 5 | PRIORITY_MAP = { 6 | "1": "CRITICAL", 7 | "2": "HIGH", 8 | "3": "MEDIUM", 9 | "4": "LOW", 10 | "5": "UNDEFINED", 11 | } 12 | -------------------------------------------------------------------------------- /lib/csv_parser.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | 4 | def get_report_data(csvfile): 5 | """Convert csv file to dict 6 | 7 | :param csvfile: CSV file to parse 8 | """ 9 | raw_data = csv.reader(csvfile, delimiter=",") 10 | report_data = [] 11 | headers = None 12 | for row in raw_data: 13 | if not headers: 14 | headers = [r.lower().replace(" ", "_") for r in row] 15 | else: 16 | report_data.append(dict(zip(headers, row))) 17 | return headers, report_data 18 | -------------------------------------------------------------------------------- /lib/cwe.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from pathlib import Path 3 | 4 | cwe_software_csv = Path(__file__).parent / "data" / "cwe_software.csv" 5 | cwe_research_csv = Path(__file__).parent / "data" / "cwe_research.csv" 6 | 7 | cwe_dict = {} 8 | 9 | if not cwe_dict: 10 | with open(cwe_software_csv, newline="") as f: 11 | reader = csv.DictReader(f) 12 | for row in reader: 13 | cwe_dict[row["CWE-ID"]] = row 14 | with open(cwe_research_csv, newline="") as f: 15 | reader = csv.DictReader(f) 16 | for row in reader: 17 | cwe_dict[row["CWE-ID"]] = row 18 | 19 | 20 | def get_all(): 21 | """ 22 | Return data about all CWE 23 | :return: dict containing all CWE data 24 | """ 25 | return cwe_dict 26 | 27 | 28 | def get(cid): 29 | """ 30 | Return CWE details for the given id 31 | :param cid: CWE id 32 | :return: CWE data 33 | """ 34 | cid = cid.upper().replace("CWE-", "") 35 | return cwe_dict.get(cid) 36 | 37 | 38 | def get_name(cid): 39 | """ 40 | Return the name for the given cwe 41 | 42 | :param cid: cwe id 43 | :return: Name 44 | """ 45 | data = get(cid) 46 | if not data: 47 | return "" 48 | name = data.get("Name") 49 | if not name.endswith("."): 50 | name = name + "." 51 | return name 52 | 53 | 54 | def get_description(cid, extended=False): 55 | """ 56 | Method to retrieve just the description for the given cwe 57 | :param cid: cwe id 58 | :param extended Boolean to indicate if extended description is required 59 | :return: Description string 60 | """ 61 | data = get(cid) 62 | if not data: 63 | return "" 64 | desc = data.get("Description") 65 | if not extended: 66 | return desc 67 | if data.get("Extended Description"): 68 | desc = desc + "\n" + data.get("Extended Description") 69 | desc = desc.replace("::TYPE:Relationship:NOTE:", "\n\nNOTE:\n") 70 | desc = desc.replace("::TYPE:Terminology:NOTE:", "\n\nNOTE:\n") 71 | desc = desc.replace("::", "") 72 | if not desc.endswith("."): 73 | desc = desc + "." 74 | return desc 75 | -------------------------------------------------------------------------------- /lib/data/CREDITS: -------------------------------------------------------------------------------- 1 | cis-k8s.yaml 2 | 3 | Source: 4 | https://github.com/aquasecurity/kube-bench/tree/master/cfg/cis-1.5 5 | 6 | License: Apache-2.0 7 | 8 | cwe_research.csv 9 | cwe_software.csv 10 | 11 | Source: http://cwe.mitre.org/data/ 12 | -------------------------------------------------------------------------------- /lib/integration/__init__.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | 3 | 4 | class GitProvider(metaclass=ABCMeta): 5 | @classmethod 6 | @abstractmethod 7 | def get_context(cls, repo_context): 8 | pass 9 | 10 | @classmethod 11 | @abstractmethod 12 | def annotate_pr(cls, repo_context, findings_file, report_summary, build_status): 13 | pass 14 | 15 | @classmethod 16 | def upload_report(cls, repo_context, findings_file, report_summary, build_status): 17 | pass 18 | 19 | @classmethod 20 | def create_status(cls, repo_context, findings_file, report_summary, build_status): 21 | pass 22 | 23 | @classmethod 24 | def manage_issues(cls, repo_context, findings_file, report_summary, build_status): 25 | pass 26 | 27 | @classmethod 28 | def to_emoji(cls, status): 29 | emoji_codes = {":white_heavy_check_mark:": "✅", ":cross_mark:": "❌"} 30 | return emoji_codes.get(status, status) 31 | -------------------------------------------------------------------------------- /lib/integration/gitlab.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | import os 5 | 6 | import requests 7 | 8 | import lib.config as config 9 | from lib.integration import GitProvider 10 | from lib.logger import LOG 11 | 12 | 13 | class GitLab(GitProvider): 14 | def get_token(self): 15 | token = config.get("GITLAB_TOKEN") 16 | if not token: 17 | token = config.get("MR_TOKEN") 18 | return token 19 | 20 | def get_context(self, repo_context): 21 | apiUrl = os.getenv("CI_API_V4_URL") 22 | if not apiUrl: 23 | apiUrl = "https://gitlab.com/api/v4" 24 | return { 25 | **repo_context, 26 | "apiUrl": apiUrl, 27 | "mergeRequestIID": os.getenv("CI_MERGE_REQUEST_IID"), 28 | "mergeRequestProjectId": os.getenv("CI_MERGE_REQUEST_PROJECT_ID"), 29 | "mergeRequestSourceBranch": os.getenv( 30 | "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" 31 | ), 32 | "mergeRequestTargetBranch": os.getenv( 33 | "CI_MERGE_REQUEST_TARGET_BRANCH_NAME" 34 | ), 35 | "commitSHA": os.getenv("CI_COMMIT_SHA"), 36 | "projectId": os.getenv("CI_PROJECT_ID"), 37 | "projectName": os.getenv("CI_PROJECT_NAME"), 38 | "projectUrl": os.getenv("CI_PROJECT_URL"), 39 | "jobUrl": os.getenv("CI_JOB_URL"), 40 | "jobId": os.getenv("CI_JOB_ID"), 41 | "jobName": os.getenv("CI_JOB_NAME"), 42 | # CI_JOB_TOKEN is only available to Silver/Enterprise plan of GitLab 43 | "jobToken": os.getenv("CI_JOB_TOKEN"), 44 | } 45 | 46 | def get_mr_notes_url(self, repo_context): 47 | gitlab_context = self.get_context(repo_context) 48 | return f"""{gitlab_context.get("apiUrl")}/projects/{gitlab_context.get("mergeRequestProjectId")}/merge_requests/{gitlab_context.get("mergeRequestIID")}/notes""" 49 | 50 | def annotate_pr(self, repo_context, findings_file, report_summary, build_status): 51 | if not findings_file: 52 | if not list(filter(lambda x: "depscan" in x, report_summary)): 53 | return 54 | else: 55 | with open(findings_file, mode="r") as fp: 56 | findings_obj = json.load(fp) 57 | findings = findings_obj.get("findings") 58 | if not findings: 59 | LOG.debug("No findings from scan available to report") 60 | return 61 | try: 62 | gitlab_context = self.get_context(repo_context) 63 | if not gitlab_context.get("mergeRequestIID") or not gitlab_context.get( 64 | "mergeRequestProjectId" 65 | ): 66 | LOG.debug( 67 | "Scan is not running as part of a merge request. Check if the pipeline is using only: [merge_requests] or rules syntax" 68 | ) 69 | return 70 | private_token = self.get_token() 71 | if not private_token: 72 | LOG.info( 73 | "To create a merge request note, create a personal access token with api scope and set it as GITLAB_TOKEN environment variable" 74 | ) 75 | return 76 | summary = "| Tool | Critical | High | Medium | Low | Status |\n" 77 | summary = summary + "| ---- | ------- | ------ | ----- | ---- | ---- |\n" 78 | for rk, rv in report_summary.items(): 79 | status_emoji = self.to_emoji(rv.get("status")) 80 | summary = f'{summary}| {rv.get("tool")} | {rv.get("critical")} | {rv.get("high")} | {rv.get("medium")} | {rv.get("low")} | {status_emoji} |\n' 81 | template = config.get("PR_COMMENT_TEMPLATE") 82 | recommendation = ( 83 | f"Please review the [scan reports]({gitlab_context.get('jobUrl')}/artifacts/browse/reports) before approving this merge request." 84 | if build_status == "fail" 85 | else "Looks good" 86 | ) 87 | apiUrl = f"{gitlab_context.get('apiUrl')}" 88 | mergeRequestIID = f"{gitlab_context.get('mergeRequestIID')}" 89 | mergeRequestProjectId = f"{gitlab_context.get('mergeRequestProjectId')}" 90 | mergeRequestSourceBranch = ( 91 | f"{gitlab_context.get('mergeRequestSourceBranch')}" 92 | ) 93 | mergeRequestTargetBranch = ( 94 | f"{gitlab_context.get('mergeRequestTargetBranch')}" 95 | ) 96 | commitSHA = f"{gitlab_context.get('commitSHA')}" 97 | projectId = f"{gitlab_context.get('projectId')}" 98 | projectName = f"{gitlab_context.get('projectName')}" 99 | projectUrl = f"{gitlab_context.get('projectUrl')}" 100 | jobUrl = f"{gitlab_context.get('jobUrl')}" 101 | jobId = f"{gitlab_context.get('jobId')}" 102 | jobName = f"{gitlab_context.get('jobName')}" 103 | jobToken = f"{gitlab_context.get('jobToken')}" 104 | 105 | body = template % dict( 106 | summary=summary, 107 | recommendation=recommendation, 108 | apiUrl=apiUrl, 109 | mergeRequestIID=mergeRequestIID, 110 | mergeRequestProjectId=mergeRequestProjectId, 111 | mergeRequestSourceBranch=mergeRequestSourceBranch, 112 | mergeRequestTargetBranch=mergeRequestTargetBranch, 113 | commitSHA=commitSHA, 114 | projectId=projectId, 115 | projectName=projectName, 116 | projectUrl=projectUrl, 117 | jobUrl=jobUrl, 118 | jobId=jobId, 119 | jobName=jobName, 120 | jobToken=jobToken, 121 | ) 122 | rr = requests.post( 123 | self.get_mr_notes_url(repo_context), 124 | headers={ 125 | "Content-Type": "application/json", 126 | "PRIVATE-TOKEN": self.get_token(), 127 | }, 128 | json={"body": body}, 129 | ) 130 | if not rr.ok: 131 | LOG.debug(rr.json()) 132 | except Exception as e: 133 | LOG.debug(e) 134 | -------------------------------------------------------------------------------- /lib/integration/provider.py: -------------------------------------------------------------------------------- 1 | from lib.integration import bitbucket, github, gitlab 2 | 3 | 4 | def get_git_provider(repo_context): 5 | if repo_context and repo_context.get("gitProvider"): 6 | gitProvider = repo_context.get("gitProvider") 7 | if gitProvider == "bitbucket": 8 | return bitbucket.Bitbucket() 9 | elif gitProvider == "gitlab": 10 | return gitlab.GitLab() 11 | elif gitProvider == "github": 12 | return github.GitHub() 13 | return None 14 | -------------------------------------------------------------------------------- /lib/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from rich.console import Console 5 | from rich.logging import RichHandler 6 | from rich.theme import Theme 7 | 8 | custom_theme = Theme({"info": "cyan", "warning": "purple4", "danger": "bold red"}) 9 | color_system = "256" 10 | if os.getenv("SHIFTLEFT_ACCESS_TOKEN") or os.getenv("SHIFTLEFT_APP"): 11 | color_system = "auto" 12 | console = Console( 13 | log_time=False, 14 | log_path=False, 15 | theme=custom_theme, 16 | width=140, 17 | color_system=color_system, 18 | ) 19 | 20 | logging.basicConfig( 21 | level=logging.INFO, 22 | format="%(message)s", 23 | datefmt="[%X]", 24 | handlers=[ 25 | RichHandler( 26 | console=console, markup=True, show_path=False, enable_link_path=False 27 | ) 28 | ], 29 | ) 30 | LOG = logging.getLogger(__name__) 31 | 32 | # Set logging level 33 | if os.getenv("SCAN_DEBUG_MODE") == "debug": 34 | LOG.setLevel(logging.DEBUG) 35 | 36 | DEBUG = logging.DEBUG 37 | -------------------------------------------------------------------------------- /lib/pyt/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | pyt is based on the now defunct [pyt project](https://github.com/python-security/pyt). This folder contains numerous scan specific modifications. 4 | -------------------------------------------------------------------------------- /lib/pyt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/lib/pyt/__init__.py -------------------------------------------------------------------------------- /lib/pyt/analysis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/lib/pyt/analysis/__init__.py -------------------------------------------------------------------------------- /lib/pyt/analysis/constraint_table.py: -------------------------------------------------------------------------------- 1 | """Global lookup table for constraints. 2 | 3 | Uses cfg node as key and operates on bitvectors in the form of ints.""" 4 | 5 | constraint_table = dict() 6 | 7 | 8 | def initialize_constraint_table(cfg_list): 9 | """Collects all given cfg nodes and initializes the table with value 0.""" 10 | for cfg in cfg_list: 11 | constraint_table.update(dict.fromkeys(cfg.nodes, 0)) 12 | 13 | 14 | def constraint_join(cfg_nodes): 15 | """Looks up all cfg_nodes and joins the bitvectors by using logical or.""" 16 | r = 0 17 | for e in cfg_nodes: 18 | r = r | constraint_table[e] 19 | return r 20 | -------------------------------------------------------------------------------- /lib/pyt/analysis/definition_chains.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | from lib.pyt.analysis.constraint_table import constraint_table 4 | from lib.pyt.core.node_types import AssignmentNode 5 | 6 | 7 | def get_constraint_nodes(node, lattice): 8 | for n in lattice.get_elements(constraint_table[node]): 9 | if n is not node: 10 | yield n 11 | 12 | 13 | def build_def_use_chain(cfg_nodes, lattice): 14 | def_use = defaultdict(list) 15 | # For every node 16 | for node in cfg_nodes: 17 | # That's a definition 18 | if isinstance(node, AssignmentNode): 19 | # Get the uses 20 | for variable in node.right_hand_side_variables: 21 | # Loop through most of the nodes before it 22 | for earlier_node in get_constraint_nodes(node, lattice): 23 | # and add them to the 'uses list' of each earlier node, when applicable 24 | # 'earlier node' here being a simplification 25 | if variable in earlier_node.left_hand_side: 26 | def_use[earlier_node].append(node) 27 | return def_use 28 | -------------------------------------------------------------------------------- /lib/pyt/analysis/fixed_point.py: -------------------------------------------------------------------------------- 1 | """This module implements the fixed point algorithm.""" 2 | from lib.pyt.analysis.constraint_table import constraint_table 3 | from lib.pyt.analysis.reaching_definitions_taint import ReachingDefinitionsTaintAnalysis 4 | 5 | max_analysis_steps = 20 6 | max_runs = 100 7 | max_none_runs = 3 8 | 9 | 10 | class FixedPointAnalysis: 11 | """Run the fix point analysis.""" 12 | 13 | def __init__(self, cfg): 14 | """Fixed point analysis. 15 | 16 | Analysis must be a dataflow analysis containing a 'fixpointmethod' 17 | method that analyses one CFG.""" 18 | self.analysis = ReachingDefinitionsTaintAnalysis(cfg) 19 | self.cfg = cfg 20 | 21 | def fixpoint_runner(self, max_analysis_steps, max_runs): 22 | """Work list algorithm that runs the fixpoint algorithm.""" 23 | q = self.cfg.nodes 24 | cnt = 0 25 | none_break_cnt = 0 26 | while q: 27 | if q[0] is None: 28 | if none_break_cnt > max_none_runs: 29 | break 30 | else: 31 | none_break_cnt = none_break_cnt + 1 32 | continue 33 | cnt = cnt + 1 34 | x_i = constraint_table[q[0]] # x_i = q[0].old_constraint 35 | self.analysis.fixpointmethod(q[0]) # y = F_i(x_1, ..., x_n); 36 | y = constraint_table[q[0]] # y = q[0].new_constraint 37 | 38 | if y != x_i: 39 | acnt = 0 40 | for node in self.analysis.dep(q[0]): # for (v in dep(v_i)) 41 | q.append(node) # q.append(v): 42 | acnt = acnt + 1 43 | if acnt > max_analysis_steps: 44 | break 45 | constraint_table[ 46 | q[0] 47 | ] = y # q[0].old_constraint = q[0].new_constraint # x_i = y 48 | q = q[1:] # q = q.tail() # The list minus the head 49 | if not q: 50 | break 51 | if cnt > max_runs: 52 | break 53 | 54 | 55 | def analyse(cfg_list): 56 | """Analyse a list of control flow graphs with a given analysis type.""" 57 | for cfg in cfg_list: 58 | analysis = FixedPointAnalysis(cfg) 59 | analysis.fixpoint_runner( 60 | max_analysis_steps=max_analysis_steps, max_runs=max_runs 61 | ) 62 | -------------------------------------------------------------------------------- /lib/pyt/analysis/lattice.py: -------------------------------------------------------------------------------- 1 | from lib.pyt.analysis.constraint_table import constraint_table 2 | from lib.pyt.core.node_types import AssignmentNode 3 | 4 | 5 | def get_lattice_elements(cfg_nodes): 6 | """Returns all assignment nodes as they are the only lattice elements 7 | in the reaching definitions analysis. 8 | """ 9 | for node in cfg_nodes: 10 | if isinstance(node, AssignmentNode): 11 | yield node 12 | 13 | 14 | class Lattice: 15 | def __init__(self, cfg_nodes): 16 | self.el2bv = dict() # Element to bitvector dictionary 17 | self.bv2el = list() # Bitvector to element list 18 | for i, e in enumerate(get_lattice_elements(cfg_nodes)): 19 | # Give each element a unique shift of 1 20 | self.el2bv[e] = 0b1 << i 21 | self.bv2el.insert(0, e) 22 | 23 | def get_elements(self, number): 24 | if number == 0: 25 | return [] 26 | 27 | elements = list() 28 | # Turn number into a binary string of length len(self.bv2el) 29 | binary_string = format(number, "0" + str(len(self.bv2el)) + "b") 30 | for i, bit in enumerate(binary_string): 31 | if bit == "1": 32 | elements.append(self.bv2el[i]) 33 | return elements 34 | 35 | def in_constraint(self, node1, node2): 36 | """Checks if node1 is in node2's constraints 37 | For instance, if node1 = 010 and node2 = 110: 38 | 010 & 110 = 010 -> has the element.""" 39 | constraint = constraint_table[node2] 40 | if constraint == 0b0: 41 | return False 42 | 43 | try: 44 | value = self.el2bv[node1] 45 | except KeyError: 46 | return False 47 | 48 | return constraint & value != 0 49 | -------------------------------------------------------------------------------- /lib/pyt/analysis/reaching_definitions_taint.py: -------------------------------------------------------------------------------- 1 | from lib.pyt.analysis.constraint_table import constraint_join, constraint_table 2 | from lib.pyt.analysis.lattice import Lattice 3 | from lib.pyt.core.node_types import AssignmentNode 4 | 5 | 6 | class ReachingDefinitionsTaintAnalysis: 7 | def __init__(self, cfg): 8 | self.cfg = cfg 9 | self.lattice = Lattice(cfg.nodes) 10 | 11 | def fixpointmethod(self, cfg_node): 12 | """The most important part of PyT, where we perform 13 | the variant of reaching definitions to find where sources reach. 14 | """ 15 | JOIN = self.join(cfg_node) 16 | # Assignment check 17 | if isinstance(cfg_node, AssignmentNode): 18 | arrow_result = JOIN 19 | 20 | # Reassignment check 21 | if cfg_node.left_hand_side not in cfg_node.right_hand_side_variables: 22 | # Get previous assignments of cfg_node.left_hand_side and remove them from JOIN 23 | arrow_result = self.arrow(JOIN, cfg_node.left_hand_side) 24 | 25 | arrow_result = arrow_result | self.lattice.el2bv[cfg_node] 26 | constraint_table[cfg_node] = arrow_result 27 | # Default case 28 | else: 29 | constraint_table[cfg_node] = JOIN 30 | 31 | def join(self, cfg_node): 32 | """Joins all constraints of the ingoing nodes and returns them. 33 | This represents the JOIN auxiliary definition from Schwartzbach.""" 34 | return constraint_join(cfg_node.ingoing) 35 | 36 | def arrow(self, JOIN, _id): 37 | """Removes all previous assignments from JOIN that have the same left hand side. 38 | This represents the arrow id definition from Schwartzbach.""" 39 | r = JOIN 40 | for node in self.lattice.get_elements(JOIN): 41 | if node.left_hand_side == _id: 42 | r = r ^ self.lattice.el2bv[node] 43 | return r 44 | 45 | def dep(self, q_1): 46 | """Represents the dep mapping from Schwartzbach.""" 47 | for node in q_1.outgoing: 48 | yield node 49 | -------------------------------------------------------------------------------- /lib/pyt/cfg/__init__.py: -------------------------------------------------------------------------------- 1 | from lib.pyt.cfg.make_cfg import make_cfg 2 | 3 | __all__ = ["make_cfg"] 4 | -------------------------------------------------------------------------------- /lib/pyt/cfg/alias_helper.py: -------------------------------------------------------------------------------- 1 | """This module contains alias helper functions for the expr_visitor module.""" 2 | 3 | 4 | def as_alias_handler(alias_list): 5 | """Returns a list of all the names that will be called.""" 6 | list_ = list() 7 | for alias in alias_list: 8 | if alias.asname: 9 | list_.append(alias.asname) 10 | else: 11 | list_.append(alias.name) 12 | return list_ 13 | 14 | 15 | def handle_aliases_in_calls(name, import_alias_mapping): 16 | """Returns either None or the handled alias. 17 | Used in add_module. 18 | """ 19 | for key, val in import_alias_mapping.items(): 20 | # e.g. Foo == Foo 21 | # e.g. Foo.Bar startswith Foo. 22 | if name == key or name.startswith(key + "."): 23 | 24 | # Replace key with val in name 25 | # e.g. StarbucksVisitor.Tea -> Eataly.Tea because 26 | # "from .nested_folder import StarbucksVisitor as Eataly" 27 | return name.replace(key, val) 28 | return None 29 | 30 | 31 | def handle_aliases_in_init_files(name, import_alias_mapping): 32 | """Returns either None or the handled alias. 33 | Used in add_module. 34 | """ 35 | for key, val in import_alias_mapping.items(): 36 | # e.g. Foo == Foo 37 | # e.g. Foo.Bar startswith Foo. 38 | if name == val or name.startswith(val + "."): 39 | 40 | # Replace val with key in name 41 | # e.g. StarbucksVisitor.Tea -> Eataly.Tea because 42 | # "from .nested_folder import StarbucksVisitor as Eataly" 43 | return name.replace(val, key) 44 | return None 45 | 46 | 47 | def handle_fdid_aliases(module_or_package_name, import_alias_mapping): 48 | """Returns either None or the handled alias. 49 | Used in add_module. 50 | fdid means from directory import directory. 51 | """ 52 | for key, val in import_alias_mapping.items(): 53 | if module_or_package_name == val: 54 | return key 55 | return None 56 | 57 | 58 | def not_as_alias_handler(names_list): 59 | """Returns a list of names ignoring any aliases.""" 60 | list_ = list() 61 | for alias in names_list: 62 | list_.append(alias.name) 63 | return list_ 64 | 65 | 66 | def retrieve_import_alias_mapping(names_list): 67 | """Creates a dictionary mapping aliases to their respective name. 68 | import_alias_names is used in module_definitions.py and visit_Call""" 69 | import_alias_names = dict() 70 | 71 | for alias in names_list: 72 | if alias.asname: 73 | import_alias_names[alias.asname] = alias.name 74 | return import_alias_names 75 | 76 | 77 | def fully_qualify_alias_labels(label, aliases): 78 | """Replace any aliases in label with the fully qualified name. 79 | 80 | Args: 81 | label -- A label : str representing a name (e.g. myos.system) 82 | aliases -- A dict of {alias: real_name} (e.g. {'myos': 'os'}) 83 | 84 | >>> fully_qualify_alias_labels('myos.mycall', {'myos':'os'}) 85 | 'os.mycall' 86 | """ 87 | for alias, full_name in aliases.items(): 88 | if label == alias: 89 | return full_name 90 | elif label.startswith(alias + "."): 91 | return full_name + label[len(alias) :] 92 | return label 93 | -------------------------------------------------------------------------------- /lib/pyt/cfg/expr_visitor_helper.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | from lib.pyt.core.node_types import ConnectToExitNode 4 | 5 | SavedVariable = namedtuple("SavedVariable", ("LHS", "RHS")) 6 | BUILTINS = ( 7 | "get", 8 | "Flask", 9 | "run", 10 | "replace", 11 | "read", 12 | "set_cookie", 13 | "make_response", 14 | "SQLAlchemy", 15 | "Column", 16 | "execute", 17 | "sessionmaker", 18 | "Session", 19 | "filter", 20 | "call", 21 | "render_template", 22 | "redirect", 23 | "url_for", 24 | "flash", 25 | "jsonify", 26 | ) 27 | MUTATORS = ( # list.append(x) taints list if x is tainted 28 | "add", 29 | "append", 30 | "extend", 31 | "insert", 32 | "update", 33 | ) 34 | 35 | 36 | def return_connection_handler(nodes, exit_node): 37 | """Connect all return statements to the Exit node.""" 38 | for function_body_node in nodes: 39 | if isinstance(function_body_node, ConnectToExitNode): 40 | if exit_node not in function_body_node.outgoing: 41 | function_body_node.connect(exit_node) 42 | -------------------------------------------------------------------------------- /lib/pyt/cfg/make_cfg.py: -------------------------------------------------------------------------------- 1 | from lib.pyt.cfg.expr_visitor import ExprVisitor 2 | 3 | 4 | class CFG: 5 | def __init__(self, nodes, blackbox_assignments, filename): 6 | self.nodes = nodes 7 | self.blackbox_assignments = blackbox_assignments 8 | self.filename = filename 9 | 10 | def __repr__(self): 11 | output = "" 12 | for x, n in enumerate(self.nodes): 13 | output = "".join((output, "Node: " + str(x) + " " + repr(n), "\n\n")) 14 | return output 15 | 16 | def __str__(self): 17 | output = "" 18 | for x, n in enumerate(self.nodes): 19 | output = "".join((output, "Node: " + str(x) + " " + str(n), "\n\n")) 20 | return output 21 | 22 | 23 | def make_cfg( 24 | tree, 25 | project_modules, 26 | local_modules, 27 | filename, 28 | module_definitions=None, 29 | allow_local_directory_imports=True, 30 | ): 31 | visitor = ExprVisitor( 32 | tree, 33 | project_modules, 34 | local_modules, 35 | filename, 36 | module_definitions, 37 | allow_local_directory_imports, 38 | ) 39 | return CFG(visitor.nodes, visitor.blackbox_assignments, filename) 40 | -------------------------------------------------------------------------------- /lib/pyt/cfg/stmt_visitor_helper.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import random 3 | from collections import namedtuple 4 | 5 | from lib.pyt.core.node_types import ( 6 | AssignmentCallNode, 7 | BBorBInode, 8 | BreakNode, 9 | ControlFlowNode, 10 | RestoreNode, 11 | ) 12 | 13 | CALL_IDENTIFIER = "~" 14 | ConnectStatements = namedtuple( 15 | "ConnectStatements", ("first_statement", "last_statements", "break_statements") 16 | ) 17 | 18 | 19 | def _get_inner_most_function_call(call_node): 20 | # Loop to inner most function call 21 | # e.g. return scrypt.inner in `foo = scrypt.outer(scrypt.inner(image_name))` 22 | old_call_node = None 23 | while call_node != old_call_node: 24 | old_call_node = call_node 25 | if isinstance(call_node, BBorBInode): 26 | call_node = call_node.inner_most_call 27 | else: 28 | try: 29 | # e.g. save_2_blah, even when there is a save_3_blah 30 | call_node = call_node.first_node 31 | except AttributeError: 32 | # No inner calls 33 | # Possible improvement: Make new node for RestoreNode's made in process_function 34 | # and make `self.inner_most_call = self` 35 | # So that we can duck type and not catch an exception when there are no inner calls. 36 | # This is what we do in BBorBInode 37 | pass 38 | 39 | return call_node 40 | 41 | 42 | def _connect_control_flow_node(control_flow_node, next_node): 43 | """Connect a ControlFlowNode properly to the next_node.""" 44 | for last in control_flow_node.last_nodes: 45 | if last is None: 46 | continue 47 | if isinstance(next_node, ControlFlowNode): 48 | last.connect(next_node.test) # connect to next if test case 49 | elif isinstance(next_node, AssignmentCallNode): 50 | call_node = next_node.call_node 51 | inner_most_call_node = _get_inner_most_function_call(call_node) 52 | last.connect(inner_most_call_node) 53 | else: 54 | last.connect(next_node) 55 | 56 | 57 | def connect_nodes(nodes): 58 | """Connect the nodes in a list linearly.""" 59 | for n, next_node in zip(nodes, nodes[1:]): 60 | if n is None: 61 | continue 62 | if isinstance(n, ControlFlowNode): 63 | _connect_control_flow_node(n, next_node) 64 | elif isinstance(next_node, ControlFlowNode): 65 | n.connect(next_node.test) 66 | elif isinstance(next_node, RestoreNode): 67 | continue 68 | elif next_node is not None and CALL_IDENTIFIER in next_node.label: 69 | continue 70 | else: 71 | n.connect(next_node) 72 | 73 | 74 | def _get_names(node, result): 75 | """Recursively finds all names.""" 76 | if isinstance(node, ast.Name): 77 | return node.id + result 78 | elif isinstance(node, ast.Subscript): 79 | return result 80 | elif isinstance(node, ast.Starred): 81 | return _get_names(node.value, result) 82 | else: 83 | if hasattr(node, "value"): 84 | return _get_names(node.value, result + "." + node.attr) 85 | else: 86 | return result 87 | 88 | 89 | def extract_left_hand_side(target): 90 | """Extract the left hand side variable from a target. 91 | 92 | Removes list indexes, stars and other left hand side elements. 93 | """ 94 | left_hand_side = _get_names(target, "") 95 | 96 | left_hand_side.replace("*", "") 97 | if "[" in left_hand_side: 98 | index = left_hand_side.index("[") 99 | left_hand_side = target[:index] 100 | 101 | return left_hand_side 102 | 103 | 104 | def get_first_node(node, node_not_to_step_past): 105 | """ 106 | This is a super hacky way of getting the first node after a statement. 107 | We do this because we visit a statement and keep on visiting and get something in return that is rarely the first node. 108 | So we loop and loop backwards until we hit the statement or there is nothing to step back to. 109 | """ 110 | ingoing = None 111 | i = 0 112 | j = 0 113 | if not node: 114 | return None 115 | current_node = node 116 | while current_node.ingoing and j < 5: 117 | # This is used because there may be multiple ingoing and loop will cause an infinite loop if we did [0] 118 | i = random.randrange(len(current_node.ingoing)) 119 | # e.g. We don't want to step past the Except of an Except basic block 120 | if current_node.ingoing[i] == node_not_to_step_past: 121 | break 122 | ingoing = current_node.ingoing 123 | current_node = current_node.ingoing[i] 124 | j = j + 1 125 | if ingoing: 126 | return ingoing[i] 127 | return current_node 128 | 129 | 130 | def get_first_statement(node_or_tuple): 131 | """Find the first statement of the provided object. 132 | 133 | Returns: 134 | The first element in the tuple if it is a tuple. 135 | The node if it is a node. 136 | """ 137 | if isinstance(node_or_tuple, tuple): 138 | return node_or_tuple[0] 139 | else: 140 | return node_or_tuple 141 | 142 | 143 | def get_last_statements(cfg_statements): 144 | """Retrieve the last statements from a cfg_statements list.""" 145 | if isinstance(cfg_statements[-1], ControlFlowNode): 146 | return cfg_statements[-1].last_nodes 147 | else: 148 | return [cfg_statements[-1]] 149 | 150 | 151 | def remove_breaks(last_statements): 152 | """Remove all break statements in last_statements.""" 153 | return [n for n in last_statements if not isinstance(n, BreakNode)] 154 | -------------------------------------------------------------------------------- /lib/pyt/cfg_analyzer.py: -------------------------------------------------------------------------------- 1 | """The comand line module of PyT.""" 2 | 3 | import os 4 | import traceback 5 | 6 | from lib.logger import LOG 7 | from lib.pyt.analysis.constraint_table import initialize_constraint_table 8 | from lib.pyt.analysis.fixed_point import analyse 9 | from lib.pyt.cfg import make_cfg 10 | from lib.pyt.core.ast_helper import generate_ast 11 | from lib.pyt.core.project_handler import get_directory_modules, get_modules 12 | from lib.pyt.vulnerabilities import find_insights, find_vulnerabilities 13 | from lib.pyt.vulnerabilities.vulnerability_helper import SanitisedVulnerability 14 | from lib.pyt.web_frameworks import FrameworkAdaptor, is_taintable_function 15 | 16 | default_blackbox_mapping_file = os.path.join( 17 | os.path.dirname(__file__), "vulnerability_definitions", "blackbox_mapping.json" 18 | ) 19 | 20 | 21 | default_trigger_word_file = os.path.join( 22 | os.path.dirname(__file__), "vulnerability_definitions", "all_sources_sinks.pyt" 23 | ) 24 | 25 | # Some framework have special files that can be ignored 26 | special_framework_path = ["management/commands"] 27 | 28 | 29 | def is_analyzable(src, path): 30 | for d in special_framework_path: 31 | if d in path: 32 | return False 33 | return True 34 | 35 | 36 | def deep_analysis(src, files): 37 | has_unsanitised_vulnerabilities = False 38 | cfg_list = list() 39 | insights = [] 40 | vulnerabilities = [] 41 | framework_route_criteria = is_taintable_function 42 | for path in sorted(files, key=os.path.dirname, reverse=True): 43 | # Check if the file path is analyzable 44 | if not is_analyzable(src, path): 45 | continue 46 | directory = os.path.dirname(path) 47 | project_modules = get_modules(directory, prepend_module_root=False) 48 | local_modules = get_directory_modules(directory) 49 | 50 | LOG.debug(f"Generating AST and CFG for {path}") 51 | try: 52 | tree = generate_ast(path) 53 | if not tree: 54 | continue 55 | except Exception as e: 56 | track = traceback.format_exc() 57 | LOG.debug(e) 58 | LOG.debug(track) 59 | try: 60 | # Should we skip insights? 61 | if not os.environ.get("SKIP_INSIGHTS"): 62 | violations = find_insights(tree, path) 63 | if violations: 64 | insights += violations 65 | cfg = make_cfg( 66 | tree, 67 | project_modules, 68 | local_modules, 69 | path, 70 | allow_local_directory_imports=True, 71 | ) 72 | cfg_list.append(cfg) 73 | except Exception as e: 74 | track = traceback.format_exc() 75 | LOG.debug(e) 76 | LOG.debug(track) 77 | 78 | try: 79 | # Taint all possible entry points 80 | LOG.debug("Determining taints") 81 | FrameworkAdaptor( 82 | cfg_list, project_modules, local_modules, framework_route_criteria 83 | ) 84 | LOG.debug("Building constraints table") 85 | initialize_constraint_table(cfg_list) 86 | LOG.debug("About to begin deep analysis") 87 | analyse(cfg_list) 88 | except Exception as e: 89 | track = traceback.format_exc() 90 | LOG.debug(e) 91 | LOG.debug(track) 92 | LOG.debug("Finding vulnerabilities from the graph") 93 | try: 94 | vulnerabilities = find_vulnerabilities( 95 | cfg_list, 96 | default_blackbox_mapping_file, 97 | default_trigger_word_file, 98 | ) 99 | except Exception as e: 100 | track = traceback.format_exc() 101 | LOG.debug(e) 102 | LOG.debug(track) 103 | if vulnerabilities: 104 | has_unsanitised_vulnerabilities = any( 105 | not isinstance(v, SanitisedVulnerability) for v in vulnerabilities 106 | ) 107 | return vulnerabilities, insights, has_unsanitised_vulnerabilities 108 | -------------------------------------------------------------------------------- /lib/pyt/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/lib/pyt/core/__init__.py -------------------------------------------------------------------------------- /lib/pyt/core/module_definitions.py: -------------------------------------------------------------------------------- 1 | """This module handles module definitions 2 | which basically is a list of module definition.""" 3 | 4 | import ast 5 | 6 | # Contains all project definitions for a program run 7 | # Only used in framework_adaptor.py, but modified here 8 | project_definitions = dict() 9 | 10 | 11 | class ModuleDefinition: 12 | """Handling of a definition.""" 13 | 14 | module_definitions = None 15 | name = None 16 | node = None 17 | path = None 18 | 19 | def __init__(self, local_module_definitions, name, parent_module_name, path): 20 | self.module_definitions = local_module_definitions 21 | self.parent_module_name = parent_module_name 22 | self.path = path 23 | 24 | if parent_module_name: 25 | if isinstance(parent_module_name, ast.alias): 26 | self.name = parent_module_name.name + "." + name 27 | else: 28 | self.name = parent_module_name + "." + name 29 | else: 30 | self.name = name 31 | 32 | def __str__(self): 33 | name = "NoName" 34 | node = "NoNode" 35 | if self.name: 36 | name = self.name 37 | if self.node: 38 | node = str(self.node) 39 | return ( 40 | "Path:" 41 | + self.path 42 | + " " 43 | + self.__class__.__name__ 44 | + ": " 45 | + ";".join((name, node)) 46 | ) 47 | 48 | 49 | class LocalModuleDefinition(ModuleDefinition): 50 | """A local definition.""" 51 | 52 | pass 53 | 54 | 55 | class ModuleDefinitions: 56 | """A collection of module definition. 57 | 58 | Adds to the project definitions list. 59 | """ 60 | 61 | def __init__( 62 | self, import_names=None, module_name=None, is_init=False, filename=None 63 | ): 64 | """Optionally set import names and module name. 65 | 66 | Module name should only be set when it is a normal import statement. 67 | """ 68 | self.import_names = import_names 69 | # module_name is sometimes ast.alias or a string 70 | self.module_name = module_name 71 | self.is_init = is_init 72 | self.filename = filename 73 | self.definitions = list() 74 | self.classes = list() 75 | self.import_alias_mapping = dict() 76 | 77 | def append_if_local_or_in_imports(self, definition): 78 | """Add definition to list. 79 | 80 | Handles local definitions and adds to project_definitions. 81 | """ 82 | if isinstance(definition, LocalModuleDefinition): 83 | self.definitions.append(definition) 84 | elif self.import_names == ["*"]: 85 | self.definitions.append(definition) 86 | elif self.import_names and definition.name in self.import_names: 87 | self.definitions.append(definition) 88 | elif ( 89 | self.import_alias_mapping 90 | and definition.name in self.import_alias_mapping.values() 91 | ): 92 | self.definitions.append(definition) 93 | 94 | if definition.parent_module_name: 95 | self.definitions.append(definition) 96 | 97 | if definition.node not in project_definitions: 98 | project_definitions[definition.node] = definition 99 | 100 | def get_definition(self, name): 101 | """Get definitions by name.""" 102 | for definition in self.definitions: 103 | if definition.name == name: 104 | return definition 105 | 106 | def set_definition_node(self, node, name): 107 | """Set definition by name.""" 108 | definition = self.get_definition(name) 109 | if definition: 110 | definition.node = node 111 | 112 | def __str__(self): 113 | module = "NoModuleName" 114 | if self.module_name: 115 | module = self.module_name 116 | 117 | if self.definitions: 118 | if isinstance(module, ast.alias): 119 | return ( 120 | 'Definitions: "' 121 | + '", "'.join([str(definition) for definition in self.definitions]) 122 | + '" and module_name: ' 123 | + module.name 124 | + " and filename: " 125 | + str(self.filename) 126 | + " and is_init: " 127 | + str(self.is_init) 128 | + "\n" 129 | ) 130 | return ( 131 | 'Definitions: "' 132 | + '", "'.join([str(definition) for definition in self.definitions]) 133 | + '" and module_name: ' 134 | + module 135 | + " and filename: " 136 | + str(self.filename) 137 | + " and is_init: " 138 | + str(self.is_init) 139 | + "\n" 140 | ) 141 | else: 142 | if isinstance(module, ast.alias): 143 | return ( 144 | "import_names is " 145 | + str(self.import_names) 146 | + " No Definitions, module_name: " 147 | + str(module.name) 148 | + " and filename: " 149 | + str(self.filename) 150 | + " and is_init: " 151 | + str(self.is_init) 152 | + "\n" 153 | ) 154 | return ( 155 | "import_names is " 156 | + str(self.import_names) 157 | + " No Definitions, module_name: " 158 | + str(module) 159 | + " and filename: " 160 | + str(self.filename) 161 | + " and is_init: " 162 | + str(self.is_init) 163 | + "\n" 164 | ) 165 | -------------------------------------------------------------------------------- /lib/pyt/core/project_handler.py: -------------------------------------------------------------------------------- 1 | """Generates a list of CFGs from a path. 2 | 3 | The module finds all python modules and generates an ast for them. 4 | """ 5 | import os 6 | from functools import lru_cache 7 | 8 | _local_modules = list() 9 | 10 | 11 | @lru_cache() 12 | def get_directory_modules(directory): 13 | """Return a list containing tuples of 14 | e.g. ('__init__', 'example/import_test_project/__init__.py') 15 | """ 16 | if _local_modules and os.path.dirname(_local_modules[0][1]) == directory: 17 | return _local_modules 18 | 19 | if not os.path.isdir(directory): 20 | # example/import_test_project/A.py -> example/import_test_project 21 | directory = os.path.dirname(directory) 22 | 23 | if directory == "": 24 | return _local_modules 25 | 26 | if os.path.exists(directory): 27 | for path in os.listdir(directory): 28 | if _is_python_file(path): 29 | # A.py -> A 30 | module_name = os.path.splitext(path)[0] 31 | _local_modules.append((module_name, os.path.join(directory, path))) 32 | 33 | return _local_modules 34 | 35 | 36 | @lru_cache() 37 | def get_modules(path, prepend_module_root=True): 38 | """Return a list containing tuples of 39 | e.g. ('test_project.utils', 'example/test_project/utils.py') 40 | """ 41 | module_root = os.path.split(path)[1] 42 | modules = list() 43 | for root, directories, filenames in os.walk(path): 44 | for filename in filenames: 45 | if _is_python_file(filename): 46 | directory = ( 47 | os.path.dirname(os.path.realpath(os.path.join(root, filename))) 48 | .split(module_root)[-1] 49 | .replace(os.sep, ".") # e.g. '/' 50 | ) 51 | directory = directory.replace(".", "", 1) 52 | 53 | module_name_parts = [] 54 | if prepend_module_root: 55 | module_name_parts.append(module_root) 56 | if directory: 57 | module_name_parts.append(directory) 58 | 59 | if filename == "__init__.py": 60 | path = root 61 | else: 62 | module_name_parts.append(os.path.splitext(filename)[0]) 63 | path = os.path.join(root, filename) 64 | 65 | modules.append((".".join(module_name_parts), path)) 66 | 67 | return modules 68 | 69 | 70 | def _is_python_file(path): 71 | if os.path.splitext(path)[1] == ".py": 72 | return True 73 | return False 74 | -------------------------------------------------------------------------------- /lib/pyt/core/transformer.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | 4 | class AsyncTransformer: 5 | """Converts all async nodes into their synchronous counterparts.""" 6 | 7 | def visit_Await(self, node): 8 | """Awaits are treated as if the keyword was absent.""" 9 | return self.visit(node.value) 10 | 11 | def visit_AsyncFunctionDef(self, node): 12 | return self.visit(ast.FunctionDef(**node.__dict__)) 13 | 14 | def visit_AsyncFor(self, node): 15 | return self.visit(ast.For(**node.__dict__)) 16 | 17 | def visit_AsyncWith(self, node): 18 | return self.visit(ast.With(**node.__dict__)) 19 | 20 | 21 | class ChainedFunctionTransformer: 22 | def visit_chain(self, node, depth=1): 23 | if ( 24 | isinstance(node.value, ast.Call) 25 | and isinstance(node.value.func, ast.Attribute) 26 | and isinstance(node.value.func.value, ast.Call) 27 | ): 28 | # Node is assignment or return with value like `b.c().d()` 29 | call_node = node.value 30 | # If we want to handle nested functions in future, depth needs fixing 31 | temp_var_id = "__chain_tmp_{}".format(depth) 32 | # AST tree is from right to left, so d() is the outer Call and b.c() is the inner Call 33 | unvisited_inner_call = ast.Assign( 34 | targets=[ast.Name(id=temp_var_id, ctx=ast.Store())], 35 | value=call_node.func.value, 36 | ) 37 | ast.copy_location(unvisited_inner_call, node) 38 | inner_calls = self.visit_chain(unvisited_inner_call, depth + 1) 39 | for inner_call_node in inner_calls: 40 | ast.copy_location(inner_call_node, node) 41 | outer_call = self.generic_visit( 42 | type(node)( 43 | value=ast.Call( 44 | func=ast.Attribute( 45 | value=ast.Name(id=temp_var_id, ctx=ast.Load()), 46 | attr=call_node.func.attr, 47 | ctx=ast.Load(), 48 | ), 49 | args=call_node.args, 50 | keywords=call_node.keywords, 51 | ), 52 | **{ 53 | field: value 54 | for field, value in ast.iter_fields(node) 55 | if field != "value" 56 | } # e.g. targets 57 | ) 58 | ) 59 | ast.copy_location(outer_call, node) 60 | ast.copy_location(outer_call.value, node) 61 | ast.copy_location(outer_call.value.func, node) 62 | return [*inner_calls, outer_call] 63 | else: 64 | return [self.generic_visit(node)] 65 | 66 | def visit_Assign(self, node): 67 | return self.visit_chain(node) 68 | 69 | def visit_Return(self, node): 70 | return self.visit_chain(node) 71 | 72 | 73 | class IfExpRewriter(ast.NodeTransformer): 74 | """Splits IfExp ternary expressions containing complex tests into multiple statements 75 | 76 | Will change 77 | 78 | a if b(c) else d 79 | 80 | into 81 | 82 | a if __if_exp_0 else d 83 | 84 | with Assign nodes in assignments [__if_exp_0 = b(c)] 85 | """ 86 | 87 | def __init__(self, starting_index=0): 88 | self._temporary_variable_index = starting_index 89 | self.assignments = [] 90 | super().__init__() 91 | 92 | def visit_IfExp(self, node): 93 | if isinstance(node.test, (ast.Name, ast.Attribute)): 94 | return self.generic_visit(node) 95 | else: 96 | temp_var_id = "__if_exp_{}".format(self._temporary_variable_index) 97 | self._temporary_variable_index += 1 98 | assignment_of_test = ast.Assign( 99 | targets=[ast.Name(id=temp_var_id, ctx=ast.Store())], 100 | value=self.visit(node.test), 101 | ) 102 | ast.copy_location(assignment_of_test, node) 103 | self.assignments.append(assignment_of_test) 104 | transformed_if_exp = ast.IfExp( 105 | test=ast.Name(id=temp_var_id, ctx=ast.Load()), 106 | body=self.visit(node.body), 107 | orelse=self.visit(node.orelse), 108 | ) 109 | ast.copy_location(transformed_if_exp, node) 110 | return transformed_if_exp 111 | 112 | def visit_FunctionDef(self, node): 113 | return node 114 | 115 | 116 | class IfExpTransformer: 117 | """Goes through module and function bodies, adding extra Assign nodes due to IfExp expressions.""" 118 | 119 | def visit_body(self, nodes): 120 | new_nodes = [] 121 | count = 0 122 | for node in nodes: 123 | rewriter = IfExpRewriter(count) 124 | possibly_transformed_node = rewriter.visit(node) 125 | if rewriter.assignments: 126 | new_nodes.extend(rewriter.assignments) 127 | count += len(rewriter.assignments) 128 | new_nodes.append(possibly_transformed_node) 129 | return new_nodes 130 | 131 | def visit_FunctionDef(self, node): 132 | transformed = ast.FunctionDef( 133 | name=node.name, 134 | args=node.args, 135 | body=self.visit_body(node.body), 136 | decorator_list=node.decorator_list, 137 | returns=node.returns, 138 | ) 139 | ast.copy_location(transformed, node) 140 | return self.generic_visit(transformed) 141 | 142 | def visit_Module(self, node): 143 | transformed = ast.Module(self.visit_body(node.body)) 144 | ast.copy_location(transformed, node) 145 | return self.generic_visit(transformed) 146 | 147 | 148 | class PytTransformer( 149 | AsyncTransformer, IfExpTransformer, ChainedFunctionTransformer, ast.NodeTransformer 150 | ): 151 | pass 152 | -------------------------------------------------------------------------------- /lib/pyt/formatters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/lib/pyt/formatters/__init__.py -------------------------------------------------------------------------------- /lib/pyt/formatters/json.py: -------------------------------------------------------------------------------- 1 | """This formatter outputs the issues in JSON.""" 2 | import json 3 | from datetime import datetime 4 | 5 | from lib.logger import LOG 6 | from lib.pyt.vulnerabilities.vulnerability_helper import ( 7 | SanitisedVulnerability, 8 | UnknownVulnerability, 9 | ) 10 | 11 | 12 | def report(vulnerabilities, insights, report_fname): 13 | """ 14 | Prints issues in JSON format. 15 | Args: 16 | vulnerabilities: list of vulnerabilities to report 17 | insights: list of insights 18 | report_fname: The output file name 19 | """ 20 | TZ_AGNOSTIC_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 21 | time_string = datetime.utcnow().strftime(TZ_AGNOSTIC_FORMAT) 22 | filtered_vulns = [] 23 | filtered_insights = [] 24 | vuln_keys = {} 25 | for vuln in vulnerabilities: 26 | if not isinstance(vuln, SanitisedVulnerability) and not isinstance( 27 | vuln, UnknownVulnerability 28 | ): 29 | avuln = vuln.as_dict() 30 | avuln_key = f"""{avuln["rule_id"]}|{avuln["source"]["line_number"]}|{avuln["source"]["path"]}|{avuln["sink"]["line_number"]}|{avuln["sink"]["path"]}""" 31 | if not vuln_keys.get(avuln_key): 32 | filtered_vulns.append(avuln) 33 | vuln_keys[avuln_key] = True 34 | for ins in insights: 35 | filtered_insights.append( 36 | { 37 | "rule_id": ins.code, 38 | "rule_name": ins.name, 39 | "short_description": ins.short_description, 40 | "description": ins.short_description, 41 | "recommendation": ins.recommendation, 42 | "cwe_category": ins.cwe_category, 43 | "owasp_category": ins.owasp_category, 44 | "severity": ins.severity, 45 | "source": { 46 | "trigger_word": ins.source.trigger_word, 47 | "line_number": ins.source.line_number, 48 | "label": ins.source.label, 49 | "path": ins.source.path, 50 | }, 51 | "sink": { 52 | "trigger_word": ins.sink.trigger_word, 53 | "line_number": ins.sink.line_number, 54 | "label": ins.sink.label, 55 | "path": ins.sink.path, 56 | }, 57 | } 58 | ) 59 | if filtered_insights: 60 | filtered_vulns += filtered_insights 61 | machine_output = {"generated_at": time_string, "vulnerabilities": filtered_vulns} 62 | try: 63 | with open(report_fname, mode="w") as fileobj: 64 | json.dump(machine_output, fileobj, indent=2) 65 | except Exception as e: 66 | LOG.debug(e) 67 | -------------------------------------------------------------------------------- /lib/pyt/helper_visitors/__init__.py: -------------------------------------------------------------------------------- 1 | from lib.pyt.helper_visitors.call_visitor import CallVisitor 2 | from lib.pyt.helper_visitors.label_visitor import LabelVisitor 3 | from lib.pyt.helper_visitors.right_hand_side_visitor import RHSVisitor 4 | from lib.pyt.helper_visitors.vars_visitor import VarsVisitor 5 | 6 | __all__ = ["CallVisitor", "LabelVisitor", "RHSVisitor", "VarsVisitor"] 7 | -------------------------------------------------------------------------------- /lib/pyt/helper_visitors/call_visitor.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import re 3 | from collections import defaultdict, namedtuple 4 | from itertools import count 5 | 6 | from lib.pyt.core.ast_helper import get_call_names_as_string 7 | from lib.pyt.helper_visitors.right_hand_side_visitor import RHSVisitor 8 | 9 | 10 | class CallVisitorResults( 11 | namedtuple( 12 | "CallVisitorResults", ("args", "kwargs", "unknown_args", "unknown_kwargs") 13 | ) 14 | ): 15 | __slots__ = () 16 | 17 | def all_results(self): 18 | for x in self.args: 19 | yield from x 20 | for x in self.kwargs.values(): 21 | yield from x 22 | yield from self.unknown_args 23 | yield from self.unknown_kwargs 24 | 25 | 26 | class CallVisitor(ast.NodeVisitor): 27 | def __init__(self, trigger_str): 28 | self.unknown_arg_visitor = RHSVisitor() 29 | self.unknown_kwarg_visitor = RHSVisitor() 30 | self.argument_visitors = defaultdict(lambda: RHSVisitor()) 31 | self._trigger_str = trigger_str 32 | 33 | def visit_Call(self, call_node): 34 | func_name = get_call_names_as_string(call_node.func) 35 | trigger_re = r"(^|\.){}$".format(re.escape(self._trigger_str)) 36 | if re.search(trigger_re, func_name): 37 | seen_starred = False 38 | for index, arg in enumerate(call_node.args): 39 | if isinstance(arg, ast.Starred): 40 | seen_starred = True 41 | if seen_starred: 42 | self.unknown_arg_visitor.visit(arg) 43 | else: 44 | self.argument_visitors[index].visit(arg) 45 | 46 | for keyword in call_node.keywords: 47 | if keyword.arg is None: 48 | self.unknown_kwarg_visitor.visit(keyword.value) 49 | else: 50 | self.argument_visitors[keyword.arg].visit(keyword.value) 51 | self.generic_visit(call_node) 52 | 53 | @classmethod 54 | def get_call_visit_results(cls, trigger_str, node): 55 | visitor = cls(trigger_str) 56 | visitor.visit(node) 57 | 58 | arg_results = [] 59 | for i in count(): 60 | try: 61 | arg_results.append(set(visitor.argument_visitors.pop(i).result)) 62 | except KeyError: 63 | break 64 | 65 | return CallVisitorResults( 66 | arg_results, 67 | {k: set(v.result) for k, v in visitor.argument_visitors.items()}, 68 | set(visitor.unknown_arg_visitor.result), 69 | set(visitor.unknown_kwarg_visitor.result), 70 | ) 71 | -------------------------------------------------------------------------------- /lib/pyt/helper_visitors/right_hand_side_visitor.py: -------------------------------------------------------------------------------- 1 | """Contains a class that finds all names. 2 | Used to find all variables on a right hand side(RHS) of assignment. 3 | """ 4 | import ast 5 | 6 | 7 | class RHSVisitor(ast.NodeVisitor): 8 | """Visitor collecting all names.""" 9 | 10 | def __init__(self): 11 | """Initialize result as list.""" 12 | self.result = list() 13 | 14 | def visit_Name(self, node): 15 | self.result.append(node.id) 16 | 17 | def visit_Call(self, node): 18 | if node.args: 19 | for arg in node.args: 20 | self.visit(arg) 21 | if node.keywords: 22 | for keyword in node.keywords: 23 | self.visit(keyword) 24 | 25 | def visit_IfExp(self, node): 26 | # The test doesn't taint the assignment 27 | self.visit(node.body) 28 | self.visit(node.orelse) 29 | 30 | @classmethod 31 | def result_for_node(cls, node): 32 | visitor = cls() 33 | visitor.visit(node) 34 | return visitor.result 35 | -------------------------------------------------------------------------------- /lib/pyt/helper_visitors/vars_visitor.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import itertools 3 | 4 | from lib.pyt.core.ast_helper import get_call_names 5 | 6 | 7 | class VarsVisitor(ast.NodeVisitor): 8 | def __init__(self): 9 | self.result = list() 10 | 11 | def visit_Name(self, node): 12 | self.result.append(node.id) 13 | 14 | def visit_BoolOp(self, node): 15 | for v in node.values: 16 | self.visit(v) 17 | 18 | def visit_BinOp(self, node): 19 | self.visit(node.left) 20 | self.visit(node.right) 21 | 22 | def visit_UnaryOp(self, node): 23 | self.visit(node.operand) 24 | 25 | def visit_Lambda(self, node): 26 | self.visit(node.body) 27 | 28 | def visit_IfExp(self, node): 29 | self.visit(node.test) 30 | self.visit(node.body) 31 | self.visit(node.orelse) 32 | 33 | def visit_Dict(self, node): 34 | for k in node.keys: 35 | if k is not None: 36 | self.visit(k) 37 | for v in node.values: 38 | self.visit(v) 39 | 40 | def visit_Set(self, node): 41 | for e in node.elts: 42 | self.visit(e) 43 | 44 | def comprehension(self, node): 45 | self.visit(node.target) 46 | self.visit(node.iter) 47 | for c in node.ifs: 48 | self.visit(c) 49 | 50 | def visit_ListComp(self, node): 51 | self.visit(node.elt) 52 | for gen in node.generators: 53 | self.comprehension(gen) 54 | 55 | def visit_SetComp(self, node): 56 | self.visit(node.elt) 57 | for gen in node.generators: 58 | self.comprehension(gen) 59 | 60 | def visit_DictComp(self, node): 61 | self.visit(node.key) 62 | self.visit(node.value) 63 | for gen in node.generators: 64 | self.comprehension(gen) 65 | 66 | def visit_GeneratorComp(self, node): 67 | self.visit(node.elt) 68 | for gen in node.generators: 69 | self.comprehension(gen) 70 | 71 | def visit_Yield(self, node): 72 | if node.value: 73 | self.visit(node.value) 74 | 75 | def visit_YieldFrom(self, node): 76 | self.visit(node.value) 77 | 78 | def visit_Compare(self, node): 79 | self.visit(node.left) 80 | for c in node.comparators: 81 | self.visit(c) 82 | 83 | def visit_Call(self, node): 84 | # This will not visit Flask in Flask(__name__) but it will visit request in `request.args.get() 85 | if not isinstance(node.func, ast.Name): 86 | self.visit(node.func) 87 | for arg_node in itertools.chain(node.args, node.keywords): 88 | arg = arg_node.value if isinstance(arg_node, ast.keyword) else arg_node 89 | if isinstance(arg, ast.Call): 90 | if isinstance(arg.func, ast.Name): 91 | # We can't just visit because we need to add 'ret_' 92 | self.result.append("ret_" + arg.func.id) 93 | elif isinstance(arg.func, ast.Attribute): 94 | # e.g. html.replace('{{ param }}', param) 95 | # func.attr is replace 96 | # func.value.id is html 97 | # We want replace 98 | self.result.append("ret_" + arg.func.attr) 99 | elif isinstance(arg.func, ast.Call): 100 | self.visit_curried_call_inside_call_args(arg) 101 | else: 102 | continue 103 | else: 104 | self.visit(arg) 105 | 106 | def visit_curried_call_inside_call_args(self, inner_call): 107 | # Curried functions aren't supported really, but we now at least have a defined behaviour. 108 | # In f(g(a)(b)(c)), inner_call is the Call node with argument c 109 | # Try to get the name of curried function g 110 | curried_func = inner_call.func.func 111 | while isinstance(curried_func, ast.Call): 112 | curried_func = curried_func.func 113 | if isinstance(curried_func, ast.Name): 114 | self.result.append("ret_" + curried_func.id) 115 | elif isinstance(curried_func, ast.Attribute): 116 | self.result.append("ret_" + curried_func.attr) 117 | 118 | # Visit all arguments except a (ignore the curried function g) 119 | not_curried = inner_call 120 | while not_curried.func is not curried_func: 121 | for arg in itertools.chain(not_curried.args, not_curried.keywords): 122 | self.visit(arg.value if isinstance(arg, ast.keyword) else arg) 123 | not_curried = not_curried.func 124 | 125 | def visit_Attribute(self, node): 126 | if not isinstance(node.value, ast.Name): 127 | self.visit(node.value) 128 | else: 129 | self.result.append(node.value.id) 130 | 131 | def slicev(self, node): 132 | if isinstance(node, ast.Slice): 133 | if node.lower: 134 | self.visit(node.lower) 135 | if node.upper: 136 | self.visit(node.upper) 137 | if node.step: 138 | self.visit(node.step) 139 | elif isinstance(node, ast.ExtSlice): 140 | if node.dims: 141 | for d in node.dims: 142 | self.visit(d) 143 | else: 144 | self.visit(node.value) 145 | 146 | def visit_Subscript(self, node): 147 | if isinstance(node.value, ast.Attribute): 148 | # foo.bar[1] 149 | self.result.append(list(get_call_names(node.value))[0]) 150 | self.visit(node.value) 151 | self.slicev(node.slice) 152 | 153 | def visit_Starred(self, node): 154 | self.visit(node.value) 155 | 156 | def visit_List(self, node): 157 | for el in node.elts: 158 | self.visit(el) 159 | 160 | def visit_Tuple(self, node): 161 | for el in node.elts: 162 | self.visit(el) 163 | -------------------------------------------------------------------------------- /lib/pyt/vulnerabilities/__init__.py: -------------------------------------------------------------------------------- 1 | import lib.pyt.vulnerabilities.rules as rules 2 | from lib.pyt.vulnerabilities.insights import find_insights 3 | from lib.pyt.vulnerabilities.vulnerabilities import find_vulnerabilities 4 | 5 | __all__ = [ 6 | "find_insights", 7 | "find_vulnerabilities", 8 | "rules", 9 | ] 10 | -------------------------------------------------------------------------------- /lib/pyt/vulnerabilities/trigger_definitions_parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | from collections import namedtuple 3 | 4 | Definitions = namedtuple("Definitions", ("sources", "sinks")) 5 | 6 | Source = namedtuple("Source", ("trigger_word", "source_type")) 7 | 8 | Rule = namedtuple( 9 | "Rule", 10 | ( 11 | "name", 12 | "code", 13 | "severity", 14 | "cwe_category", 15 | "owasp_category", 16 | "sources", 17 | "sinks", 18 | "message_format", 19 | ), 20 | ) 21 | 22 | 23 | class Sink: 24 | def __init__( 25 | self, 26 | sink_type, 27 | trigger, 28 | *, 29 | unlisted_args_propagate=True, 30 | arg_dict=None, 31 | sanitisers=None, 32 | ): 33 | self.sink_type = sink_type 34 | self._trigger = trigger 35 | self.sanitisers = sanitisers or [] 36 | self.arg_list_propagates = not unlisted_args_propagate 37 | 38 | if trigger[-1] != "(": 39 | if self.arg_list_propagates or arg_dict: 40 | return 41 | 42 | arg_dict = {} if arg_dict is None else arg_dict 43 | self.arg_position_to_kwarg = { 44 | position: name 45 | for name, position in arg_dict.items() 46 | if position is not None 47 | } 48 | self.kwarg_list = set(arg_dict.keys()) 49 | 50 | def arg_propagates(self, index): 51 | kwarg = self.get_kwarg_from_position(index) 52 | return self.kwarg_propagates(kwarg) 53 | 54 | def kwarg_propagates(self, keyword): 55 | in_list = keyword in self.kwarg_list 56 | return self.arg_list_propagates == in_list 57 | 58 | def get_kwarg_from_position(self, index): 59 | return self.arg_position_to_kwarg.get(index) 60 | 61 | def __str__(self): 62 | return f"Sink: Type: {self.sink_type}, Trigger: {self._trigger}" 63 | 64 | @property 65 | def all_arguments_propagate_taint(self): 66 | if self.kwarg_list: 67 | return False 68 | return True 69 | 70 | @property 71 | def call(self): 72 | if self._trigger[-1] == "(": 73 | return self._trigger[:-1] 74 | return None 75 | 76 | @property 77 | def trigger_word(self): 78 | return self._trigger 79 | 80 | @classmethod 81 | def from_json(cls, sink_type, key, data): 82 | return cls(sink_type=sink_type, trigger=key, **data) 83 | 84 | 85 | def parse(trigger_word_file): 86 | """Parse the file for source and sink definitions. 87 | 88 | Returns: 89 | A definitions tuple with sources and sinks. 90 | """ 91 | with open(trigger_word_file, mode="r", encoding="utf-8") as fd: 92 | triggers_dict = json.load(fd) 93 | sources = [] 94 | sinks = [] 95 | for st, sv in triggers_dict["sources"].items(): 96 | for tw in sv: 97 | sources.append(Source(tw, st)) 98 | for sink_type, trigger_obj in triggers_dict["sinks"].items(): 99 | for trigger, data in trigger_obj.items(): 100 | sinks.append(Sink.from_json(sink_type, trigger, data)) 101 | return Definitions(sources, sinks) 102 | 103 | 104 | def parse_rules(taint_config_file): 105 | """Parse taint config to produce rules 106 | 107 | Returns: 108 | List of rules 109 | """ 110 | with open(taint_config_file, mode="r", encoding="utf-8") as fd: 111 | taints_dict = json.load(fd) 112 | rules = [] 113 | for r in taints_dict.get("rules"): 114 | rules.append( 115 | Rule( 116 | r["name"], 117 | r["code"], 118 | r["severity"], 119 | r["cwe_category"], 120 | r["owasp_category"], 121 | r["sources"], 122 | r["sinks"], 123 | r["message_format"], 124 | ) 125 | ) 126 | return rules 127 | -------------------------------------------------------------------------------- /lib/pyt/vulnerability_definitions/blackbox_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "does_not_propagate": [ 3 | "url_for", 4 | "Post.query.paginate", 5 | "get_safe_redirect_to", 6 | "generate_thumbnail_url", 7 | "sanitize_name", 8 | "user_avatar_path_from_ids", 9 | "create_user_messages", 10 | "mark_sanitized", 11 | "reverse", 12 | "clean", 13 | "linkify", 14 | "bleach", 15 | "Filter", 16 | "BleachSanitizerFilter", 17 | "filter", 18 | "objects.get", 19 | "messages.warning", 20 | "messages.info", 21 | "messages.success", 22 | "messages.debug", 23 | "messages.error", 24 | "ast.literal_eval", 25 | "safe_join", 26 | "check_password_hash", 27 | "safe_str_cmp", 28 | "strftime", 29 | "replace", 30 | "fetch_mysql_df", 31 | "select", 32 | "os.path.exists", 33 | "xhtml_escape", 34 | "url_escape", 35 | "json_encode", 36 | "has_perm", 37 | "guess_type", 38 | "escape_uri_path", 39 | "len", 40 | "subprocess.run", 41 | "scrypt.outer", 42 | "scrypt.hash", 43 | "scrypt.encrypt", 44 | "scrypt.first_inner", 45 | "scrypt.inner", 46 | "scrypt.other_inner", 47 | "scrypt.second_inner", 48 | "TOTP", 49 | "escape", 50 | "secure_filename", 51 | "validate_arguments", 52 | "environ_property", 53 | "UserSerializer", 54 | "UserDeserializer", 55 | "URLSafeSerializer", 56 | "URLSafeTimedSerializer", 57 | "TimestampSigner", 58 | "sign", 59 | "validate", 60 | "unsign", 61 | "timestamp_to_datetime", 62 | "int", 63 | "float", 64 | "double", 65 | "pop", 66 | "AES.new", 67 | "aes.encrypt", 68 | "Encryption.unpad", 69 | "aes.decrypt", 70 | "base64_encode", 71 | "base64_decode", 72 | "localize", 73 | "basename", 74 | "getsize", 75 | "create", 76 | "delete", 77 | "filter", 78 | "first" 79 | ], 80 | "propagates": [ 81 | "os.path.join", 82 | "gzip.decompress", 83 | "format", 84 | "bytes", 85 | "decodestring", 86 | "base64.decodestring", 87 | "urlsafe_b64decode", 88 | "query_db", 89 | "Path", 90 | "pickle.loads", 91 | "RequestContext", 92 | "request.POST.get", 93 | "request.get", 94 | "django.template.RequestContext", 95 | "request.POST.dict", 96 | "jsonify", 97 | "unescape", 98 | "bind_arguments", 99 | "import_string", 100 | "objects.raw", 101 | "base64.b64decode", 102 | "serializers.serialize", 103 | "json.loads", 104 | "urlparse.unquote", 105 | "pickle.dumps", 106 | "push", 107 | "instance.exports", 108 | "Instance", 109 | "Module.from_file", 110 | "linker.instantiate", 111 | "loads_unsafe", 112 | "start_response", 113 | "dict", 114 | "copy", 115 | "values", 116 | "file", 117 | "add_message", 118 | "update" 119 | ], 120 | "safe_decorators": [ 121 | "user_must_be_authorized", 122 | "user_passes_test", 123 | "login_required", 124 | "permission_required", 125 | "filter", 126 | "register.filter", 127 | "stringfilter", 128 | "roles_required", 129 | "roles_accepted", 130 | "auth_token_required", 131 | "http_auth_required", 132 | "talisman", 133 | "jwt_required", 134 | "is_authenticated", 135 | "requires", 136 | "permission_classes", 137 | "IsAuthenticated", 138 | "authorize", 139 | "contextmanager", 140 | "command", 141 | "option", 142 | "argument" 143 | ], 144 | "sensitive_data_list": [ 145 | "key", "token", "secret", "request", "response", "cookie", "req", "res", "password", "passwd", "private", "certificate", "cert", "hash", 146 | "oauth2", "access_token", "refresh_token", "jwt", "cookie", "session", "cvv", "cvc", "card", "address", "mobile", "pin", "vulnerability", "disability", 147 | "postcode", "paypal", "uuid", "patient", "gender", "heart", "blood", "cholestrol", "hba1c" 148 | ], 149 | "sensitive_allowed_log_levels": [ 150 | "trace", 151 | "debug" 152 | ], 153 | "safe_path_list": [ 154 | "util", "models", "test", "setup.py", "__init__.py", "settings" 155 | ] 156 | } 157 | -------------------------------------------------------------------------------- /lib/pyt/vulnerability_definitions/test_positions.pyt: -------------------------------------------------------------------------------- 1 | { 2 | "sources": [ 3 | "request.args.get(", 4 | "make_taint(" 5 | ], 6 | "sinks": { 7 | "normal(": {}, 8 | "execute(": { 9 | "unlisted_args_propagate": false, 10 | "arg_dict": { 11 | "text": 0 12 | } 13 | }, 14 | "run(": { 15 | "arg_dict": { 16 | "non_propagating": 2, 17 | "something_else": 3 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /lib/pyt/vulnerability_definitions/test_triggers.pyt: -------------------------------------------------------------------------------- 1 | { 2 | "sources": [ 3 | "input" 4 | ], 5 | "sinks": { 6 | "eval(": { 7 | "sanitisers": [ 8 | "sanitise" 9 | ] 10 | }, 11 | "horse(": { 12 | "sanitisers": [ 13 | "japan", 14 | "host", 15 | "kost" 16 | ] 17 | }, 18 | "valmue": {} 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/pyt/web_frameworks/__init__.py: -------------------------------------------------------------------------------- 1 | from lib.pyt.web_frameworks.framework_adaptor import FrameworkAdaptor, _get_func_nodes 2 | from lib.pyt.web_frameworks.framework_helper import ( 3 | is_django_view_function, 4 | is_flask_route_function, 5 | is_function_with_leading_, 6 | is_taintable_function, 7 | ) 8 | 9 | __all__ = [ 10 | "FrameworkAdaptor", 11 | "is_django_view_function", 12 | "is_flask_route_function", 13 | "is_taintable_function", 14 | "is_function_with_leading_", 15 | "_get_func_nodes", # Only used in framework_helper_test 16 | ] 17 | -------------------------------------------------------------------------------- /lib/pyt/web_frameworks/framework_adaptor.py: -------------------------------------------------------------------------------- 1 | """A generic framework adaptor that leaves route criteria to the caller.""" 2 | 3 | import ast 4 | 5 | from lib.pyt.cfg import make_cfg 6 | from lib.pyt.core.ast_helper import Arguments 7 | from lib.pyt.core.module_definitions import project_definitions 8 | from lib.pyt.core.node_types import AssignmentNode, TaintedNode 9 | 10 | 11 | class FrameworkAdaptor: 12 | """An engine that uses the template pattern to find all 13 | entry points in a framework and then taints their arguments. 14 | """ 15 | 16 | def __init__(self, cfg_list, project_modules, local_modules, is_route_function): 17 | self.cfg_list = cfg_list 18 | self.project_modules = project_modules 19 | self.local_modules = local_modules 20 | self.is_route_function = is_route_function 21 | self.run() 22 | 23 | def get_func_cfg_with_tainted_args(self, definition): 24 | """Build a function cfg and return it, with all arguments tainted.""" 25 | func_cfg = make_cfg( 26 | definition.node, 27 | self.project_modules, 28 | self.local_modules, 29 | definition.path, 30 | definition.module_definitions, 31 | ) 32 | 33 | args = Arguments(definition.node.args) 34 | if args: 35 | function_entry_node = func_cfg.nodes[0] 36 | function_entry_node.outgoing = list() 37 | first_node_after_args = func_cfg.nodes[1] 38 | first_node_after_args.ingoing = list() 39 | 40 | # We are just going to give all the tainted args the lineno of the def 41 | definition_lineno = definition.node.lineno 42 | 43 | # Taint all the arguments 44 | for i, arg in enumerate(args): 45 | node_type = TaintedNode 46 | if i == 0 and arg == "self": 47 | node_type = AssignmentNode 48 | 49 | arg_node = node_type( 50 | label=arg, 51 | left_hand_side=arg, 52 | ast_node=None, 53 | right_hand_side_variables=[], 54 | line_number=definition_lineno, 55 | path=definition.path, 56 | ) 57 | function_entry_node.connect(arg_node) 58 | # 1 and not 0 so that Entry Node remains first in the list 59 | func_cfg.nodes.insert(1, arg_node) 60 | arg_node.connect(first_node_after_args) 61 | 62 | return func_cfg 63 | 64 | def find_route_functions_taint_args(self): 65 | """Find all route functions and taint all of their arguments. 66 | 67 | Yields: 68 | CFG of each route function, with args marked as tainted. 69 | """ 70 | for definition in _get_func_nodes(): 71 | if self.is_route_function(definition.node): 72 | yield self.get_func_cfg_with_tainted_args(definition) 73 | 74 | def run(self): 75 | """Run find_route_functions_taint_args on each CFG.""" 76 | function_cfgs = list() 77 | # for _ in self.cfg_list: 78 | function_cfgs.extend(self.find_route_functions_taint_args()) 79 | self.cfg_list.extend(function_cfgs) 80 | 81 | 82 | def _get_func_nodes(): 83 | """Get all function nodes.""" 84 | return [ 85 | definition 86 | for definition in project_definitions.values() 87 | if isinstance(definition.node, ast.FunctionDef) 88 | ] 89 | -------------------------------------------------------------------------------- /lib/pyt/web_frameworks/framework_helper.py: -------------------------------------------------------------------------------- 1 | """Provides helper functions that help with determining if a function is a route function.""" 2 | import ast 3 | import json 4 | import os 5 | 6 | from lib.pyt.core.ast_helper import get_call_names 7 | 8 | default_blackbox_mapping_file = os.path.join( 9 | os.path.dirname(__file__), 10 | "..", 11 | "vulnerability_definitions", 12 | "blackbox_mapping.json", 13 | ) 14 | safe_decorators = [] 15 | with open(default_blackbox_mapping_file) as fp: 16 | bb_mapping = json.load(fp) 17 | safe_decorators = bb_mapping.get("safe_decorators") 18 | 19 | 20 | def is_django_view_function(ast_node): 21 | if len(ast_node.args.args): 22 | first_arg_name = ast_node.args.args[0].arg 23 | return first_arg_name == "request" 24 | return False 25 | 26 | 27 | def is_flask_route_function(ast_node): 28 | """Check whether function uses a route decorator.""" 29 | for decorator in ast_node.decorator_list: 30 | if isinstance(decorator, ast.Call): 31 | if _get_last_of_iterable(get_call_names(decorator.func)) == "route": 32 | return True 33 | return False 34 | 35 | 36 | def is_taintable_function(ast_node): 37 | """Returns only functions without a sanitization decorator""" 38 | for decorator in ast_node.decorator_list: 39 | if isinstance(decorator, ast.Call): 40 | if _get_last_of_iterable(get_call_names(decorator.func)) in safe_decorators: 41 | return False 42 | # Flask route and Django tag 43 | if _get_last_of_iterable(get_call_names(decorator.func)) in [ 44 | "route", 45 | "errorhandler", 46 | "simple_tag", 47 | "inclusion_tag", 48 | "to_end_tag", 49 | "expose", 50 | "view_config", 51 | "template", 52 | "get", 53 | "post", 54 | "put", 55 | "delete", 56 | "middleware", 57 | "api_view", 58 | "action", 59 | "csrf_exempt", 60 | "deserialise_with", 61 | "marshal_with", 62 | "before", 63 | "csrf_protect", 64 | "requires_csrf_token", 65 | "xframe_options_exempt", 66 | "xframe_options_deny", 67 | "xframe_options_sameorigin", 68 | "before_first_request", 69 | "receiver", 70 | "require_http_methods", 71 | "application", 72 | "command", 73 | "option", 74 | "group", 75 | "argument", 76 | ]: 77 | return True 78 | # Ignore database functions 79 | if len(ast_node.args.args): 80 | first_arg_name = ast_node.args.args[0].arg 81 | if first_arg_name == "self" and len(ast_node.args.args) > 1: 82 | first_arg_name = ast_node.args.args[1].arg 83 | # Common view functions such as django, starlette, falcon 84 | if first_arg_name in ["req", "request", "context", "scope", "environ"]: 85 | return True 86 | # Ignore dao classes due to potential FP 87 | if first_arg_name in ["conn", "connection", "cls", "session", "session_cls"]: 88 | return False 89 | # Ignore internal functions prefixed with _ 90 | if is_function_with_leading_(ast_node): 91 | return False 92 | # Ignore known validation and sanitization functions 93 | for n in ["valid", "sanitize", "sanitise", "is_", "set_", "assert"]: 94 | if n in ast_node.name: 95 | return False 96 | # Should we limit the scan only to web routes? 97 | web_route_only = os.environ.get("WEB_ROUTE_ONLY", False) 98 | if web_route_only: 99 | return False 100 | return True 101 | 102 | 103 | def is_function_with_leading_(ast_node): 104 | if ast_node.name.startswith("_"): 105 | return True 106 | return False 107 | 108 | 109 | def _get_last_of_iterable(iterable): 110 | """Get last element of iterable.""" 111 | item = None 112 | for item in iterable: 113 | pass 114 | return item 115 | -------------------------------------------------------------------------------- /lib/remediate.py: -------------------------------------------------------------------------------- 1 | from lib.cis import get_rule 2 | from lib.pyt.vulnerabilities.rules import rules_message_map 3 | 4 | IAC_LINKS = "\n\n## Documentation\n\n- [AWS Terraform](https://registry.terraform.io/providers/hashicorp/aws/latest/docs)\n- [Azure Terraform](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs)\n- [Google Cloud Terraform](https://registry.terraform.io/providers/hashicorp/google/latest/docs)" 5 | 6 | 7 | def get_help( 8 | rule_id, rule_obj=None, tool_name=None, owasp_category=None, cwe_category=None 9 | ): 10 | """ 11 | Method to find remediation text for the given rule, tool and categories 12 | 13 | :param rule_id: Rule id 14 | :param rule_obj: Rule object from the SARIF file 15 | :param tool_name: Full name of the tool 16 | :param owasp_category: OWASP category 17 | :param cwe_category: CWE category 18 | 19 | :return: Help text in markdown format 20 | """ 21 | desc = "" 22 | if rules_message_map.get(rule_id): 23 | return rules_message_map.get(rule_id) 24 | cis_rule = get_rule(rule_id) 25 | if cis_rule: 26 | cis_desc = cis_rule.get("text", "").strip() 27 | if cis_desc and not cis_desc.endswith("."): 28 | cis_desc = cis_desc + "." 29 | rem_text = cis_rule.get( 30 | "remediation", 31 | f"Refer to the provider documentation for the configuration options available.{IAC_LINKS}", 32 | ) 33 | rationale_text = cis_rule.get("rationale", "") 34 | if rationale_text: 35 | rationale_text += "\n" 36 | desc = f"""CIS Benchmark: **{cis_rule.get("id", "")}**\n\n{cis_desc}\n\n{rationale_text}## Remediation\n\n{rem_text}""" 37 | if cis_rule.get("help_url"): 38 | help_urls = "\n- ".join(cis_rule.get("help_url")) 39 | desc = desc + f"""\n\n## Additional information\n\n- {help_urls}""" 40 | return desc 41 | else: 42 | desc = rule_obj.get("fullDescription", {}).get("text") 43 | if desc: 44 | desc = desc.replace("'", "`") 45 | helpUri = rule_obj.get("helpUri") 46 | if helpUri and "slscan" not in helpUri: 47 | desc += "\n\n## Additional information\n\n" 48 | if rule_obj.get("name"): 49 | desc += f"""**[{rule_obj.get("name")}]({helpUri})**""" 50 | else: 51 | desc += f"**[{rule_id}]({helpUri})**" 52 | if "CKV_" in rule_id: 53 | desc += IAC_LINKS 54 | return desc 55 | -------------------------------------------------------------------------------- /lib/telemetry.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | import lib.config as config 4 | from lib.logger import LOG 5 | 6 | 7 | def track(track_obj): 8 | """ 9 | Method to send a track message to the telemetry api 10 | :param track_obj: 11 | :return: 12 | """ 13 | # Check if telemetry is disabled 14 | disable_telemetry = config.get("DISABLE_TELEMETRY", False) 15 | if ( 16 | disable_telemetry == "true" 17 | or disable_telemetry == "1" 18 | or not config.get("TELEMETRY_URL") 19 | ): 20 | disable_telemetry = True 21 | else: 22 | disable_telemetry = False 23 | if track_obj and not disable_telemetry: 24 | try: 25 | track_obj["tool"] = "@ShiftLeft/scan" 26 | requests.post(config.get("TELEMETRY_URL"), json=track_obj) 27 | except Exception: 28 | LOG.debug("Unable to send telemetry") 29 | -------------------------------------------------------------------------------- /lib/xml_parser.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from defusedxml.ElementTree import parse 4 | 5 | from lib.constants import PRIORITY_MAP 6 | from lib.utils import find_path_prefix 7 | 8 | 9 | def get_report_data(xmlfile, file_path_list=[], working_dir=""): 10 | """Convert xml file to dict 11 | 12 | :param xmlfile: xml file to parse 13 | :param file_path_list: Full file path for any manipulation 14 | :param working_dir: Working directory 15 | """ 16 | issues = [] 17 | metrics = {} 18 | file_ref = {} 19 | file_name_prefix = "" 20 | if not file_path_list: 21 | file_path_list = [] 22 | et = parse(xmlfile) 23 | root = et.getroot() 24 | # Check if this is a checkstyle xml 25 | if root.tag.lower() == "checkstyle".lower(): 26 | return parse_checkstyle(root, file_path_list, working_dir) 27 | for child in root: 28 | issue = {} 29 | if child.tag.lower() == "BugInstance".lower(): 30 | issue = child.attrib 31 | if "priority" in child.attrib: 32 | priority = child.attrib["priority"] 33 | if priority in PRIORITY_MAP: 34 | issue["issue_severity"] = PRIORITY_MAP.get(priority, priority) 35 | if "cweid" in child.attrib and child.attrib["cweid"]: 36 | issue["test_id"] = "CWE-" + child.attrib["cweid"] 37 | elif "type" in child.attrib and child.attrib["type"]: 38 | issue["test_id"] = child.attrib["type"] 39 | for ele in child.iter(): 40 | if ele.tag.lower() == "ShortMessage".lower(): 41 | issue["title"] = ele.text 42 | if ele.tag.lower() == "LongMessage".lower(): 43 | issue["description"] = ele.text 44 | if ele.tag.lower() == "Message".lower(): 45 | issue["description"] = issue["description"] + " \n" + ele.text 46 | if ele.tag.lower() == "SourceLine".lower() and ( 47 | ele.attrib.get("synthetic") == "true" 48 | or ele.attrib.get("primary") == "true" 49 | ): 50 | issue["line"] = ele.attrib["start"] 51 | fname = ele.attrib["sourcepath"] 52 | if fname in file_ref: 53 | fname = file_ref[fname] 54 | else: 55 | if not file_name_prefix: 56 | file_name_prefix = find_path_prefix(working_dir, fname) 57 | if file_path_list: 58 | # FIXME: This logic is too slow. 59 | # Tools like find-sec-bugs are not reliably reporting the full path 60 | # so such a lookup is required 61 | for tf in file_path_list: 62 | if tf.endswith(fname): 63 | file_ref[fname] = tf 64 | fname = tf 65 | break 66 | elif file_name_prefix: 67 | fname = os.path.join(file_name_prefix, fname) 68 | issue["filename"] = fname 69 | issues.append(issue) 70 | if child.tag.lower() == "FindBugsSummary".lower(): 71 | metrics = {"summary": child.attrib} 72 | 73 | return issues, metrics 74 | 75 | 76 | def parse_checkstyle(root, file_path_list, working_dir): 77 | """Parse checkstyle xml""" 78 | issues = [] 79 | metrics = {} 80 | for child in root: 81 | issue = {} 82 | if child.tag.lower() == "file": 83 | issue["filename"] = child.attrib["name"] 84 | for ele in child.iter(): 85 | if ele.tag.lower() == "error": 86 | issue["line"] = ele.attrib["line"] 87 | issue["issue_severity"] = ele.attrib["severity"] 88 | issue["test_id"] = ele.attrib["source"] 89 | issue["title"] = ele.attrib["message"] 90 | issues.append(issue) 91 | return issues, metrics 92 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | black 4 | bandit 5 | flake8 6 | pytest 7 | pytest-cov 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | sarif-om 2 | jschema_to_python 3 | defusedxml 4 | gitpython 5 | git+https://github.com/ShiftLeftSecurity/scan-reports.git 6 | appthreat-depscan 7 | requests 8 | urllib3 9 | bandit 10 | ansible-lint 11 | pipenv 12 | checkov==2.4.6 13 | yamllint 14 | rich 15 | PyGithub>=1.59 16 | PyYAML 17 | blint 18 | packaging 19 | -------------------------------------------------------------------------------- /starlark/scan.star: -------------------------------------------------------------------------------- 1 | # vi:syntax=python 2 | 3 | load("github.com/mesosphere/dispatch-catalog/starlark/stable/git@0.0.7", "git_checkout_dir") 4 | load("github.com/mesosphere/dispatch-catalog/starlark/stable/pipeline@0.0.7", "storage_resource") 5 | 6 | __doc__ = """ 7 | # Scan 8 | 9 | Provides methods for running [Scan](https://www.shiftleft.io/scan/) on your CI. 10 | 11 | To import, add the following to your Dispatchfile: 12 | 13 | ``` 14 | load("github.com/shiftleftsecurity/sast-scan/starlark/scan@master", "sast_scan") 15 | ``` 16 | 17 | For help, please visit https://appthreat.com/en/latest/integrations/dispatch/ 18 | """ 19 | 20 | def sast_scan(task_name, git_name, image="shiftleft/scan:latest", src=None, extra_scan_options=None, **kwargs): 21 | """ 22 | Runs a SAST Scan using the provided image on a given directory. 23 | 24 | #### Parameters 25 | - *task_name* : name of the task to be created 26 | - *git_name* : input git resource name 27 | - *image* : image (with tag) of the Shiftleft Scan 28 | - *src* : Optional string to override the `src` directory to run the scan. Defaults to given git resource directory. 29 | - *extra_scan_options* : Optional array containing flag names (and values) to be passed to scan command 30 | """ 31 | if not src: 32 | src = git_checkout_dir(git_name) 33 | output_name = storage_resource("storage-{}".format(task_name)) 34 | if not extra_scan_options: 35 | extra_scan_options=[] 36 | task(task_name, inputs=[git_name], outputs=[output_name], steps=[ 37 | k8s.corev1.Container( 38 | name="sast-scan-shiftleft-{}".format(git_name), 39 | image=image, 40 | command=[ 41 | "scan", 42 | "--build", 43 | "--src={}".format(src), 44 | "--out_dir={}".format("$(resources.outputs.{}.path)".format(output_name)), 45 | ] + extra_scan_options, 46 | **kwargs 47 | )]) 48 | return output_name 49 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/test/__init__.py -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ["PMD_CMD"] = "/opt/pmd-bin/bin/run.sh pmd" 4 | os.environ["APP_SRC_DIR"] = "/usr/local/src" 5 | os.environ["TOOLS_CONFIG_DIR"] = "/usr/local/src" 6 | os.environ["SPOTBUGS_HOME"] = "/opt/spotbugs" 7 | -------------------------------------------------------------------------------- /test/data/.sastscan.baseline: -------------------------------------------------------------------------------- 1 | { 2 | "baseline_fingerprints": { 3 | "scanPrimaryLocationHash": ["foo"], 4 | "scanTagsHash": ["bar"] 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /test/data/.sastscanrc: -------------------------------------------------------------------------------- 1 | { 2 | "build_break_rules": { 3 | "default": { "max_critical": 0, "max_high": 0, "max_medium": 0 } 4 | }, 5 | "scan_tools_args_map": { 6 | "go": ["echo", "hello", "world"] 7 | }, 8 | "scan_type": "credscan,java" 9 | } 10 | -------------------------------------------------------------------------------- /test/data/dep_check-report.json: -------------------------------------------------------------------------------- 1 | {"reportSchema": "1.1","scanInfo": {"engineVersion": "5.2.4","dataSource": [{"name": "NVD CVE Checked","timestamp": "2020-01-09T20:44:04"},{"name": "NVD CVE Modified","timestamp": "2020-01-09T19:03:00"},{"name": "VersionCheckOn","timestamp": "2020-01-09T20:44:04"}]},"projectInfo": {"name": "","reportDate": "2020-01-09T20:44:16.156773Z","credits": {"NVD": "This report contains data retrieved from the National Vulnerability Database: http://nvd.nist.gov","NPM": "This report may contain data retrieved from the NPM Public Advisories: https://www.npmjs.com/advisories","RETIREJS": "This report may contain data retrieved from the RetireJS community: https://retirejs.github.io/retire.js/","OSSINDEX": "This report may contain data retrieved from the Sonatype OSS Index: https://ossindex.sonatype.org"}},"dependencies": []} -------------------------------------------------------------------------------- /test/data/issue-259.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/test/data/issue-259.php -------------------------------------------------------------------------------- /test/data/njsscan-report.json: -------------------------------------------------------------------------------- 1 | { 2 | "errors": [], 3 | "nodejs": { 4 | "express_open_redirect": { 5 | "files": [ 6 | { 7 | "file_path": "/Users/prabhu/work/NodeGoat/app/routes/index.js", 8 | "match_lines": [ 9 | 72, 10 | 72 11 | ], 12 | "match_position": [ 13 | 16, 14 | 42 15 | ], 16 | "match_string": " return res.redirect(req.query.url);" 17 | } 18 | ], 19 | "metadata": { 20 | "cwe": "CWE-601: URL Redirection to Untrusted Site ('Open Redirect')", 21 | "description": "Untrusted user input in redirect() can result in Open Redirect vulnerability.", 22 | "owasp": "A1: Injection", 23 | "severity": "ERROR" 24 | } 25 | }, 26 | "node_insecure_random_generator": { 27 | "files": [ 28 | { 29 | "file_path": "/Users/prabhu/work/NodeGoat/app/routes/session.js", 30 | "match_lines": [ 31 | 13, 32 | 13 33 | ], 34 | "match_position": [ 35 | 34, 36 | 45 37 | ], 38 | "match_string": " var stocks = Math.floor((Math.random() * 40) + 1);" 39 | }, 40 | { 41 | "file_path": "/Users/prabhu/work/NodeGoat/app/routes/session.js", 42 | "match_lines": [ 43 | 14, 44 | 14 45 | ], 46 | "match_position": [ 47 | 33, 48 | 44 49 | ], 50 | "match_string": " var funds = Math.floor((Math.random() * 40) + 1);" 51 | }, 52 | { 53 | "file_path": "/Users/prabhu/work/NodeGoat/app/data/user-dao.js", 54 | "match_lines": [ 55 | 59, 56 | 59 57 | ], 58 | "match_position": [ 59 | 31, 60 | 42 61 | ], 62 | "match_string": " var day = (Math.floor(Math.random() * 10) + today.getDay()) % 29;" 63 | }, 64 | { 65 | "file_path": "/Users/prabhu/work/NodeGoat/app/data/user-dao.js", 66 | "match_lines": [ 67 | 60, 68 | 60 69 | ], 70 | "match_position": [ 71 | 33, 72 | 44 73 | ], 74 | "match_string": " var month = (Math.floor(Math.random() * 10) + today.getMonth()) % 12;" 75 | }, 76 | { 77 | "file_path": "/Users/prabhu/work/NodeGoat/app/data/user-dao.js", 78 | "match_lines": [ 79 | 61, 80 | 61 81 | ], 82 | "match_position": [ 83 | 30, 84 | 41 85 | ], 86 | "match_string": " var year = Math.ceil(Math.random() * 30) + today.getFullYear();" 87 | } 88 | ], 89 | "metadata": { 90 | "cwe": "CWE-327: Use of a Broken or Risky Cryptographic Algorithm", 91 | "description": "crypto.pseudoRandomBytes()/Math.random() is a cryptographically weak random number generator.", 92 | "owasp": "A9: Using Components with Known Vulnerabilities", 93 | "severity": "WARNING" 94 | } 95 | }, 96 | "node_timing_attack": { 97 | "files": [ 98 | { 99 | "file_path": "/Users/prabhu/work/NodeGoat/app/routes/session.js", 100 | "match_lines": [ 101 | 176, 102 | 178 103 | ], 104 | "match_position": [ 105 | 9, 106 | 25 107 | ], 108 | "match_string": " if (password !== verify) {\n\n errors.verifyError = \"Password must match\";\n\n return false;" 109 | } 110 | ], 111 | "metadata": { 112 | "cwe": "CWE-208: Observable Timing Discrepancy", 113 | "description": "String comparisons using '===', '!==', '!=' and '==' is vulnerable to timing attacks. More info: https://snyk.io/blog/node-js-timing-attack-ccc-ctf/", 114 | "owasp": "A9: Using Components with Known Vulnerabilities", 115 | "severity": "WARNING" 116 | } 117 | } 118 | }, 119 | "templates": {} 120 | } -------------------------------------------------------------------------------- /test/data/pmd-report.csv: -------------------------------------------------------------------------------- 1 | "Problem","Package","File","Priority","Line","Description","Rule set","Rule" 2 | "1","","/app/CWE-190/ComparisonWithWiderType.java","1","10","Do not use the short type","Performance","AvoidUsingShortType" 3 | "2","","/app/CWE-190/ComparisonWithWiderType.java","1","31","Do not use the short type","Performance","AvoidUsingShortType" 4 | -------------------------------------------------------------------------------- /test/data/retire-report.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/test/data/retire-report.json -------------------------------------------------------------------------------- /test/data/source-go-ignore.json: -------------------------------------------------------------------------------- 1 | { 2 | "Golang errors": {}, 3 | "Issues": [ 4 | { 5 | "severity": "HIGH", 6 | "confidence": "MEDIUM", 7 | "cwe": { 8 | "ID": "338", 9 | "URL": "https://cwe.mitre.org/data/definitions/338.html" 10 | }, 11 | "rule_id": "G404", 12 | "details": "Use of weak random number generator (math/rand instead of crypto/rand)", 13 | "file": "dnscrypt-proxy/dnscrypt-proxy/crypto.go", 14 | "code": "86 /* nolint:go */ \n\n88 rand.Read(xpad[:])" 15 | }, 16 | { 17 | "severity": "MEDIUM", 18 | "confidence": "HIGH", 19 | "cwe": { 20 | "ID": "", 21 | "URL": "" 22 | }, 23 | "rule_id": "G307", 24 | "details": "Deferring unsafe method \"*os.File\" on type \"Close\"", 25 | "file": "dnscrypt-proxy/dnscrypt-proxy/systemd_linux.go", 26 | "code": "22 //nolint\n23 defer file.Close()" 27 | }, 28 | { 29 | "severity": "MEDIUM", 30 | "confidence": "HIGH", 31 | "cwe": { 32 | "ID": "276", 33 | "URL": "https://cwe.mitre.org/data/definitions/276.html" 34 | }, 35 | "rule_id": "G301", 36 | "details": "Expect directory permissions to be 0750 or less", 37 | "file": "dnscrypt-proxy/dnscrypt-proxy/pidfile.go", 38 | "code": "5 // scan:ignore os.MkdirAll(filepath.Dir(*pidFile), 0755)" 39 | }, 40 | { 41 | "severity": "MEDIUM", 42 | "confidence": "HIGH", 43 | "cwe": { 44 | "ID": "22", 45 | "URL": "https://cwe.mitre.org/data/definitions/22.html" 46 | }, 47 | "rule_id": "G304", 48 | "details": "Potential file inclusion via variable", 49 | "file": "dnscrypt-proxy/dnscrypt-proxy/common.go", 50 | "code": "12 ioutil.ReadFile(filename) /* sl:ignore */" 51 | }, 52 | { 53 | "severity": "MEDIUM", 54 | "confidence": "HIGH", 55 | "cwe": { 56 | "ID": "276", 57 | "URL": "https://cwe.mitre.org/data/definitions/276.html" 58 | }, 59 | "rule_id": "G302", 60 | "details": "Expect file permissions to be 0600 or less", 61 | "file": "dnscrypt-proxy/dnscrypt-proxy/logger.go", 62 | "code": "17 #nosec:b103 os.OpenFile(fileName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)" 63 | } 64 | ], 65 | "Stats": { 66 | "files": 39, 67 | "lines": 7010, 68 | "nosec": 0, 69 | "found": 5 70 | } 71 | } -------------------------------------------------------------------------------- /test/data/source-kt-report.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /test/data/staticcheck-report.json: -------------------------------------------------------------------------------- 1 | {"code":"ST1005","severity":"error","location":{"file":"/Users/guest/go/kube-score/cmd/kube-score/main.go","line":156,"column":10},"end":{"file":"/Users/guest/go/kube-score/cmd/kube-score/main.go","line":156,"column":86},"message":"error strings should not be capitalized"} 2 | {"code":"ST1005","severity":"error","location":{"file":"/Users/guest/go/kube-score/cmd/kube-score/main.go","line":161,"column":10},"end":{"file":"/Users/guest/go/kube-score/cmd/kube-score/main.go","line":165,"column":61},"message":"error strings should not be capitalized"} 3 | {"code":"ST1005","severity":"error","location":{"file":"/Users/guest/go/kube-score/cmd/kube-score/main.go","line":161,"column":10},"end":{"file":"/Users/guest/go/kube-score/cmd/kube-score/main.go","line":165,"column":61},"message":"error strings should not end with punctuation or a newline"} 4 | {"code":"SA1006","severity":"error","location":{"file":"/Users/guest/go/kube-score/renderer/human/human.go","line":39,"column":3},"end":{"file":"/Users/guest/go/kube-score/renderer/human/human.go","line":39,"column":75},"message":"printf-style function with dynamic format string and no further arguments should use print-style function instead"} 5 | {"code":"SA1006","severity":"error","location":{"file":"/Users/guest/go/kube-score/renderer/human/human.go","line":108,"column":4},"end":{"file":"/Users/guest/go/kube-score/renderer/human/human.go","line":108,"column":76},"message":"printf-style function with dynamic format string and no further arguments should use print-style function instead"} 6 | {"code":"S1002","severity":"error","location":{"file":"/Users/guest/go/kube-score/score/security/security.go","line":58,"column":43},"end":{"file":"/Users/guest/go/kube-score/score/security/security.go","line":58,"column":79},"message":"should omit comparison to bool constant, can be simplified to !*sec.ReadOnlyRootFilesystem"} 7 | {"code":"S1002","severity":"error","location":{"file":"/Users/guest/go/kube-score/scorecard/scorecard.go","line":61,"column":6},"end":{"file":"/Users/guest/go/kube-score/scorecard/scorecard.go","line":61,"column":24},"message":"should omit comparison to bool constant, can be simplified to !o.Skipped"} 8 | -------------------------------------------------------------------------------- /test/data/taint-php-report.json: -------------------------------------------------------------------------------- 1 | [{"severity":"error","line_from":25,"line_to":25,"type":"TaintedInput","message":"Detected tainted shell in path: $_GET -> $_GET['username'] (CommandExecution\/CommandExec-1.php:25:23) -> call to shell_exec (CommandExecution\/CommandExec-1.php:25:23) -> shell_exec#1","file_name":"CommandExecution\/CommandExec-1.php","file_path":"\/app\/CommandExecution\/CommandExec-1.php","snippet":" echo shell_exec($_GET[\"username\"]);\r","selected_text":"$_GET[\"username\"]","from":1121,"to":1138,"snippet_from":1099,"snippet_to":1141,"column_from":23,"column_to":40,"error_level":-2,"shortcode":205,"link":"https:\/\/psalm.dev\/205"},{"severity":"error","line_from":25,"line_to":25,"type":"TaintedInput","message":"Detected tainted shell in path: $_GET -> $_GET['typeBox'] (CommandExecution\/CommandExec-2.php:22:16) -> $target (CommandExecution\/CommandExec-2.php:22:7) -> call to str_replace (CommandExecution\/CommandExec-2.php:24:71) -> str_replace#3 (CommandExecution\/CommandExec-2.php:24:71) -> str_replace (CommandExecution\/CommandExec-2.php:24:17) -> $target (CommandExecution\/CommandExec-2.php:24:7) -> call to shell_exec (CommandExecution\/CommandExec-2.php:25:23) -> shell_exec#1","file_name":"CommandExecution\/CommandExec-2.php","file_path":"\/app\/CommandExecution\/CommandExec-2.php","snippet":" echo shell_exec($target);\r","selected_text":"$target","from":1216,"to":1223,"snippet_from":1194,"snippet_to":1226,"column_from":23,"column_to":30,"error_level":-2,"shortcode":205,"link":"https:\/\/psalm.dev\/205"},{"severity":"error","line_from":39,"line_to":39,"type":"TaintedInput","message":"Detected tainted shell in path: $_GET -> $_GET['typeBox'] (CommandExecution\/CommandExec-3.php:22:16) -> $target (CommandExecution\/CommandExec-3.php:22:7) -> call to str_replace (CommandExecution\/CommandExec-3.php:38:71) -> str_replace#3 (CommandExecution\/CommandExec-3.php:38:71) -> str_replace (CommandExecution\/CommandExec-3.php:38:17) -> $target (CommandExecution\/CommandExec-3.php:38:7) -> call to shell_exec (CommandExecution\/CommandExec-3.php:39:23) -> shell_exec#1","file_name":"CommandExecution\/CommandExec-3.php","file_path":"\/app\/CommandExecution\/CommandExec-3.php","snippet":" echo shell_exec($target);\r","selected_text":"$target","from":1401,"to":1408,"snippet_from":1379,"snippet_to":1411,"column_from":23,"column_to":30,"error_level":-2,"shortcode":205,"link":"https:\/\/psalm.dev\/205"},{"severity":"error","line_from":44,"line_to":44,"type":"TaintedInput","message":"Detected tainted shell in path: $_GET -> $_GET['typeBox'] (CommandExecution\/CommandExec-4.php:29:16) -> $target (CommandExecution\/CommandExec-4.php:29:7) -> call to str_replace (CommandExecution\/CommandExec-4.php:43:71) -> str_replace#3 (CommandExecution\/CommandExec-4.php:43:71) -> str_replace (CommandExecution\/CommandExec-4.php:43:17) -> $target (CommandExecution\/CommandExec-4.php:43:7) -> call to shell_exec (CommandExecution\/CommandExec-4.php:44:23) -> shell_exec#1","file_name":"CommandExecution\/CommandExec-4.php","file_path":"\/app\/CommandExecution\/CommandExec-4.php","snippet":" echo shell_exec($target);\r","selected_text":"$target","from":1549,"to":1556,"snippet_from":1527,"snippet_to":1559,"column_from":23,"column_to":30,"error_level":-2,"shortcode":205,"link":"https:\/\/psalm.dev\/205"},{"severity":"error","line_from":32,"line_to":32,"type":"TaintedInput","message":"Detected tainted text in path: $_GET -> $_GET['file'] (FileInclusion\/pages\/lvl1.php:25:20) -> include#1 (FileInclusion\/pages\/lvl2.php:32:24)","file_name":"FileInclusion\/pages\/lvl2.php","file_path":"\/app\/FileInclusion\/pages\/lvl2.php","snippet":" @include($secure2);\r","selected_text":"$secure2","from":1164,"to":1172,"snippet_from":1141,"snippet_to":1175,"column_from":24,"column_to":32,"error_level":-2,"shortcode":205,"link":"https:\/\/psalm.dev\/205"},{"severity":"error","line_from":33,"line_to":33,"type":"TaintedInput","message":"Detected tainted html in path: $_GET -> $_GET['file'] (FileInclusion\/pages\/lvl2.php:25:22) -> $secure2 (FileInclusion\/pages\/lvl2.php:25:11) -> call to str_replace (FileInclusion\/pages\/lvl2.php:27:77) -> str_replace#3 (FileInclusion\/pages\/lvl2.php:27:77) -> str_replace (FileInclusion\/pages\/lvl2.php:27:22) -> $secure2 (FileInclusion\/pages\/lvl2.php:27:11) -> call to str_replace (FileInclusion\/pages\/lvl2.php:28:73) -> str_replace#3 (FileInclusion\/pages\/lvl2.php:28:73) -> str_replace (FileInclusion\/pages\/lvl2.php:28:22) -> $secure2 (FileInclusion\/pages\/lvl2.php:28:11) -> concat (FileInclusion\/pages\/lvl2.php:33:19) -> concat (FileInclusion\/pages\/lvl2.php:33:19) -> call to echo (FileInclusion\/pages\/lvl2.php:33:19) -> echo#1","file_name":"FileInclusion\/pages\/lvl2.php","file_path":"\/app\/FileInclusion\/pages\/lvl2.php","snippet":" echo\"
\".$secure2.\"<\/h5><\/b><\/div> \"; \r","selected_text":"\"
\".$secure2.\"<\/h5><\/b><\/div> \"","from":1194,"to":1251,"snippet_from":1176,"snippet_to":1256,"column_from":19,"column_to":76,"error_level":-2,"shortcode":205,"link":"https:\/\/psalm.dev\/205"},{"severity":"error","line_from":22,"line_to":22,"type":"TaintedInput","message":"Detected tainted html in path: $_GET -> $_GET['username'] (XSS\/XSS_level1.php:22:23) -> concat (XSS\/XSS_level1.php:22:7) -> call to echo (XSS\/XSS_level1.php:22:7) -> echo#1","file_name":"XSS\/XSS_level1.php","file_path":"\/app\/XSS\/XSS_level1.php","snippet":"\techo(\"Your name is \".$_GET[\"username\"])?>","selected_text":"\"Your name is \".$_GET[\"username\"]","from":648,"to":681,"snippet_from":642,"snippet_to":684,"column_from":7,"column_to":40,"error_level":-2,"shortcode":205,"link":"https:\/\/psalm.dev\/205"}] 2 | -------------------------------------------------------------------------------- /test/integration/test_bitbucket.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from lib.integration import bitbucket 4 | 5 | 6 | def test_context(): 7 | os.environ["BITBUCKET_REPO_OWNER"] = "test" 8 | os.environ["BITBUCKET_WORKSPACE"] = "foo" 9 | os.environ["BITBUCKET_REPO_UUID"] = "uuid123" 10 | os.environ["BITBUCKET_REPO_FULL_NAME"] = "test/bar" 11 | os.environ["BITBUCKET_PR_ID"] = "pr-123" 12 | os.environ["BITBUCKET_PR_DESTINATION_BRANCH"] = "main" 13 | context = bitbucket.Bitbucket().get_context({"foo": "bar"}) 14 | assert context["foo"] == "bar" 15 | 16 | 17 | def test_reports_url(): 18 | url = bitbucket.Bitbucket().get_reports_url( 19 | {"repositoryName": "bar", "revisionId": "123", "repoFullname": "test/bar"} 20 | ) 21 | assert ( 22 | url 23 | == "http://api.bitbucket.org/2.0/repositories/test/bar/commit/123/reports/shiftleft-scan" 24 | ) 25 | 26 | 27 | def test_pr_comments_url(): 28 | url = bitbucket.Bitbucket().get_pr_comments_url( 29 | { 30 | "repositoryName": "bar", 31 | "revisionId": "123", 32 | "prID": "pr-123", 33 | "repoFullname": "test/bar", 34 | } 35 | ) 36 | assert ( 37 | url 38 | == "https://api.bitbucket.org/2.0/repositories/test/bar/pullrequests/pr-123/comments" 39 | ) 40 | 41 | 42 | def test_emoji(): 43 | emoji = bitbucket.Bitbucket().to_emoji("foo") 44 | assert emoji == "foo" 45 | emoji = bitbucket.Bitbucket().to_emoji(":white_heavy_check_mark:") 46 | assert emoji == "✅" 47 | emoji = bitbucket.Bitbucket().to_emoji(":cross_mark:") 48 | assert emoji == "❌" 49 | -------------------------------------------------------------------------------- /test/integration/test_github.py: -------------------------------------------------------------------------------- 1 | from lib.integration import github 2 | 3 | 4 | def test_context(): 5 | context = github.GitHub().get_context({"foo": "bar"}) 6 | assert context["foo"] == "bar" 7 | -------------------------------------------------------------------------------- /test/integration/test_gitlab.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from lib.integration import gitlab 4 | 5 | 6 | def test_context(): 7 | os.environ["CI_MERGE_REQUEST_IID"] = "test" 8 | os.environ["CI_MERGE_REQUEST_PROJECT_ID"] = "test" 9 | context = gitlab.GitLab().get_context({"foo": "bar"}) 10 | assert context["foo"] == "bar" 11 | 12 | 13 | def test_reports_url(): 14 | url = gitlab.GitLab().get_mr_notes_url( 15 | {"repositoryName": "bar", "revisionId": "123"} 16 | ) 17 | assert url == "https://gitlab.com/api/v4/projects/test/merge_requests/test/notes" 18 | 19 | 20 | def test_emoji(): 21 | emoji = gitlab.GitLab().to_emoji("foo") 22 | assert emoji == "foo" 23 | emoji = gitlab.GitLab().to_emoji(":white_heavy_check_mark:") 24 | assert emoji == "✅" 25 | emoji = gitlab.GitLab().to_emoji(":cross_mark:") 26 | assert emoji == "❌" 27 | -------------------------------------------------------------------------------- /test/integration/test_provider.py: -------------------------------------------------------------------------------- 1 | from lib.integration import bitbucket, gitlab, provider 2 | 3 | 4 | def test_get(): 5 | prov = provider.get_git_provider({}) 6 | assert not prov 7 | prov = provider.get_git_provider({"gitProvider": "bitbucket"}) 8 | assert prov 9 | assert isinstance(prov, bitbucket.Bitbucket) 10 | prov = provider.get_git_provider({"gitProvider": "gitlab"}) 11 | assert prov 12 | assert isinstance(prov, gitlab.GitLab) 13 | -------------------------------------------------------------------------------- /test/pyt/test_ast.py: -------------------------------------------------------------------------------- 1 | from lib.pyt.core.ast_helper import generate_ast_from_code, has_import_like 2 | 3 | 4 | def test_ast_imports(): 5 | tree = generate_ast_from_code( 6 | """ 7 | import pymongo 8 | import ssl 9 | 10 | client = pymongo.MongoClient() 11 | """ 12 | ) 13 | assert has_import_like("pymongo", tree) 14 | 15 | tree = generate_ast_from_code( 16 | """ 17 | import pymongo 18 | import ssl 19 | 20 | client = pymongo.MongoClient('example.com', ssl=False, ssl_cert_reqs=ssl.CERT_NONE) 21 | """ 22 | ) 23 | assert has_import_like("pymongo", tree) 24 | 25 | tree = generate_ast_from_code( 26 | """ 27 | import pymongo 28 | import ssl 29 | 30 | client = pymongo.MongoClient('mongodb://example.com/?ssl=true') 31 | """ 32 | ) 33 | assert has_import_like("pymongo", tree) 34 | 35 | 36 | def test_ast_imports_from(): 37 | tree = generate_ast_from_code( 38 | """ 39 | from pymongo import MongoClient 40 | import ssl 41 | 42 | client = MongoClient('mongodb://example.com/?ssl=true') 43 | """ 44 | ) 45 | assert has_import_like("pymongo", tree) 46 | -------------------------------------------------------------------------------- /test/pyt/test_cfg.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from lib.pyt.analysis.constraint_table import initialize_constraint_table 4 | from lib.pyt.analysis.fixed_point import analyse 5 | from lib.pyt.cfg import make_cfg 6 | from lib.pyt.core.ast_helper import generate_ast_from_code 7 | from lib.pyt.vulnerabilities import find_vulnerabilities 8 | from lib.pyt.web_frameworks import FrameworkAdaptor, is_taintable_function 9 | 10 | default_blackbox_mapping_file = os.path.join( 11 | os.path.dirname(__file__), 12 | "..", 13 | "..", 14 | "lib", 15 | "pyt", 16 | "vulnerability_definitions", 17 | "blackbox_mapping.json", 18 | ) 19 | 20 | 21 | default_trigger_word_file = os.path.join( 22 | os.path.dirname(__file__), 23 | "..", 24 | "..", 25 | "lib", 26 | "pyt", 27 | "vulnerability_definitions", 28 | "all_sources_sinks.pyt", 29 | ) 30 | 31 | 32 | def ret_vulnerabilities(tree, cfg_list): 33 | cfg = make_cfg(tree, None, None, "", allow_local_directory_imports=True) 34 | cfg_list = [cfg] 35 | FrameworkAdaptor(cfg_list, None, None, is_taintable_function) 36 | initialize_constraint_table(cfg_list) 37 | analyse(cfg_list) 38 | return find_vulnerabilities( 39 | cfg_list, default_blackbox_mapping_file, default_trigger_word_file 40 | ) 41 | 42 | 43 | def test_data_leak_sinks_1(): 44 | cfg_list = [] 45 | tree = generate_ast_from_code( 46 | """ 47 | from flask import Flask, request, redirect 48 | 49 | app = Flask(__name__) 50 | 51 | SECRET_STRING = '' 52 | app.config['SECRET_KEY'] = os.environ.get("SECRET_KEY", "") 53 | 54 | def log(thing: str): 55 | print(thing) 56 | 57 | 58 | @app.route('/') 59 | def index(): 60 | cookie = request.cookies.get('MyCookie') 61 | if cookie != SECRET_STRING: 62 | return redirect('/login') 63 | # Pretend this is a log 64 | log(cookie) 65 | return 'You made it!' 66 | 67 | 68 | @app.route('/login', methods=['POST']) 69 | def login(): 70 | username = request.args.get('username') 71 | password = request.args.get('password') 72 | log(password) 73 | resp = redirect('/') 74 | if username == 'admin' and password == 'password': 75 | # One minute long session 76 | resp.headers['Set-Cookie'] = f'MyCookie={SECRET_STRING}; Max-Age=60' 77 | return resp 78 | 79 | 80 | if __name__ == '__main__': 81 | app.run() 82 | """ 83 | ) 84 | vulnerabilities = ret_vulnerabilities(tree, cfg_list) 85 | assert vulnerabilities 86 | -------------------------------------------------------------------------------- /test/test_aggregate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import json 3 | import os 4 | import tempfile 5 | from pathlib import Path 6 | 7 | import lib.aggregate as aggregate 8 | 9 | 10 | def find_test_data(): 11 | data_dir = Path(__file__).parent / "data" 12 | return [p.as_posix() for p in data_dir.glob("*.sarif")] 13 | 14 | 15 | def test_aggregate(): 16 | test_sarif_files = find_test_data() 17 | run_data_list = [] 18 | for sf in test_sarif_files: 19 | with open(sf, mode="r") as report_file: 20 | report_data = json.loads(report_file.read()) 21 | run_data_list += report_data["runs"] 22 | 23 | with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as afile: 24 | aggregate.jsonl_aggregate(run_data_list, afile.name) 25 | afile.close() 26 | with open(afile.name, "r") as outfile: 27 | data = outfile.read() 28 | assert data 29 | os.unlink(afile.name) 30 | -------------------------------------------------------------------------------- /test/test_analysis.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import tempfile 4 | from pathlib import Path 5 | 6 | import lib.analysis as analysis 7 | 8 | 9 | def find_test_data(): 10 | data_dir = Path(__file__).parent / "data" 11 | return [p.as_posix() for p in data_dir.glob("*.sarif")] 12 | 13 | 14 | def find_test_depscan_data(): 15 | data_dir = Path(__file__).parent / "data" 16 | return [p.as_posix() for p in data_dir.glob("depscan-report-java.json")] 17 | 18 | 19 | def find_test_depscan_nodejs_data(): 20 | data_dir = Path(__file__).parent / "data" 21 | return [p.as_posix() for p in data_dir.glob("depscan-report-nodejs.json")] 22 | 23 | 24 | def test_summary(): 25 | test_sarif_files = find_test_data() 26 | report_summary, build_status = analysis.summary( 27 | test_sarif_files, depscan_files=None 28 | ) 29 | assert len(report_summary.keys()) == 7 30 | for k, v in report_summary.items(): 31 | if k == "findsecbugs": 32 | assert v["status"] == ":cross_mark:" 33 | elif k == "nodejsscan": 34 | assert v["status"] == ":white_heavy_check_mark:" 35 | assert build_status == "fail" 36 | 37 | 38 | def test_calculate_depscan_metrics(): 39 | test_depscan_files = find_test_depscan_data() 40 | with open(test_depscan_files[0]) as fp: 41 | dep_data = analysis.get_depscan_data(fp) 42 | metrics, required_pkgs_found = analysis.calculate_depscan_metrics(dep_data) 43 | assert metrics 44 | assert metrics["critical"] == 29 45 | assert metrics["optional_critical"] == 26 46 | assert required_pkgs_found 47 | 48 | 49 | def test_summary_with_depscan(): 50 | test_sarif_files = find_test_data() 51 | test_depscan_files = find_test_depscan_data() 52 | report_summary, build_status = analysis.summary( 53 | test_sarif_files, depscan_files=test_depscan_files 54 | ) 55 | assert len(report_summary.keys()) == 8 56 | for k, v in report_summary.items(): 57 | if k == "depscan-java": 58 | assert v == { 59 | "status": ":cross_mark:", 60 | "tool": "Dependency Scan (java)", 61 | "critical": 29, 62 | "high": 25, 63 | "medium": 5, 64 | "low": 0, 65 | } 66 | if k in ("findsecbugs", "depscan-java"): 67 | assert v["status"] == ":cross_mark:" 68 | elif k == "nodejsscan": 69 | assert v["status"] == ":white_heavy_check_mark:" 70 | assert build_status == "fail" 71 | 72 | 73 | def test_summary_with_depscan_usage(): 74 | test_sarif_files = [] 75 | test_depscan_files = find_test_depscan_nodejs_data() 76 | report_summary, build_status = analysis.summary( 77 | test_sarif_files, depscan_files=test_depscan_files 78 | ) 79 | assert len(report_summary.keys()) == 1 80 | for k, v in report_summary.items(): 81 | if k == "depscan-nodejs": 82 | assert v == { 83 | "status": ":white_heavy_check_mark:", 84 | "tool": "Dependency Scan (nodejs)", 85 | "critical": 1, 86 | "high": 20, 87 | "medium": 9, 88 | "low": 3, 89 | } 90 | assert build_status == "pass" 91 | 92 | 93 | def test_summary_with_agg(): 94 | test_sarif_files = find_test_data() 95 | with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as afile: 96 | report_summary, build_status = analysis.summary( 97 | test_sarif_files, depscan_files=None, aggregate_file=afile.name 98 | ) 99 | assert len(report_summary.keys()) == 7 100 | afile.close() 101 | with open(afile.name, "r") as outfile: 102 | data = outfile.read() 103 | assert data 104 | os.unlink(afile.name) 105 | 106 | 107 | def test_summary_strict(): 108 | test_sarif_files = find_test_data() 109 | report_summary, build_status = analysis.summary( 110 | test_sarif_files, 111 | depscan_files=None, 112 | aggregate_file=None, 113 | override_rules={ 114 | "max_critical": 0, 115 | "max_high": 0, 116 | "max_medium": 0, 117 | "max_low": 0, 118 | }, 119 | ) 120 | assert len(report_summary.keys()) == 7 121 | for k, v in report_summary.items(): 122 | assert v["status"] == ":cross_mark:" 123 | assert build_status == "fail" 124 | 125 | 126 | def test_summary_depscan_strict(): 127 | test_sarif_files = [] 128 | test_depscan_files = find_test_depscan_data() 129 | report_summary, build_status = analysis.summary( 130 | test_sarif_files, 131 | depscan_files=test_depscan_files, 132 | aggregate_file=None, 133 | override_rules={ 134 | "depscan": { 135 | "max_critical": 0, 136 | "max_required_critical": 0, 137 | "max_high": 0, 138 | "max_required_high": 0, 139 | "max_medium": 0, 140 | "max_required_medium": 0, 141 | "max_low": 0, 142 | "max_required_low": 0, 143 | } 144 | }, 145 | ) 146 | assert len(report_summary.keys()) == 1 147 | for k, v in report_summary.items(): 148 | assert v["critical"] == 29 149 | assert v["status"] == ":cross_mark:" 150 | assert build_status == "fail" 151 | 152 | 153 | def test_summary_depscan_relaxed(): 154 | test_sarif_files = [] 155 | test_depscan_files = find_test_depscan_data() 156 | report_summary, build_status = analysis.summary( 157 | test_sarif_files, 158 | depscan_files=test_depscan_files, 159 | aggregate_file=None, 160 | override_rules={ 161 | "depscan": { 162 | "max_critical": 30, 163 | "max_required_critical": 30, 164 | "max_high": 30, 165 | "max_required_high": 30, 166 | "max_medium": 30, 167 | "max_required_medium": 30, 168 | "max_low": 30, 169 | "max_required_low": 30, 170 | } 171 | }, 172 | ) 173 | assert len(report_summary.keys()) == 1 174 | for k, v in report_summary.items(): 175 | assert v["critical"] == 29 176 | assert v["status"] == ":white_heavy_check_mark:" 177 | assert build_status == "pass" 178 | -------------------------------------------------------------------------------- /test/test_cis.py: -------------------------------------------------------------------------------- 1 | from lib.cis import get_cis_rules, get_rule 2 | 3 | 4 | def test_k8s_all(): 5 | data = get_cis_rules() 6 | assert data 7 | 8 | 9 | def test_k8s_rule(): 10 | data = get_rule("DefaultServiceAccount") 11 | assert data 12 | 13 | 14 | def test_aws_rule(): 15 | data = get_rule("SecurityGroupUnrestrictedIngress22") 16 | assert data 17 | -------------------------------------------------------------------------------- /test/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import lib.config as config 4 | 5 | 6 | def test_scan_tools_map(): 7 | test_src = "/app" 8 | test_reports_dir = "/app/reports" 9 | test_report_fname_prefix = "/app/reports/tool-report" 10 | 11 | for k, v in config.scan_tools_args_map.items(): 12 | if isinstance(v, list): 13 | default_cmd = " ".join(v) % dict( 14 | src=test_src, 15 | src_or_file=test_src, 16 | reports_dir=test_reports_dir, 17 | report_fname_prefix=test_report_fname_prefix, 18 | type=k, 19 | ) 20 | assert k 21 | assert "%(src)s" not in default_cmd 22 | elif isinstance(v, dict): 23 | for cmd_key, cmd_val in v.items(): 24 | assert cmd_key 25 | default_cmd = " ".join(cmd_val) % dict( 26 | src=test_src, 27 | src_or_file=test_src, 28 | reports_dir=test_reports_dir, 29 | report_fname_prefix=test_report_fname_prefix, 30 | type=k, 31 | ) 32 | assert "%(src)s" not in default_cmd 33 | 34 | 35 | def test_override(): 36 | build_break_rules = config.get("build_break_rules").copy() 37 | golang_cmd = config.get("scan_tools_args_map").get("go") 38 | assert list(golang_cmd.keys()) == ["source-go", "staticcheck"] 39 | test_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") 40 | config.set("SAST_SCAN_SRC_DIR", test_data_dir) 41 | config.reload() 42 | # Test if we are able to override the whole dict 43 | new_rules = config.get("build_break_rules") 44 | assert build_break_rules != new_rules 45 | # Test if we are able to override a command 46 | golang_cmd = config.get("scan_tools_args_map").get("go") 47 | assert golang_cmd[0] == "echo" 48 | assert config.get("scan_type") == "credscan,java" 49 | 50 | 51 | def test_baseline(): 52 | test_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") 53 | config.set("SAST_SCAN_SRC_DIR", test_data_dir) 54 | config.reload() 55 | fps = config.get_suppress_fingerprints("") 56 | assert fps == { 57 | "scanPrimaryLocationHash": ["foo"], 58 | "scanTagsHash": ["bar"], 59 | "scanFileHash": [], 60 | } 61 | -------------------------------------------------------------------------------- /test/test_context.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | 4 | import lib.context as context 5 | 6 | 7 | def test_find_repo(): 8 | curr_rep_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") 9 | repo_details = context.find_repo_details(curr_rep_dir) 10 | assert len(repo_details.keys()) > 1 11 | assert ( 12 | repo_details["repositoryUri"] 13 | == "https://github.com/ShiftLeftSecurity/sast-scan" 14 | ) 15 | 16 | 17 | def test_env_detection(): 18 | os.environ["COMMIT_SHA"] = "123" 19 | os.environ["BRANCH"] = "develop" 20 | importlib.reload(context) 21 | 22 | repo_details = context.find_repo_details(None) 23 | assert repo_details["revisionId"] == "123" 24 | assert repo_details["branch"] == "develop" 25 | 26 | 27 | def test_sanitize(): 28 | u = context.sanitize_url("https://username:password@website.com/foo/bar") 29 | assert u == "https://website.com/foo/bar" 30 | u = context.sanitize_url("https://git-ci-token:password-123@website.com/foo/bar") 31 | assert u == "https://website.com/foo/bar" 32 | u = context.sanitize_url("") 33 | assert u == "" 34 | u = context.sanitize_url("git@gitexample.com/foo/bar") 35 | assert u == "git@gitexample.com/foo/bar" 36 | 37 | 38 | def test_bot(): 39 | is_bot = context.is_bot("foo@bar.com") 40 | assert not is_bot 41 | is_bot = context.is_bot("Renovate Bot ") 42 | assert is_bot 43 | is_bot = context.is_bot("dependabot-preview[bot] <123242134@sdfdsfdsdf.com>") 44 | assert is_bot 45 | -------------------------------------------------------------------------------- /test/test_csv.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import lib.csv_parser as csv_parser 4 | 5 | 6 | def test_pmd_parse(): 7 | with open( 8 | os.path.join( 9 | os.path.dirname(os.path.realpath(__file__)), 10 | "data", 11 | "pmd-report.csv", 12 | ) 13 | ) as rf: 14 | headers, report_data = csv_parser.get_report_data(rf) 15 | assert len(headers) == 8 16 | assert len(report_data) == 2 17 | -------------------------------------------------------------------------------- /test/test_cwe.py: -------------------------------------------------------------------------------- 1 | from lib.cwe import get, get_description 2 | 3 | 4 | def test_cwe_get(): 5 | data = get("cwe-115") 6 | assert data 7 | assert ( 8 | data["Description"] 9 | == "The software misinterprets an input, whether from an attacker or another product, in a security-relevant fashion." 10 | ) 11 | assert data["Extended Description"] == "" 12 | 13 | 14 | def test_cwe_get_desc(): 15 | data = get_description("cwe-78") 16 | assert data 17 | -------------------------------------------------------------------------------- /test/test_inspect.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | 4 | import pytest 5 | 6 | import lib.inspect as inspect 7 | 8 | 9 | @pytest.fixture 10 | def test_sarif_files(): 11 | curr_dir = os.path.dirname(os.path.abspath(__file__)) 12 | return [ 13 | os.path.join(curr_dir, "data", "gosec-report.sarif"), 14 | os.path.join(curr_dir, "data", "staticcheck-report.sarif"), 15 | ] 16 | 17 | 18 | def test_convert(test_sarif_files): 19 | with tempfile.NamedTemporaryFile(delete=False) as fp: 20 | inspect.convert_sarif("demo-app", {}, test_sarif_files, fp.name) 21 | data = fp.read() 22 | assert data 23 | -------------------------------------------------------------------------------- /test/test_issue.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from lib.issue import Issue 6 | 7 | 8 | def test_get_code(): 9 | curr_dir = os.path.dirname(os.path.abspath(__file__)) 10 | source_file = os.path.join(curr_dir, "data", "issue-259.php") 11 | issue = Issue(lineno=12) 12 | issue.fname = source_file 13 | text = issue.get_code() 14 | 15 | assert text == "11 \tpublic function test_convert_to_utf8()\n12 \t{\n" 16 | -------------------------------------------------------------------------------- /test/test_utils.py: -------------------------------------------------------------------------------- 1 | import lib.utils as utils 2 | 3 | 4 | def test_get_workspace(): 5 | d = utils.get_workspace( 6 | {"repositoryUri": "https://github.com/AppThreat/WebGoat", "branch": "develop"} 7 | ) 8 | assert d == "https://github.com/AppThreat/WebGoat/blob/develop" 9 | d = utils.get_workspace({"repositoryUri": "", "branch": "develop"}) 10 | assert not d 11 | d = utils.get_workspace( 12 | { 13 | "repositoryUri": "https://gitlab.com/prabhu3/helloshiftleft", 14 | "branch": "develop", 15 | } 16 | ) 17 | assert d == "https://gitlab.com/prabhu3/helloshiftleft/-/blob/develop" 18 | d = utils.get_workspace( 19 | { 20 | "repositoryUri": "https://gitlab.com/prabhu3/helloshiftleft", 21 | "branch": "", 22 | "revisionId": "fd302c3938a3c58908839ceaf48c2ce8176353f0", 23 | } 24 | ) 25 | assert ( 26 | d 27 | == "https://gitlab.com/prabhu3/helloshiftleft/-/blob/fd302c3938a3c58908839ceaf48c2ce8176353f0" 28 | ) 29 | d = utils.get_workspace( 30 | { 31 | "repositoryUri": "https://dev.azure.com/appthreat/aio", 32 | "branch": "develop", 33 | "revisionId": "fd302c3938a3c58908839ceaf48c2ce8176353f0", 34 | } 35 | ) 36 | assert ( 37 | d == "https://dev.azure.com/appthreat/aio?_a=contents&version=GBdevelop&path=" 38 | ) 39 | 40 | 41 | def test_filter_ignored_dirs(): 42 | d = utils.filter_ignored_dirs([]) 43 | assert d == [] 44 | d = utils.filter_ignored_dirs([".git", "foo", "node_modules", "tmp"]) 45 | assert d == ["foo"] 46 | d = utils.filter_ignored_dirs([".git", ".idea", "node_modules", "tmp"]) 47 | assert d == [] 48 | d = utils.filter_ignored_dirs([".foo", ".bar"]) 49 | assert d == [] 50 | 51 | 52 | def test_filter_ignored_files(): 53 | d = utils.is_ignored_file("", "") 54 | assert not d 55 | d = utils.is_ignored_file("", "foo.log") 56 | assert d 57 | d = utils.is_ignored_file("", "bar.d.ts") 58 | assert d 59 | d = utils.is_ignored_file("", "bar.tar.gz") 60 | assert d 61 | d = utils.is_ignored_file("", "bar.java") 62 | assert not d 63 | d = utils.is_ignored_file("", "bar.min.js") 64 | assert d 65 | d = utils.is_ignored_file("", "bar.min.css") 66 | assert d 67 | d = utils.is_ignored_file("", ".babelrc.js") 68 | assert d 69 | d = utils.is_ignored_file("", ".eslintrc.js") 70 | assert d 71 | -------------------------------------------------------------------------------- /test/test_xml.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import lib.xml_parser as xml_parser 4 | 5 | 6 | def test_findsec_parse(): 7 | with open( 8 | os.path.join( 9 | os.path.dirname(os.path.realpath(__file__)), 10 | "data", 11 | "findsecbugs-report.xml", 12 | ) 13 | ) as rf: 14 | issues, metrics = xml_parser.get_report_data(rf) 15 | assert len(issues) == 85 16 | assert len(metrics.keys()) == 1 17 | assert issues[0]["issue_severity"] == "HIGH" 18 | assert issues[0]["test_id"] == "CWE-78" 19 | 20 | 21 | def test_checkstyle_parse(): 22 | with open( 23 | os.path.join( 24 | os.path.dirname(os.path.realpath(__file__)), 25 | "data", 26 | "source-kt-report.xml", 27 | ) 28 | ) as rf: 29 | issues, metrics = xml_parser.get_report_data(rf) 30 | assert len(issues) == 1 31 | assert issues[0] == { 32 | "filename": "/app/app/src/main/java/owasp/sat/agoat/DownloadInvoiceService.kt", 33 | "line": "37", 34 | "issue_severity": "warning", 35 | "test_id": "detekt.UnreachableCode", 36 | "title": "This expression is followed by unreachable code which should either be used or removed.", 37 | } 38 | -------------------------------------------------------------------------------- /tools_config/detekt-config.yml: -------------------------------------------------------------------------------- 1 | config: 2 | validation: true 3 | # when writing own rules with new properties, exclude the property path e.g.: 'my_rule_set,.*>.*>[my_property]' 4 | excludes: '' 5 | 6 | processors: 7 | active: true 8 | exclude: 9 | - 'DetektProgressListener' 10 | # - 'FunctionCountProcessor' 11 | # - 'PropertyCountProcessor' 12 | # - 'ClassCountProcessor' 13 | # - 'PackageCountProcessor' 14 | # - 'KtFileCountProcessor' 15 | 16 | console-reports: 17 | active: false 18 | exclude: 19 | - 'ProjectStatisticsReport' 20 | - 'ComplexityReport' 21 | - 'NotificationReport' 22 | # - 'FindingsReport' 23 | - 'FileBasedFindingsReport' 24 | 25 | complexity: 26 | active: true 27 | ComplexCondition: 28 | active: true 29 | threshold: 6 30 | ComplexInterface: 31 | active: false 32 | threshold: 10 33 | includeStaticDeclarations: false 34 | includePrivateDeclarations: false 35 | CyclomaticComplexMethod: 36 | active: true 37 | threshold: 10 38 | ignoreSingleWhenExpression: false 39 | ignoreSimpleWhenEntries: false 40 | ignoreNestingFunctions: false 41 | nestingFunctions: [run, let, apply, with, also, use, forEach, isNotNull, ifNull] 42 | LabeledExpression: 43 | active: false 44 | ignoredLabels: [] 45 | LargeClass: 46 | active: true 47 | threshold: 600 48 | LongMethod: 49 | active: true 50 | threshold: 80 51 | LongParameterList: 52 | active: true 53 | functionThreshold: 8 54 | constructorThreshold: 8 55 | ignoreDefaultParameters: false 56 | ignoreDataClasses: true 57 | ignoreAnnotated: [] 58 | MethodOverloading: 59 | active: false 60 | threshold: 8 61 | NestedBlockDepth: 62 | active: true 63 | threshold: 6 64 | StringLiteralDuplication: 65 | active: false 66 | excludes: ['**/test/**', '**/androidTest/**', '**/commonTest/**', '**/jvmTest/**', '**/jsTest/**', '**/iosTest/**'] 67 | threshold: 10 68 | ignoreAnnotation: true 69 | excludeStringsWithLessThan5Characters: true 70 | ignoreStringsRegex: '$^' 71 | TooManyFunctions: 72 | active: true 73 | excludes: ['**/test/**', '**/androidTest/**', '**/commonTest/**', '**/jvmTest/**', '**/jsTest/**', '**/iosTest/**'] 74 | thresholdInFiles: 20 75 | thresholdInClasses: 20 76 | thresholdInInterfaces: 20 77 | thresholdInObjects: 20 78 | thresholdInEnums: 20 79 | ignoreDeprecated: false 80 | ignorePrivate: false 81 | ignoreOverridden: false 82 | 83 | performance: 84 | active: true 85 | ArrayPrimitive: 86 | active: true 87 | ForEachOnRange: 88 | active: true 89 | excludes: ['**/test/**', '**/androidTest/**', '**/commonTest/**', '**/jvmTest/**', '**/jsTest/**', '**/iosTest/**'] 90 | SpreadOperator: 91 | active: true 92 | excludes: ['**/test/**', '**/androidTest/**', '**/commonTest/**', '**/jvmTest/**', '**/jsTest/**', '**/iosTest/**'] 93 | UnnecessaryTemporaryInstantiation: 94 | active: true 95 | 96 | potential-bugs: 97 | active: true 98 | Deprecation: 99 | active: false 100 | EqualsAlwaysReturnsTrueOrFalse: 101 | active: true 102 | EqualsWithHashCodeExist: 103 | active: true 104 | ExplicitGarbageCollectionCall: 105 | active: true 106 | HasPlatformType: 107 | active: false 108 | IgnoredReturnValue: 109 | active: false 110 | restrictToConfig: true 111 | returnValueAnnotations: ['*.CheckReturnValue', '*.CheckResult'] 112 | ImplicitDefaultLocale: 113 | active: false 114 | ImplicitUnitReturnType: 115 | active: false 116 | allowExplicitReturnType: true 117 | InvalidRange: 118 | active: true 119 | IteratorHasNextCallsNextMethod: 120 | active: true 121 | IteratorNotThrowingNoSuchElementException: 122 | active: true 123 | LateinitUsage: 124 | active: false 125 | excludes: ['**/test/**', '**/androidTest/**', '**/commonTest/**', '**/jvmTest/**', '**/jsTest/**', '**/iosTest/**'] 126 | ignoreAnnotated: [] 127 | ignoreOnClassesPattern: '' 128 | MapGetWithNotNullAssertionOperator: 129 | active: false 130 | UnconditionalJumpStatementInLoop: 131 | active: false 132 | UnnecessaryNotNullOperator: 133 | active: false 134 | UnnecessarySafeCall: 135 | active: false 136 | UnreachableCode: 137 | active: true 138 | UnsafeCallOnNullableType: 139 | active: true 140 | UnsafeCast: 141 | active: false 142 | UselessPostfixExpression: 143 | active: false 144 | WrongEqualsTypeParameter: 145 | active: true 146 | -------------------------------------------------------------------------------- /tools_config/io.shiftleft.scan.appdata.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | io.shiftleft.scan 4 | 5 | scan 6 | ShiftLeft Scan is a free open-source security tool for modern DevOps teams 7 | 8 | FSFAP 9 | Apache-2.0 10 | 11 | 12 |

13 | Scan is a free open-source security tool for modern DevOps teams. With an integrated multi-scanner based design, Scan can detect various kinds of security flaws in your application and infrastructure code in a single fast scan. The kind of flaws detected are: 14 |

15 |
    16 |
  • Credentials Scanning to detect accidental secret leaks
    • 17 |
  • Static Analysis Security Testing (SAST) for a range of languages and frameworks
    • 18 |
  • Open-source dependencies audit
    • 19 |
  • Licence violation checks
    • 20 |
21 | 22 | 23 | utilities-terminal 24 | 25 | 26 | Development 27 | Building 28 | 29 | https://qwiet.ai 30 | ShiftLeftSecurity 31 | 32 | 33 | scan 34 | 35 | 36 | -------------------------------------------------------------------------------- /tools_config/phpstan.neon.dist: -------------------------------------------------------------------------------- 1 | parameters: 2 | level: 8 3 | reportUnmatchedIgnoredErrors: false 4 | excludes_analyse: 5 | - %currentWorkingDirectory%/docs/* 6 | - %currentWorkingDirectory%/examples/* 7 | - %currentWorkingDirectory%/tests/* 8 | - %currentWorkingDirectory%/vendor/* 9 | -------------------------------------------------------------------------------- /tools_config/rules-pmd.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | Custom ruleset for java application 8 | 9 | .*/.mvn/.* 10 | .*/test/.* 11 | .*/.sfdx/.* 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 3 33 | 34 | 35 | 36 | 37 | 3 38 | 39 | 40 | 41 | 42 | 43 | 8 44 | 45 | 46 | 47 | 48 | 8 49 | 50 | 51 | 52 | 53 | 54 | 8 55 | 56 | 57 | 58 | 59 | 8 60 | 61 | 62 | 63 | 64 | 8 65 | 66 | 67 | 68 | 69 | 8 70 | 71 | 72 | 73 | 74 | 8 75 | 76 | 77 | 78 | 79 | 3 80 | 81 | 82 | 83 | 84 | 8 85 | 86 | 87 | 88 | 89 | 3 90 | 91 | 92 | 93 | 94 | 8 95 | 96 | 97 | 8 98 | 99 | 100 | -------------------------------------------------------------------------------- /tools_config/scan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShiftLeftSecurity/sast-scan/ae74004b82a41e800c6bc0b6ed05d48da9b8fc6f/tools_config/scan.png -------------------------------------------------------------------------------- /tools_config/scan.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /tools_config/spotbugs/exclude.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /tools_config/spotbugs/include.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /ubuntu_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # This script fulfills all requirements in an Ubuntu 22.04 environment to build the AppImage for sast-scan 5 | # It might work in other debian based environments but it has not been tested. 6 | # Furthermore, the AppImage Process does add ubuntu repositories for 22.04 so it is not guaranteed to work outside 7 | # of that environment (or derivatives). 8 | # AppImage generation is rather slow by nature and this script installs a set of packages, we recommend running it 9 | # within the confines of an LXC container or similar. 10 | 11 | # Install all dependencies that come from the distro package manager. 12 | sudo apt-get update -y 13 | sudo apt-get install -y --no-install-recommends python3 python3-dev \ 14 | python3-pip python3-setuptools patchelf desktop-file-utils \ 15 | libgdk-pixbuf2.0-dev php php-curl php-zip php-bcmath php-json \ 16 | php-pear php-mbstring php-dev php-xml wget curl git unzip \ 17 | adwaita-icon-theme libfuse2 squashfs-tools zsync 18 | 19 | # Build the cache folder if missing 20 | mkdir -p appimage-builder-cache 21 | 22 | # Download latest AppImage Builder Tool 23 | wget https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-"${ARCH}".AppImage 24 | chmod +x appimagetool-"${ARCH}".AppImage 25 | ./appimagetool-"${ARCH}".AppImage --appimage-extract 26 | ln -s /squashfs-root/AppRun ./appimage-builder-cache/appimagetool 27 | 28 | # Download latest AppImage Builder Runtime 29 | wget https://github.com/AppImage/AppImageKit/releases/download/continuous/runtime-"${ARCH}" -O ./appimage-builder-cache/runtime-"${ARCH}" 30 | chmod +x ./appimage-builder-cache/runtime-"${ARCH}" 31 | 32 | # Install appimage-builder python package, this version is forked to account for arm64 architecture name missmatches 33 | # which cause issues when building an AppImage for ubuntu 34 | # (see https://github.com/AppImageCrafters/appimage-builder/pull/318) 35 | python3 -m pip install git+https://github.com/perrito666/appimage-builder.git 36 | 37 | # This variable is expected by AppImage builder 38 | export UPDATE_INFO="gh-releases-zsync|ShiftLeftSecurity|sast-scan|latest|*${ARCH}.AppImage.zsync" 39 | 40 | # Add the newly installed tools to the path 41 | export PATH="${PWD}/AppDir/usr/bin:${PWD}/AppDir/usr/bin/nodejs/bin:${PWD}/appimage-builder-cache:${PATH}" 42 | 43 | # This is the script to build X86_68/amd64 AppImages, AppImage can't cross build ootb so we do not. 44 | 45 | # Uncomment this for debugging purposes, this will reuse the same folder, it is quite unstable as AppImage 46 | # modifies the files to depend on relative envs so after that process ran these will no longer work outside of it. 47 | #export KEEP_BUILD_ARTIFACTS=true 48 | 49 | # Build App Image for this arch 50 | appimage-builder --recipe "${BUILDER_SCRIPT}" --skip-test 51 | 52 | -------------------------------------------------------------------------------- /ubuntu_build_arm64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | export ARCH=aarch64 5 | export BUILDER_SCRIPT=appimage-builder-arm64.yml 6 | 7 | ./ubuntu_build.sh -------------------------------------------------------------------------------- /ubuntu_build_x86_64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | export ARCH=x86_64 5 | export BUILDER_SCRIPT=appimage-builder.yml 6 | 7 | ./ubuntu_build.sh --------------------------------------------------------------------------------