├── .clang-format ├── .github └── workflows │ ├── ci.yml │ └── coverity.yml ├── .gitignore ├── .pylintrc ├── LICENSE.txt ├── Makefile ├── README.md ├── read_binary_api ├── scripts ├── install-deps.sh └── travis.sh ├── setup.py ├── shlibvischeck-common ├── shlibvischeck-debian ├── shlibvischeck ├── __init__.py ├── __main__.py ├── analysis │ ├── __init__.py │ ├── debian.py │ ├── elf.py │ ├── header.py │ └── package.py └── common │ ├── __init__.py │ ├── error.py │ └── process.py ├── src └── read_header_api.cc └── tests ├── basic ├── a.ref ├── ab.ref ├── b.ref ├── run.sh ├── xyz.c └── xyz.h ├── debian ├── libbz2-1.0.ref └── run.sh └── only ├── other.h ├── out.ref ├── run.sh ├── xyz.c └── xyz.h /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: LLVM 2 | --- 3 | Language: Cpp 4 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # TODO: 2 | # * proper deploy (with dependencies, etc.) 3 | 4 | name: CI 5 | on: 6 | push: 7 | paths-ignore: 8 | - 'LICENSE.txt' 9 | - 'README.md' 10 | - '.gitignore' 11 | pull_request: 12 | paths-ignore: 13 | - 'LICENSE.txt' 14 | - 'README.md' 15 | - '.gitignore' 16 | jobs: 17 | Baseline: 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, ubuntu-latest] 22 | cxx: [g++, clang++] 23 | py: [python3.6, python3.7, python3] 24 | exclude: 25 | - os: ubuntu-22.04 26 | py: python3.6 27 | runs-on: ${{ matrix.os }} 28 | env: 29 | CXX: ${{ matrix.cxx }} 30 | PYTHON: ${{ matrix.py }} 31 | steps: 32 | - uses: actions/checkout@v2 33 | - name: Install deps 34 | run: | 35 | sudo add-apt-repository ppa:deadsnakes/ppa 36 | sudo apt-get update 37 | scripts/install-deps.sh 38 | - name: Run tests 39 | run: scripts/travis.sh 40 | Pylint: 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v2 44 | - name: Install deps 45 | run: | 46 | scripts/install-deps.sh 47 | sudo apt-get install pylint 48 | - name: Run tests 49 | run: make pylint 50 | CSA: 51 | runs-on: ubuntu-latest 52 | env: 53 | CXX: clang 54 | steps: 55 | - uses: actions/checkout@v2 56 | - name: Install deps 57 | run: | 58 | scripts/install-deps.sh 59 | sudo apt-get install clang-tools 60 | - name: Run tests 61 | run: scan-build --keep-going --status-bugs scripts/travis.sh 62 | Asan: 63 | runs-on: ubuntu-latest 64 | env: 65 | CXX: clang++ 66 | ASAN: 1 67 | steps: 68 | - uses: actions/checkout@v2 69 | - name: Install deps 70 | run: scripts/install-deps.sh 71 | - name: Run tests 72 | run: scripts/travis.sh 73 | UBsan: 74 | runs-on: ubuntu-latest 75 | env: 76 | CXX: clang++ 77 | UBSAN: 1 78 | steps: 79 | - uses: actions/checkout@v2 80 | - name: Install deps 81 | run: scripts/install-deps.sh 82 | - name: Run tests 83 | run: scripts/travis.sh 84 | Valgrind: 85 | runs-on: ubuntu-latest 86 | env: 87 | VALGRIND: 1 88 | steps: 89 | - uses: actions/checkout@v2 90 | - name: Install deps 91 | run: | 92 | scripts/install-deps.sh 93 | sudo apt-get install valgrind 94 | - name: Run tests 95 | run: scripts/travis.sh 96 | Coverage: 97 | needs: Baseline 98 | runs-on: ubuntu-latest 99 | environment: secrets 100 | env: 101 | COVERAGE: 1 102 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 103 | steps: 104 | - uses: actions/checkout@v2 105 | - name: Install deps 106 | run: | 107 | scripts/install-deps.sh 108 | sudo python3 -m pip install codecov 109 | - name: Run tests and upload coverage 110 | env: 111 | PYTHON: coverage run -a 112 | run: scripts/travis.sh 113 | -------------------------------------------------------------------------------- /.github/workflows/coverity.yml: -------------------------------------------------------------------------------- 1 | name: Coverity 2 | on: 3 | schedule: 4 | # Run on Mondays 5 | - cron: '0 5 * * MON' 6 | jobs: 7 | Coverity: 8 | runs-on: ubuntu-latest 9 | environment: secrets 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Install deps 13 | run: scripts/install-deps.sh 14 | - uses: vapier/coverity-scan-action@v0 15 | with: 16 | project: yugr%2FShlibVisibilityChecker 17 | token: ${{ secrets.COVERITY_SCAN_TOKEN }} 18 | email: ${{ secrets.EMAIL }} 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python files 2 | build 3 | dist 4 | .*.swp 5 | __pycache__ 6 | *.egg-info 7 | 8 | # C++ binaries 9 | bin/* 10 | 11 | # GDB temp files 12 | .gdb_history 13 | 14 | # Test temp files 15 | api.txt 16 | abi.txt 17 | api_abi.diff 18 | *.so 19 | codecov.bash 20 | out.log 21 | *.gcov 22 | .coverage 23 | *.gcda 24 | *.gcno 25 | coverage.xml 26 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | disable = trailing-whitespace, # In copyrights 4 | invalid-name, # Short variables like 'v' 5 | unused-wildcard-import, # For 'from ... import ...' 6 | fixme, # TODO/FIXME 7 | unspecified-encoding, # Rely on default UTF-8 8 | too-many-locals, too-many-branches, too-many-boolean-expressions 9 | 10 | [FORMAT] 11 | 12 | indent-string = ' ' 13 | 14 | [IMPORT] 15 | 16 | allow-wildcard-with-all = yes 17 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018-2022 Yury Gribov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2018-2022 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | 8 | $(shell mkdir -p bin) 9 | 10 | CXX ?= g++ 11 | LLVM_CONFIG ?= llvm-config 12 | DESTDIR ?= /usr/local 13 | 14 | CXXFLAGS = $(shell $(LLVM_CONFIG) --cflags) -std=c++11 -g -Wall -Wextra -Werror 15 | LDFLAGS = $(shell $(LLVM_CONFIG) --ldflags) -Wl,--warn-common 16 | 17 | ifneq (,$(COVERAGE)) 18 | DEBUG = 1 19 | CXXFLAGS += --coverage -DNDEBUG 20 | LDFLAGS += --coverage 21 | endif 22 | ifeq (,$(DEBUG)) 23 | CXXFLAGS += -O2 24 | LDFLAGS += -Wl,-O2 25 | else 26 | CXXFLAGS += -O0 27 | endif 28 | ifneq (,$(ASAN)) 29 | CXXFLAGS += -fsanitize=address -fsanitize-address-use-after-scope -U_FORTIFY_SOURCE -fno-common -D_GLIBCXX_DEBUG -D_GLIBCXX_SANITIZE_VECTOR 30 | LDFLAGS += -fsanitize=address 31 | endif 32 | ifneq (,$(UBSAN)) 33 | ifneq (,$(shell $(CXX) --version | grep clang)) 34 | # Isan is clang-only... 35 | CXXFLAGS += -fsanitize=undefined,integer -fno-sanitize-recover=undefined,integer 36 | LDFLAGS += -fsanitize=undefined,integer -fno-sanitize-recover=undefined,integer 37 | else 38 | CXXFLAGS += -fsanitize=undefined -fno-sanitize-recover=undefined 39 | LDFLAGS += -fsanitize=undefined -fno-sanitize-recover=undefined 40 | endif 41 | endif 42 | 43 | all: bin/read_header_api 44 | 45 | # TODO: install Python as well 46 | install: 47 | mkdir -p $(DESTDIR) 48 | install bin/read_header_api $(DESTDIR)/bin 49 | install read_binary_api $(DESTDIR)/bin 50 | 51 | check: 52 | tests/basic/run.sh 53 | tests/debian/run.sh 54 | tests/only/run.sh 55 | 56 | pylint: 57 | pylint shlibvischeck 58 | 59 | bin/read_header_api: bin/read_header_api.o Makefile bin/FLAGS 60 | $(CXX) $(LDFLAGS) -o $@ $(filter %.o, $^) -lclang 61 | 62 | bin/%.o: src/%.cc Makefile bin/FLAGS 63 | $(CXX) $(CXXFLAGS) -o $@ -c $< 64 | 65 | bin/FLAGS: FORCE 66 | if test x"$(CFLAGS) $(CXXFLAGS) $(LDFLAGS)" != x"$$(cat $@)"; then \ 67 | echo "$(CFLAGS) $(CXXFLAGS) $(LDFLAGS)" > $@; \ 68 | fi 69 | 70 | clean: 71 | rm -rf bin/* build dist *.egg-info 72 | find -name \*.gcov -o -name \*.gcno -o -name \*.gcda | xargs rm -rf 73 | find -name .coverage -o -name \*.xml | xargs rm -rf 74 | 75 | .PHONY: check all install clean pylint FORCE 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License](http://img.shields.io/:license-MIT-blue.svg)](https://github.com/yugr/ShlibVisibilityChecker/blob/master/LICENSE.txt) 2 | [![Build Status](https://github.com/yugr/ShlibVisibilityChecker/actions/workflows/ci.yml/badge.svg)](https://github.com/yugr/ShlibVisibilityChecker/actions) 3 | [![codecov](https://codecov.io/gh/yugr/ShlibVisibilityChecker/branch/master/graph/badge.svg)](https://codecov.io/gh/yugr/ShlibVisibilityChecker) 4 | [![Total alerts](https://img.shields.io/lgtm/alerts/g/yugr/ShlibVisibilityChecker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/yugr/ShlibVisibilityChecker/alerts/) 5 | [![Coverity Scan](https://scan.coverity.com/projects/yugr-ShlibVisibilityChecker/badge.svg)](https://scan.coverity.com/projects/yugr-ShlibVisibilityChecker) 6 | 7 | # What's this? 8 | 9 | ShlibVisibilityChecker is a small tool which locates internal symbols 10 | that are unnecessarily exported from shared libraries. 11 | Such symbols are undesirable because they cause 12 | * slower startup time (due to [slower relocation processing by dynamic linker](https://lwn.net/Articles/341309/), see a [real-world example](https://lore.kernel.org/lkml/CAKwvOdk0nxxUATg2jEKgx4HutXCMXcW92SX3DT+uCTgqBwQHBg@mail.gmail.com/) for Linux kernel) 13 | * performance slowdown (due to indirect function calls, compiler's inability 14 | to optimize exportable functions e.g. inline them, effective turnoff of `--gc-sections`) 15 | * leak of implementation details 16 | (if some clients start to use private functions instead of regular APIs) 17 | * bugs due to runtime symbol clashing 18 | - [crash in Apache due to symbol clash with libasn1](https://github.com/DCIT/perl-CryptX/issues/68) 19 | - more real-world examples in [Flameeyes blog](https://flameeyes.blog/2008/02/09/flex-and-linking-conflicts-or-a-possible-reason-why-php-and-recode-are-so-crashy/) 20 | 21 | ShlibVisibilityChecker compares APIs declared in public headers 22 | against APIs exported from shared libraries and warns about discrepancies. 23 | In majority of cases such symbols are internal library symbols which should be hidden 24 | (in rare cases these are internal symbols which are used by other libraries or executables 25 | in the same package and `shlibvischeck-debian` tries hard to not report such cases). 26 | 27 | Such discrepancies should then be fixed by recompiling package 28 | with `-fvisibility=hidden` (see [here](https://gcc.gnu.org/wiki/Visibility) for details). 29 | A typical fix, for a typical Autoconf project can be found 30 | [here](https://github.com/cacalabs/libcaca/issues/33#issuecomment-387656546). 31 | 32 | ShlibVisibilityChecker _not_ meant to be 100% precise but rather provide assistance in locating packages 33 | which may benefit the most from visibility annotations (and to understand how bad the situation 34 | with visibility is in modern distros). 35 | 36 | # How to use 37 | 38 | To check a raw package, i.e. a bunch of headers and shared libs, 39 | collect source and binary interfaces and compare them: 40 | ``` 41 | $ bin/read_header_api --only-args /usr/include/xcb/* > api.txt 42 | $ ./read_binary_api --permissive /usr/lib/x86_64-linux-gnu/libxcb*.so > abi.txt 43 | $ vimdiff api.txt abi.txt # Or `comm -13 api.txt abi.txt' 44 | ``` 45 | 46 | Another useful scenario is locating symbols that are exported from 47 | Debian package's shared libraries but are not declared in it's headers. 48 | The main tool for this is a `shlibvischeck-debian` script. 49 | 50 | To apply it to a package, run 51 | ``` 52 | $ shlibvischeck-debian libacl1 53 | The following exported symbols in package 'libacl1' are private: 54 | __acl_extended_file 55 | __acl_from_xattr 56 | __acl_to_xattr 57 | __bss_start 58 | _edata 59 | _end 60 | _fini 61 | _init 62 | closed 63 | head 64 | high_water_alloc 65 | next_line 66 | num_dir_handles 67 | walk_tree 68 | ``` 69 | To skip autogenerated symbols like `_init` or `_edata` (caused by [ld linker scripts](https://sourceware.org/ml/binutils/2018-04/msg00326.html) and [libgcc startup files](https://gcc.gnu.org/ml/gcc-help/2018-04/msg00097.html)) add `--permissive`. 70 | 71 | You can also check visibility issues in arbitrary set of headers and libraries: 72 | ``` 73 | $ shlibvischeck-common --permissive --cflags="-I/usr/include -I$AUDIT_INSTALL/include -I/usr/lib/llvm-5.0/lib/clang/5.0.0/include" $AUDIT_INSTALL/include/*.h $AUDIT_INSTALL/lib/*.so* 74 | ``` 75 | 76 | # How to install 77 | 78 | Build-time prerequisites are `python3` (`setuptools` module), `clang`, 79 | `llvm`, `libclang-dev`, `g++` and `make`. 80 | Run-time dependencies are `python3` (`python-magic` module), `pkg-config` and `aptitude`. 81 | To install everything on Ubuntu, run 82 | ``` 83 | $ sudo apt-get install python3 clang llvm libclang-dev g++ make pkg-config aptitude 84 | $ sudo python3 -mensurepip 85 | $ sudo pip3 install setuptools python-magic 86 | ``` 87 | (you could also use script `scripts/install-deps.sh`). 88 | 89 | You also need to enable access to Ubuntu source packages via 90 | ``` 91 | $ sudo sed -Ei 's/^# *deb-src /deb-src /' /etc/apt/sources.list 92 | $ sudo apt-get update 93 | ``` 94 | 95 | Python and binary components are built separately: 96 | ``` 97 | $ make clean all && make install 98 | $ ./setup.py build && pip3 install . 99 | ``` 100 | 101 | During analysis `shlibvischeck-debian` installs new Debian packages so it's recommended to run it under chroot or in VM. 102 | There are many instructions on setting up chroot e.g. [this one](https://github.com/yugr/debian_pkg_test). 103 | 104 | # Where to find packages 105 | 106 | A list of packages for analysis can be obtained from [Debian rating](https://popcon.debian.org/by_vote): 107 | ``` 108 | $ curl https://popcon.debian.org/by_vote | awk '/^[0-9]+ +lib/{print $2}' > by_vote 109 | $ shlibvischeck-debian $(head -500 by_vote | tr '\n' ' ') 110 | ``` 111 | 112 | # How to fix a package 113 | 114 | Once you found a problematic package, you can fix it by restricting visibility of internal symbols. 115 | The best way to control symbol visibility in a package is to 116 | * hide all symbols by default by adding `-fvisibility=hidden` to `CFLAGS` in project buildscripts 117 | (`Makefile.in` or `CMakeLists.txt`) 118 | * explicitly annotate publicly visible functions with 119 | `__attribute__((visibility("default")))` 120 | 121 | See [fix in libcaca](https://github.com/cacalabs/libcaca/pull/34/files) 122 | for example. 123 | 124 | # Issues and limitations 125 | 126 | At the moment tool works only on Debian-based systems (e.g. Ubuntu). 127 | This should be fine as buildscripts are the same across all distros 128 | so detecting issues on Ubuntu would serve everyone else too. 129 | 130 | An important design issue is that the tool can not detect symbols which are used indirectly 131 | i.e. not through an API but through `dlsym` or explicit out-of-header prototype declaration 132 | in source file. This happens in plugins or tightly interconnected shlibs within the same project. 133 | Such cases should hopefully be rare. 134 | 135 | ShlibVisibilityChecker is a heuristic tool so it will not be able to analyze all packages. 136 | Current success rate is around 60%. 137 | Major reasons for errors are 138 | * badly-structured headers i.e. the ones which do not \#include all their dependencies 139 | (e.g. `libatasmart` [fails to include `stddef.h`](https://github.com/Rupan/libatasmart/issues/1) 140 | and `tdb` [fails to include `sys/types.h`](https://bugzilla.samba.org/show_bug.cgi?id=13398)). 141 | * internal headers which should not be \#included directly (e.g. `lzma/container.h`) 142 | * experimental headers which require custom macro definitions (not listed in 143 | pkgconfig) (e.g. `dpkg/macros.h` requires `LIBDPKG_VOLATILE_API`) 144 | * missing dependencies (e.g. `libverto-dev` uses Glib headers but does not declare this) 145 | 146 | Other issues: 147 | * TODOs are scattered all over the codebase 148 | * would be interesting to go over dependent packages and check if they use invalid symbols 149 | 150 | # Trophies 151 | 152 | The tool found huge number of packages that lacked visibility annotations (in practice every second package 153 | has spurious exports). Here are some which I tried to fix: 154 | 155 | * Bzip2: [Hide unused symbols in libbz2](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=896750) 156 | * Expat: [Private symbols exported from shared library](https://github.com/libexpat/libexpat/issues/195) (fixed) 157 | * Libaudit: [Exported private symbols in audit-userspace](https://www.redhat.com/archives/linux-audit/2018-April/msg00119.html) (partially fixed) 158 | * Gdbm: [sr #347: Add visibility annotations to hide private symbols](https://puszcza.gnu.org.ua/support/index.php?347) 159 | * Libnfnetfilter: [\[RFC\]\[PATCH\] Hide private symbols in libnfnetlink](https://marc.info/?l=netfilter-devel&m=152481166515881) (fixed) 160 | * Libarchive: [Hide private symbols in libarchive.so](https://github.com/libarchive/libarchive/issues/1017) ([fixed](https://github.com/libarchive/libarchive/pull/1751)) 161 | * Libcaca: [Hide private symbols in libcaca](https://github.com/cacalabs/libcaca/issues/33) (fixed) 162 | * Libgmp: [Building gmp with -fvisibility=hidden](https://gmplib.org/list-archives/gmp-discuss/2018-April/006229.html) 163 | * Vorbis: [Remove private symbols from Vorbis shared libs](https://github.com/xiph/vorbis/issues/43) 164 | 165 | More perspective packages (from Debian top-100): libpopt1, libgpg-error0, libxml2, libwrap0, libpcre3, libkeyutils1, libedit2, liblcms2-2. 166 | -------------------------------------------------------------------------------- /read_binary_api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Copyright 2018-2021 Yury Gribov 4 | # 5 | # Use of this source code is governed by MIT license that can be 6 | # found in the LICENSE.txt file. 7 | 8 | # Reads binary APIs from a list of shlibs. 9 | 10 | import os 11 | import os.path 12 | import argparse 13 | 14 | import shlibvischeck.common.error as e 15 | import shlibvischeck.analysis.elf as elf 16 | 17 | def main(): 18 | me = os.path.basename(__file__) 19 | e.set_basename(me) 20 | e.set_throw_on_error() 21 | parser = argparse.ArgumentParser(description="Reads binary APIs from a list of shlibs.", 22 | formatter_class=argparse.RawDescriptionHelpFormatter, 23 | epilog="""\ 24 | Examples: 25 | $ {0} /usr/lib/x86_64-linux-gnu/libaa.so.1 26 | aa_ansi_format 27 | aa_attrs 28 | ... 29 | """.format(me)) 30 | parser.add_argument('--verbose', '-v', 31 | help="Print diagnostic info.", 32 | action='count', default=0) 33 | parser.add_argument('--permissive', '-p', 34 | help="Ignore dummy symbols introduced by ld (_edata, __bss_start, etc.) " 35 | "and libgcc (_init, _fini).", 36 | dest='permissive', action='store_true', default=False) 37 | parser.add_argument('--no-permissive', 38 | help="Disable --permissive (default).", 39 | dest='permissive', action='store_false') 40 | parser.add_argument('--export', '-e', 41 | help="List exported symbols (default).", 42 | dest='export', action='store_true', default=True) 43 | parser.add_argument('--import', '-i', 44 | help="List imported symbols.", 45 | dest='export', action='store_false') 46 | parser.add_argument('shlibs', 47 | help="Analyzed shared library.", metavar='SHLIB...', 48 | nargs=argparse.REMAINDER, default=[]) 49 | 50 | args = parser.parse_args() 51 | 52 | all_names = set() 53 | for shlib in args.shlibs: 54 | names = elf.read_binary_api(shlib, args.export, args.permissive) 55 | all_names.update(names) 56 | for name in sorted(all_names): 57 | print(name) 58 | 59 | if __name__ == '__main__': 60 | main() 61 | -------------------------------------------------------------------------------- /scripts/install-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) 2021-2022 Yury Gribov 6 | # 7 | # Use of this source code is governed by The MIT License (MIT) 8 | # that can be found in the LICENSE.txt file. 9 | 10 | set -eu 11 | set -x 12 | 13 | PYTHON=${PYTHON:-python3} 14 | 15 | sudo apt-get -y install llvm libclang-dev $PYTHON pkg-config aptitude 16 | sudo apt-get -y install $PYTHON-pip || true 17 | # distutils is needed by pip 18 | sudo apt-get -y install $PYTHON-distutils || true 19 | sudo $PYTHON -m pip install setuptools wheel python-magic 20 | 21 | # shlibvischeck-debian needs source repos 22 | sudo sed -Ei 's/^# *deb-src /deb-src /' /etc/apt/sources.list 23 | sudo apt-get update 24 | -------------------------------------------------------------------------------- /scripts/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) 2018-2022 Yury Gribov 6 | # 7 | # Use of this source code is governed by The MIT License (MIT) 8 | # that can be found in the LICENSE.txt file. 9 | 10 | set -eu 11 | set -x 12 | 13 | cd $(dirname $0)/.. 14 | 15 | PYTHON=${PYTHON:-python3} 16 | 17 | # Build 18 | 19 | make "$@" clean all 20 | $PYTHON ./setup.py build 21 | $PYTHON ./setup.py bdist_wheel 22 | 23 | # Run tests 24 | 25 | export ASAN_OPTIONS='detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:strict_string_checks=1' 26 | 27 | if test -n "${VALGRIND:-}"; then 28 | cp -r bin bin-real 29 | for f in $(find bin -type f -a -executable); do 30 | cat > $f < codecov.bash 49 | bash codecov.bash -Z -X gcov 50 | fi 51 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright (c) 2020 Yury Gribov 6 | # 7 | # Use of this source code is governed by The MIT License (MIT) 8 | # that can be found in the LICENSE.txt file. 9 | 10 | import setuptools 11 | import os 12 | 13 | with open(os.path.join(os.path.dirname(__file__), 'README.md'), 'r') as f: 14 | long_description = f.read() 15 | 16 | setuptools.setup( 17 | name='shlibvischeck', 18 | version='0.1', 19 | author='Yury Gribov', 20 | author_email='tetra2005@gmail.com', 21 | description="Tool for locating internal symbols unnecessarily exported from shared libraries.", 22 | long_description=long_description, 23 | long_description_content_type='text/markdown', 24 | url='https://github.com/yugr/ShlibVisibilityChecker', 25 | packages=setuptools.find_packages(), 26 | scripts=['shlibvischeck-debian', 'shlibvischeck-common', 'read_binary_api'], 27 | install_requires=['python-magic'], 28 | classifiers=[ 29 | 'Programming Language :: Python :: 3', 30 | 'License :: OSI Approved :: MIT License', 31 | 'Operating System :: OS Independent', 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /shlibvischeck-common: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright 2020 Yury Gribov 6 | # 7 | # Use of this source code is governed by MIT license that can be 8 | # found in the LICENSE.txt file. 9 | 10 | # Analyzes difference between public and binary interfaces in package files. 11 | # Command-line equivalent: 12 | # $ read_header_api --cflags="-I/usr/include -I$AUDIT_INSTALL/include -I/usr/lib/llvm-5.0/lib/clang/5.0.0/include" $AUDIT_INSTALL/include/*.h > public_api.txt 13 | # $ read_binary_api --permissive $AUDIT_INSTALL/lib/*.so* > exported_api.txt 14 | # $ comm -13 public_api.txt exported_api.txt 15 | 16 | import os 17 | import os.path 18 | import argparse 19 | 20 | import shlibvischeck.common.error as e 21 | from shlibvischeck.analysis.package import * 22 | 23 | def main(): 24 | me = os.path.basename(__file__) 25 | e.set_basename(me) 26 | e.set_throw_on_error() 27 | 28 | parser = argparse.ArgumentParser(description="Analyzes difference between public and binary interfaces in package files.", 29 | formatter_class=argparse.RawDescriptionHelpFormatter, 30 | epilog="""\ 31 | Examples: 32 | $ {0} *.h *.so* 33 | """.format(me)) 34 | parser.add_argument('--verbose', '-v', 35 | help="Print diagnostic info.", 36 | action='count', default=0) 37 | parser.add_argument('--permissive', '-p', 38 | help="Ignore dummy symbols introduced by ld (_edata, __bss_start, etc.) " 39 | "and libgcc (_init, _fini).", 40 | dest='permissive', action='store_true', default=False) 41 | parser.add_argument('--no-permissive', 42 | help="Disable --permissive (default).", 43 | dest='permissive', action='store_false') 44 | parser.add_argument('--cflags', '-f', 45 | help="Compiler flags to use when parsing headers (may be specified more than once for separate trials).", 46 | action='append', default=[]) 47 | parser.add_argument('rest', 48 | nargs=argparse.REMAINDER, default=[]) 49 | 50 | args = parser.parse_args() 51 | 52 | spurious_syms = analyze_package('', args.rest, args.cflags, args.permissive, args.verbose) 53 | if spurious_syms: 54 | print("The following exported symbols are private:\n %s" 55 | % ('\n '.join(spurious_syms))) 56 | else: 57 | print("No private exports") 58 | 59 | if __name__ == '__main__': 60 | main() 61 | -------------------------------------------------------------------------------- /shlibvischeck-debian: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # The MIT License (MIT) 4 | # 5 | # Copyright 2018-2021 Yury Gribov 6 | # 7 | # Use of this source code is governed by MIT license that can be 8 | # found in the LICENSE.txt file. 9 | 10 | # Analyzes difference between public and binary interfaces in Debian packages. 11 | 12 | import os 13 | import os.path 14 | import re 15 | import argparse 16 | 17 | import shlibvischeck.common.error as e 18 | from shlibvischeck.analysis.debian import * 19 | 20 | def is_bad_pkg(pkg): 21 | return re.search('^(libreoffice|glibc)', pkg) 22 | 23 | def main(): 24 | me = os.path.basename(__file__) 25 | e.set_basename(me) 26 | e.set_throw_on_error() 27 | 28 | parser = argparse.ArgumentParser(description="Analyzes difference between public and binary interfaces in Debian packages.", 29 | formatter_class=argparse.RawDescriptionHelpFormatter, 30 | epilog="""\ 31 | Examples: 32 | $ {0} libacl1 33 | """.format(me)) 34 | parser.add_argument('--verbose', '-v', 35 | help="Print diagnostic info.", 36 | action='count', default=0) 37 | parser.add_argument('--permissive', '-p', 38 | help="Ignore dummy symbols introduced by ld (_edata, __bss_start, etc.) " 39 | "and libgcc (_init, _fini).", 40 | dest='permissive', action='store_true', default=False) 41 | parser.add_argument('--no-permissive', 42 | help="Disable --permissive (default).", 43 | dest='permissive', action='store_false') 44 | parser.add_argument('rest', 45 | nargs=argparse.REMAINDER, default=[]) 46 | 47 | args = parser.parse_args() 48 | 49 | total = fails = 0 50 | failed_packages = [] 51 | processed_pkgs = set() 52 | 53 | for pkg in args.rest: 54 | src = get_pkg_attribute(pkg, 'Package', True, True)[0] 55 | # TODO: handle errors 56 | if src in processed_pkgs or is_bad_pkg(src): 57 | continue 58 | processed_pkgs.add(src) 59 | 60 | total += 1 61 | print("Processing package '%s'..." % pkg) 62 | 63 | success = False 64 | try: 65 | if analyze_debian_package(pkg, args.permissive, args.verbose): 66 | success = True 67 | except Exception: 68 | import traceback 69 | traceback.print_exc() 70 | finally: 71 | if not success: 72 | print("Failed to analyze package '%s'." % pkg) 73 | fails += 1 74 | failed_packages.append(src) 75 | 76 | if fails > 0: 77 | print("%d packages failed (out of %d): %s" % (fails, total, 78 | ' '.join(failed_packages))) 79 | 80 | if __name__ == '__main__': 81 | main() 82 | -------------------------------------------------------------------------------- /shlibvischeck/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | -------------------------------------------------------------------------------- /shlibvischeck/__main__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | -------------------------------------------------------------------------------- /shlibvischeck/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | -------------------------------------------------------------------------------- /shlibvischeck/analysis/debian.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020-2022 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | 8 | """ 9 | APIs for analyzing Debian packages 10 | """ 11 | 12 | import os 13 | import os.path 14 | import re 15 | 16 | from shlibvischeck.common.error import error 17 | from shlibvischeck.common.process import * 18 | from shlibvischeck.analysis.package import * 19 | 20 | __all__ = ['get_pkg_attribute', 'analyze_debian_package'] 21 | 22 | def _get_installed_packages(): 23 | """ Returns installed Debian packages. """ 24 | pkgs = set() 25 | _, out, _ = run('dpkg --get-selections') 26 | for l in out.split('\n'): 27 | name = re.split(r'\s+', l)[0] 28 | name = re.sub(r':.*', '', name) 29 | pkgs.add(name) 30 | return pkgs 31 | 32 | def get_pkg_attribute(pkg, attr, src, last): 33 | """ Returns attribute of Debian package. """ 34 | # Parse e.g. "Depends: libc6-dev | libc-dev, libacl1 (= 2.2.52-3), libattr1-dev (>= 1:2.4.46-8)" 35 | cmd = 'showsrc' if src else 'show' 36 | _, out, _ = run(f'apt-cache -q {cmd} {pkg}') 37 | lines = list(filter(lambda l: l.startswith(attr + ':'), out.split('\n'))) 38 | if last: 39 | lines = [lines[-1]] 40 | vals = [] 41 | for l in lines: 42 | l = re.sub(r'^[^:]*: *', '', l) 43 | l = re.sub(r'\([^)]*\)', '', l) 44 | l = l.strip() 45 | vals += re.split(r' *, *', l) 46 | return sorted(v for v in vals if v) 47 | 48 | def _get_pkg_files(pkg): 49 | """ Returns list of files that belong to a package. """ 50 | # Avoid large files and dummy /. 51 | _, out, _ = run(f'dpkg -L {pkg}') 52 | return list(filter(lambda f: not re.search(r'(\.gz|\.html?|\.)$', f), 53 | out.split('\n'))) 54 | 55 | _stdinc = ['', 56 | '-include stdint.h -include stddef.h', 57 | '-include stdio.h', 58 | '-include sys/types.h'] 59 | 60 | def _get_cflags(files, v=0): 61 | """ Collect CFLAGS from pkgconfigs. """ 62 | 63 | cflags = [] 64 | 65 | def add_variants(f): 66 | for lang in ['', ' -x c++']: 67 | for i in range(len(_stdinc) + 1): 68 | cflags.append(' '.join([lang, f] + _stdinc[:i])) 69 | 70 | for pc in filter(lambda pkg: pkg.endswith('.pc'), files): 71 | stem, _ = os.path.splitext(os.path.basename(pc)) 72 | os.environ['PKG_CONFIG_PATH'] = os.path.dirname(pc) 73 | _, out, _ = run(f'pkg-config --print-errors --cflags {stem}') 74 | out = out.strip() # Trailing \n 75 | if v: 76 | print(f'pkg-config output:\n{out}') 77 | if out: 78 | add_variants(out) 79 | # In case no .pc files are present 80 | if not cflags: 81 | add_variants('') 82 | 83 | return cflags 84 | 85 | def analyze_debian_package(pkg, permissive, v=0): 86 | """ Retuns erroneously exported private package symbolds. """ 87 | 88 | for exe in ['aptitude', 'apt-cache', 'pkg-config']: 89 | if not is_runnable(exe): 90 | error(f"'{exe}' not installed") 91 | 92 | # Install all binary packages 93 | def is_good_package(p): 94 | return (not p.endswith('-udeb') 95 | and not p.endswith('-doc') 96 | and not p.endswith('-dbg') 97 | and 'mingw-w64' not in p) 98 | pkgs = get_pkg_attribute(pkg, 'Binary', True, True) 99 | pkgs = list(filter(is_good_package, pkgs)) 100 | dev_pkgs = list(filter(lambda p: p.endswith('-dev'), pkgs)) 101 | if v > 0: 102 | print(f"{pkg} packages: {' '.join(pkgs)}") 103 | print(f"{pkg} dev packages: {' '.join(dev_pkgs)}") 104 | 105 | installed_pkgs = _get_installed_packages() 106 | 107 | files = [] 108 | for p in pkgs: # Install single package at a time, in case some of them are missing 109 | if p in installed_pkgs: 110 | if v > 0: 111 | print(f"Not installing package '{p}' (already installed)") 112 | else: 113 | if v > 0: 114 | print(f"Installing package '{p}'...") 115 | # Use Aptitude to automatically resolve conflicts. 116 | run(f'sudo aptitude install -y -q=2 {p}') # sudo apt-get install -qq -y $p 117 | files += _get_pkg_files(p) 118 | 119 | # TODO: use dedicated flags for files from each package 120 | cflags = _get_cflags(files, v) 121 | 122 | spurious_syms = analyze_package(pkg, files, cflags, permissive, v) 123 | if spurious_syms: 124 | print(f"The following exported symbols in package '{pkg}' are private:\n " 125 | + '\n '.join(spurious_syms)) 126 | else: 127 | print(f"No private exports in package '{pkg}'") 128 | 129 | return True 130 | -------------------------------------------------------------------------------- /shlibvischeck/analysis/elf.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright 2020-2022 Yury Gribov 4 | # 5 | # Use of this source code is governed by MIT license that can be 6 | # found in the LICENSE.txt file. 7 | 8 | """ 9 | APIs for analyzing ELF files 10 | """ 11 | 12 | import re 13 | import subprocess 14 | 15 | from shlibvischeck.common.process import * 16 | from shlibvischeck.common.error import error 17 | 18 | __all__ = ['read_binary_api'] 19 | 20 | def readelf(filename): 21 | """ Returns symbol table of ELF file, """ 22 | 23 | # --dyn-syms does not always work for some reason so dump all symtabs 24 | with subprocess.Popen(["readelf", "-sW", filename], 25 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: 26 | out, err = p.communicate() 27 | out = out.decode() 28 | err = err.decode() 29 | if p.returncode != 0 or err: 30 | error(f"readelf failed with retcode {p.returncode}: {err}") 31 | 32 | toc = None 33 | syms = [] 34 | for line in out.splitlines(): 35 | line = line.strip() 36 | if not line: 37 | # Next symtab 38 | toc = None 39 | continue 40 | words = re.split(r' +', line) 41 | if line.startswith('Num'): # Header? 42 | if toc is not None: 43 | error("multiple headers in output of readelf") 44 | toc = {} 45 | for i, n in enumerate(words): 46 | # Colons are different across readelf versions so get rid of them. 47 | n = n.replace(':', '') 48 | toc[i] = n 49 | elif toc is not None: 50 | sym = {k: (words[i] if i < len(words) else '') for i, k in toc.items()} 51 | name = sym['Name'] 52 | if '@' in name: 53 | sym['Default'] = '@@' in name 54 | name, ver = re.split(r'@+', name) 55 | sym['Name'] = name 56 | sym['Version'] = ver 57 | else: 58 | sym['Default'] = True 59 | sym['Version'] = None 60 | syms.append(sym) 61 | 62 | if toc is None: 63 | error(f"failed to analyze {filename}") 64 | 65 | return syms 66 | 67 | # These symbols are exported by nearly all shlibs 68 | # due to bugs in older (?) GNU toolchains. 69 | _spurious_syms = {'__bss_start', '_edata', '_init', '_fini', '_etext', '__etext', '_end'} 70 | 71 | def read_binary_api(filename, export, disallow_spurious, v=0): 72 | """ Returns functions exported from shlib. """ 73 | 74 | syms = readelf(filename) 75 | 76 | output_syms = [] 77 | for s in syms: 78 | name = s['Name'] 79 | ndx = s['Ndx'] 80 | is_defined = ndx != 'UND' 81 | is_exported = s['Bind'] != 'LOCAL' and s['Type'] != 'NOTYPE' 82 | allow_versioned = not export # If symbol is imported, we do not consider it's version 83 | if (name 84 | and ndx != 'ABS' 85 | and (is_defined and is_exported if export else not is_defined) 86 | and (not disallow_spurious or name not in _spurious_syms) 87 | and (s['Version'] is None or allow_versioned or s['Default'])): 88 | output_syms.append(name) 89 | output_syms = sorted(output_syms) 90 | 91 | if v > 0: 92 | print("%s symbols in %s:\n %s" 93 | % ("Exported" if export else "Imported", 94 | filename, 95 | '\n '.join(output_syms))) 96 | 97 | return output_syms 98 | -------------------------------------------------------------------------------- /shlibvischeck/analysis/header.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright 2020-2022 Yury Gribov 4 | # 5 | # Use of this source code is governed by MIT license that can be 6 | # found in the LICENSE.txt file. 7 | 8 | """ 9 | APIs for analyzing C headers 10 | """ 11 | 12 | import os 13 | import os.path 14 | import re 15 | 16 | from shlibvischeck.common.process import * 17 | from shlibvischeck.common.error import error 18 | 19 | __all__ = ['read_header_api'] 20 | 21 | def read_header_api(hdr, whitelist, cflags, v=0): 22 | """ Returns functions declared in header 23 | (and included headers from whitelist). """ 24 | 25 | if not cflags: 26 | cflags = [''] 27 | 28 | # Is this a helper header and so not intended for direct inclusion? 29 | is_helper = 'private' in hdr # E.g. json_object_private.h 30 | hdr_base = os.path.basename(hdr) 31 | for filename in whitelist: 32 | with open(filename, errors='ignore') as f: 33 | txt = f.read() 34 | if re.search(fr'^\s*#\s*include\s+[<"].*{hdr_base}[>"]', txt, re.M): 35 | is_helper = True 36 | 37 | errors = [] 38 | syms = [] 39 | 40 | for f in cflags: 41 | cmd = ['read_header_api', '--only', ' '.join(whitelist), '--cflags', f, hdr] 42 | rc, out, err = run(cmd, fatal=False) 43 | if rc == 0: 44 | syms = out.split('\n') 45 | break 46 | errors.append((' '.join(cmd), out, err)) 47 | 48 | if not syms and not is_helper: 49 | msgs = ["failed to parse:"] 50 | for cmd, out, err in errors: 51 | msgs.append(f"compiling '{cmd}':") 52 | msgs.append(err) 53 | error('\n'.join(msgs)) 54 | 55 | if v > 0 and syms: 56 | print("Public functions in header %s:\n %s" 57 | % (hdr, '\n '.join(syms))) 58 | 59 | return syms 60 | -------------------------------------------------------------------------------- /shlibvischeck/analysis/package.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020-2022 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | 8 | """ 9 | APIs for working with Debian packages 10 | """ 11 | 12 | import os 13 | import os.path 14 | import re 15 | import magic 16 | 17 | from shlibvischeck.analysis.header import * 18 | from shlibvischeck.analysis.elf import * 19 | from shlibvischeck.common.error import error 20 | from shlibvischeck.common.process import run 21 | 22 | __all__ = ['analyze_package'] 23 | 24 | def _get_paths(lang): 25 | """ Returns standard compiler paths. """ 26 | p = _get_paths.cache.get(lang) 27 | if p is not None: 28 | return p 29 | # cc = 'clang-5.0' 30 | cc = 'gcc' 31 | _, _, err = run(f'{cc} -E -v -x {lang} /dev/null') 32 | in_paths = False 33 | paths = [] 34 | for l in err.split('\n'): 35 | if 'search starts here' in l: 36 | in_paths = True 37 | if 'End of search list' in l: 38 | break 39 | if in_paths and l[0] == ' ': 40 | paths.append('-I' + l[1:]) 41 | p = ' '.join(paths) 42 | _get_paths.cache[lang] = p 43 | return p 44 | 45 | _get_paths.cache = {} 46 | 47 | def analyze_package(pkg, files, cflags, permissive, v=0): 48 | """ Returns erroneously exported private symbols in package shlibs. """ 49 | 50 | shlibs = [] 51 | hdrs = [] 52 | elfs = [] 53 | 54 | for f in files: 55 | if os.path.isfile(f): 56 | _, ext = os.path.splitext(f) 57 | if ext in {'.h', '.hpp', '.H'}: 58 | hdrs.append(f) 59 | typ = magic.from_file(f) 60 | if (re.search(r'^ELF.*shared object', typ) 61 | # Need more checks to avoid PIEs 62 | and '/bin/' not in f): 63 | shlibs.append(f) 64 | if re.search(r'^ELF.*(shared object|executable)', typ): 65 | elfs.append(f) 66 | 67 | if not hdrs: 68 | error("failed to locate headers") 69 | if not shlibs: 70 | error("failed to locate any shared libs") 71 | 72 | if v > 0: 73 | print(f"Package headers: {' '.join(hdrs)}") 74 | print(f"Package shlibs: {' '.join(hdrs)}") 75 | 76 | # Need to add std paths 77 | c_paths = _get_paths('c') 78 | cxx_paths = _get_paths('c++') 79 | full_cflags = list(map(lambda f: (cxx_paths if 'c++' in f else c_paths) + ' ' + f, cflags)) 80 | 81 | # Collect header interfaces 82 | public_api = set() 83 | for h in hdrs: 84 | public_api.update(read_header_api(h, hdrs, full_cflags, v)) 85 | 86 | # Collect binary interfaces 87 | exported_api = set() 88 | for f in shlibs: 89 | exported_api.update(read_binary_api(f, True, permissive, v)) 90 | if v > 0: 91 | print("All exported APIs in package '%s':\n %s" % (pkg, '\n '.join(exported_api))) 92 | 93 | # Ignore private interfaces used by other modules in same package 94 | imported_api = set() 95 | for f in elfs: 96 | imported_api.update(read_binary_api(f, False, permissive)) 97 | if v > 0: 98 | print("All imported APIs in package '%s':\n %s" % (pkg, '\n '.join(imported_api))) 99 | exported_api.difference_update(imported_api) 100 | 101 | # TODO: for C++ we may want to ignore vtables, RTTI and C++ default methods 102 | # (as libclang does not report them). 103 | # TODO: warn about missing APIs? 104 | 105 | return sorted(list(exported_api - public_api)) 106 | -------------------------------------------------------------------------------- /shlibvischeck/common/__init__.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | -------------------------------------------------------------------------------- /shlibvischeck/common/error.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020-2022 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | 8 | """ 9 | Error message APIs 10 | """ 11 | 12 | import sys 13 | import os.path 14 | 15 | _except = False 16 | _me = os.path.basename(sys.argv[0]) 17 | 18 | def warn(msg): 19 | "Emit nicely-formatted warning" 20 | sys.stderr.write(f"{_me}: warning: {msg}\n") 21 | 22 | def error(msg): 23 | "Emit nicely-formatted error and abort" 24 | sys.stderr.write(f"{_me}: error: {msg}\n") 25 | if _except: 26 | raise RuntimeError 27 | sys.exit(1) 28 | 29 | def set_basename(name): 30 | "Set basename for error/warning reports" 31 | global _me # pylint: disable=global-statement 32 | _me = name 33 | 34 | def set_throw_on_error(v=True): 35 | "Set behavior on error" 36 | global _except # pylint: disable=global-statement 37 | _except = v 38 | -------------------------------------------------------------------------------- /shlibvischeck/common/process.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 2020-2022 Yury Gribov 4 | # 5 | # Use of this source code is governed by The MIT License (MIT) 6 | # that can be found in the LICENSE.txt file. 7 | 8 | """ 9 | APIs for handling processes 10 | """ 11 | 12 | import subprocess 13 | 14 | from shlibvischeck.common.error import error 15 | 16 | __all__ = ['run', 'is_runnable'] 17 | 18 | def run(cmd, fatal=True): 19 | """ Simple wrapper for subprocess. """ 20 | 21 | if isinstance(cmd, str): 22 | cmd = cmd.split(' ') 23 | # print(cmd) 24 | with subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE, 25 | stderr=subprocess.PIPE) as p: 26 | out, err = p.communicate() 27 | out = out.decode() 28 | err = err.decode() 29 | if fatal and p.returncode != 0: 30 | error(f"'{cmd}' failed:\n{out}{err}") 31 | return p.returncode, out, err 32 | 33 | def is_runnable(name): 34 | """ Check if program is present in path. """ 35 | rc, _, _ = run([name, "--help"], fatal=False) 36 | return rc == 0 37 | -------------------------------------------------------------------------------- /src/read_header_api.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (c) 2018-2022 Yury Gribov 5 | * 6 | * Use of this source code is governed by The MIT License (MIT) 7 | * that can be found in the LICENSE.txt file. 8 | */ 9 | 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | struct Symbol { 25 | std::string Name, MangledName; 26 | }; 27 | 28 | struct InterfaceInfo { 29 | int Verbose; 30 | const std::string &Root; 31 | const std::set OnlyHdrs; 32 | std::vector Syms; 33 | bool HasClasses; 34 | InterfaceInfo(int Verbose, const std::string &Root, 35 | const std::set &OnlyHdrs) 36 | : Verbose(Verbose), Root(Root), OnlyHdrs(OnlyHdrs), HasClasses(false) {} 37 | }; 38 | 39 | static std::string ToStr(CXString CXS) { 40 | const char *CStr = clang_getCString(CXS); 41 | std::string Str(CStr); 42 | clang_disposeString(CXS); 43 | return Str; 44 | } 45 | 46 | static std::string RealPath(const char *p) { 47 | char *tmp = realpath(p, 0); 48 | if (!tmp) { 49 | fprintf(stderr, "File %s not found\n", p); 50 | exit(1); 51 | } 52 | std::string Res(tmp); 53 | free(tmp); 54 | return Res; 55 | } 56 | 57 | static std::string ParseLoc(CXSourceLocation Loc, unsigned *Line, 58 | bool Expansion) { 59 | CXFile F; 60 | (Expansion ? clang_getExpansionLocation 61 | : clang_getSpellingLocation)(Loc, &F, Line, 0, 0); 62 | return RealPath(ToStr(clang_getFileName(F)).c_str()); 63 | } 64 | 65 | static bool IsUnderRoot(const std::string &Filename, const std::string &Root) { 66 | return 0 == Filename.compare(0, Root.size(), Root); 67 | } 68 | 69 | static enum CXChildVisitResult collectDecls(CXCursor C, CXCursor Parent, 70 | CXClientData Data) { 71 | (void)Parent; 72 | InterfaceInfo *Info = (InterfaceInfo *)Data; 73 | 74 | #if 0 75 | { 76 | std::string KindStr = ToStr(clang_getCursorKindSpelling(clang_getCursorKind(C))); 77 | fprintf(stderr, "Got %s\n", KindStr.c_str()); 78 | } 79 | #endif 80 | 81 | // Skip host includes 82 | CXSourceLocation Loc = clang_getCursorLocation(C); 83 | unsigned Line; 84 | std::string ExpFilename = ParseLoc(Loc, &Line, true); 85 | std::string SpellFilename = ParseLoc(Loc, &Line, false); 86 | if (!IsUnderRoot(SpellFilename, Info->Root)) { 87 | if (Info->Verbose) 88 | fprintf(stderr, "note: skipping declaration at %s:%u (not under %s)\n", 89 | SpellFilename.c_str(), Line, Info->Root.c_str()); 90 | return CXChildVisit_Continue; 91 | } else if (!Info->OnlyHdrs.empty() && !Info->OnlyHdrs.count(SpellFilename)) { 92 | if (Info->Verbose) 93 | fprintf(stderr, 94 | "note: skipping declaration at %s:%u (not in list of headers)\n", 95 | SpellFilename.c_str(), Line); 96 | return CXChildVisit_Continue; 97 | } 98 | 99 | #if 0 100 | fprintf(stderr, "In file %s:%u\n", ExpFilename.c_str(), Line); 101 | #endif 102 | 103 | switch (C.kind) { 104 | // TODO: handle C++ methods 105 | // TODO: ignore static functions/vars 106 | case CXCursor_FunctionDecl: 107 | case CXCursor_CXXMethod: 108 | case CXCursor_Constructor: 109 | case CXCursor_Destructor: 110 | case CXCursor_VarDecl: { 111 | std::string Name = ToStr(clang_getCursorSpelling(C)), 112 | MangledName = ToStr(clang_Cursor_getMangling(C)); 113 | if (Info->Verbose) 114 | fprintf(stderr, "note: found symbol %s (expanded in %s, spelled in %s)\n", 115 | Name.c_str(), ExpFilename.c_str(), SpellFilename.c_str()); 116 | Info->Syms.push_back({Name, MangledName}); 117 | break; 118 | } 119 | case CXCursor_ClassDecl: 120 | Info->HasClasses = true; 121 | return CXChildVisit_Recurse; 122 | case CXCursor_Namespace: 123 | return CXChildVisit_Recurse; 124 | default: 125 | break; 126 | } 127 | 128 | return CXChildVisit_Continue; 129 | } 130 | 131 | static void usage(const char *prog) { 132 | printf("\ 133 | Usage: %s [OPT]... HDR...\n\ 134 | Print APIs provided by header(s).\n\ 135 | \n\ 136 | Options:\n\ 137 | -h, --help Print this help and exit.\n\ 138 | -v, --verbose Enable debug prints.\n\ 139 | -c FLAGS, --cflags FLAGS Specify CFLAGS to use.\n\ 140 | -r ROOT, --root ROOT Only consider symbols which are defined\n\ 141 | in files under ROOT.\n\ 142 | --only A.H,B.H,... Report only functions in these headers.\n\ 143 | --only-args Report only functions in HDRs\n\ 144 | ", 145 | prog); 146 | exit(0); 147 | } 148 | 149 | int main(int argc, char *argv[]) { 150 | const char *me = basename((char *)argv[0]); 151 | 152 | std::string Flags, Root; 153 | std::set OnlyHdrs; 154 | int Verbose = 0; 155 | bool OnlyArgs = false; 156 | while (1) { 157 | static struct option long_opts[] = { 158 | {"verbose", no_argument, 0, 'v'}, 159 | {"cflags", required_argument, 0, 'c'}, 160 | {"root", required_argument, 0, 'r'}, 161 | {"help", required_argument, 0, 'h'}, 162 | #define OPT_ONLY 1 163 | {"only", required_argument, 0, OPT_ONLY}, 164 | #define OPT_ONLY_ARGS 2 165 | {"only-args", no_argument, 0, OPT_ONLY_ARGS}, 166 | }; 167 | 168 | int opt_index = 0; 169 | int c = getopt_long(argc, argv, "vc:r:h", long_opts, &opt_index); 170 | 171 | if (c == -1) 172 | break; 173 | 174 | switch (c) { 175 | case 'v': 176 | ++Verbose; 177 | break; 178 | case 'c': 179 | Flags = optarg; 180 | break; 181 | case 'r': 182 | if (access(optarg, F_OK) == -1) { 183 | fprintf(stderr, "Directory %s does not exist\n", optarg); 184 | exit(1); 185 | } 186 | Root = RealPath(optarg); 187 | break; 188 | case 'h': 189 | usage(me); 190 | break; 191 | case OPT_ONLY: { 192 | char *hdrs = optarg; 193 | char *rest = NULL; 194 | while (char *hdr = strtok_r(hdrs, " \t,", &rest)) { 195 | hdrs = NULL; 196 | OnlyHdrs.insert(RealPath(hdr)); 197 | } 198 | break; 199 | } 200 | case OPT_ONLY_ARGS: 201 | OnlyArgs = true; 202 | OnlyHdrs.clear(); 203 | break; 204 | default: 205 | abort(); 206 | } 207 | } 208 | 209 | std::vector FlagsArray; 210 | for (size_t End = 0; End < std::string::npos;) { 211 | size_t Begin = Flags.find_first_not_of(" \t", End); 212 | if (Begin == std::string::npos) 213 | break; 214 | End = Flags.find_first_of(" \t", Begin + 1); 215 | if (End != std::string::npos) 216 | Flags[End++] = 0; 217 | FlagsArray.push_back(&Flags[Begin]); 218 | } 219 | 220 | if (optind >= argc) { 221 | fprintf(stderr, "error: no headers specified in command line..."); 222 | return 0; 223 | } 224 | 225 | if (OnlyArgs) { 226 | for (int I = optind; I < argc; ++I) 227 | OnlyHdrs.insert(RealPath(argv[I])); 228 | } 229 | 230 | std::set Names; 231 | for (int I = optind; I < argc; ++I) { 232 | std::string Hdr = argv[I]; 233 | 234 | CXIndex Idx = clang_createIndex(0, 0); 235 | CXTranslationUnit Unit = clang_parseTranslationUnit( 236 | Idx, Hdr.c_str(), FlagsArray.empty() ? 0 : &FlagsArray[0], 237 | FlagsArray.size(), 0, 0, CXTranslationUnit_None); 238 | if (!Unit) { 239 | fprintf(stderr, "error: failed to read file %s\n", Hdr.c_str()); 240 | exit(1); 241 | } 242 | 243 | bool AnyError = false; 244 | for (unsigned J = 0, N = clang_getNumDiagnostics(Unit); J != N; ++J) { 245 | CXDiagnostic D = clang_getDiagnostic(Unit, J); 246 | clang_disposeDiagnostic(D); 247 | CXDiagnosticSeverity S = clang_getDiagnosticSeverity(D); 248 | if (S >= CXDiagnostic_Error || (S >= CXDiagnostic_Warning && Verbose)) { 249 | std::string Msg = ToStr( 250 | clang_formatDiagnostic(D, clang_defaultDiagnosticDisplayOptions())); 251 | fprintf(stderr, "%s\n", Msg.c_str()); 252 | } 253 | AnyError |= S >= CXDiagnostic_Error; 254 | } 255 | 256 | if (AnyError) 257 | return 1; 258 | 259 | std::string HdrRoot = Root; 260 | if (HdrRoot.empty()) { 261 | size_t J = Hdr.find("/include/"); 262 | if (J == std::string::npos) 263 | fprintf(stderr, "Failed to determine include root for %s\n", 264 | Hdr.c_str()); 265 | else 266 | HdrRoot = Hdr.substr(0, J) + "/include"; 267 | } 268 | 269 | InterfaceInfo Info(Verbose, HdrRoot, OnlyHdrs); 270 | clang_visitChildren(clang_getTranslationUnitCursor(Unit), collectDecls, 271 | (CXClientData)&Info); 272 | 273 | if (Info.HasClasses) 274 | fprintf(stderr, "warning: C++ is not fully supported, interface info may " 275 | "be incomplete\n"); 276 | 277 | if (Verbose) { 278 | fprintf(stderr, "APIs exported from %s:\n", Hdr.c_str()); 279 | for (auto &Sym : Info.Syms) { 280 | fprintf(stderr, " %s (%s)\n", Sym.MangledName.c_str(), 281 | Sym.Name.c_str()); 282 | } 283 | } 284 | 285 | for (auto &Sym : Info.Syms) 286 | Names.insert(Sym.MangledName); 287 | 288 | clang_disposeTranslationUnit(Unit); 289 | clang_disposeIndex(Idx); 290 | } 291 | 292 | for (auto &Name : Names) 293 | printf("%s\n", Name.c_str()); 294 | 295 | return 0; 296 | } 297 | -------------------------------------------------------------------------------- /tests/basic/a.ref: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yugr/ShlibVisibilityChecker/5fd18f9191c175a70055be66eef128352a6eebdd/tests/basic/a.ref -------------------------------------------------------------------------------- /tests/basic/ab.ref: -------------------------------------------------------------------------------- 1 | b 2 | -------------------------------------------------------------------------------- /tests/basic/b.ref: -------------------------------------------------------------------------------- 1 | a 2 | b 3 | -------------------------------------------------------------------------------- /tests/basic/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2021-2022 Yury Gribov 4 | # 5 | # The MIT License (MIT) 6 | # 7 | # Use of this source code is governed by MIT license that can be 8 | # found in the LICENSE.txt file. 9 | 10 | # This is a simple test for ShlibVisibilityChecker functionality. 11 | 12 | set -eu 13 | 14 | cd $(dirname $0) 15 | 16 | if test -n "${GITHUB_ACTIONS:-}"; then 17 | set -x 18 | fi 19 | 20 | CFLAGS='-g -O2 -Wall -Wextra -Werror -shared -fPIC' 21 | 22 | ROOT=$PWD/../.. 23 | 24 | $ROOT/bin/read_header_api -r. xyz.h > api.txt 25 | 26 | errors=0 27 | for name_flags in 'a;-DA' 'b;-DB' 'ab;-DA -DB'; do 28 | name=${name_flags%;*} 29 | flags=${name_flags#*;} 30 | 31 | ${CC:-gcc} $CFLAGS $flags -shared -fPIC xyz.c -o libxyz.so 32 | ${PYTHON:-python3} $ROOT/read_binary_api --permissive libxyz.so > abi.txt 33 | 34 | (comm -3 api.txt abi.txt || true) > api_abi.diff 35 | if ! diff -q $name.ref api_abi.diff; then 36 | echo >&2 "Invalid results for test $name" 37 | diff $name.ref api_abi.diff >&2 38 | errors=1 39 | fi 40 | done 41 | 42 | if test $errors = 0; then 43 | echo SUCCESS 44 | else 45 | echo FAIL 46 | fi 47 | -------------------------------------------------------------------------------- /tests/basic/xyz.c: -------------------------------------------------------------------------------- 1 | /* * Copyright 2021-2022 Yury Gribov 2 | * 3 | * The MIT License (MIT) 4 | *▫ 5 | * Use of this source code is governed by MIT license that can be 6 | * found in the LICENSE.txt file. 7 | */ 8 | 9 | #ifdef A 10 | void a() {} 11 | #endif 12 | 13 | #ifdef B 14 | void b() {} 15 | #endif 16 | -------------------------------------------------------------------------------- /tests/basic/xyz.h: -------------------------------------------------------------------------------- 1 | /* * Copyright 2021-2022 Yury Gribov 2 | * 3 | * The MIT License (MIT) 4 | *▫ 5 | * Use of this source code is governed by MIT license that can be 6 | * found in the LICENSE.txt file. 7 | */ 8 | 9 | #ifndef XYZ_H 10 | #define XYZ_H 11 | 12 | extern void a(); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /tests/debian/libbz2-1.0.ref: -------------------------------------------------------------------------------- 1 | Processing package 'libbz2-1.0'... 2 | The following exported symbols in package 'libbz2-1.0' are private: 3 | BZ2_blockSort 4 | BZ2_bsInitWrite 5 | BZ2_bz__AssertH__fail 6 | BZ2_compressBlock 7 | BZ2_crc32Table 8 | BZ2_decompress 9 | BZ2_hbAssignCodes 10 | BZ2_hbCreateDecodeTables 11 | BZ2_hbMakeCodeLengths 12 | BZ2_indexIntoF 13 | BZ2_rNums 14 | -------------------------------------------------------------------------------- /tests/debian/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2021-2022 Yury Gribov 4 | # 5 | # The MIT License (MIT) 6 | # 7 | # Use of this source code is governed by MIT license that can be 8 | # found in the LICENSE.txt file. 9 | 10 | # This is a test for Debian package functionality: 11 | # verify that libbz2 has extra symbols in its inteface. 12 | 13 | set -eu 14 | 15 | cd $(dirname $0) 16 | 17 | if test -n "${GITHUB_ACTIONS:-}"; then 18 | set -x 19 | fi 20 | 21 | CFLAGS='-g -O2 -Wall -Wextra -Werror -shared -fPIC' 22 | 23 | T=$PWD 24 | ROOT=$T/../.. 25 | PATH=$ROOT/bin:$PATH 26 | 27 | # Debian #896750 will likely never be fixed 28 | PKGS=libbz2-1.0 29 | 30 | errors=0 31 | for pkg in $PKGS; do 32 | ${PYTHON:-python3} $ROOT/shlibvischeck-debian --permissive $pkg > out.log 33 | 34 | if ! diff -q $pkg.ref out.log; then 35 | echo >&2 "Invalid results for package" 36 | diff $pkg.ref out.log >&2 37 | errors=1 38 | fi 39 | done 40 | 41 | if test $errors = 0; then 42 | echo SUCCESS 43 | else 44 | echo FAIL 45 | fi 46 | -------------------------------------------------------------------------------- /tests/only/other.h: -------------------------------------------------------------------------------- 1 | /* * Copyright 2022 Yury Gribov 2 | * 3 | * The MIT License (MIT) 4 | *▫ 5 | * Use of this source code is governed by MIT license that can be 6 | * found in the LICENSE.txt file. 7 | */ 8 | 9 | #ifndef XYZ_H 10 | #define XYZ_H 11 | 12 | extern void b(); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /tests/only/out.ref: -------------------------------------------------------------------------------- 1 | b 2 | -------------------------------------------------------------------------------- /tests/only/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2022 Yury Gribov 4 | # 5 | # The MIT License (MIT) 6 | # 7 | # Use of this source code is governed by MIT license that can be 8 | # found in the LICENSE.txt file. 9 | 10 | # This is a simple test for --only/--only-args functionality: 11 | # function b() is not considered as belonging to libxyz interface. 12 | 13 | set -eu 14 | 15 | cd $(dirname $0) 16 | 17 | if test -n "${GITHUB_ACTIONS:-}"; then 18 | set -x 19 | fi 20 | 21 | CFLAGS='-g -O2 -Wall -Wextra -Werror -shared -fPIC' 22 | 23 | ROOT=$PWD/../.. 24 | 25 | errors=0 26 | ${CC:-gcc} $CFLAGS -shared -fPIC xyz.c -o libxyz.so 27 | ${PYTHON:-python3} $ROOT/read_binary_api --permissive libxyz.so > abi.txt 28 | 29 | $ROOT/bin/read_header_api -r. --only xyz.h xyz.h > api.txt 30 | (comm -3 api.txt abi.txt || true) > api_abi.diff 31 | if ! diff -q out.ref api_abi.diff; then 32 | echo >&2 "Invalid results for --only test" 33 | diff $name.ref api_abi.diff >&2 34 | errors=1 35 | fi 36 | 37 | $ROOT/bin/read_header_api -r. --only-args xyz.h xyz.h > api.txt 38 | (comm -3 api.txt abi.txt || true) > api_abi.diff 39 | if ! diff -q out.ref api_abi.diff; then 40 | echo >&2 "Invalid results for --only test" 41 | diff $name.ref api_abi.diff >&2 42 | errors=1 43 | fi 44 | 45 | if test $errors = 0; then 46 | echo SUCCESS 47 | else 48 | echo FAIL 49 | fi 50 | -------------------------------------------------------------------------------- /tests/only/xyz.c: -------------------------------------------------------------------------------- 1 | /* * Copyright 2022 Yury Gribov 2 | * 3 | * The MIT License (MIT) 4 | *▫ 5 | * Use of this source code is governed by MIT license that can be 6 | * found in the LICENSE.txt file. 7 | */ 8 | 9 | void a() {} 10 | 11 | void b() {} 12 | -------------------------------------------------------------------------------- /tests/only/xyz.h: -------------------------------------------------------------------------------- 1 | /* * Copyright 2022 Yury Gribov 2 | * 3 | * The MIT License (MIT) 4 | *▫ 5 | * Use of this source code is governed by MIT license that can be 6 | * found in the LICENSE.txt file. 7 | */ 8 | 9 | #ifndef XYZ_H 10 | #define XYZ_H 11 | 12 | #include "other.h" 13 | 14 | extern void a(); 15 | 16 | #endif 17 | --------------------------------------------------------------------------------