├── .dockerignore ├── .github └── workflows │ └── release.yaml ├── .gitignore ├── 3.7.Dockerfile ├── 3.7.ignore_some_tests.py ├── 3.7.patches ├── 01_python_ssl_module_add_android_certificates └── series ├── 3.7.sh └── README.md /.dockerignore: -------------------------------------------------------------------------------- 1 | output 2 | .git 3 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | # Run this Action on creating a new tag. 2 | on: 3 | push: 4 | tags: 5 | - 'v*' 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: Build project 12 | run: bash -x 3.7.sh 13 | - name: Create Release 14 | id: create_release 15 | uses: actions/create-release@v1 16 | env: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | with: 19 | tag_name: ${{ github.ref }} 20 | release_name: Release ${{ github.ref }} 21 | draft: false 22 | prerelease: true 23 | - name: Upload Release Asset 24 | uses: actions/upload-release-asset@v1 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | with: 28 | upload_url: ${{ steps.create_release.outputs.upload_url }} 29 | asset_path: ./output/3.7.zip 30 | asset_name: 3.7.zip 31 | asset_content_type: application/zip 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | output 3 | download-cache 4 | .vscode 5 | -------------------------------------------------------------------------------- /3.7.Dockerfile: -------------------------------------------------------------------------------- 1 | # The toolchain container encodes environment 2 | # downloads essential dependencies. 3 | FROM ubuntu:18.04 as toolchain 4 | ENV DEBIAN_FRONTEND=noninteractive 5 | RUN apt-get update -qq && apt-get -qq install unzip 6 | 7 | # Install toolchains: Android NDK & Java JDK. 8 | WORKDIR /opt/ndk 9 | ADD download-cache/android-ndk-r20b-linux-x86_64.zip . 10 | RUN unzip -q android-ndk-r20b-linux-x86_64.zip && rm android-ndk-r20b-linux-x86_64.zip 11 | ENV NDK /opt/ndk/android-ndk-r20b 12 | WORKDIR /opt/jdk 13 | ADD download-cache/OpenJDK8U-jdk_x64_linux_hotspot_8u242b08.tar.gz . 14 | ENV JAVA_HOME /opt/jdk/jdk8u242-b08/ 15 | ENV PATH "/opt/jdk/jdk8u242-b08/bin:${PATH}" 16 | 17 | # Store output here; the directory structure corresponds to our Android app template. 18 | ENV APPROOT /opt/python-build/approot 19 | # Do our Python build work here 20 | ENV BUILD_HOME "/opt/python-build" 21 | ENV PYTHON_INSTALL_DIR="$BUILD_HOME/built/python" 22 | WORKDIR /opt/python-build 23 | 24 | # Configure build variables 25 | ENV HOST_TAG="linux-x86_64" 26 | ARG TARGET_ABI_SHORTNAME 27 | ENV TARGET_ABI_SHORTNAME $TARGET_ABI_SHORTNAME 28 | ARG ANDROID_API_LEVEL 29 | ENV ANDROID_API_LEVEL $ANDROID_API_LEVEL 30 | ENV JNI_LIBS $APPROOT/app/libs/${TARGET_ABI_SHORTNAME} 31 | ARG TOOLCHAIN_TRIPLE 32 | ENV TOOLCHAIN_TRIPLE $TOOLCHAIN_TRIPLE 33 | ENV TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/$HOST_TAG 34 | ARG COMPILER_TRIPLE 35 | ENV COMPILER_TRIPLE=$COMPILER_TRIPLE 36 | ENV AR=$TOOLCHAIN/bin/$TOOLCHAIN_TRIPLE-ar \ 37 | AS=$TOOLCHAIN/bin/$TOOLCHAIN_TRIPLE-as \ 38 | CC=$TOOLCHAIN/bin/${COMPILER_TRIPLE}-clang \ 39 | CXX=$TOOLCHAIN/bin/${COMPILER_TRIPLE}-clang++ \ 40 | LD=$TOOLCHAIN/bin/$TOOLCHAIN_TRIPLE-ld \ 41 | RANLIB=$TOOLCHAIN/bin/$TOOLCHAIN_TRIPLE-ranlib \ 42 | STRIP=$TOOLCHAIN/bin/$TOOLCHAIN_TRIPLE-strip \ 43 | READELF=$TOOLCHAIN/bin/$TOOLCHAIN_TRIPLE-readelf \ 44 | CFLAGS="-fPIC -Wall -O0 -g" 45 | 46 | # We build sqlite using a tarball from Ubuntu. We need to patch config.sub & config.guess so 47 | # autoconf can accept our weird TOOLCHAIN_TRIPLE value. It requires tcl8.6-dev and build-essential 48 | # because the compile process build and executes some commands on the host as part of the build process. 49 | # We hard-code avoid_version=yes into libtool so that libsqlite3.so is the SONAME. 50 | FROM toolchain as build_sqlite 51 | RUN apt-get update -qq && apt-get -qq install make autoconf autotools-dev tcl8.6-dev build-essential 52 | ADD download-cache/sqlite3_3.11.0.orig.tar.xz . 53 | RUN cd sqlite3-3.11.0 && autoreconf && cp -f /usr/share/misc/config.sub . && cp -f /usr/share/misc/config.guess . 54 | RUN cd sqlite3-3.11.0 && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$BUILD_HOME/built/sqlite" 55 | RUN cd sqlite3-3.11.0 && sed -i -E 's,avoid_version=no,avoid_version=yes,' ltmain.sh libtool 56 | RUN cd sqlite3-3.11.0 && make install 57 | 58 | # Install bzip2 & lzma libraries, for stdlib's _bzip2 and _lzma modules. 59 | FROM toolchain as build_xz 60 | RUN apt-get update -qq && apt-get -qq install make 61 | ADD download-cache/xz-5.2.4.tar.gz . 62 | ENV LIBXZ_INSTALL_DIR="$BUILD_HOME/built/xz" 63 | RUN mkdir -p "$LIBXZ_INSTALL_DIR" 64 | RUN cd xz-5.2.4 && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBXZ_INSTALL_DIR" 65 | RUN cd xz-5.2.4 && make install 66 | 67 | FROM toolchain as build_bz2 68 | RUN apt-get update -qq && apt-get -qq install make 69 | ENV LIBBZ2_INSTALL_DIR="$BUILD_HOME/built/libbz2" 70 | ADD download-cache/bzip2-1.0.8.tar.gz . 71 | RUN mkdir -p "$LIBBZ2_INSTALL_DIR" && \ 72 | cd bzip2-1.0.8 && \ 73 | sed -i -e 's,[.]1[.]0.8,,' -e 's,[.]1[.]0,,' -e 's,ln -s,#ln -s,' -e 's,rm -f libbz2.so,#rm -f libbz2.so,' -e 's,^CC=,#CC=,' Makefile-libbz2_so 74 | RUN cd bzip2-1.0.8 && make -f Makefile-libbz2_so 75 | RUN mkdir -p "${LIBBZ2_INSTALL_DIR}/lib" 76 | RUN cp bzip2-1.0.8/libbz2.so "${LIBBZ2_INSTALL_DIR}/lib" 77 | RUN mkdir -p "${LIBBZ2_INSTALL_DIR}/include" 78 | RUN cp bzip2-1.0.8/bzlib.h "${LIBBZ2_INSTALL_DIR}/include" 79 | 80 | # libffi is required by ctypes 81 | FROM toolchain as build_libffi 82 | RUN apt-get update -qq && apt-get -qq install file make 83 | ADD download-cache/libffi-3.3.tar.gz . 84 | ENV LIBFFI_INSTALL_DIR="$BUILD_HOME/built/libffi" 85 | RUN mkdir -p "$LIBFFI_INSTALL_DIR" 86 | RUN cd libffi-3.3 && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBFFI_INSTALL_DIR" 87 | RUN cd libffi-3.3 && make install 88 | 89 | FROM toolchain as build_openssl 90 | # OpenSSL requires libfindlibs-libs-perl. make is nice, too. 91 | RUN apt-get update -qq && apt-get -qq install libfindbin-libs-perl make 92 | ADD download-cache/openssl-1.1.1d.tar.gz . 93 | ARG OPENSSL_BUILD_TARGET 94 | RUN cd openssl-1.1.1d && ANDROID_NDK_HOME="$NDK" ./Configure ${OPENSSL_BUILD_TARGET} -D__ANDROID_API__="$ANDROID_API_LEVEL" --prefix="$BUILD_HOME/built/openssl" --openssldir="$BUILD_HOME/built/openssl" 95 | RUN cd openssl-1.1.1d && make SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so' 96 | RUN cd openssl-1.1.1d && make install SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so' 97 | 98 | # This build container builds Python, rubicon-java, and any dependencies. 99 | FROM toolchain as build_python 100 | RUN apt-get update -qq && apt-get -qq install python3.7 pkg-config zip quilt 101 | 102 | # Get libs & vars 103 | COPY --from=build_openssl /opt/python-build/built/openssl /opt/python-build/built/openssl 104 | COPY --from=build_bz2 /opt/python-build/built/libbz2 /opt/python-build/built/libbz2 105 | COPY --from=build_xz /opt/python-build/built/xz /opt/python-build/built/xz 106 | COPY --from=build_libffi /opt/python-build/built/libffi /opt/python-build/built/libffi 107 | COPY --from=build_sqlite /opt/python-build/built/sqlite /opt/python-build/built/sqlite 108 | 109 | ENV OPENSSL_INSTALL_DIR=/opt/python-build/built/openssl 110 | ENV LIBBZ2_INSTALL_DIR="$BUILD_HOME/built/libbz2" 111 | ENV LIBXZ_INSTALL_DIR="$BUILD_HOME/built/xz" 112 | RUN mkdir -p "$JNI_LIBS" && cp -a "$OPENSSL_INSTALL_DIR"/lib/*.so "$LIBBZ2_INSTALL_DIR"/lib/*.so /opt/python-build/built/libffi/lib/*.so /opt/python-build/built/xz/lib/*.so /opt/python-build/built/sqlite/lib/*.so "$JNI_LIBS" 113 | ENV PKG_CONFIG_PATH="/opt/python-build/built/libffi/lib/pkgconfig:/opt/python-build/built/xz/lib/pkgconfig" 114 | 115 | # Download & patch Python 116 | ADD download-cache/Python-3.7.6.tar.xz . 117 | # Modify ./configure so that, even though this is Linux, it does not append .1.0 to the .so file. 118 | RUN sed -i -e 's,INSTSONAME="$LDLIBRARY".$SOVERSION,,' Python-3.7.6/configure 119 | # Apply a C extensions linker hack; already fixed in Python 3.8+; see https://github.com/python/cpython/commit/254b309c801f82509597e3d7d4be56885ef94c11 120 | RUN sed -i -e s,'libraries or \[\],\["python3.7m"] + libraries if libraries else \["python3.7m"\],' Python-3.7.6/Lib/distutils/extension.py 121 | # Apply a hack to get the NDK library paths into the Python build. TODO(someday): Discuss with e.g. Kivy and see how to remove this. 122 | RUN sed -i -e "s# dirs = \[\]# dirs = \[os.environ.get('SYSROOT_INCLUDE'), os.environ.get('SYSROOT_LIB')\]#" Python-3.7.6/setup.py 123 | # Apply a hack to get the sqlite include path into setup.py. TODO(someday): Discuss with upstream Python if we can use pkg-config for sqlite. 124 | RUN sed -i -E 's,sqlite_inc_paths = [[][]],sqlite_inc_paths = ["/opt/python-build/built/sqlite/include"],' Python-3.7.6/setup.py 125 | # Apply a hack to make platform.py stop looking for a libc version. 126 | RUN sed -i -e "s#Linux#DisabledLinuxCheck#" Python-3.7.6/Lib/platform.py 127 | 128 | # Build Python, pre-configuring some values so it doesn't check if those exist. 129 | ENV SYSROOT_LIB=${TOOLCHAIN}/sysroot/usr/lib/${TOOLCHAIN_TRIPLE}/${ANDROID_API_LEVEL}/ \ 130 | SYSROOT_INCLUDE=${NDK}/sysroot/usr/include/ 131 | RUN cd Python-3.7.6 && LDFLAGS="$(pkg-config --libs-only-L libffi) $(pkg-config --libs-only-L liblzma) -L${LIBBZ2_INSTALL_DIR}/lib -L$OPENSSL_INSTALL_DIR/lib" \ 132 | CFLAGS="${CFLAGS} -I${LIBBZ2_INSTALL_DIR}/include $(pkg-config --cflags-only-I libffi) $(pkg-config --cflags-only-I liblzma) " \ 133 | ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --enable-shared \ 134 | --enable-ipv6 ac_cv_file__dev_ptmx=yes \ 135 | --with-openssl=$OPENSSL_INSTALL_DIR \ 136 | ac_cv_file__dev_ptc=no --without-ensurepip ac_cv_little_endian_double=yes \ 137 | --prefix="$PYTHON_INSTALL_DIR" \ 138 | ac_cv_func_setuid=no ac_cv_func_seteuid=no ac_cv_func_setegid=no ac_cv_func_getresuid=no ac_cv_func_setresgid=no ac_cv_func_setgid=no ac_cv_func_sethostname=no ac_cv_func_setresuid=no ac_cv_func_setregid=no ac_cv_func_setreuid=no ac_cv_func_getresgid=no ac_cv_func_setregid=no ac_cv_func_clock_settime=no ac_cv_header_termios_h=no ac_cv_func_sendfile=no ac_cv_header_spawn_h=no ac_cv_func_posix_spawn=no \ 139 | ac_cv_func_setlocale=no ac_cv_working_tzset=no ac_cv_member_struct_tm_tm_zone=no ac_cv_func_sched_setscheduler=no 140 | # Override ./configure results to futher force Python not to use some libc calls that trigger blocked syscalls. 141 | # TODO(someday): See if HAVE_INITGROUPS has another way to disable it. 142 | RUN cd Python-3.7.6 && sed -i -E 's,#define (HAVE_CHROOT|HAVE_SETGROUPS|HAVE_INITGROUPS) 1,,' pyconfig.h 143 | # Adjust timemodule.c to perform data validation for mktime(). The libc call is supposed to do its own 144 | # validation, but on one Android 8.1 device, it doesn't. We leverage the existing AIX-related check in timemodule.c. 145 | RUN cd Python-3.7.6 && sed -i -E 's,#ifdef _AIX,#if defined(_AIX) || defined(__ANDROID__),' Modules/timemodule.c 146 | # Override posixmodule.c assumption that fork & exec exist & work. 147 | RUN cd Python-3.7.6 && sed -i -E 's,#define.*(HAVE_EXECV|HAVE_FORK).*1,,' Modules/posixmodule.c 148 | # Copy libbz2 into the SYSROOT_LIB. This is the IMHO the easiest way for setup.py to find it. 149 | RUN cp "${LIBBZ2_INSTALL_DIR}/lib/libbz2.so" $SYSROOT_LIB 150 | # Compile Python. We can still remove some tests from the test suite before `make install`. 151 | RUN cd Python-3.7.6 && make 152 | 153 | # Modify stdlib & test suite before `make install`. 154 | 155 | # Apply a hack to ssl.py so it looks at the Android certificate store. 156 | ADD 3.7.patches Python-3.7.6/patches 157 | RUN cd Python-3.7.6 && quilt push 158 | # Apply a hack to ctypes so that it loads libpython.so, even though this isn't Windows. 159 | RUN sed -i -e 's,pythonapi = PyDLL(None),pythonapi = PyDLL("libpython3.7m.so"),' Python-3.7.6/Lib/ctypes/__init__.py 160 | # Hack the test suite so that when it tries to remove files, if it can't remove them, the error passes silently. 161 | # To see if ths is still an issue, run `test_bdb`. 162 | RUN sed -i -e "s#NotADirectoryError#NotADirectoryError, OSError#" Python-3.7.6/Lib/test/support/__init__.py 163 | # Ignore some tests 164 | ADD 3.7.ignore_some_tests.py . 165 | RUN python3.7 3.7.ignore_some_tests.py $(find Python-3.7.6/Lib/test -iname '*.py') $(find Python-3.7.6/Lib/distutils/tests -iname '*.py') $(find Python-3.7.6/Lib/unittest/test/ -iname '*.py') $(find Python-3.7.6/Lib/lib2to3/tests -iname '*.py') 166 | # Skip test_multiprocessing in test_venv.py. Not sure why this fails yet. 167 | RUN cd Python-3.7.6 && sed -i -e 's,def test_multiprocessing,def skip_test_multiprocessing,' Lib/test/test_venv.py 168 | # Skip test_faulthandler & test_signal & test_threadsignals. Signal delivery on Android is not super reliable. 169 | RUN cd Python-3.7.6 && rm Lib/test/test_faulthandler.py Lib/test/test_signal.py Lib/test/test_threadsignals.py 170 | # In test_cmd_line.py: 171 | # - test_empty_PYTHONPATH_issue16309() fails. I think it is because it assumes PYTHONHOME is set; 172 | # if we can fix our dependency on that variable for Python subprocesses, we'd be better off. 173 | # - test_stdout_flush_at_shutdown() fails. The situation is that the test assumes you can't 174 | # close() a FD (stdout) that's already been closed; however, seemingly, on Android, you can. 175 | RUN cd Python-3.7.6 && sed -i -e 's,def test_empty_PYTHONPATH_issue16309,def skip_test_empty_PYTHONPATH_issue16309,' Lib/test/test_cmd_line.py 176 | RUN cd Python-3.7.6 && sed -i -e 's,def test_stdout_flush_at_shutdown,def skip_test_stdout_flush_at_shutdown,' Lib/test/test_cmd_line.py 177 | # TODO(someday): restore asyncio tests & fix them 178 | RUN cd Python-3.7.6 && rm -rf Lib/test/test_asyncio 179 | # TODO(someday): restore subprocess tests & fix them 180 | RUN cd Python-3.7.6 && rm Lib/test/test_subprocess.py 181 | # TODO(someday): Restore test_httpservers tests. They depend on os.setuid() existing, and they have 182 | # little meaning in Android. 183 | RUN cd Python-3.7.6 && rm Lib/test/test_httpservers.py 184 | # TODO(someday): restore xmlrpc tests & fix them; right now they hang forever. 185 | RUN cd Python-3.7.6 && rm Lib/test/test_xmlrpc.py 186 | # TODO(someday): restore wsgiref tests & fix them; right now they hang forever. 187 | RUN cd Python-3.7.6 && rm Lib/test/test_wsgiref.py 188 | 189 | # Install Python. 190 | RUN cd Python-3.7.6 && make install 191 | RUN cp -a $PYTHON_INSTALL_DIR/lib/libpython3.7m.so "$JNI_LIBS" 192 | ENV ASSETS_DIR $APPROOT/app/src/main/assets/ 193 | RUN mkdir -p "$ASSETS_DIR" && cd "$PYTHON_INSTALL_DIR" && zip -0 -q "$ASSETS_DIR"/pythonhome.${TARGET_ABI_SHORTNAME}.zip -r . 194 | 195 | # Download & install rubicon-java. 196 | ARG RUBICON_JAVA_VERSION=0.2020-02-27.0 197 | ADD download-cache/${RUBICON_JAVA_VERSION}.tar.gz . 198 | RUN cd rubicon-java-${RUBICON_JAVA_VERSION} && \ 199 | LDFLAGS='-landroid -llog' PYTHON_CONFIG=$PYTHON_INSTALL_DIR/bin/python3-config make 200 | RUN mv rubicon-java-${RUBICON_JAVA_VERSION}/dist/librubicon.so $JNI_LIBS 201 | RUN mkdir -p /opt/python-build/app/libs/ && mv rubicon-java-${RUBICON_JAVA_VERSION}/dist/rubicon.jar $APPROOT/app/libs/ 202 | RUN cd rubicon-java-${RUBICON_JAVA_VERSION} && zip -0 -q "$ASSETS_DIR"/rubicon.zip -r rubicon 203 | 204 | RUN apt-get update -qq && apt-get -qq install rsync 205 | -------------------------------------------------------------------------------- /3.7.ignore_some_tests.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | 4 | LEADING_SPACES_RE = re.compile("^( +)") 5 | 6 | 7 | def fix(filename): 8 | # Don't apply these hacks to script_helper.py directly. 9 | if filename.endswith("script_helper.py"): 10 | return 11 | 12 | with open(filename) as fd: 13 | try: 14 | contents = fd.read() 15 | except UnicodeDecodeError: 16 | # We're going to hope that we don't have to process 17 | # any non-UTF-8 files. 18 | return 19 | 20 | matching_lines = [] 21 | splitted = contents.split("\n") 22 | for i, line in enumerate(splitted): 23 | if ( 24 | # Skip test_extension_init within test_extension because we currently hack 25 | # distutils to add -lpython3.7m when building any dynamic module. 26 | "# others arguments have defaults" in line 27 | # The following skips one test in test_dir_util, which fails because 28 | # on Android, a directory gets made as 02700 not 0700. It doesn't matter 29 | # much for us. 30 | or "# Get and set the current umask value for testing mode bits." in line 31 | # Skip a specific zipimport-related test :( 32 | or "then check that the filter works on individual files" in line 33 | # The following avoid executing subprocesses via tests. 34 | or "subprocess.run(" in line 35 | or "subprocess.check_output(" in line 36 | or "subprocess.check_call(" in line 37 | or " spawn(" in line 38 | or "platform.popen(" in line 39 | or "os.popen(" in line 40 | or "os.spawnl(" in line 41 | or "with Popen(" in line 42 | # pydoc start_server() is failing. Not fully sure why. 43 | or "pydoc._start_server" in line 44 | # some tests find out that we're bad at passing 100% of UNIX signals to Python; sorry! 45 | or "= self.decide_itimer_count()" in line 46 | # some tests try to make a socket with no params; somehow this is not OK on Android! 47 | # or "socket.socket()" in line 48 | # one test tries to do os.chdir('/') to get the top of the filesystem tree, then os.listdir(). This will not work. 49 | or " self.assertEqual(set(os.listdir()), set(os.listdir(os.sep)))" in line 50 | # os.get_terminal_size() doesn't work for now 51 | or ( 52 | " os.get_terminal_size()" in line 53 | and not " os.get_terminal_size()'" in line 54 | ) 55 | # process exit codes are weird 56 | or " self.assertEqual(exitcode, self.exitcode)" in line 57 | or " os.spawnv(" in line 58 | # Disable the group module's beliefs that all gr_name values are strings; 59 | # on Android, somehow, they're None. 60 | or "self.assertIsInstance(value.gr_name, str)" in line 61 | # Similar for pwd (password file) module 62 | or "self.assertIsInstance(e.pw_gecos, str)" in line 63 | # test_socketserver.py has a test for signals & UNIX sockets interactions; this test hangs on Android. 64 | # Skip for now. 65 | or "test.support.get_attribute(signal, 'pthread_kill')" in line 66 | ): 67 | matching_lines.append(i) 68 | 69 | # If there is nothing to do, we do nothing. 70 | if not matching_lines: 71 | return 72 | 73 | # Some lines try to spawn subprocesses, so we mod those out. 74 | out_lines = [] 75 | # If the file doesn't import unittest, add that to the top. 76 | if "import unittest\n" not in contents: 77 | out_lines.append("import unittest") 78 | 79 | for i, line in enumerate(splitted): 80 | if i in matching_lines: 81 | # Find indent level 82 | match = LEADING_SPACES_RE.match(line) 83 | if match: 84 | num_spaces = len(match.group(0)) 85 | line = ( 86 | " " * num_spaces 87 | + 'raise unittest.SkipTest("Skipping this test for Python within an Android app")' 88 | + "\n" 89 | + line 90 | ) 91 | out_lines.append(line) 92 | 93 | with open(filename, "w") as fd: 94 | fd.write("\n".join(out_lines)) 95 | 96 | 97 | def main(): 98 | filenames = sys.argv[1:] 99 | for filename in filenames: 100 | fix(filename) 101 | 102 | 103 | if __name__ == "__main__": 104 | main() 105 | -------------------------------------------------------------------------------- /3.7.patches/01_python_ssl_module_add_android_certificates: -------------------------------------------------------------------------------- 1 | Index: Python-3.7.6/Lib/ssl.py 2 | =================================================================== 3 | --- Python-3.7.6.orig/Lib/ssl.py 4 | +++ Python-3.7.6/Lib/ssl.py 5 | @@ -488,6 +488,17 @@ class SSLContext(_SSLContext): 6 | if sys.platform == "win32": 7 | for storename in self._windows_cert_stores: 8 | self._load_windows_store_certs(storename, purpose) 9 | + if os.path.exists('/etc/security/cacerts'): 10 | + certs = [] 11 | + for basename in os.listdir('/etc/security/cacerts'): 12 | + with open('/etc/security/cacerts/' + basename) as fd: 13 | + s = fd.read() 14 | + if 'END CERTIFICATE' not in s: 15 | + continue 16 | + lines = s.split('\n') 17 | + line_end_certificate = [i for i, line in enumerate(lines) if 'END CERTIFICATE' in line][0] 18 | + certs.append('\n'.join(lines[0:line_end_certificate+1])) 19 | + self.load_verify_locations(None, None, '\n'.join(certs)) 20 | self.set_default_verify_paths() 21 | 22 | if hasattr(_SSLContext, 'minimum_version'): 23 | Index: Python-3.7.6/Lib/test/test_ssl.py 24 | =================================================================== 25 | --- Python-3.7.6.orig/Lib/test/test_ssl.py 26 | +++ Python-3.7.6/Lib/test/test_ssl.py 27 | @@ -1601,6 +1601,7 @@ class ContextTests(unittest.TestCase): 28 | @unittest.skipIf(sys.platform == "win32", "not-Windows specific") 29 | @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars") 30 | def test_load_default_certs_env(self): 31 | + raise unittest.SkipTest("Skipping this test for Python within an Android app") 32 | ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) 33 | with support.EnvironmentVarGuard() as env: 34 | env["SSL_CERT_DIR"] = CAPATH 35 | -------------------------------------------------------------------------------- /3.7.patches/series: -------------------------------------------------------------------------------- 1 | 01_python_ssl_module_add_android_certificates 2 | -------------------------------------------------------------------------------- /3.7.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script only uses bash features available in bash <= 3, 3 | # so that it works the same on macOS and GNU/Linux. 4 | 5 | # Set bash strict mode. 6 | set -eou pipefail 7 | 8 | # Check dependencies. 9 | function require() { 10 | local FOUND="no" 11 | which "$1" >/dev/null && FOUND="yes" 12 | if [ "$FOUND" = "no" ]; then 13 | echo "Missing dependency: $1 14 | 15 | Please install it. One of following might work, depending on your system: 16 | 17 | $ sudo apt-get install $1 18 | $ brew install $1 19 | 20 | Exiting." 21 | exit 1 22 | fi 23 | } 24 | 25 | # We require `perl` because that is the the program that provides 26 | # shasum; we use shasum because it's available easily on macOS & 27 | # GNU/Linux. 28 | for dependency in curl cut docker grep perl python3 shasum zip; do 29 | require "$dependency" 30 | done 31 | 32 | # Extract image ID from `docker build` output. Used by `build_one_abi`. 33 | IMAGE_NAME_TEMPFILE="$(mktemp)" 34 | function extract_image_name() { 35 | tee >(tail -n1 | grep '^Successfully built ' | cut -d' ' -f3 >"$IMAGE_NAME_TEMPFILE") 36 | } 37 | 38 | function build_one_abi() { 39 | TARGET_ABI_SHORTNAME="$1" 40 | PYTHON_VERSION="$2" 41 | # Using ANDROID_API_LEVEL=21 for two reasons: 42 | # 43 | # - >= 21 gives us a `localeconv` libc function (admittedly a 44 | # non-working one), which makes compiling (well, linking) Python 45 | # easier. 46 | # 47 | # - 64-bit architectures only start existing at API level 21. 48 | ANDROID_API_LEVEL=21 49 | # Compute the compiler name & binutils prefix name. See also: 50 | # https://developer.android.com/ndk/guides/other_build_systems 51 | 52 | case "${TARGET_ABI_SHORTNAME}" in 53 | armeabi-v7a) 54 | TOOLCHAIN_TRIPLE="arm-linux-androideabi" 55 | COMPILER_TRIPLE="armv7a-linux-androideabi${ANDROID_API_LEVEL}" 56 | ;; 57 | arm64-v8a) 58 | TOOLCHAIN_TRIPLE="aarch64-linux-android" 59 | COMPILER_TRIPLE="${TOOLCHAIN_TRIPLE}${ANDROID_API_LEVEL}" 60 | ;; 61 | x86) 62 | TOOLCHAIN_TRIPLE="i686-linux-android" 63 | COMPILER_TRIPLE="${TOOLCHAIN_TRIPLE}${ANDROID_API_LEVEL}" 64 | ;; 65 | x86_64) 66 | TOOLCHAIN_TRIPLE="x86_64-linux-android" 67 | COMPILER_TRIPLE="${TOOLCHAIN_TRIPLE}${ANDROID_API_LEVEL}" 68 | ;; 69 | esac 70 | 71 | # Compute OpenSSL build target name. We avoid using OpenSSL's built-in 72 | # Android support because it does not seem to give us any benefits. 73 | case "${TARGET_ABI_SHORTNAME}" in 74 | armeabi-v7a) 75 | OPENSSL_BUILD_TARGET="linux-generic32" 76 | ;; 77 | arm64-v8a) 78 | OPENSSL_BUILD_TARGET="linux-aarch64" 79 | ;; 80 | x86) 81 | OPENSSL_BUILD_TARGET="linux-x86" 82 | ;; 83 | x86_64) 84 | OPENSSL_BUILD_TARGET="linux-x86_64" 85 | ;; 86 | esac 87 | 88 | docker build --build-arg COMPILER_TRIPLE="${COMPILER_TRIPLE}" --build-arg OPENSSL_BUILD_TARGET="$OPENSSL_BUILD_TARGET" --build-arg TARGET_ABI_SHORTNAME="$TARGET_ABI_SHORTNAME" --build-arg TOOLCHAIN_TRIPLE="$TOOLCHAIN_TRIPLE" --build-arg ANDROID_API_LEVEL="$ANDROID_API_LEVEL" -f "${PYTHON_VERSION}".Dockerfile . | extract_image_name 89 | local IMAGE_NAME 90 | IMAGE_NAME="$(cat $IMAGE_NAME_TEMPFILE)" 91 | if [ -z "$IMAGE_NAME" ]; then 92 | echo 'Unable to find image name. Did Docker build succeed? Aborting.' 93 | exit 1 94 | fi 95 | 96 | # Extract the build artifacts we need to create our zip file. 97 | docker run -v "${PWD}"/output/"${PYTHON_VERSION}"/:/mnt/ --rm --entrypoint rsync "$IMAGE_NAME" -a /opt/python-build/approot/. /mnt/. 98 | # Extract pyconfig.h for debugging ./configure strangeness. 99 | docker run -v "${PWD}"/output/"${PYTHON_VERSION}"/:/mnt/ --rm --entrypoint rsync "$IMAGE_NAME" -a /opt/python-build/built/python/include/python"${PYTHON_VERSION}"m/pyconfig.h /mnt/ 100 | fix_permissions 101 | } 102 | 103 | function download() { 104 | # Pass -L to follow redirects. 105 | echo "Downloading $2" 106 | curl -L "$1" -o "$2" 107 | } 108 | 109 | # Store a bash associative array of URLs we download, and their expected SHA256 sum. 110 | function download_urls() { 111 | echo "Preparing downloads..." 112 | URLS_AND_SHA256=( 113 | "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u242-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u242b08.tar.gz=f39b523c724d0e0047d238eb2bb17a9565a60574cf651206c867ee5fc000ab43" 114 | "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip=8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c" 115 | "https://www.openssl.org/source/openssl-1.1.1d.tar.gz=1e3a91bc1f9dfce01af26026f856e064eab4c8ee0a8f457b5ae30b40b8b711f2" 116 | "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz=72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056" 117 | "https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tar.xz=55a2cce72049f0794e9a11a84862e9039af9183603b78bc60d89539f82cf533f" 118 | "https://tukaani.org/xz/xz-5.2.4.tar.gz=b512f3b726d3b37b6dc4c8570e137b9311e7552e8ccbab4d39d47ce5f4177145" 119 | "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz"="ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" 120 | "http://archive.ubuntu.com/ubuntu/pool/main/s/sqlite3/sqlite3_3.11.0.orig.tar.xz"="79fb8800b8744337d5317270899a5a40612bb76f81517e131bf496c26b044490" 121 | "https://github.com/paulproteus/rubicon-java/archive/0.2020-02-27.0.tar.gz=b698c1f5fd3f8d825ed88e1a782f1aaa58f6d27404edc43fdb7dd117ab4c8f28" 122 | ) 123 | local DOWNLOAD_CACHE="$PWD/download-cache" 124 | local DOWNLOAD_CACHE_TMP="$PWD/download-cache.tmp" 125 | for url_and_sha256 in "${URLS_AND_SHA256[@]}" ; do 126 | url="${url_and_sha256%%=*}" 127 | sha256="${url_and_sha256##*=}" 128 | expected_filename="$(echo "$url" | tr '/' '\n' | tail -n1)" 129 | # Check existing file. 130 | if [ -f "${DOWNLOAD_CACHE}/${expected_filename}" ] ; then 131 | echo "Using ${expected_filename} from download-cache/" 132 | continue 133 | fi 134 | 135 | # Download. 136 | rm -rf download-cache.tmp && mkdir -p download-cache.tmp 137 | cd download-cache.tmp && download "$url" "$expected_filename" && cd .. 138 | local OK="no" 139 | shasum -a 256 "${DOWNLOAD_CACHE_TMP}/${expected_filename}" | grep -q "$sha256" && OK="yes" 140 | if [ "$OK" = "yes" ] ; then 141 | mkdir -p "$DOWNLOAD_CACHE" 142 | mv "${DOWNLOAD_CACHE_TMP}/${expected_filename}" "${DOWNLOAD_CACHE}/${expected_filename}" 143 | rmdir "${DOWNLOAD_CACHE_TMP}" 144 | else 145 | echo "Checksum mismatch while downloading: $url" 146 | echo "" 147 | echo "Maybe your Internet connection got disconnected during the download. Please re-run the script." 148 | echo "Aborting." 149 | exit 1 150 | fi 151 | done 152 | } 153 | 154 | fix_permissions() { 155 | USER_AND_GROUP="$(id -u):$(id -g)" 156 | # When using Docker on Linux, the `rsync` command creates files owned by root. 157 | # Compute the user ID and group ID of this script on the non-Docker side, and ask 158 | # Docker to adjust permissions accordingly. 159 | if [ -d output ] ; then 160 | docker run -v "${PWD}":/mnt/ --rm --entrypoint chown ubuntu:18.04 -R "$USER_AND_GROUP" /mnt/output/ 161 | fi 162 | } 163 | 164 | function main() { 165 | echo 'Starting Docker builds.' 166 | 167 | # Clear the output directory. 168 | fix_permissions 169 | rm -rf ./output/3.7 170 | mkdir -p output/3.7 171 | 172 | # Allow TARGET_ABIs to be overridden by argv. 173 | TARGET_ABIS="${@:-x86 x86_64 armeabi-v7a arm64-v8a}" 174 | for TARGET_ABI_SHORTNAME in $TARGET_ABIS; do 175 | build_one_abi "$TARGET_ABI_SHORTNAME" "3.7" 176 | done 177 | 178 | # Make a ZIP file. 179 | fix_permissions 180 | cd output/3.7/app && zip -q -0 -r ../../3.7.zip . && cd ../../.. 181 | } 182 | 183 | download_urls 184 | main "$@" 185 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This contains scripts for building Python on Android. It allows you 2 | to generate a Python support ZIP file, which is a file you an unpack 3 | over an Android app to get Python support in that Android app. 4 | 5 | Table of contents: 6 | 7 | - Generating a Python support ZIP file 8 | 9 | - Maintaining these scripts 10 | 11 | - Creating a sample app for Android 12 | 13 | ## Generating a Python support ZIP file 14 | 15 | If you have `docker` installed, you can `git clone` this repository 16 | and run `./3.7.sh`. This will run for about 45 minutes, then create a 17 | Python support ZIP file at `output/3.7.zip`. 18 | 19 | You can run e.g. `3.7.sh x86` to rebuild the Python support code for 20 | one or more Android ABIs. This allows faster iteration. 21 | 22 | ## Maintaining these scripts 23 | 24 | The `3.7.sh` script downloads some source code, then passes control to 25 | `docker` which runs `3.7.Dockerfile`. This configures dependencies, 26 | patches Python, and does the build. 27 | 28 | The shell script does nearly all of the downloading up-front. This 29 | allows the Docker-based build process to make the best use possible of 30 | the Docker cache. The Dockerfile does include some `apt-get` calls, 31 | which I consider an acceptable compromise of this design goal. 32 | 33 | The Dockerfile patches the source code using `sed`, a custom Python 34 | script called `3.7.ignore_some_tests.py`, and patches that we apply 35 | using `quilt`. 36 | 37 | It uses `sed` when making changes that I do not intend to send 38 | upstream. It is easy to use `sed` to make one-line changes to 39 | various files, and these changes are resilient to the lines 40 | moving around slightly. 41 | 42 | The `3.7.ignore_some_tests.py` makes a lot of changes to the Python 43 | test suite, focusing on removing tests that do not make sense within 44 | the context of an Android app. Most of these relate to disabling the 45 | use of Python subprocesses to run parts of the test suite. Launching 46 | subprocesses works properly within an Android app on some API 47 | versions. However, the `libpython` that we build requires setting the 48 | `PYTHONHOME` environment variable at the moment, so it was easier to 49 | disable these tests than to ensure that variable is threaded through 50 | appropriately. Another difficulty is that in more recent versions of 51 | Android, launching subprocesses [requires additional work to comply 52 | with new sandboxing 53 | restrictions.](https://www.reddit.com/r/androiddev/comments/b2inbu/psa_android_q_blocks_executing_binaries_in_your/) 54 | Because there are a lot of tests that needed to be changed, and at the 55 | moment I don't plan to upstream this, I consider this similar to the 56 | use of `sed`, but more powerful. 57 | 58 | It also uses a patch which is added to the Python source tree using 59 | `quilt`. This is a patch which allows Python to use the Android system 60 | certificates to validate TLS/SSL connections. It will probably make 61 | sense to upstream this after some revision; however, it will not 62 | necessarily land in the Python 3.7 branch even when upstreamed. To 63 | learn more about using quilt, read [this documentation about 64 | quilt.](https://www.yoctoproject.org/docs/1.8/dev-manual/dev-manual.html#using-a-quilt-workflow) If we need more patches to Python that are substantial and 65 | may be upstreamed, relying more on `quilt` might be wise. 66 | 67 | If you attempt to run the full Python standard library test suite, it 68 | should all pass. Note that the Docker-based build also manually 69 | removes some parts of the Python standard library test suite to 70 | accommodate this goal! You can find an app to run the Python standard 71 | library test suite in 72 | [Python-Android-sample-apps](https://github.com/paulproteus/Python-Android-sample-apps). 73 | 74 | ## Creating a sample app for Android 75 | 76 | In these steps, you will: 77 | 78 | - Download the Android SDK. 79 | - Download/configure an appropriate version of Java. 80 | - Configure an Android emulator. 81 | - Generate a Python-based Android app using cookiecutter. 82 | - Download a Python Android support ZIP file, and add that to your app. (You can build it yourself if you prefer.) 83 | - Run the app on the Android emulator. 84 | 85 | This will require approximately 5GB of disk space and downloads. It 86 | will require about 30 minutes of time. I have tested these instructions 87 | on macOS and Ubuntu 18.04. 88 | 89 | ### Downloading the Android SDK 90 | 91 | On macOS, run the following commands. 92 | 93 | ``` 94 | $ mkdir -p ~/android/sdk && cd ~/android/sdk 95 | $ curl -O https://dl.google.com/android/repository/sdk-tools-darwin-4333796.zip 96 | $ unzip sdk*zip 97 | ``` 98 | 99 | If you’re on Linux, you’d need to use a different URL, 100 | e.g. https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip 101 | . 102 | 103 | These URLs have existed since approximately 2017, and they have a 104 | built-in autoupdater, so I expect them to keep working for quite a few 105 | years longer. 106 | 107 | We need the `skins` folder from the JetBrains Android Studio IDE. Run the following 108 | commands to extract them. 109 | 110 | ``` 111 | $ git clone --depth 1 --no-checkout https://github.com/JetBrains/android ~/android/sdk/android-ide-git 112 | $ cd ~/android/sdk/android-ide-git 113 | $ git archive HEAD:artwork/resources/device-art-resources -o device-art-resources.zip 114 | $ mkdir ~/android/sdk/skins 115 | $ cd ~/android/sdk/skins 116 | $ unzip ~/android/sdk/android-ide-git/device-art-resources.zip 117 | $ rm -rf ~/android/sdk/android-ide-git 118 | ``` 119 | 120 | ### Download/configure an appropriate version of Java 121 | 122 | Ensure you have Java 8. Look at the output of this command. 123 | 124 | ``` 125 | $ java -version 126 | ``` 127 | 128 | If macOS shows a pop-up explaining that Java is not installed, 129 | offering "More Info" and "OK," click "OK." 130 | 131 | On macOS, if you don’t have Java, or the version is not Java 8, run 132 | these commands: 133 | 134 | ``` 135 | $ brew tap adoptopenjdk/openjdk 136 | $ brew cask install adoptopenjdk8 137 | ``` 138 | 139 | See also: https://stackoverflow.com/a/55775566 140 | 141 | ### Configure the Android SDK 142 | 143 | ``` 144 | $ export ANDROID_SDK_ROOT="${HOME}/android/sdk" 145 | $ PATH="$PATH:${ANDROID_SDK_ROOT}/tools/bin:${ANDROID_SDK_ROOT}/emulator:${ANDROID_SDK_ROOT}/platform-tools" 146 | $ mkdir -p ~/.android 147 | $ touch ~/.android/repositories.cfg 148 | $ sdkmanager --update 149 | $ sdkmanager --licenses 150 | $ sdkmanager 'platforms;android-28' 'system-images;android-28;default;x86' 'emulator' 'platform-tools' 151 | ``` 152 | 153 | ### Configure an Android emulator 154 | 155 | Open a **new** terminal window/tab and run the following. 156 | 157 | ``` 158 | $ export ANDROID_SDK_ROOT="${HOME}/android/sdk" 159 | $ PATH="$PATH:${ANDROID_SDK_ROOT}/tools/bin:${ANDROID_SDK_ROOT}/emulator:${ANDROID_SDK_ROOT}/platform-tools" 160 | $ avdmanager --verbose create avd --name robotfriend --abi x86 --package 'system-images;android-28;default;x86' --device pixel 161 | $ echo 'disk.dataPartition.size=4096M' >> $HOME/.android/avd/robotfriend.avd/config.ini 162 | $ echo 'hw.keyboard=yes' >> $HOME/.android/avd/robotfriend.avd/config.ini 163 | $ echo 'skin.dynamic=yes' >> $HOME/.android/avd/robotfriend.avd/config.ini 164 | $ echo 'skin.name=pixel_3a' >> $HOME/.android/avd/robotfriend.avd/config.ini 165 | $ echo 'skin.path=skins/pixel_3a' >> $HOME/.android/avd/robotfriend.avd/config.ini 166 | $ echo 'showDeviceFrame=yes' >> $HOME/.android/avd/robotfriend.avd/config.ini 167 | $ emulator @robotfriend 168 | ``` 169 | 170 | The emulator command will open an Android emulator, and will block your terminal window. 171 | 172 | Note: If you find your emulator lacks Internet access, and you are OK using a third-party DNS server, you can run this: 173 | 174 | ``` 175 | $ emulator @robotFriend -dns-server 1.1.1.1,8.8.8.8 176 | ``` 177 | 178 | ### Generate a Python-based Android app with cookiecutter 179 | 180 | In your original terminal, run the following commands. 181 | 182 | ``` 183 | $ python3 -m pip install --user cookiecutter 184 | $ mkdir -p ~/projects/beeware-sample-app 185 | $ cd ~/projects/beeware-sample-app 186 | $ python3 -m cookiecutter https://github.com/paulproteus/cookiecutter-beeware-android 187 | ``` 188 | 189 | Now, in a web browser, visit this URL: 190 | 191 | https://drive.google.com/uc?export=download&id=1Bsr_3VMkEez5VWHq2tjjwl8xwHpffIcb 192 | 193 | And download 3.7.zip. 194 | 195 | Back in a terminal, run: 196 | 197 | 198 | ``` 199 | $ cd MyApp # or whatever you said for project_name above 200 | $ unzip ~/Downloads/3.7.zip 201 | ``` 202 | 203 | Finally, we need to create some sample Python code. 204 | 205 | I'm assuming you called your app `my_app` in the earlier sections. 206 | Create a file called `app/src/main/assets/python/my_app/__init__.py` 207 | with the following content: 208 | 209 | ```python 210 | from rubicon.java import JavaClass, JavaInterface 211 | 212 | IPythonApp = JavaInterface('org/beeware/android/IPythonApp') 213 | 214 | class Application(IPythonApp): 215 | def onCreate(self): 216 | print('called Python onCreate()') 217 | 218 | def onStart(self): 219 | print('called Python onStart()') 220 | 221 | def onResume(self): 222 | print('called Python onResume()') 223 | ``` 224 | 225 | Create another file called `app/src/main/assets/my_app/__main__.py` 226 | with the following content. 227 | 228 | ```python 229 | from . import Application 230 | from rubicon.java import JavaClass 231 | 232 | activity_class = JavaClass('org/beeware/android/MainActivity') 233 | app = Application() 234 | activity_class.setPythonApp(app) 235 | print('Python app launched & stored in Android Activity class') 236 | ``` 237 | 238 | ### Run the app on the Android emulator 239 | 240 | Run this in a terminal. 241 | 242 | ``` 243 | $ ./gradlew installDebug 244 | ``` 245 | 246 | After about 3 minutes of waiting, the command should successfully 247 | exit. Note that this command will be faster any future times you run 248 | it. 249 | 250 | In the emulator, find the circle icon at the bottom, next to the back 251 | icon. Drag the circle icon up, and look for MyApp. Click it. 252 | 253 | After about 10 seconds, you will see your app name visible. This means 254 | the app is launched. 255 | 256 | Now, let’s look through the Android log to find evidence that our app 257 | launched. We’re looking for “called Python onCreate()” in the 258 | following output. 259 | 260 | ``` 261 | $ adb logcat -d | grep -i python 262 | ``` 263 | 264 | TA-DA! It works. 265 | 266 | You may notice that the Android image looks somewhat unstyled. This is 267 | the fastest to download Android image; it contains all of fully open 268 | source Android APIs, but it lacks Google’s additional APIs. I’ve 269 | tested it, and the app displays properly this way. Based on my 270 | research I expect that all APIs we will wrap will continue to work 271 | properly with this Android image. 272 | --------------------------------------------------------------------------------