Rebuild python docker image (#5475)

* Rebuild python docker image

* Update docker image

* Change call to build_and_run_docker.sh

* Set up python tests for different versions

* Fix comments

kokoro/linux/dockerfile/test/python_jessie/Dockerfile

+FROM debian:jessie
+
+# Install dependencies.  We start with the basic ones require to build protoc
+# and the C++ build
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  git \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  parallel \
+  time \
+  wget \
+  && apt-get clean
+
+# Install python dependencies
+RUN apt-get update && apt-get install -y \
+  python-setuptools \
+  python-all-dev \
+  python3-all-dev \
+  python-pip
+
+# Install Python packages from PyPI
+RUN pip install --upgrade pip==10.0.1
+RUN pip install virtualenv
+RUN pip install six==1.10.0 twisted==17.5.0
+
+# Install pip and virtualenv for Python 3.4
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4
+RUN python3.4 -m pip install virtualenv

kokoro/linux/dockerfile/test/python_stretch/Dockerfile

+FROM debian:stretch
+
+# Install dependencies.  We start with the basic ones require to build protoc
+# and the C++ build
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  git \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  parallel \
+  time \
+  wget \
+  && apt-get clean
+
+# Install Python 2.7
+RUN apt-get update && apt-get install -y python2.7 python-all-dev
+RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
+
+# Install python dependencies
+RUN apt-get update && apt-get install -y \
+  python-setuptools \
+  python-pip
+
+# Add Debian 'testing' repository
+RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
+
+# Install Python3
+RUN apt-get update && apt-get -t testing install -y \
+  python3.5 \
+  python3.6 \
+  python3.7 \
+  python3-all-dev
+
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.5
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.6
+RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7

kokoro/linux/python27/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_jessie
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python27"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python27/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python27/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python27_cpp/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_jessie
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python27_cpp"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python27_cpp/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python27_cpp/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python33/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_jessie
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python33"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python33/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python33/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python33_cpp/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_jessie
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python33_cpp"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python33_cpp/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python33_cpp/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python34/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_jessie
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python34"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python34/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python34/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python34_cpp/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_jessie
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python34_cpp"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python34_cpp/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python34_cpp/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python35/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_stretch
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python35"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python35/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python35/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python35_cpp/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_stretch
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python35_cpp"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python35_cpp/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python35_cpp/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python36/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_stretch
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python36"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python36/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python36/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python36_cpp/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_stretch
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python36_cpp"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python36_cpp/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python36_cpp/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python37/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_stretch
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python37"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python37/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python37/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python37_cpp/build.sh

+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python_stretch
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="python37_cpp"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/python37_cpp/continuous.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python37_cpp/presubmit.cfg

+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_cpp/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

python/setup.py

@@ -262,6 +262,9 @@ if __name__ == '__main__':
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         ],
       namespace_packages=['google'],
       packages=find_packages(

tests.sh

@@ -220,6 +220,38 @@ build_python() {
   cd ..
 }
 
+build_python_version() {
+  internal_build_cpp
+  cd python
+  envlist=$1
+  tox -e $envlist
+  cd ..
+}
+
+build_python27() {
+  build_python_version py27-python
+}
+
+build_python33() {
+  build_python_version py33-python
+}
+
+build_python34() {
+  build_python_version py34-python
+}
+
+build_python35() {
+  build_python_version py35-python
+}
+
+build_python36() {
+  build_python_version py36-python
+}
+
+build_python37() {
+  build_python_version py37-python
+}
+
 build_python_cpp() {
   internal_build_cpp
   export LD_LIBRARY_PATH=../src/.libs # for Linux
@@ -234,6 +266,40 @@ build_python_cpp() {
   cd ..
 }
 
+build_python_cpp_version() {
+  internal_build_cpp
+  export LD_LIBRARY_PATH=../src/.libs # for Linux
+  export DYLD_LIBRARY_PATH=../src/.libs # for OS X
+  cd python
+  envlist=$1
+  tox -e $envlist
+  cd ..
+}
+
+build_python27_cpp() {
+  build_python_cpp_version py27-cpp
+}
+
+build_python33_cpp() {
+  build_python_cpp_version py33-cpp
+}
+
+build_python34_cpp() {
+  build_python_cpp_version py34-cpp
+}
+
+build_python35_cpp() {
+  build_python_cpp_version py35-cpp
+}
+
+build_python36_cpp() {
+  build_python_cpp_version py36-cpp
+}
+
+build_python37_cpp() {
+  build_python_cpp_version py37-cpp
+}
+
 build_python_compatibility() {
   internal_build_cpp
   # Use the unit-tests extraced from 2.5.0 to test the compatibilty.