Merge pull request #1595 from thomasvl/objc_travis_tweaks

Automated testing tweaks for ObjC

.travis.yml

@@ -8,7 +8,7 @@ os:
   - linux
   - osx
 # The Objective C build needs Xcode 7.0 or later.
-osx_image: xcode7.2
+osx_image: xcode7.3
 script:
   - ./tests.sh $CONFIG
 env:
@@ -23,6 +23,11 @@ env:
   - CONFIG=javanano_jdk7
   - CONFIG=javanano_oracle7
   - CONFIG=javascript
+  # iOS build log was starting to choke travis UI, so split to cover the
+  # Xcode Debug and Release Configurations independently.
+  - CONFIG=objectivec_ios_debug
+  - CONFIG=objectivec_ios_release
+  - CONFIG=objectivec_osx
   - CONFIG=python
   - CONFIG=python_cpp
   - CONFIG=ruby19
@@ -54,12 +59,13 @@ matrix:
     # which doesn't work on OS X.
     - os: osx
       env: CONFIG=golang
-  # Add into the matrix OS X tests of Objective C (needs Xcode, so it won't
-  # work on other platforms). These are split so it doesn't take as long to run.
-  include:
-    - os: osx
-      env: CONFIG=objectivec_ios
-    - os: osx
+    # OS X/iOS tests of Objective C (needs Xcode, so it won't work on other
+    # platforms).
+    - os: linux
+      env: CONFIG=objectivec_ios_debug
+    - os: linux
+      env: CONFIG=objectivec_ios_release
+    - os: linux
       env: CONFIG=objectivec_osx
   allow_failures:
     # These currently do not work on OS X but are being worked on by @haberman.
@@ -71,14 +77,11 @@ matrix:
     # we moved to an OS X image that is 10.11.
     - os: osx
       env: CONFIG=python_cpp
-    # xctool 0.2.8 seems to have a bug where it randomly kills tests saying
-    # they failed.
-    #   https://github.com/facebook/xctool/issues/619
-    #   https://github.com/google/protobuf/issues/1232
-    # travis updated their images to include 0.2.8:
-    #   https://blog.travis-ci.com/2016-03-23-xcode-image-updates
-    # Mark the iOS test as flakey so these failures don't turn things red.
+    # Mark the iOS test as flakey as xcodebuild some times fails to start the
+    # iOS Simulator.
+    - os: osx
+      env: CONFIG=objectivec_ios_debug
     - os: osx
-      env: CONFIG=objectivec_ios
+      env: CONFIG=objectivec_ios_release
 notifications:
   email: false

objectivec/DevTools/full_mac_build.sh

@@ -37,6 +37,10 @@ OPTIONS:
          Skip the invoke of Xcode to test the runtime on both iOS and OS X.
    --skip-xcode-ios
          Skip the invoke of Xcode to test the runtime on iOS.
+   --skip-xcode-debug
+         Skip the Xcode Debug configuration.
+   --skip-xcode-release
+         Skip the Xcode Release configuration.
    --skip-xcode-osx
          Skip the invoke of Xcode to test the runtime on OS X.
    --skip-objc-conformance
@@ -66,8 +70,8 @@ wrapped_make() {
 }
 
 NUM_MAKE_JOBS=$(/usr/sbin/sysctl -n hw.ncpu)
-if [[ "${NUM_MAKE_JOBS}" -lt 4 ]] ; then
-  NUM_MAKE_JOBS=4
+if [[ "${NUM_MAKE_JOBS}" -lt 2 ]] ; then
+  NUM_MAKE_JOBS=2
 fi
 
 DO_AUTOGEN=no
@@ -76,6 +80,8 @@ REGEN_DESCRIPTORS=no
 CORE_ONLY=no
 DO_XCODE_IOS_TESTS=yes
 DO_XCODE_OSX_TESTS=yes
+DO_XCODE_DEBUG=yes
+DO_XCODE_RELEASE=yes
 DO_OBJC_CONFORMANCE_TESTS=yes
 while [[ $# != 0 ]]; do
   case "${1}" in
@@ -109,6 +115,12 @@ while [[ $# != 0 ]]; do
     --skip-xcode-osx )
       DO_XCODE_OSX_TESTS=no
       ;;
+    --skip-xcode-debug )
+      DO_XCODE_DEBUG=no
+      ;;
+    --skip-xcode-release )
+      DO_XCODE_RELEASE=no
+      ;;
     --skip-objc-conformance )
       DO_OBJC_CONFORMANCE_TESTS=no
       ;;
@@ -151,8 +163,12 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then
         -project objectivec/ProtocolBuffers_iOS.xcodeproj
         -scheme ProtocolBuffers
     )
-  "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean
-  "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean
+    if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean
+    fi
+    if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean
+    fi
   fi
   if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then
     XCODEBUILD_CLEAN_BASE_OSX=(
@@ -160,8 +176,12 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then
         -project objectivec/ProtocolBuffers_OSX.xcodeproj
         -scheme ProtocolBuffers
     )
-  "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean
-  "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean
+    if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean
+    fi
+    if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+      "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean
+    fi
   fi
 fi
 
@@ -222,20 +242,20 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
           -destination "platform=iOS Simulator,name=iPad Air,OS=9.0" # 64bit
       )
       ;;
-    7.3* )
+    7.2* )
       XCODEBUILD_TEST_BASE_IOS+=(
           -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.3" # 64bit
+          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
           -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPad Air,OS=9.3" # 64bit
+          -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
       )
       ;;
-    7.* )
+    7.3* )
       XCODEBUILD_TEST_BASE_IOS+=(
           -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
+          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.3" # 64bit
           -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
+          -destination "platform=iOS Simulator,name=iPad Air,OS=9.3" # 64bit
       )
       ;;
     * )
@@ -243,10 +263,14 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
       exit 2
       ;;
   esac
-  header "Doing Xcode iOS build/tests - Debug"
-  "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test
-  header "Doing Xcode iOS build/tests - Release"
-  "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test
+  if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+    header "Doing Xcode iOS build/tests - Debug"
+    "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test
+  fi
+  if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+    header "Doing Xcode iOS build/tests - Release"
+    "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test
+  fi
   # Don't leave the simulator in the developer's face.
   killall "${IOS_SIMULATOR_NAME}"
 fi
@@ -258,13 +282,18 @@ if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then
       # Since the ObjC 2.0 Runtime is required, 32bit OS X isn't supported.
       -destination "platform=OS X,arch=x86_64" # 64bit
   )
-  header "Doing Xcode OS X build/tests - Debug"
-  "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test
-  header "Doing Xcode OS X build/tests - Release"
-  "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test
+  if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
+    header "Doing Xcode OS X build/tests - Debug"
+    "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test
+  fi
+  if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
+    header "Doing Xcode OS X build/tests - Release"
+    "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test
+  fi
 fi
 
 if [[ "${DO_OBJC_CONFORMANCE_TESTS}" == "yes" ]] ; then
+  header "Running ObjC Conformance Tests"
   cd conformance
   wrapped_make -j "${NUM_MAKE_JOBS}" test_objc
   cd ..

tests.sh

@@ -197,59 +197,30 @@ internal_install_python_deps() {
   fi
 }
 
-internal_objectivec_common () {
-  # Make sure xctool is up to date. Adapted from
-  #  http://docs.travis-ci.com/user/osx-ci-environment/
-  # We don't use a before_install because we test multiple OSes.
-  brew update
-  brew outdated xctool || brew upgrade xctool
+build_objectivec_ios() {
   # Reused the build script that takes care of configuring and ensuring things
-  # are up to date. Xcode and conformance tests will be directly invoked.
+  # are up to date.  The OS X test runs the objc conformance test, so skip it
+  # here.
+  # Note: travis has xctool installed, and we've looked at using it in the past
+  # but it has ended up proving unreliable (bugs), an they are removing build
+  # support in favor of xcbuild (or just xcodebuild).
   objectivec/DevTools/full_mac_build.sh \
-      --core-only --skip-xcode --skip-objc-conformance
+      --core-only --skip-xcode-osx --skip-objc-conformance "$@"
 }
 
-internal_xctool_debug_and_release() {
-  # Always use -reporter plain to avoid escape codes in output (makes travis
-  # logs easier to read).
-  xctool -reporter plain -configuration Debug "$@"
-  xctool -reporter plain -configuration Release "$@"
+build_objectivec_ios_debug() {
+  build_objectivec_ios --skip-xcode-release
 }
 
-build_objectivec_ios() {
-  internal_objectivec_common
-  # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool
-  # doesn't support >1 destination, so we have to build first and then run the
-  # tests one destination at a time.
-  internal_xctool_debug_and_release \
-    -project objectivec/ProtocolBuffers_iOS.xcodeproj \
-    -scheme ProtocolBuffers \
-    -sdk iphonesimulator \
-    build-tests
-  IOS_DESTINATIONS=(
-    "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-    "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
-    "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-    "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
-  )
-  for i in "${IOS_DESTINATIONS[@]}" ; do
-    internal_xctool_debug_and_release \
-      -project objectivec/ProtocolBuffers_iOS.xcodeproj \
-      -scheme ProtocolBuffers \
-      -sdk iphonesimulator \
-      -destination "${i}" \
-      run-tests
-  done
+build_objectivec_ios_release() {
+  build_objectivec_ios --skip-xcode-debug
 }
 
 build_objectivec_osx() {
-  internal_objectivec_common
-  internal_xctool_debug_and_release \
-    -project objectivec/ProtocolBuffers_OSX.xcodeproj \
-    -scheme ProtocolBuffers \
-    -destination "platform=OS X,arch=x86_64" \
-    test
-  cd conformance && make test_objc && cd ..
+  # Reused the build script that takes care of configuring and ensuring things
+  # are up to date.
+  objectivec/DevTools/full_mac_build.sh \
+      --core-only --skip-xcode-ios
 }
 
 build_python() {
@@ -330,6 +301,8 @@ Usage: $0 { cpp |
             javanano_jdk7 |
             javanano_oracle7 |
             objectivec_ios |
+            objectivec_ios_debug |
+            objectivec_ios_release |
             objectivec_osx |
             python |
             python_cpp |