compiler: Upgrade to protobuf 33.1 (#12534)

kannanjgithub · web-flow · commit 58ae5f808cf8 · 2025-12-05T16:42:43.000+05:30
diff --git a/buildscripts/kokoro/macos.sh b/buildscripts/kokoro/macos.sh
@@ -1,12 +1,17 @@
 #!/bin/bash
 set -veux -o pipefail
+CMAKE_VERSION=3.31.10
 
 if [[ -f /VERSION ]]; then
   cat /VERSION
 fi
 
 readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)"
 
+DOWNLOAD_DIR=/tmp/source
+mkdir -p ${DOWNLOAD_DIR}
+curl -Ls https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-macos-universal.tar.gz | tar xz -C ${DOWNLOAD_DIR}
+
 # We had problems with random tests timing out because it took seconds to do
 # trivial (ns) operations. The Kokoro Mac machines have 2 cores with 4 logical
 # threads, so Gradle should be using 4 workers by default.
@@ -15,7 +20,9 @@ export GRADLE_FLAGS="${GRADLE_FLAGS:-} --max-workers=2"
 . "$GRPC_JAVA_DIR"/buildscripts/kokoro/kokoro.sh
 trap spongify_logs EXIT
 
-export -n JAVA_HOME
-export PATH="$(/usr/libexec/java_home -v"1.8.0")/bin:${PATH}"
+brew install --cask temurin@8
+export PATH="$(/usr/libexec/java_home -v"1.8.0")/bin:${DOWNLOAD_DIR}/cmake-${CMAKE_VERSION}-macos-universal/CMake.app/Contents/bin:${PATH}"
+export JAVA_HOME="$(/usr/libexec/java_home -v"1.8.0")"
+brew install maven
 
 "$GRPC_JAVA_DIR"/buildscripts/kokoro/unix.sh
diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat
@@ -25,7 +25,7 @@ cd "%WORKSPACE%"
 
 SET TARGET_ARCH=x86_32
 SET FAIL_ON_WARNINGS=true
-SET PROTOBUF_VER=26.1
+SET PROTOBUF_VER=33.1
 SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
 SET VC_PROTOBUF_LIBS=/LIBPATH:%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
 SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
@@ -66,14 +66,16 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
       set lib=!lib:~2!
       @rem remove spaces
       set lib=!lib: =!
-      @rem Because protobuf is specified as libprotobuf and elsewhere
-      if !lib! NEQ protobuf (
+      set libprefix=!lib:~0,4!
+      if !libprefix!==absl (
         set lib=!lib!.lib
-        if "!libs_list!"=="" (
-          set libs_list=!lib!
-        ) else (
-          set libs_list=!libs_list!,!lib!
-        )
+      ) else (
+        set lib=lib!lib!.lib
+      )
+      if "!libs_list!"=="" (
+        set libs_list=!lib!
+      ) else (
+        set libs_list=!libs_list!,!lib!
       )
     )
   )
diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat
@@ -24,7 +24,7 @@ cd "%WORKSPACE%"
 
 SET TARGET_ARCH=x86_64
 SET FAIL_ON_WARNINGS=true
-SET PROTOBUF_VER=26.1
+SET PROTOBUF_VER=33.1
 SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
 SET VC_PROTOBUF_LIBS=/LIBPATH:%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
 SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
@@ -50,14 +50,16 @@ for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
       set lib=!lib:~2!
       @rem remove spaces
       set lib=!lib: =!
-      @rem Because protobuf is specified as libprotobuf and elsewhere
-      if !lib! NEQ protobuf (
+      set libprefix=!lib:~0,4!
+      if !libprefix!==absl (
         set lib=!lib!.lib
-        if "!libs_list!"=="" (
-          set libs_list=!lib!
-        ) else (
-          set libs_list=!libs_list!,!lib!
-        )
+      ) else (
+        set lib=lib!lib!.lib
+      )
+      if "!libs_list!"=="" (
+        set libs_list=!lib!
+      ) else (
+        set libs_list=!libs_list!,!lib!
       )
     )
   )
diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat
@@ -1,7 +1,7 @@
 choco install -y pkgconfiglite
 choco install -y openjdk --version=17.0
 set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-17\bin"
-set PROTOBUF_VER=26.1
+set PROTOBUF_VER=33.1
 set ABSL_VERSION=20250127.1
 set CMAKE_NAME=cmake-3.26.3-windows-x86_64
 
@@ -30,7 +30,6 @@ del protobuf.zip
 powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1
 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1
 del absl.zip
-rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
 move abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
 mkdir protobuf-%PROTOBUF_VER%\build
 pushd protobuf-%PROTOBUF_VER%\build
@@ -51,7 +50,7 @@ for /f "tokens=4 delims=\" %%a in ("%VCINSTALLDIR%") do (
 for /f "tokens=1 delims=." %%a in ("%VisualStudioVersion%") do (
   SET visual_studio_major_version=%%a
 )
-cmake -DABSL_MSVC_STATIC_RUNTIME=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% .. || exit /b 1
+cmake -DCMAKE_CXX_STANDARD=17 -DABSL_MSVC_STATIC_RUNTIME=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% .. || exit /b 1
 cmake --build . --config Release --target install || exit /b 1
 popd
 goto :eof
diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh
@@ -3,54 +3,33 @@
 # Build protoc
 set -evux -o pipefail
 
-PROTOBUF_VERSION=26.1
+PROTOBUF_VERSION=33.1
 ABSL_VERSION=20250127.1
-CMAKE_VERSION=3.26.3
 
 # ARCH is x86_64 bit unless otherwise specified.
 ARCH="${ARCH:-x86_64}"
 DOWNLOAD_DIR=/tmp/source
 INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH"
 BUILDSCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
-mkdir -p $DOWNLOAD_DIR
-cd "$DOWNLOAD_DIR"
-
-# Start with a sane default
-NUM_CPU=4
-if [[ $(uname) == 'Linux' ]]; then
-    NUM_CPU=$(nproc)
-fi
-if [[ $(uname) == 'Darwin' ]]; then
-    NUM_CPU=$(sysctl -n hw.ncpu)
-fi
 
-# Make protoc
-# Can't check for presence of directory as cache auto-creates it.
-if [ -f ${INSTALL_DIR}/bin/protoc ]; then
-  echo "Not building protobuf. Already built"
-# TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date
-else
-  if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then
-    curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz
-    curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz
-    rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp"
-    mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp"
+function build_and_install() {
+  if [[ "$1" == "abseil" ]]; then
+    TESTS_OFF_ARG=ABSL_BUILD_TEST_HELPERS
+  else
+    TESTS_OFF_ARG=protobuf_BUILD_TESTS  
   fi
-  # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first
-  rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
-  mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
-  pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
-  # install here so we don't need sudo
   if [[ "$(uname -s)" == "Darwin" ]]; then
     cmake .. \
-      -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \
-      -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 \
+      -DCMAKE_CXX_STANDARD=17 -D${TESTS_OFF_ARG}=OFF -DBUILD_SHARED_LIBS=OFF \
+      -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+      -DCMAKE_PREFIX_PATH="$INSTALL_DIR" \
       -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
       -B. || exit 1
   elif [[ "$ARCH" == x86* ]]; then
     CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \
-      -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \
-      -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 \
+      -DCMAKE_CXX_STANDARD=17 -D${TESTS_OFF_ARG}=OFF -DBUILD_SHARED_LIBS=OFF \
+      -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+      -DCMAKE_PREFIX_PATH="$INSTALL_DIR" \
       -B. || exit 1
   else
     if [[ "$ARCH" == aarch_64 ]]; then
@@ -66,17 +45,56 @@ else
       exit 1
     fi
     cmake .. \
-      -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \
-      -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DABSL_INTERNAL_AT_LEAST_CXX17=0 \
+      -DCMAKE_CXX_STANDARD=17 -D${TESTS_OFF_ARG}=OFF -DBUILD_SHARED_LIBS=OFF \
+      -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+      -DCMAKE_PREFIX_PATH="$INSTALL_DIR" \
       -Dcrosscompile_ARCH="$GCC_ARCH" \
       -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake \
       -B. || exit 1
   fi
   export CMAKE_BUILD_PARALLEL_LEVEL="$NUM_CPU"
   cmake --build . || exit 1
+  # install here so we don't need sudo
   cmake --install . || exit 1
-  [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib"
+}
+
+mkdir -p $DOWNLOAD_DIR
+cd "$DOWNLOAD_DIR"
+
+# Start with a sane default
+NUM_CPU=4
+if [[ $(uname) == 'Linux' ]]; then
+    NUM_CPU=$(nproc)
+fi
+if [[ $(uname) == 'Darwin' ]]; then
+    NUM_CPU=$(sysctl -n hw.ncpu)
+fi
+export CMAKE_BUILD_PARALLEL_LEVEL="$NUM_CPU"
+
+# Make protoc
+# Can't check for presence of directory as cache auto-creates it.
+if [ -f ${INSTALL_DIR}/bin/protoc ]; then
+  echo "Not building protobuf. Already built"
+# TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date
+else
+  if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then
+    curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz
+    curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz
+  fi
+  # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first
+  rm -rf "$DOWNLOAD_DIR/abseil-cpp-${ABSL_VERSION}/build"
+  mkdir "$DOWNLOAD_DIR/abseil-cpp-${ABSL_VERSION}/build"
+  pushd "$DOWNLOAD_DIR/abseil-cpp-${ABSL_VERSION}/build"
+  build_and_install "abseil"
+  popd 
+  
+  rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
+  mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
+  pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
+  build_and_install "protobuf"
   popd
+  
+  [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib"  
 fi
 
 # If /tmp/protobuf exists then we just assume it's a symlink created by us.
@@ -94,3 +112,4 @@ export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cfla
 export LIBRARY_PATH=/tmp/protobuf/lib
 export LD_LIBRARY_PATH=/tmp/protobuf/lib
 EOF
+
diff --git a/compiler/build.gradle b/compiler/build.gradle
@@ -102,7 +102,7 @@ model {
         all {
             if (toolChain in Gcc || toolChain in Clang) {
                 cppCompiler.define("GRPC_VERSION", version)
-                cppCompiler.args "--std=c++14"
+                cppCompiler.args "--std=c++17"
                 addEnvArgs("CXXFLAGS", cppCompiler.args)
                 addEnvArgs("CPPFLAGS", cppCompiler.args)
                 if (project.hasProperty('buildUniversal') &&
@@ -132,7 +132,7 @@ model {
             } else if (toolChain in VisualCpp) {
                 usingVisualCpp = true
                 cppCompiler.define("GRPC_VERSION", version)
-                cppCompiler.args "/EHsc", "/MT"
+                cppCompiler.args "/EHsc", "/MT", "/std:c++17"
                 if (rootProject.hasProperty('vcProtobufInclude')) {
                     cppCompiler.args "/I${rootProject.vcProtobufInclude}"
                 } 
