Merge branch 'master' into add-replay-capture-1

infra/base-images/base-builder/Dockerfile

@@ -47,6 +47,16 @@ RUN export PYTHON_DEPS="\
     rm -rf /usr/local/lib/python3.8/test && \
     apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888
 
+
+ENV CCACHE_VERSION 4.10.2
+RUN cd /tmp && curl -OL https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz && \
+    tar -xvf ccache-$CCACHE_VERSION.tar.xz && cd ccache-$CCACHE_VERSION && \
+    mkdir build && cd build && \
+    export LDFLAGS='-lpthread' && \
+    cmake -D CMAKE_BUILD_TYPE=Release .. && \
+    make -j && make install && \
+    rm -rf /tmp/ccache-$CCACHE_VERSION /tmp/ccache-$CCACHE_VERSION.tar.xz
+
 # Install six for Bazel rules.
 RUN unset CFLAGS CXXFLAGS && pip3 install -v --no-cache-dir \
     six==1.15.0 && rm -rf /tmp/*
@@ -181,4 +191,13 @@ COPY llvmsymbol.diff $SRC
 COPY detect_repo.py /opt/cifuzz/
 COPY bazel.bazelrc /root/.bazelrc
 
+# Set up ccache binary and cache directory.
+# /ccache/bin will contain the compiler wrappers, and /ccache/cache will
+# contain the actual cache, which can be saved.
+# To use this, set PATH=/ccache/bin:$PATH.
+RUN mkdir -p /ccache/bin && mkdir -p /ccache/cache && \
+    ln -s /usr/local/bin/ccache /ccache/bin/clang && \
+    ln -s /usr/local/bin/ccache /ccache/bin/clang++
+ENV CCACHE_DIR /ccache/cache
+
 CMD ["compile"]

infra/base-images/base-clang/Dockerfile

@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \
 RUN apt-get update && apt-get install -y git && \
     git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
     cd fuzz-introspector && \
-    git checkout 3ec7681506e00a936247103cb51ea4dc1538c930 && \
+    git checkout 5924aea8bcfe1fbdac9dc815adff91d3ee51f52b && \
     git submodule init && \
     git submodule update && \
     apt-get autoremove --purge -y git && \

infra/build/functions/base_images.py

@@ -76,7 +76,8 @@ def run_build(steps, images, tags=None, build_version=MAJOR_TAG):
           'machineType': 'E2_HIGHCPU_32'
       },
   }
-  return build_lib.run_build(steps,
+  return build_lib.run_build('',
+                             steps,
                              credentials,
                              BASE_PROJECT,
                              TIMEOUT,

infra/build/functions/build_and_run_coverage.py

@@ -316,7 +316,11 @@ def get_fuzz_introspector_steps(  # pylint: disable=too-many-locals, too-many-ar
   env.append(f'PROJECT_NAME={project.name}')
 
   build_steps.append(
-      build_project.get_compile_step(project, build, env, config.parallel))
+      build_project.get_compile_step(project,
+                                     build,
+                                     env,
+                                     config.parallel,
+                                     allow_failure=True))
 
   # Upload the report.
   upload_report_url = bucket.get_upload_url('inspector-report')

infra/build/functions/build_project.py

@@ -61,6 +61,10 @@
 LOCAL_BUILD_LOG_PATH = '/workspace/build.log'
 BUILD_SUCCESS_MARKER = '/workspace/build.succeeded'
 
+_CACHED_IMAGE = ('us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/'
+                 '{name}-ofg-cached-{sanitizer}')
+_CACHED_SANITIZERS = ('address', 'coverage')
+
 
 @dataclass
 class Config:
@@ -163,6 +167,8 @@ def __init__(self, name, project_yaml, dockerfile):
     else:
       self.main_repo = ''
 
+    self.cached_sanitizer = None
+
   @property
   def sanitizers(self):
     """Returns processed sanitizers."""
@@ -172,8 +178,14 @@ def sanitizers(self):
   @property
   def image(self):
     """Returns the docker image for the project."""
+    if self.cached_sanitizer:
+      return self.cached_image(self.cached_sanitizer)
+
     return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}'
 
+  def cached_image(self, sanitizer):
+    return _CACHED_IMAGE.format(name=self.name, sanitizer=sanitizer)
+
 
 def get_last_step_id(steps):
   """Returns the id of the last step in |steps|."""
@@ -243,7 +255,12 @@ def get_env(fuzzing_language, build):
   return list(sorted([f'{key}={value}' for key, value in env_dict.items()]))
 
 
-def get_compile_step(project, build, env, parallel, upload_build_logs=None):
+def get_compile_step(project,
+                     build,
+                     env,
+                     parallel,
+                     upload_build_logs=None,
+                     allow_failure=False):
   """Returns the GCB step for compiling |projects| fuzzers using |env|. The type
   of build is specified by |build|."""
   failure_msg = (
@@ -277,7 +294,7 @@ def get_compile_step(project, build, env, parallel, upload_build_logs=None):
       'id': get_id('compile', build),
   }
 
-  if upload_build_logs:
+  if upload_build_logs or allow_failure:
     # The failure will be reported in a subsequent step.
     compile_step['allowFailure'] = True
 
@@ -308,30 +325,37 @@ def get_build_steps(  # pylint: disable=too-many-locals, too-many-statements, to
     project_yaml,
     dockerfile,
     config,
-    additional_env=None):
+    additional_env=None,
+    use_caching=False):
   """Returns build steps for project."""
 
   project = Project(project_name, project_yaml, dockerfile)
-
   if project.disabled:
     logging.info('Project "%s" is disabled.', project.name)
     return []
 
   timestamp = get_datetime_now().strftime('%Y%m%d%H%M')
-  build_steps = build_lib.get_project_image_steps(
-      project.name,
-      project.image,
-      project.fuzzing_language,
-      config=config,
-      architectures=project.architectures,
-      experiment=config.experiment)
+  if use_caching:
+    # Use cached built image.
+    build_steps = []
+  else:
+    build_steps = build_lib.get_project_image_steps(
+        project.name,
+        project.image,
+        project.fuzzing_language,
+        config=config,
+        architectures=project.architectures,
+        experiment=config.experiment)
 
   # Sort engines to make AFL first to test if libFuzzer has an advantage in
   # finding bugs first since it is generally built first.
   for fuzzing_engine in sorted(project.fuzzing_engines):
     # Sort sanitizers and architectures so order is determinisitic (good for
     # tests).
     for sanitizer in sorted(project.sanitizers):
+      if use_caching and sanitizer in _CACHED_SANITIZERS:
+        project.cached_sanitizer = sanitizer
+
       # Build x86_64 before i386.
       for architecture in reversed(sorted(project.architectures)):
         build = Build(fuzzing_engine, sanitizer, architecture)

infra/build/functions/request_introspector_build.py

@@ -32,8 +32,7 @@ def get_build_steps(project_name, image_project, base_images_project):
   project_yaml_contents, dockerfile_lines = request_build.get_project_data(
       project_name)
   return build_and_run_coverage.get_fuzz_introspector_steps(
-      project_name, project_yaml_contents, dockerfile_lines, image_project,
-      base_images_project, build_config)
+      project_name, project_yaml_contents, dockerfile_lines, build_config)
 
 
 def request_introspector_build(event, context):

infra/build/functions/target_experiment.py

@@ -30,7 +30,8 @@
 
 def run_experiment(project_name, target_name, args, output_path, errlog_path,
                    build_output_path, upload_corpus_path, upload_coverage_path,
-                   experiment_name, upload_reproducer_path, tags):
+                   experiment_name, upload_reproducer_path, tags,
+                   use_cached_image):
   config = build_project.Config(testing=True,
                                 test_image_suffix='',
                                 repo=build_project.DEFAULT_OSS_FUZZ_REPO,
@@ -68,7 +69,8 @@ def run_experiment(project_name, target_name, args, output_path, errlog_path,
                                         project_yaml,
                                         dockerfile_contents,
                                         config,
-                                        additional_env=jcc_env)
+                                        additional_env=jcc_env,
+                                        use_caching=use_cached_image)
 
   build = build_project.Build('libfuzzer', 'address', 'x86_64')
   local_output_path = '/workspace/output.log'
@@ -209,6 +211,9 @@ def run_experiment(project_name, target_name, args, output_path, errlog_path,
   env = build_project.get_env(project_yaml['language'], build)
   env.extend(jcc_env)
 
+  if use_cached_image:
+    project.cached_sanitizer = 'coverage'
+
   steps.append(
       build_project.get_compile_step(project, build, env, config.parallel))
 
@@ -330,12 +335,15 @@ def main():
                       nargs='*',
                       help='Tags for cloud build.',
                       default=[])
+  parser.add_argument('--use_cached_image',
+                      action='store_true',
+                      help='Use cached images post build.')
   args = parser.parse_args()
 
   run_experiment(args.project, args.target, args.args, args.upload_output_log,
                  args.upload_err_log, args.upload_build_log, args.upload_corpus,
                  args.upload_coverage, args.experiment_name,
-                 args.upload_reproducer, args.tags)
+                 args.upload_reproducer, args.tags, args.use_cached_image)
 
 
 if __name__ == '__main__':

infra/build/request_build.sh

@@ -19,6 +19,8 @@ if [ "$2" = "fuzzing" ]; then
   topic=request-build
 elif [ "$2" = "coverage" ]; then
   topic=request-coverage-build
+elif [ "$2" = "introspector" ]; then
+  topic=request-introspector-build
 else
   echo "Invalid build type $2."
   exit 1

infra/cifuzz/config_utils.py

@@ -129,7 +129,7 @@ def __init__(self):
     self.build_integration_path = (
         constants.DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH)
 
-    self.parallel_fuzzing = os.environ.get('PARALLEL_FUZZING')
+    self.parallel_fuzzing = environment.get_bool('PARALLEL_FUZZING', False)
     self.extra_environment_variables = _get_extra_environment_variables()
     self.output_sarif = environment.get_bool('OUTPUT_SARIF', False)
 

infra/experimental/chronos/README.md

@@ -9,12 +9,12 @@ infra/experimental/chronos/prepare-recompile "$PROJECT" "$FUZZ_TARGET" "$FUZZING
 python infra/helper.py build_image "$PROJECT"
 # AddressSanitizer.
 docker run -ti --entrypoint="/bin/sh" --env SANITIZER="address" --name "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*"
-docker commit "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan"
+docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan"
 docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan"
 
 # Coverage measurement.
 docker run -ti --entrypoint="/bin/sh" --env SANITIZER="coverage" --name "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*"
-docker commit "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov"
+docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov"
 docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov"
 ```
 

infra/experimental/chronos/cloudbuild.yaml

@@ -40,8 +40,12 @@ steps:
 - name: 'gcr.io/cloud-builders/docker'
   args:
   - commit
+  - '--change'
+  - 'CMD ["/usr/local/bin/recompile"]'
+  - '--change'
+  - 'ENTRYPOINT ["/bin/sh", "-c"]'
   - ${_PROJECT}-origin-asan
-  - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-asan
+  - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address
 - name: 'gcr.io/cloud-builders/docker'
   args:
   - run
@@ -54,11 +58,15 @@ steps:
 - name: 'gcr.io/cloud-builders/docker'
   args:
   - commit
+  - '--change'
+  - 'CMD ["/usr/local/bin/recompile"]'
+  - '--change'
+  - 'ENTRYPOINT ["/bin/sh", "-c"]'
   - ${_PROJECT}-origin-cov
-  - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-cov
+  - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage
 images:
-- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-asan
-- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-cov
+- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address
+- us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage
 timeout: 1800s
 logsBucket: oss-fuzz-gcb-logs
 tags:

projects/apache-commons-collections/project.yaml

@@ -2,7 +2,7 @@ fuzzing_engines:
 - libfuzzer
 homepage: https://commons.apache.org/proper/commons-collections/
 language: jvm
-main_repo: https://gitbox.apache.org/repos/asf/commons-codellections.git
+main_repo: https://github.com/apache/commons-collections.git
 primary_contact: "[email protected]"
 sanitizers:
 - address
@@ -16,4 +16,4 @@ vendor_ccs:
   - "[email protected]"
   - "[email protected]"
   - "[email protected]"
-  - "[email protected]"
+  - "[email protected]"

projects/bitcoin-core/build.sh

@@ -52,7 +52,7 @@ export CPPFLAGS="-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG -DBOOST_M
 (
   cd depends
   sed -i --regexp-extended '/.*rm -rf .*extract_dir.*/d' ./funcs.mk  # Keep extracted source
-  make HOST=$BUILD_TRIPLET DEBUG=1 NO_QT=1 NO_BDB=1 NO_ZMQ=1 NO_UPNP=1 NO_NATPMP=1 NO_USDT=1 \
+  make HOST=$BUILD_TRIPLET DEBUG=1 NO_QT=1 NO_BDB=1 NO_ZMQ=1 NO_USDT=1 \
        AR=llvm-ar NM=llvm-nm RANLIB=llvm-ranlib STRIP=llvm-strip \
        -j$(nproc)
 )

projects/boost-json/Dockerfile

@@ -14,9 +14,7 @@
 #
 ################################################################################
 
-FROM gcr.io/oss-fuzz-base/base-builder@sha256:19782f7fe8092843368894dbc471ce9b30dd6a2813946071a36e8b05f5b1e27e
-# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors
-#RUN apt-get update && apt-get install -y g++
+FROM gcr.io/oss-fuzz-base/base-builder
 
 RUN git clone --depth 1 --single-branch --branch master https://github.com/boostorg/boost.git
 RUN pwd

projects/boost-json/build.sh

@@ -15,14 +15,21 @@
 #
 ################################################################################
 
-./bootstrap.sh --with-libraries=json
+./bootstrap.sh --with-toolset=clang
 
 echo "using clang : ossfuzz : $CXX : <compileflags>\"$CXXFLAGS\" <linkflags>\"$CXXFLAGS\" <linkflags>\"${LIB_FUZZING_ENGINE}\" ;" >user-config.jam
 
-./b2 --user-config=user-config.jam --toolset=clang-ossfuzz --prefix=$WORK/stage --with-json link=static install
+./b2 --user-config=user-config.jam                              \
+     --toolset=clang-ossfuzz                                    \
+     --prefix=$WORK/stage                                       \
+     --with-json                                                \
+     include=/usr/local/include/x86_64-unknown-linux-gnu/c++/v1 \
+     link=static                                                \
+     install
 
 for i in libs/json/fuzzing/*.cpp; do
    fuzzer=$(basename $i .cpp)
    $CXX $CXXFLAGS -pthread libs/json/fuzzing/$fuzzer.cpp -I $WORK/stage/include/ $WORK/stage/lib/*.a $LIB_FUZZING_ENGINE -o $OUT/$fuzzer
+   zip -q -r -j $OUT/${fuzzer}_seed_corpus.zip libs/json/fuzzing/old_crashes
 done
 

projects/boost/project.yaml

@@ -6,6 +6,7 @@ auto_ccs:
    - "[email protected]"
    - "[email protected]"
    - "[email protected]"
+   - "[email protected]"
    - "[email protected]"
 main_repo: 'https://github.com/boostorg/boost.git'
 

projects/bson-rust/build.sh

@@ -20,3 +20,4 @@ fuzz_release=fuzz/target/x86_64-unknown-linux-gnu/release
 cp $fuzz_release/deserialize $OUT/
 cp $fuzz_release/iterate $OUT/
 cp $fuzz_release/raw_deserialize $OUT/
+cp $fuzz_release/raw_deserialize_utf8_lossy $OUT/

projects/containerd/project.yaml

@@ -9,6 +9,7 @@ auto_ccs :
   - "[email protected]"
   - "[email protected]"
   - "[email protected]"
+  - "[email protected]"
 language: go
 fuzzing_engines:
   - libfuzzer

projects/data-encoding/Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+FROM gcr.io/oss-fuzz-base/base-builder-rust
+RUN apt-get update && \
+    apt-get install -y make autoconf automake libtool curl cmake python
+
+RUN git clone https://github.com/ia0/data-encoding
+WORKDIR $SRC/data-encoding
+
+COPY build.sh $SRC/