From 69211171f8f94c3142a20a8f491d023967b6b21d Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 9 Dec 2022 14:49:16 +0000 Subject: [PATCH] libgit2: Remove libgit2 from fuzzers Signed-off-by: Paulo Gomes --- tests/fuzz/Dockerfile.builder | 8 +-- tests/fuzz/compile_native_go_fuzzer | 62 ---------------------- tests/fuzz/oss_fuzz_build.sh | 80 ----------------------------- tests/fuzz/oss_fuzz_postbuild.sh | 33 ------------ tests/fuzz/oss_fuzz_prebuild.sh | 51 ------------------ tests/fuzz/oss_fuzz_run.sh | 2 +- 6 files changed, 5 insertions(+), 231 deletions(-) delete mode 100755 tests/fuzz/compile_native_go_fuzzer delete mode 100755 tests/fuzz/oss_fuzz_build.sh delete mode 100755 tests/fuzz/oss_fuzz_postbuild.sh diff --git a/tests/fuzz/Dockerfile.builder b/tests/fuzz/Dockerfile.builder index a09a8e6e..293a261d 100644 --- a/tests/fuzz/Dockerfile.builder +++ b/tests/fuzz/Dockerfile.builder @@ -1,9 +1,9 @@ FROM gcr.io/oss-fuzz-base/base-builder-go -RUN apt-get update && apt-get install -y cmake pkg-config +ENV SRC=$GOPATH/src/github.com/fluxcd/source-controller +ENV FLUX_CI=true -COPY ./ $GOPATH/src/github.com/fluxcd/source-controller/ -COPY ./tests/fuzz/oss_fuzz_build.sh $SRC/build.sh -COPY tests/fuzz/compile_native_go_fuzzer /usr/local/bin/ +COPY ./ $SRC +RUN wget https://raw.githubusercontent.com/google/oss-fuzz/master/projects/fluxcd/build.sh -O $SRC/build.sh WORKDIR $SRC diff --git a/tests/fuzz/compile_native_go_fuzzer b/tests/fuzz/compile_native_go_fuzzer deleted file mode 100755 index 447c7477..00000000 --- a/tests/fuzz/compile_native_go_fuzzer +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -eux -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -# This is a copy of the upstream script which is only needed to link -# additional static libraries. Orignal source: -# -# https://github.com/google/oss-fuzz/blob/9e8dd47cb902545efc60a5580126adc36d70bae3/infra/base-images/base-builder/compile_native_go_fuzzer - -function build_native_go_fuzzer() { - fuzzer=$1 - function=$2 - path=$3 - tags="-tags gofuzz" - - if [[ $SANITIZER == *coverage* ]]; then - current_dir=$(pwd) - mkdir $OUT/rawfuzzers || true - cd $abs_file_dir - go test -c -run $fuzzer -o $OUT/$fuzzer -cover - cp "${fuzzer_filename}" "${OUT}/rawfuzzers/${fuzzer}" - cd $current_dir - else - go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir - # TODO: upstream support for linking $ADDITIONAL_LIBS - $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer \ - $ADDITIONAL_LIBS - fi -} - - -path=$1 -function=$2 -fuzzer=$3 -tags="-tags gofuzz" - -# Get absolute path. -abs_file_dir=$(go list $tags -f {{.Dir}} $path) - -# TODO(adamkorcz): Get rid of "-r" flag here. -fuzzer_filename=$(grep -r -l --include='*.go' -s "$function" "${abs_file_dir}") - -# Test if file contains a line with "func $function" and "testing.F". -if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ] -then - build_native_go_fuzzer $fuzzer $function $abs_file_dir -else - echo "Could not find the function: func ${function}(f *testing.F)" -fi diff --git a/tests/fuzz/oss_fuzz_build.sh b/tests/fuzz/oss_fuzz_build.sh deleted file mode 100755 index 45c2e278..00000000 --- a/tests/fuzz/oss_fuzz_build.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2022 The Flux authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euxo pipefail - -# This file aims for: -# - Dynamically discover and build all fuzz tests within the repository. -# - Work for both local make fuzz-smoketest and the upstream oss-fuzz. - -GOPATH="${GOPATH:-/root/go}" -GO_SRC="${GOPATH}/src" -PROJECT_PATH="github.com/fluxcd/source-controller" - -# install_deps installs all dependencies needed for upstream oss-fuzz. -# Unfortunately we can't pin versions here, as we want to always -# have the latest, so that we can reproduce errors occuring upstream. -install_deps(){ - if ! command -v go-118-fuzz-build &> /dev/null; then - go install github.com/AdamKorcz/go-118-fuzz-build@latest - fi -} - -install_deps - -cd "${GO_SRC}/${PROJECT_PATH}" - -# Ensure any project-specific requirements are catered for ahead of -# the generic build process. -if [ -f "tests/fuzz/oss_fuzz_prebuild.sh" ]; then - . tests/fuzz/oss_fuzz_prebuild.sh -fi - -modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g') - -for module in ${modules}; do - - cd "${GO_SRC}/${PROJECT_PATH}/${module}" - - test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "") - if [ -z "${test_files}" ]; then - continue - fi - - go get github.com/AdamKorcz/go-118-fuzz-build/testing - - # Iterate through all Go Fuzz targets, compiling each into a fuzzer. - for file in ${test_files}; do - # If the subdir is a module, skip this file, as it will be handled - # at the next iteration of the outer loop. - if [ -f "$(dirname "${file}")/go.mod" ]; then - continue - fi - - targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}") - for target_name in ${targets}; do - # Transform module path into module name (e.g. git/libgit2 to git_libgit2). - module_name="$(echo ${module} | tr / _)_" - # Compose fuzzer name based on the lowercase version of the func names. - # The module name is added after the fuzz prefix, for better discoverability. - fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]' | sed "s;fuzz_;fuzz_${module_name//._/};g") - target_dir=$(dirname "${file}") - - echo "Building ${file}.${target_name} into ${fuzzer_name}" - compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}" - done - done -done diff --git a/tests/fuzz/oss_fuzz_postbuild.sh b/tests/fuzz/oss_fuzz_postbuild.sh deleted file mode 100755 index 2b155772..00000000 --- a/tests/fuzz/oss_fuzz_postbuild.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2022 The Flux authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euxo pipefail - -# This file is executed by upstream oss-fuzz after its building process. -# Use it for unsetting any environment variables that may impact other building -# processes. - -if [[ -n "${PRE_LIB_FUZZING_ENGINE}" ]]; then - export LIB_FUZZING_ENGINE="${PRE_LIB_FUZZING_ENGINE}" -fi - -unset TARGET_DIR -unset CGO_ENABLED -unset LIBRARY_PATH -unset PKG_CONFIG_PATH -unset CGO_CFLAGS -unset CGO_LDFLAGS -unset PRE_LIB_FUZZING_ENGINE diff --git a/tests/fuzz/oss_fuzz_prebuild.sh b/tests/fuzz/oss_fuzz_prebuild.sh index ac6314f5..308407ca 100755 --- a/tests/fuzz/oss_fuzz_prebuild.sh +++ b/tests/fuzz/oss_fuzz_prebuild.sh @@ -23,54 +23,3 @@ set -euxo pipefail # for traversing into ascending dirs, therefore we copy those contents here: mkdir -p controllers/testdata/crd cp config/crd/bases/*.yaml controllers/testdata/crd/ - -# libgit2, cmake and pkg-config are requirements to support libgit2. -LIBGIT2_TAG="${LIBGIT2_TAG:-v0.4.0}" - -# Avoid updating apt get and installing dependencies, if they are already in place. -if (! command -v cmake &> /dev/null) || (! command -v pkg-config &> /dev/null) then - apt-get update && apt-get install -y cmake pkg-config -fi - -export TARGET_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}" - -# For most cases, libgit2 will already be present. -# The exception being at the oss-fuzz integration. -if [ ! -d "${TARGET_DIR}" ]; then - curl --connect-timeout 2 --retry 3 --retry-delay 1 --retry-max-time 30 \ - -o output.tar.gz -LO "https://github.com/fluxcd/golang-with-libgit2/releases/download/${LIBGIT2_TAG}/linux-$(uname -m)-libgit2-only.tar.gz" - - DIR=linux-libgit2-only - NEW_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}" - INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}" - - mkdir -p ./build/libgit2 - - tar -xf output.tar.gz - rm output.tar.gz - mv "${DIR}" "${LIBGIT2_TAG}" - mv "${LIBGIT2_TAG}/" "./build/libgit2" - - # Update the prefix paths included in the .pc files. - # This will make it easier to update to the location in which they will be used. - find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {} -fi - -export CGO_ENABLED=1 -export LIBRARY_PATH="${TARGET_DIR}/lib" -export PKG_CONFIG_PATH="${TARGET_DIR}/lib/pkgconfig" -export CGO_CFLAGS="-I${TARGET_DIR}/include" -export CGO_LDFLAGS="$(pkg-config --libs --static --cflags libgit2)" - -# Temporary hack whilst libgit2 is still in use. -# Enables the fuzzing compilation to link libgit2. -# -# After building the fuzzers, the value of -# LIB_FUZZING_ENGINE is reset to what it was before -# it to avoid side effects onto other repositories. -# -# For context refer to: -# https://github.com/google/oss-fuzz/pull/9063 -export PRE_LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE}" - -export LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE} -Wl,--start-group ${TARGET_DIR}/lib/libgit2.a" diff --git a/tests/fuzz/oss_fuzz_run.sh b/tests/fuzz/oss_fuzz_run.sh index 4c87f489..12912e51 100755 --- a/tests/fuzz/oss_fuzz_run.sh +++ b/tests/fuzz/oss_fuzz_run.sh @@ -17,4 +17,4 @@ set -euxo pipefail # run each fuzzer once to ensure they are working properly -find /out -type f -name "fuzz*" -exec echo {} -runs=1 \; | bash -e +find /out -type f -iname "fuzz*" -exec echo {} -runs=1 \; | bash -e