Skip to content

Commit

Permalink
ofg: prepare use of cached images (#12693)
Browse files Browse the repository at this point in the history
Ref: google/oss-fuzz-gen#696

---------

Signed-off-by: David Korczynski <[email protected]>
Co-authored-by: Oliver Chang <[email protected]>
  • Loading branch information
DavidKorczynski and oliverchang authored Nov 7, 2024
1 parent edfa673 commit e3ccf89
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 17 deletions.
18 changes: 15 additions & 3 deletions infra/build/functions/build_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,8 @@ def get_docker_build_step(image_names,
directory,
use_buildkit_cache=False,
src_root='oss-fuzz',
architecture='x86_64'):
architecture='x86_64',
cache_image=''):
"""Returns the docker build step."""
assert len(image_names) >= 1
directory = os.path.join(src_root, directory)
Expand All @@ -404,6 +405,9 @@ def get_docker_build_step(image_names,
_make_image_name_architecture_specific(image_name, architecture)
for image_name in image_names
]
if cache_image:
args.extend(['--build-arg', f'CACHE_IMAGE={cache_image}'])

for image_name in image_names:
args.extend(['--tag', image_name])

Expand Down Expand Up @@ -437,7 +441,8 @@ def get_project_image_steps( # pylint: disable=too-many-arguments
language,
config,
architectures=None,
experiment=False):
experiment=False,
cache_image=None):
"""Returns GCB steps to build OSS-Fuzz project image."""
if architectures is None:
architectures = []
Expand All @@ -453,9 +458,16 @@ def get_project_image_steps( # pylint: disable=too-many-arguments
if config.test_image_suffix:
steps.extend(get_pull_test_images_steps(config.test_image_suffix))
src_root = 'oss-fuzz' if not experiment else '.'

steps.append({
'name': 'ubuntu',
'args': ['bash', '-c', f'cat {src_root}/projects/{name}/Dockerfile'],
})

docker_build_step = get_docker_build_step([image],
os.path.join('projects', name),
src_root=src_root)
src_root=src_root,
cache_image=cache_image)
steps.append(docker_build_step)
srcmap_step_id = get_srcmap_step_id()
steps.extend([{
Expand Down
33 changes: 19 additions & 14 deletions infra/build/functions/build_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,13 +185,12 @@ def sanitizers(self):
@property
def image(self):
"""Returns the docker image for the project."""
if self.cached_sanitizer:
return self.cached_image(self.cached_sanitizer)

return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}'

def cached_image(self, sanitizer):
return _CACHED_IMAGE.format(name=self.real_name, sanitizer=sanitizer)
@property
def cached_image(self):
return _CACHED_IMAGE.format(name=self.real_name,
sanitizer=self.cached_sanitizer)


def get_last_step_id(steps):
Expand Down Expand Up @@ -354,17 +353,23 @@ def get_build_steps_for_project(project,
return []

timestamp = get_datetime_now().strftime('%Y%m%d%H%M')

# If we use caching, then we need to use the right name. We assume that
# there is only a single sanitizer.
if use_caching:
# Use cached built image.
build_steps = []
project.cached_sanitizer = project.sanitizers[0]
cache_image = project.cached_image
else:
build_steps = build_lib.get_project_image_steps(
project.name,
project.image,
project.fuzzing_language,
config=config,
architectures=project.architectures,
experiment=config.experiment)
cache_image = None

build_steps = build_lib.get_project_image_steps(
project.name,
project.image,
project.fuzzing_language,
config=config,
architectures=project.architectures,
experiment=config.experiment,
cache_image=cache_image)

# Sort engines to make AFL first to test if libFuzzer has an advantage in
# finding bugs first since it is generally built first.
Expand Down

0 comments on commit e3ccf89

Please sign in to comment.