Skip to content

Commit

Permalink
install deps for windows based with hashes
Browse files Browse the repository at this point in the history
  • Loading branch information
s-westphal committed Jun 19, 2024
1 parent 4624ccb commit c3b2bd0
Show file tree
Hide file tree
Showing 22 changed files with 2,314 additions and 93 deletions.
13 changes: 11 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ jobs:
python-version: '3.9'
- name: Verify requirements
run: |
./travis/compile_requirements.sh ubuntu_requirements
./travis/compile_requirements.sh ubuntu_requirements ubuntu
- name: Upload requirements to GitHub artifacts
uses: actions/upload-artifact@v4
with:
Expand Down Expand Up @@ -126,7 +126,7 @@ jobs:
virtualenv "${HOME}/INSTALL"
- name: Verify requirements
run: |
./travis/compile_requirements.sh osx_requirements
./travis/compile_requirements.sh osx_requirements osx
- name: Upload requirements to GitHub artifacts
uses: actions/upload-artifact@v4
with:
Expand Down Expand Up @@ -195,6 +195,15 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: '3.9'
- name: Verify requirements
run: |
.\travis\compile_requirements.bat windows_requirements
- name: Upload requirements to GitHub artifacts
uses: actions/upload-artifact@v4
with:
name: windows-requirements
path: windows_requirements/
retention-days: 3
- name: Build installers
shell: bash
run: |
Expand Down
4 changes: 4 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,15 @@ RUN ${VIRTUAL_ENV}/bin/python -m pip install --no-deps --no-index \

RUN ${VIRTUAL_ENV}/bin/python -m pip install --require-hashes \
-r grr/client/requirements/ubuntu.txt
RUN ${VIRTUAL_ENV}/bin/python -m pip install --require-hashes \
-r grr/client/requirements/ubuntu_only.txt
RUN ${VIRTUAL_ENV}/bin/python -m pip install --no-deps --no-index \
-e grr/client

RUN ${VIRTUAL_ENV}/bin/python -m pip install --require-hashes \
-r grr/server/requirements/ubuntu.txt
RUN ${VIRTUAL_ENV}/bin/python -m pip install --require-hashes \
-r grr/server/requirements/ubuntu_only.txt
RUN ${VIRTUAL_ENV}/bin/python -m pip install --no-deps --no-index \
-e grr/server

Expand Down
322 changes: 322 additions & 0 deletions api_client/python/requirements/windows.txt

Large diffs are not rendered by default.

179 changes: 102 additions & 77 deletions appveyor/windows_templates/build_windows_templates.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python
"""Script to build windows templates."""

import argparse
Expand All @@ -11,54 +10,67 @@
import subprocess
import sys
import time

from typing import Callable
from typing import Callable, Optional, List

parser = argparse.ArgumentParser(description="Build windows templates.")

parser.add_argument(
"--build_dir", default=r"C:\grrbuild", help="GRR build directory.")
"--build_dir", default=r"C:\grrbuild", help="GRR build directory."
)

parser.add_argument(
"--grr_src",
default=r"C:\grrbuild\grr",
help="Location of the grr src code. If it doesn't exist "
" at this path we'll try to check it out from github.")
help=(
"Location of the grr src code. If it doesn't exist "
" at this path we'll try to check it out from github."
),
)

parser.add_argument(
"--output_dir",
default=r"C:\grrbuild\output",
help="Destination directory for the templates.")
help="Destination directory for the templates.",
)

parser.add_argument(
"--test_repack_install",
action="store_true",
default=False,
help="Test repacking by calling repack on the template after building,"
"then try and install the result. For use by integration tests. If you use "
"this option you must run as admin.")
help=(
"Test repacking by calling repack on the template after building,then"
" try and install the result. For use by integration tests. If you use"
" this option you must run as admin."
),
)

parser.add_argument(
"--wheel_dir",
default=None,
help="A directory that will be passed to pip as the wheel-dir parameter.")
help="A directory that will be passed to pip as the wheel-dir parameter.",
)

parser.add_argument(
"--expect_service_running",
dest="expect_service_running",
action="store_true",
help="Triggers whether after installation the GRR service should be "
"running or not. Used for testing the installation.")
help=(
"Triggers whether after installation the GRR service should be "
"running or not. Used for testing the installation."
),
)
parser.add_argument(
"--noexpect_service_running",
dest="expect_service_running",
action="store_false")
action="store_false",
)
parser.set_defaults(expect_service_running=True)

parser.add_argument(
"--config",
default="",
help="Path to the config file to be used when building templates.")
help="Path to the config file to be used when building templates.",
)


args = parser.parse_args()
Expand All @@ -79,13 +91,16 @@ def _FileRetryLoop(path: str, f: Callable[[], None]) -> None:
return
except OSError as e:
attempts += 1
if (e.errno == errno.EACCES and
attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS):
if (
e.errno == errno.EACCES
and attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS
):
# The currently installed GRR process may stick around for a few
# seconds after the service is terminated (keeping the contents of
# the installation directory locked).
logging.info("Permission-denied error while trying to process %s.",
path)
logging.info(
"Permission-denied error while trying to process %s.", path
)
time.sleep(1)
else:
raise
Expand All @@ -99,27 +114,6 @@ def _Rename(src: str, dst: str) -> None:
_FileRetryLoop(src, lambda: os.rename(src, dst))


def _RmTreePseudoTransactional(path: str) -> None:
"""Removes `path`.
Makes sure that either `path` is gone or that it is still present as
it was.
Args:
path: The path to remove.
"""
temp_path = f"{path}_orphaned_{int(time.time())}"
logging.info("Trying to rename %s -> %s.", path, temp_path)

_Rename(path, temp_path)

try:
logging.info("Trying to remove %s.", temp_path)
_RmTree(temp_path)
except: # pylint: disable=bare-except
logging.info("Failed to remove %s. Ignoring.", temp_path, exc_info=True)


def _VerboseCheckCall(params):
logging.info("Running: %s", params)

Expand All @@ -141,8 +135,9 @@ def SetupVars(self):

self.virtualenv64 = os.path.join(args.build_dir, "python_64")
self.grr_client_build64 = "grr_client_build"
self.virtualenv_python64 = os.path.join(self.virtualenv64,
r"Scripts\python.exe")
self.virtualenv_python64 = os.path.join(
self.virtualenv64, r"Scripts\python.exe"
)

self.git = r"git"

Expand Down Expand Up @@ -180,55 +175,76 @@ def Clean(self):
def GitCheckoutGRR(self):
os.chdir(args.build_dir)
subprocess.check_call(
[self.git, "clone", "https://github.com/google/grr.git"])
[self.git, "clone", "https://github.com/google/grr.git"]
)

def MakeProtoSdist(self):
os.chdir(os.path.join(args.grr_src, "grr/proto"))
subprocess.check_call([
self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
"--dist-dir=%s" % args.build_dir
self.virtualenv_python64,
"setup.py",
"sdist",
"--formats=zip",
"--dist-dir=%s" % args.build_dir,
])
return glob.glob(
os.path.join(args.build_dir, "grr_response_proto-*.zip")
).pop()
).pop(), [os.path.join(args.grr_src, "grr", "proto", "requirements", "windows.txt")]

def MakeCoreSdist(self):
os.chdir(os.path.join(args.grr_src, "grr/core"))
subprocess.check_call([
self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
"--dist-dir=%s" % args.build_dir, "--no-sync-artifacts"
self.virtualenv_python64,
"setup.py",
"sdist",
"--formats=zip",
"--dist-dir=%s" % args.build_dir,
"--no-sync-artifacts",
])
return glob.glob(
os.path.join(args.build_dir, "grr_response_core-*.zip")
).pop()
).pop(), [os.path.join(args.grr_src, "grr", "core", "requirements", "windows.txt")]

def MakeClientSdist(self):
os.chdir(os.path.join(args.grr_src, "grr/client/"))
subprocess.check_call([
self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
"--dist-dir=%s" % args.build_dir
self.virtualenv_python64,
"setup.py",
"sdist",
"--formats=zip",
"--dist-dir=%s" % args.build_dir,
])
return glob.glob(
os.path.join(args.build_dir, "grr_response_client-*.zip")
).pop()
).pop(), [
os.path.join(args.grr_src, "grr", "client", "requirements", "windows.txt"),
os.path.join(args.grr_src, "grr", "client", "requirements", "windows_only.txt"),
]

def MakeClientBuilderSdist(self):
os.chdir(os.path.join(args.grr_src, "grr/client_builder/"))
subprocess.check_call([
self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
"--dist-dir=%s" % args.build_dir
self.virtualenv_python64,
"setup.py",
"sdist",
"--formats=zip",
"--dist-dir=%s" % args.build_dir,
])
return glob.glob(
os.path.join(args.build_dir, "grr_response_client_builder-*.zip")
).pop()
).pop(), [os.path.join(args.grr_src, "grr", "client_builder", "requirements", "windows.txt")]

def InstallGRR(self, path):
def InstallGRR(self, path: str, requirements_files: Optional[List[str]] = None):
"""Installs GRR."""
if requirements_files:
for req_file in requirements_files:
install_requirements_cmd = ["pip", "install", "--require-hashes", "-r", req_file]
subprocess.check_call(install_requirements_cmd)

cmd64 = ["pip", "install"]
cmd64 = ["pip", "install", "--no-deps", "--no-index"]

if args.wheel_dir:
cmd64 += ["--no-index", r"--find-links=file:///%s" % args.wheel_dir]
cmd64 += [r"--find-links=file:///%s" % args.wheel_dir]

cmd64.append(path)

Expand All @@ -242,8 +258,12 @@ def BuildTemplates(self):
"""
if args.config:
build_args = [
"--verbose", "--config", args.config, "build", "--output",
args.output_dir
"--verbose",
"--config",
args.config,
"build",
"--output",
args.output_dir,
]
else:
build_args = ["--verbose", "build", "--output", args.output_dir]
Expand All @@ -268,9 +288,11 @@ def _WixToolsPath(self) -> str:
def _RepackTemplates(self):
"""Repack templates with a dummy config."""
dummy_config = os.path.join(
args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml")
template_amd64 = glob.glob(os.path.join(args.output_dir,
"*_amd64*.zip")).pop()
args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml"
)
template_amd64 = glob.glob(
os.path.join(args.output_dir, "*_amd64*.zip")
).pop()

fleetspeak_config = os.path.join(
args.grr_src,
Expand Down Expand Up @@ -345,8 +367,9 @@ def _CheckInstallSuccess(self):
raise RuntimeError("Install failed, no files at: %s" % self.install_path)

try:
output = subprocess.check_output(["sc", "query", self.service_name],
encoding="utf-8")
output = subprocess.check_output(
["sc", "query", self.service_name], encoding="utf-8"
)
service_running = "RUNNING" in output
except subprocess.CalledProcessError as e:
output = e.output
Expand All @@ -370,13 +393,15 @@ def _CheckInstallSuccess(self):
if self.expect_service_running:
if not service_running:
raise RuntimeError(
"GRR service not running after install, sc query output: %s" %
output)
"GRR service not running after install, sc query output: %s"
% output
)
else:
if service_running:
raise RuntimeError(
"GRR service running after install with expect_service_running == "
"False, sc query output: %s" % output)
"False, sc query output: %s" % output
)

def _InstallInstallers(self):
"""Install the installer built by RepackTemplates."""
Expand Down Expand Up @@ -404,15 +429,15 @@ def Build(self):

if not os.path.exists(args.grr_src):
self.GitCheckoutGRR()
proto_sdist = self.MakeProtoSdist()
core_sdist = self.MakeCoreSdist()
client_sdist = self.MakeClientSdist()
client_builder_sdist = self.MakeClientBuilderSdist()

self.InstallGRR(proto_sdist)
self.InstallGRR(core_sdist)
self.InstallGRR(client_sdist)
self.InstallGRR(client_builder_sdist)
proto_sdist, proto_requirements = self.MakeProtoSdist()
core_sdist, core_requirements = self.MakeCoreSdist()
client_sdist, client_requirements = self.MakeClientSdist()
client_builder_sdist, client_builder_requirements = self.MakeClientBuilderSdist()

self.InstallGRR(proto_sdist, proto_requirements)
self.InstallGRR(core_sdist, core_requirements)
self.InstallGRR(client_sdist, client_requirements)
self.InstallGRR(client_builder_sdist, client_builder_requirements)
self.BuildTemplates()
if args.test_repack_install:
self._RepackTemplates()
Expand Down
1 change: 0 additions & 1 deletion grr/client/requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,3 @@ absl-py==1.4.0
pytsk3==20230125
libfsntfs-python==20230606
fleetspeak-client-bin==0.1.13
xattr==0.9.7
Loading

0 comments on commit c3b2bd0

Please sign in to comment.