diff --git a/404.html b/404.html index 583579c..3649f50 100644 --- a/404.html +++ b/404.html @@ -4,7 +4,7 @@ gtag('js', new Date()); gtag('config', 'G-M5YNDNX7VT'); -
ActuatedActuated logo

404

This page could not be found.

\ No newline at end of file + }

404

This page could not be found.

\ No newline at end of file diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog.json b/_next/data/fmY28ZvLLPLCazi-OKnDH/blog.json deleted file mode 100644 index ae92255..0000000 --- a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog.json +++ /dev/null @@ -1 +0,0 @@ -{"pageProps":{"posts":[{"slug":"how-secure-are-containers-vs-microvms-ci","fileName":"2024-10-23-how-secure-are-containers-vs-microvms-ci.md","title":"How secure are containers & Kubernetes vs. microVMs for self-hosted CI?","description":"How secure are your self-hosted CI runners? We compare running CI jobs on VMs, containers, and Kubernetes versus one-shot, ephemeral microVMs.","tags":["security","containers","gitlab","kubernetes","multiarch"],"author_img":"alex","image":"/images/2024-10-containers-vs-microvms/background.png","date":"2024-10-23"},{"slug":"actuated-com-two-year-review","fileName":"2024-08-06-actuated-com-two-year-review.md","title":"We're now actuated.com","description":"We take a look back to our launch in November 2022, what we've learned since and what's next for actuated.com.","tags":["announcement","review","microvms"],"author_img":"alex","image":"/images/2024-08-dotcom/background.png","date":"2024-08-06"},{"slug":"actuated-for-gitlab","fileName":"2024-07-26-actuated-for-gitlab.md","title":"Secure your GitLab jobs with microVMs and Actuated","description":"Learn how microVMs provide more flexibility and security for CI workloads, by removing the need for privileged containers or Docker In Docker.","tags":["gitlab","security","docker"],"author_img":"welteki","image":"/images/2024-07-gitlab/background.png","date":"2024-07-26"},{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"},{"slug":"burst-billing-capacity","fileName":"2024-05-01-burst-billing-capacity.md","title":"Introducing burst billing and capacity for GitHub Actions","description":"Actuated now offers burst billing and capacity for customers with spiky and unpredictable CI/CD workloads.","tags":["githubactions","cicd","burst","capacity","billing"],"author_img":"alex","image":"/images/2024-05-burst/background.png","date":"2024-04-25"},{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"},{"slug":"gpus-for-github-actions","fileName":"2024-03-12-gpus-for-github-actions.md","title":"Accelerate GitHub Actions with dedicated GPUs","description":"You can now accelerate GitHub Actions with dedicated GPUs for machine learning and AI use-cases.","tags":["ai","ml","githubactions","openai","transcription","machinelearning"],"author_img":"alex","image":"/images/2024-03-gpus/background.png","date":"2024-03-12"},{"slug":"cncf-arm-march-update","fileName":"2024-03-04-cncf-arm-march-update.md","title":"The state of Arm CI for the CNCF","description":"After running almost 400k build minutes for top-tier CNCF projects, we give an update on the sponsored Arm CI program.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-cncf-update/background.png","date":"2024-03-04"},{"slug":"right-sizing-vms-github-actions","fileName":"2024-03-01-right-sizing-vms-github-actions.md","title":"Right sizing VMs for GitHub Actions","description":"How do you pick the right VM size for your GitHub Actions runners? We wrote a custom tool to help you find out.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-right-sizing/background.png","date":"2024-03-01"},{"slug":"local-caching-for-github-actions","fileName":"2024-02-23-local-caching-for-github-actions.md","title":"Testing the impact of a local cache for building Discourse","description":"We compare the impact of switching Discourse's GitHub Actions from self-hosted runners and a hosted cache, to a local cache with S3.","tags":["s3","githubactions","cache","latency"],"author_img":"welteki","image":"/images/2024-02-local-caching-for-github-actions/background.png","date":"2024-02-23"},{"slug":"custom-sizes-bpf-kvm","fileName":"2023-12-04-custom-sizes-bpf-kvm.md","title":"December Boost: Custom Job Sizes, eBPF Support & KVM Acceleration","description":"You can now request custom amounts of RAM and vCPU for jobs, run eBPF within jobs, and use KVM acceleration.","tags":["ebpf","cloudnative","opensource"],"author_img":"alex","image":"/images/2023-12-scheduling-bpf/background-bpf.png","date":"2023-12-04"},{"slug":"arm-ci-cncf-ampere","fileName":"2023-10-25-arm-ci-cncf-ampere.md","title":"Announcing managed Arm CI for CNCF projects","description":"Ampere Computing and The Cloud Native Computing Foundation are sponsoring a pilot of actuated's managed Arm CI for CNCF projects.","tags":["cloudnative","arm","opensource"],"author_img":"alex","image":"/images/2023-10-cncf/background.png","date":"2023-10-25"},{"slug":"firecracker-container-lab","fileName":"2023-09-05-firecracker-container-lab.md","title":"Grab your lab coat - we're building a microVM from a container","description":"No more broken tutorials, build a microVM from a container, boot it, access the Internet","tags":["firecracker","lab","tutorial"],"author_img":"alex","image":"/images/2023-09-firecracker-lab/background.png","date":"2023-09-05"},{"slug":"develop-a-great-go-cli","fileName":"2023-08-22-develop-a-great-go-cli.md","title":"How to develop a great CLI with Go","description":"Alex shares his insights from building half a dozen popular Go CLIs. Which can you apply to your projects?","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-08-great-cli/background.png","date":"2023-08-22"},{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"},{"slug":"amd-zenbleed-update-now","fileName":"2023-07-31-amd-zenbleed-update-now.md","title":"Update your AMD hosts now to mitigate the Zenbleed exploit","description":"Learn how to update the microcode on your AMD CPU to avoid the Zenbleed exploit.","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-07-zenbleed/background.png","date":"2023-07-31"},{"slug":"automate-packer-qemu-image-builds","fileName":"2023-07-25-automate-packer-qemu-image-builds.md","title":"Automate Packer Images with QEMU and Actuated","description":"Learn how to automate Packer images using QEMU and nested virtualisation through actuated.","tags":["images","packer","qemu","kvm"],"author_img":"welteki","image":"/images/2023-07-packer/background.png","date":"2023-07-25"},{"slug":"github-actions-usage-cli","fileName":"2023-06-16-github-actions-usage-cli.md","title":"Understand your usage of GitHub Actions","description":"Learn how you or your team is using GitHub Actions across your personal account or organisation.","tags":["costoptimization","analytics","githubactions","opensource","golang","cli"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-actions-usage/background.png"},{"slug":"secure-microvm-ci-gitlab","fileName":"2023-06-14-secure-microvm-ci-gitlab.md","title":"Secure CI for GitLab with Firecracker microVMs","description":"Learn how actuated for GitLab CI can help you secure your CI/CD pipelines with Firecracker.","tags":["security","gitlab"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-gitlab-preview/background.png"},{"slug":"faster-nix-builds","fileName":"2023-06-12-faster-nix-builds.md","title":"Faster Nix builds with GitHub Actions and actuated","description":"Speed up your Nix project builds on GitHub Actions with runners powered by Firecracker.","tags":["cicd","githubactions","nix","nixos","faasd","openfaas"],"author_img":"welteki","image":"/images/2023-06-faster-nix-builds/background.png","date":"2023-06-12"},{"slug":"faster-self-hosted-cache","fileName":"2023-05-24-faster-self-hosted-cache.md","title":"Fixing the cache latency for self-hosted GitHub Actions","description":"The cache for GitHub Actions can speed up CI/CD pipelines. But what about when it slows you down?","tags":["cicd","githubactions","cache","latency","yarn"],"author_img":"alex","image":"/images/2023-05-faster-cache/background.png","date":"2023-05-24"},{"slug":"oidc-proxy-for-openfaas","fileName":"2023-05-05-oidc-proxy-for-openfaas.md","title":"Keyless deployment to OpenFaaS with OIDC and GitHub Actions","description":"We're announcing a new OIDC proxy for OpenFaaS for keyless deployments from GitHub Actions.","author":"Alex Ellis","tags":["oidc","githubactions","security","federation","iam"],"author_img":"alex","image":"/images/2023-05-openfaas-oidc-proxy/background.png","date":"2023-05-05"},{"slug":"managing-github-actions","fileName":"2023-03-31-managing-github-actions.md","title":"Lessons learned managing GitHub Actions and Firecracker","description":"Alex shares lessons from building a managed service for GitHub Actions with Firecracker.","author":"Alex Ellis","tags":["baremetal","githubactions","saas","lessons","github"],"author_img":"alex","image":"/images/2023-03-lessons-learned/background.jpg","date":"2023-03-31"},{"slug":"how-to-run-multi-arch-builds-natively","fileName":"2023-03-24-how-to-run-multi-arch-builds-natively.md","title":"How to split up multi-arch Docker builds to run natively","description":"QEMU is a convenient way to publish containers for multiple architectures, but it can be incredibly slow. Native is much faster.","author":"Alex Ellis","tags":["baremetal","githubactions","multiarch","arm"],"author_img":"alex","image":"/images/2023-split-native/background.jpg","date":"2023-03-24"},{"slug":"case-study-bring-your-own-bare-metal-to-actions","fileName":"2023-03-10-case-study-bring-your-own-bare-metal-to-actions.md","title":"Bring Your Own Metal Case Study with GitHub Actions","description":"See how BYO bare-metal made a 6 hour GitHub Actions build complete 25x faster.","author":"Alex Ellis","tags":["baremetal","githubactions","equinixmetal","macmini","xeon"],"author_img":"alex","image":"/images/2023-03-vpp/background.jpg","date":"2023-03-10"},{"slug":"kvm-in-github-actions","fileName":"2023-02-17-kvm-in-github-actions.md","title":"How to run KVM guests in your GitHub Actions","description":"From building cloud images, to running NixOS tests and the android emulator, we look at how and why you'd want to run a VM in GitHub Actions.","author":"Han Verstraete","tags":["virtualization","kvm","githubactions","nestedvirt","cicd"],"author_img":"welteki","image":"/images/2023-02-17-kvm-in-github-actions/nested-firecracker.png","date":"2023-02-17"},{"slug":"caching-in-github-actions","fileName":"2023-02-10-caching-in-github-actions.md","title":"Make your builds run faster with Caching for GitHub Actions","description":"Learn how we made a Golang project build 4x faster using GitHub's built-in caching mechanism.","author":"Han Verstraete","tags":["github","actions","caching","golang"],"author_img":"welteki","image":"/images/2023-02-10-caching-in-github-actions/background.png","date":"2023-02-10"},{"slug":"multi-arch-docker-github-actions","fileName":"2023-02-01-multi-arch-docker-github-actions.md","title":"The efficient way to publish multi-arch containers from GitHub Actions","description":"Learn how to publish container images for both Arm and Intel machines from GitHub Actions.","author":"Alex Ellis","tags":["security","oss","multiarch"],"author_img":"alex","image":"/images/2023-02-multi-arch/architecture.jpg","date":"2023-02-01"},{"slug":"sbom-in-github-actions","fileName":"2023-01-25-sbom-in-github-actions.md","title":"How to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions","description":"Learn how to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions in a few easy steps.","author":"Alex Ellis","tags":["security","oss","supplychain","sbom"],"author_img":"alex","image":"/images/2023-jan-sbom/list.jpg","date":"2023-01-25"},{"slug":"is-the-self-hosted-runner-safe-github-actions","fileName":"2023-01-20-is-the-self-hosted-runner-safe-github-actions.md","title":"Is the GitHub Actions self-hosted runner safe for Open Source?","description":"GitHub warns against using self-hosted Actions runners for public repositories - but why? And are there alternatives?","author":"Alex Ellis","tags":["security","oss"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-20"},{"slug":"native-arm64-for-github-actions","fileName":"2023-01-17-native-arm64-for-github-actions.md","title":"How to make GitHub Actions 22x faster with bare-metal Arm","description":"GitHub doesn't provide hosted Arm runners, so how can you use native Arm runners safely & securely?","author":"Alex Ellis","tags":["cicd","githubactions","arm","arm64","multiarch"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-17"},{"slug":"blazing-fast-ci-with-microvms","fileName":"2022-11-10-blazing-fast-ci-with-microvms.md","title":"Blazing fast CI with MicroVMs","description":"I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.","author":"Alex Ellis","tags":["cicd","bare-metal","kubernetes","DevOps","linux","firecracker"],"author_img":"alex","image":"/images/2022-11-10-blazing-fast-ci-with-microvms/actuated-pilot.png","canonical":"https://blog.alexellis.io/blazing-fast-ci-with-microvms/","date":"2022-11-10"}]},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/y73HVNpk3Hj6pdIAHkKda/blog.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog.json new file mode 100644 index 0000000..ba198a9 --- /dev/null +++ b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog.json @@ -0,0 +1 @@ +{"pageProps":{"posts":[{"slug":"how-secure-are-containers-vs-microvms-ci","fileName":"2024-10-23-how-secure-are-containers-vs-microvms-ci.md","title":"How secure are containers & Kubernetes vs. microVMs for self-hosted CI?","description":"How secure are your self-hosted CI runners? We compare running CI jobs on VMs, containers, and Kubernetes versus one-shot, ephemeral microVMs.","tags":["security","containers","gitlab","kubernetes","multiarch"],"author_img":"alex","image":"/images/2024-10-containers-vs-microvms/background.png","date":"2024-10-23"},{"slug":"actuated-com-two-year-review","fileName":"2024-08-06-actuated-com-two-year-review.md","title":"We're now actuated.com","description":"We take a look back to our launch in November 2022, what we've learned since and what's next for actuated.com.","tags":["announcement","review","microvms"],"author_img":"alex","image":"/images/2024-08-dotcom/background.png","date":"2024-08-06"},{"slug":"actuated-for-gitlab","fileName":"2024-07-26-actuated-for-gitlab.md","title":"Secure your GitLab jobs with microVMs and Actuated","description":"Learn how microVMs provide more flexibility and security for CI workloads, by removing the need for privileged containers or Docker In Docker.","tags":["gitlab","security","docker"],"author_img":"welteki","image":"/images/2024-07-gitlab/background.png","date":"2024-07-26"},{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"rollup":true,"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"},{"slug":"burst-billing-capacity","fileName":"2024-05-01-burst-billing-capacity.md","title":"Introducing burst billing and capacity for GitHub Actions","description":"Actuated now offers burst billing and capacity for customers with spiky and unpredictable CI/CD workloads.","tags":["githubactions","cicd","burst","capacity","billing"],"author_img":"alex","image":"/images/2024-05-burst/background.png","date":"2024-04-25"},{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"rollup":true,"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"},{"slug":"gpus-for-github-actions","fileName":"2024-03-12-gpus-for-github-actions.md","title":"Accelerate GitHub Actions with dedicated GPUs","description":"You can now accelerate GitHub Actions with dedicated GPUs for machine learning and AI use-cases.","tags":["ai","ml","githubactions","openai","transcription","machinelearning"],"author_img":"alex","image":"/images/2024-03-gpus/background.png","date":"2024-03-12"},{"slug":"cncf-arm-march-update","fileName":"2024-03-04-cncf-arm-march-update.md","title":"The state of Arm CI for the CNCF","description":"After running almost 400k build minutes for top-tier CNCF projects, we give an update on the sponsored Arm CI program.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-cncf-update/background.png","date":"2024-03-04"},{"slug":"right-sizing-vms-github-actions","fileName":"2024-03-01-right-sizing-vms-github-actions.md","title":"Right sizing VMs for GitHub Actions","description":"How do you pick the right VM size for your GitHub Actions runners? We wrote a custom tool to help you find out.","tags":["efficiency","githubactions","metering"],"author_img":"alex","image":"/images/2024-03-right-sizing/background.png","date":"2024-03-01"},{"slug":"local-caching-for-github-actions","fileName":"2024-02-23-local-caching-for-github-actions.md","title":"Testing the impact of a local cache for building Discourse","description":"We compare the impact of switching Discourse's GitHub Actions from self-hosted runners and a hosted cache, to a local cache with S3.","tags":["s3","githubactions","cache","latency"],"author_img":"welteki","image":"/images/2024-02-local-caching-for-github-actions/background.png","date":"2024-02-23"},{"slug":"custom-sizes-bpf-kvm","fileName":"2023-12-04-custom-sizes-bpf-kvm.md","title":"December Boost: Custom Job Sizes, eBPF Support & KVM Acceleration","description":"You can now request custom amounts of RAM and vCPU for jobs, run eBPF within jobs, and use KVM acceleration.","tags":["ebpf","cloudnative","opensource"],"author_img":"alex","image":"/images/2023-12-scheduling-bpf/background-bpf.png","date":"2023-12-04"},{"slug":"arm-ci-cncf-ampere","fileName":"2023-10-25-arm-ci-cncf-ampere.md","title":"Announcing managed Arm CI for CNCF projects","description":"Ampere Computing and The Cloud Native Computing Foundation are sponsoring a pilot of actuated's managed Arm CI for CNCF projects.","tags":["cloudnative","arm","opensource"],"author_img":"alex","image":"/images/2023-10-cncf/background.png","date":"2023-10-25"},{"slug":"firecracker-container-lab","fileName":"2023-09-05-firecracker-container-lab.md","title":"Grab your lab coat - we're building a microVM from a container","description":"No more broken tutorials, build a microVM from a container, boot it, access the Internet","tags":["firecracker","lab","tutorial"],"author_img":"alex","image":"/images/2023-09-firecracker-lab/background.png","date":"2023-09-05"},{"slug":"develop-a-great-go-cli","fileName":"2023-08-22-develop-a-great-go-cli.md","title":"How to develop a great CLI with Go","description":"Alex shares his insights from building half a dozen popular Go CLIs. Which can you apply to your projects?","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-08-great-cli/background.png","date":"2023-08-22"},{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"rollup":true,"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"},{"slug":"amd-zenbleed-update-now","fileName":"2023-07-31-amd-zenbleed-update-now.md","title":"Update your AMD hosts now to mitigate the Zenbleed exploit","description":"Learn how to update the microcode on your AMD CPU to avoid the Zenbleed exploit.","tags":["images","packer","qemu","kvm"],"author_img":"alex","image":"/images/2023-07-zenbleed/background.png","date":"2023-07-31"},{"slug":"automate-packer-qemu-image-builds","fileName":"2023-07-25-automate-packer-qemu-image-builds.md","title":"Automate Packer Images with QEMU and Actuated","description":"Learn how to automate Packer images using QEMU and nested virtualisation through actuated.","tags":["images","packer","qemu","kvm"],"author_img":"welteki","image":"/images/2023-07-packer/background.png","date":"2023-07-25"},{"slug":"github-actions-usage-cli","fileName":"2023-06-16-github-actions-usage-cli.md","title":"Understand your usage of GitHub Actions","description":"Learn how you or your team is using GitHub Actions across your personal account or organisation.","tags":["costoptimization","analytics","githubactions","opensource","golang","cli"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-actions-usage/background.png"},{"slug":"secure-microvm-ci-gitlab","fileName":"2023-06-14-secure-microvm-ci-gitlab.md","title":"Secure CI for GitLab with Firecracker microVMs","description":"Learn how actuated for GitLab CI can help you secure your CI/CD pipelines with Firecracker.","tags":["security","gitlab"],"author_img":"alex","date":"2023-06-16","image":"/images/2023-06-gitlab-preview/background.png"},{"slug":"faster-nix-builds","fileName":"2023-06-12-faster-nix-builds.md","title":"Faster Nix builds with GitHub Actions and actuated","description":"Speed up your Nix project builds on GitHub Actions with runners powered by Firecracker.","tags":["cicd","githubactions","nix","nixos","faasd","openfaas"],"author_img":"welteki","image":"/images/2023-06-faster-nix-builds/background.png","date":"2023-06-12"},{"slug":"faster-self-hosted-cache","fileName":"2023-05-24-faster-self-hosted-cache.md","title":"Fixing the cache latency for self-hosted GitHub Actions","description":"The cache for GitHub Actions can speed up CI/CD pipelines. But what about when it slows you down?","tags":["cicd","githubactions","cache","latency","yarn"],"author_img":"alex","image":"/images/2023-05-faster-cache/background.png","date":"2023-05-24"},{"slug":"oidc-proxy-for-openfaas","fileName":"2023-05-05-oidc-proxy-for-openfaas.md","title":"Keyless deployment to OpenFaaS with OIDC and GitHub Actions","description":"We're announcing a new OIDC proxy for OpenFaaS for keyless deployments from GitHub Actions.","author":"Alex Ellis","tags":["oidc","githubactions","security","federation","iam"],"author_img":"alex","image":"/images/2023-05-openfaas-oidc-proxy/background.png","date":"2023-05-05"},{"slug":"managing-github-actions","fileName":"2023-03-31-managing-github-actions.md","title":"Lessons learned managing GitHub Actions and Firecracker","description":"Alex shares lessons from building a managed service for GitHub Actions with Firecracker.","author":"Alex Ellis","tags":["baremetal","githubactions","saas","lessons","github"],"author_img":"alex","image":"/images/2023-03-lessons-learned/background.jpg","date":"2023-03-31"},{"slug":"how-to-run-multi-arch-builds-natively","fileName":"2023-03-24-how-to-run-multi-arch-builds-natively.md","title":"How to split up multi-arch Docker builds to run natively","description":"QEMU is a convenient way to publish containers for multiple architectures, but it can be incredibly slow. Native is much faster.","author":"Alex Ellis","tags":["baremetal","githubactions","multiarch","arm"],"author_img":"alex","image":"/images/2023-split-native/background.jpg","date":"2023-03-24"},{"slug":"case-study-bring-your-own-bare-metal-to-actions","fileName":"2023-03-10-case-study-bring-your-own-bare-metal-to-actions.md","title":"Bring Your Own Metal Case Study with GitHub Actions","description":"See how BYO bare-metal made a 6 hour GitHub Actions build complete 25x faster.","author":"Alex Ellis","tags":["baremetal","githubactions","equinixmetal","macmini","xeon"],"author_img":"alex","image":"/images/2023-03-vpp/background.jpg","date":"2023-03-10"},{"slug":"kvm-in-github-actions","fileName":"2023-02-17-kvm-in-github-actions.md","title":"How to run KVM guests in your GitHub Actions","description":"From building cloud images, to running NixOS tests and the android emulator, we look at how and why you'd want to run a VM in GitHub Actions.","author":"Han Verstraete","tags":["virtualization","kvm","githubactions","nestedvirt","cicd"],"author_img":"welteki","image":"/images/2023-02-17-kvm-in-github-actions/nested-firecracker.png","date":"2023-02-17"},{"slug":"caching-in-github-actions","fileName":"2023-02-10-caching-in-github-actions.md","title":"Make your builds run faster with Caching for GitHub Actions","description":"Learn how we made a Golang project build 4x faster using GitHub's built-in caching mechanism.","author":"Han Verstraete","tags":["github","actions","caching","golang"],"author_img":"welteki","image":"/images/2023-02-10-caching-in-github-actions/background.png","date":"2023-02-10"},{"slug":"multi-arch-docker-github-actions","fileName":"2023-02-01-multi-arch-docker-github-actions.md","title":"The efficient way to publish multi-arch containers from GitHub Actions","description":"Learn how to publish container images for both Arm and Intel machines from GitHub Actions.","author":"Alex Ellis","tags":["security","oss","multiarch"],"author_img":"alex","image":"/images/2023-02-multi-arch/architecture.jpg","date":"2023-02-01"},{"slug":"sbom-in-github-actions","fileName":"2023-01-25-sbom-in-github-actions.md","title":"How to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions","description":"Learn how to add a Software Bill of Materials (SBOM) to your containers with GitHub Actions in a few easy steps.","author":"Alex Ellis","tags":["security","oss","supplychain","sbom"],"author_img":"alex","image":"/images/2023-jan-sbom/list.jpg","date":"2023-01-25"},{"slug":"is-the-self-hosted-runner-safe-github-actions","fileName":"2023-01-20-is-the-self-hosted-runner-safe-github-actions.md","title":"Is the GitHub Actions self-hosted runner safe for Open Source?","description":"GitHub warns against using self-hosted Actions runners for public repositories - but why? And are there alternatives?","author":"Alex Ellis","tags":["security","oss"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-20"},{"slug":"native-arm64-for-github-actions","fileName":"2023-01-17-native-arm64-for-github-actions.md","title":"How to make GitHub Actions 22x faster with bare-metal Arm","description":"GitHub doesn't provide hosted Arm runners, so how can you use native Arm runners safely & securely?","author":"Alex Ellis","tags":["cicd","githubactions","arm","arm64","multiarch"],"author_img":"alex","image":"/images/2023-native-arm64-for-oss/in-progress-dashboard.png","date":"2023-01-17"},{"slug":"blazing-fast-ci-with-microvms","fileName":"2022-11-10-blazing-fast-ci-with-microvms.md","title":"Blazing fast CI with MicroVMs","description":"I saw an opportunity to fix self-hosted runners for GitHub Actions. Actuated is now in pilot and aims to solve most if not all of the friction.","author":"Alex Ellis","tags":["cicd","bare-metal","kubernetes","DevOps","linux","firecracker"],"author_img":"alex","image":"/images/2022-11-10-blazing-fast-ci-with-microvms/actuated-pilot.png","canonical":"https://blog.alexellis.io/blazing-fast-ci-with-microvms/","date":"2022-11-10"}]},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/actuated-com-two-year-review.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/actuated-com-two-year-review.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/actuated-com-two-year-review.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/actuated-com-two-year-review.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/actuated-for-gitlab.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/actuated-for-gitlab.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/actuated-for-gitlab.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/actuated-for-gitlab.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/amd-zenbleed-update-now.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/amd-zenbleed-update-now.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/amd-zenbleed-update-now.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/amd-zenbleed-update-now.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/arm-ci-cncf-ampere.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/arm-ci-cncf-ampere.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/arm-ci-cncf-ampere.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/arm-ci-cncf-ampere.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/automate-packer-qemu-image-builds.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/automate-packer-qemu-image-builds.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/automate-packer-qemu-image-builds.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/automate-packer-qemu-image-builds.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/blazing-fast-ci-with-microvms.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/blazing-fast-ci-with-microvms.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/blazing-fast-ci-with-microvms.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/blazing-fast-ci-with-microvms.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/burst-billing-capacity.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/burst-billing-capacity.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/burst-billing-capacity.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/burst-billing-capacity.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/caching-in-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/caching-in-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/caching-in-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/caching-in-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/calyptia-case-study-arm.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/calyptia-case-study-arm.json similarity index 99% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/calyptia-case-study-arm.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/calyptia-case-study-arm.json index c4ef422..cfdee20 100644 --- a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/calyptia-case-study-arm.json +++ b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/calyptia-case-study-arm.json @@ -1 +1 @@ -{"pageProps":{"post":{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","contentHtml":"

This is a case-study, and guest article by Patrick Stephens, Tech Lead of Infrastructure at Calyptia.

\n

Introduction

\n

Different architecture builds can be slow using the Github Actions hosted runners due to emulation of the non-native architecture for the build. This blog shows a simple way to make use of self-hosted runners for dedicated builds but in a secure and easy to maintain fashion.

\n

Calyptia maintains the OSS and Cloud Native Computing Foundation (CNCF) graduated Fluent projects including Fluent Bit. We then add value to the open-source core by providing commercial services and enterprise-level features.

\n
\n

Fluent Bit is a Fast and Lightweight Telemetry Agent for Logs, Metrics, and Traces for Linux, macOS, Windows, and BSD family operating systems. It has been made with a strong focus on performance to allow the collection and processing of telemetry data from different sources without complexity.

\n

It was originally created by Eduardo Silva and is now an independent project.

\n
\n

To learn about Fluent Bit, the Open Source telemetry agent that Calyptia maintains, check out their docs.

\n

The Problem

\n

One of the best things about Fluent Bit is that we provide native packages (RPMs and DEBs) for a myriad of supported targets (various Linux, macOS and Windows), however to do this is also one of the hardest things to support due to the complexity of building and testing across all these targets.

\n

When PRs are provided we would like to ensure they function across the targets but doing so can take a very long time (hours) and consume a lot of resources (that must be paid for). This means that these long running jobs are only done via exception (manually labelling a PR or on full builds for releases) leading to issues only discovered when a full build & test is done, e.g. during the release process so blocking the release until it is fixed.

\n

The long build time problem came to a head when we discovered we could no longer build for Amazon Linux 2023 (AL2023) because the build time exceeded the 6 hour limit for a single job on Github. We had to disable the AL2023 target for releases which means users cannot then update to the latest release leading to missing features or security problems: See the issue here

\n

In addition to challenges in the OSS, there are also challenges on the commercial side. Here, we are seeing issues with extended build times for ARM64 targets because our CI is based on Github Actions and currently only AMD64 (also called x86-64 or x64) runners are provided for builds. This slows down development and can mean bugs are not caught as early as possible.

\n

Why not use self-hosted runners?

\n

One way to speed up builds is to provide self-hosted ARM64 runners.

\n

Unfortunately, runners pose security implications, particularly for public repositories. In fact, Github recommends against using self-hosted runners: About self-hosted runners - GitHub Docs

\n

In addition to security concerns, there are also infrastructure implications for using self-hosted runners. We have to provide the infrastructure around deploying and managing the self-hosted runners, installing an agent, configuring it for jobs, etc. From a perspective of OSS we want anything we do to be simple and easy for maintenance purposes.

\n

Any change we make needs to be compatible with downstream forks as well. We do not want to break builds for existing users, particularly for those who are contributors as well to the open source project. Therefore we need a solution that does not impact them.

\n

There are various tools that can help with managing self-hosted runners, https://jonico.github.io/awesome-runners/ provides a good curated list. I performed an evaluation of some of the recommended tools but the solution would be non-trivial and require some effort to maintain.

\n

Our considerations

\n

We have the following high level goals in a rough priority order:

\n\n

The solution

\n

At Kubecon EU 2023 I met up with Alex Ellis from Actuated (and of OpenFaaS fame) in-person and we wanted to put Alex and his technology to the test, to see if the Actuated technology could fix the problems we see with our build process.

\n

To understand what Actuated is then it is best to refer to their documentation with this specific blog post being a good overview of why we considered adopting it. We're not the only CNCF project that Alex's team was able to help. He describes how he helped Parca and Network Service Mesh to slash their build teams by using native Arm hardware.

\n

A quick TLDR; though would be that Actuated provides an agent you install which then automatically creates ephemeral VMs on the host for each build job. Actuated seemed to tick the various boxes (see the considerations above) we had for it but never trust a vendor until you’ve tried it yourself!

\n

Quote from Alex:

\n
\n

\"Actuated aims to give teams the closest possible experience to managed runners, but with native arm support flat rate billing, and secure VM-level isolation. Since Calyptia adopted actuated, we’ve also shipped an SSH debug experience (like you’d find with CircleCI) and detailed reports and insights on usage across repos, users and organisations.\"

\n
\n

To use Actuated, you have to provision a machine with the Actuated agent, which is trivial and well documented: https://docs.actuated.com/install-agent/.

\n

We deployed an Ampere Altra Q80 server with 256GB of RAM and 80 cores ARM64 machine via Equinix (Equinix donates resources to the CNCF which we use for Fluent Bit so this satisfies the cost side of things) and installed the Actuated agent on it per the Actuated docs.

\n

The update required to start using Actuated in OSS Fluent Bit is a one-liner. (Thanks in part to my excellent work refactoring the CI workflows, or so I like to think. You can see the actual PR here for the change: https://github.com/fluent/fluent-bit/pull/7527.)

\n

The following is the code required to start using Actuated:

\n
-    runs-on: ubuntu-latest\n+    runs-on: ${{ (contains(matrix.distro, 'arm' ) & 'actuated-arm64') || 'ubuntu-latest' }}\n
\n

For most people, the change will be much simpler:

\n
-    runs-on: ubuntu-latest\n+    runs-on: actuated\n
\n

In Github Actions parlance, the code above translates to “if we are doing an ARM build, then use the Actuated runner; otherwise, use the default Github Hosted (AMD64) Ubuntu runner”.

\n

In the real code, I added an extra check so that we only use Actuated runners for the official source repo which means any forks will also carry on running as before on the Github Hosted runner.

\n

With this very simple change, all the ARM64 builds that used to take hours to complete now finish in minutes. In addition, we can actually build the AL2023 ARM64 target to satisfy those users too. A simple change gave us a massive boost to performance and also provided a missing target.

\n

To demonstrate this is not specific to Equinix hosts or in some fashion difficult to manage in heterogeneous infrastructure (e.g. various hosts/VMs from different providers), we also replicated this for all our commercial offerings using a bare-metal Hetzner host. The process was identical: install the agent and make the runs-on code change as above to use Actuated. Massive improvements in build time were seen again as expected.

\n

The usage of bare-metal (or cloud) hosts providers is invisible and only a choice of which provider you want to put the agent on. In our case we have a mixed set up with no difference in usage or maintenance.

\n

Challenges building containers

\n

The native package (RPM/DEB) building described above was quite simple to integrate via the existing workflows we had.

\n

Building the native packages is done via a process that runs a target-specific container for each of the builds, e.g. we run a CentOS container to build for that target. This allows a complete build to be run on any Linux-compatible machine with a container runtime either in CI or locally. For ARM builds, we were using QEMU emulation for ARM builds hence the slowdown as this has to emulate instructions between architectures.

\n

Container builds are the primary commercial area for improvement as we provide a SAAS solution running on K8S. Container builds were also a trickier proposition for OSS as we were using a single job to build all architectures using the docker/build-push-action. The builds were incredibly slow for ARM and also atomic, which means if you received a transient issue in one of the architecture builds, you would have to repeat the whole lot.

\n

As an example: https://github.com/fluent/fluent-bit/blob/master/.github/workflows/call-build-images.yaml

\n
      - name: Build the production images\n        id: build_push\n        uses: docker/build-push-action@v4\n        with:\n          file: ./dockerfiles/Dockerfile\n          context: .\n          tags: ${{ steps.meta.outputs.tags }}\n          labels: ${{ steps.meta.outputs.labels }}\n          platforms: linux/amd64, linux/arm64, linux/arm/v7\n          target: production\n          # Must be disabled to provide legacy format images from the registry\n          provenance: false\n          push: true\n          load: false\n          build-args: |\n            FLB_NIGHTLY_BUILD=${{ inputs.unstable }}\n            RELEASE_VERSION=${{ inputs.version }}\n
\n

The build step above is a bit more complex to tease out into separate components: we need to run single architecture builds for each target then provide a multi-arch manifest that links them together.

\n

We reached out to Alex on a good way to modify this to work within a split build per architecture approach. The Actuated team has been very responsive on these types of questions along with proactive monitoring of our build queue and runners.

\n

Within Calyptia we have followed the approach Docker provided here and suggested by the Actuated team: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners

\n

Based on what we learned, we recommend the following process is followed:

\n

Build each architecture and push by digest in a set of parallel matrix jobs.\nCapture the output digest of each build.\nCreate the multi-arch manifest made up of each digest we have pushed in step 1 using the artefact from step 2.

\n

This approach provides two key benefits. First, it allows us to run on dedicated runners per-arch. Second, if a job fails we only need to repeat the single job, instead of having to rebuild all architectures.

\n

The new approach reduced the time for the release process for the Calyptia Core K8S Operator from more than an hour to minutes. Additionally, because we can do this so quickly, we now build all architectures for every change rather than just on release. This helps developers who are running ARM locally for development as they have containers always available.

\n

The example time speed up for the Calyptia Core K8S operator process was replicated across all the other components. A very good bang for your buck!

\n

For us, the actuated subscription fee has been of great value. Initially we tested the waters on the Basic Plan, but soon upgraded when we saw more areas where we could use it. The cost for us has been offset against a massive improvement in CI time and development time plus reducing the infrastructure costs of managing the self-hosted runners.

\n

Lessons learned

\n

The package updates were seamless really, however we did encounter some issues with the ecosystem (not with actuated), when refactoring and updating our container builds. The issues with the container builds are covered below to help anyone else with the same problems.

\n

Provenance is now enabled by default

\n

We were using v3 of Docker’s docker/build-push-action, but they made a breaking change which caused us a headache. They changed the default in v4 to create the various extra artifacts for provenance (e.g. SBOMs) which did have a few extra side effects both at the time and even now.

\n

If you do not disable this then it will push manifest lists rather than images so you will subsequently get an error message when you try to create a manifest list of another manifest list.

\n

Separately this also causes issues for older docker clients or organisations that need the legacy Docker schema format from a registry: using it means only OCI format schemas are pushed. This impacted both OSS and our commercial offerings: https://github.com/fluent/fluent-bit/issues/7748.

\n

It meant people on older OS’s or with requirements on only consuming Docker schema (e.g. maybe an internal mirror registry only supports that) could not pull the images.

\n

Invalid timestamps for gcr.io with manifests

\n

A funny problem found with our cloud-run deployments for Calyptia Core SAAS offering was that pushing the manifests to (Google Container Registry) gcr.io meant they ended up with a zero-epoch timestamp. This messed up some internal automation for us when we tried to get the latest version.

\n

To resolve this we just switched back to doing a single architecture build as we do not need multi-arch manifests for cloud-run. Internally we still have multi-arch images in ghcr.io for internal use anyway, this is purely the promotion to gcr.io.

\n

Manifests cannot use sub-paths

\n

This was a fun one: when specifying images to make up your manifest they must be in the same registry of course!

\n

Now, we tend to use sub-paths a lot to handle specific use cases for ghcr.io but unfortunately you cannot use them when trying to construct a manifest.

\n

OK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/product:tag-amd64\nNOK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/amd64/product:tag

\n

As with all good failures, the tooling let me make a broken manifest at build time but unfortunately trying to pull it meant a failure at runtime.

\n

Actuated container registry mirror

\n

All Github hosted runners provide default credentials to authenticate with docker.io for pulling public images. When running on a self-hosted runner you need to authenticate for this otherwise you will hit rate limits and builds may fail as they cannot download required base images.

\n

Actuated provide a registry mirror and Github Action to simplify this so make sure you set it up: https://docs.actuated.com/tasks/registry-mirror/

\n

As part of this, ensure it is set up for anything that uses images (e.g. we run integration tests on KIND that failed as the cluster could not download its images) and that it is done after any buildx config as it creates a dedicated buildx builder for the mirror usage.

\n

Actuated support

\n

The Actuated team helped us in two ways: the first was that we were able to enable Arm builds for our OSS projects and our commercial products, when they timed out with hosted runners. The second way was where our costs were getting out of hand on GitHub’s larger hosted runners: Actuated not only reduced the build time, but the billing model is flat-rate, meaning our costs are now fixed, rather than growing.

\n

As we made suggestions or collaborated with the Actuated team, they updated the documentation, including our suggestions on smoothing out the onboarding of new build servers and new features for the CLI.

\n

The more improvements we’ve made, the more we’ve seen. Next on our list is getting the runtime of a Go release down from 26 minutes by bringing it over to actuated.

\n

Conclusion

\n

Alex Ellis: We've learned a lot working with Patrick and Calyptia and are pleased to see that they were able to save money, whilst getting much quicker, and safer Open Source and commercial builds.

\n

We value getting feedback and suggestions from customers, and Patrick continues to provide plenty of them.

\n

If you'd like to learn more about actuated, reach out to speak to our team by clicking \"Sign-up\" and filling out the form. We'll be in touch to arrange a call.

","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"}},"__N_SSG":true} \ No newline at end of file +{"pageProps":{"post":{"slug":"calyptia-case-study-arm","fileName":"2023-08-11-calyptia-case-study-arm.md","contentHtml":"

This is a case-study, and guest article by Patrick Stephens, Tech Lead of Infrastructure at Calyptia.

\n

Introduction

\n

Different architecture builds can be slow using the Github Actions hosted runners due to emulation of the non-native architecture for the build. This blog shows a simple way to make use of self-hosted runners for dedicated builds but in a secure and easy to maintain fashion.

\n

Calyptia maintains the OSS and Cloud Native Computing Foundation (CNCF) graduated Fluent projects including Fluent Bit. We then add value to the open-source core by providing commercial services and enterprise-level features.

\n
\n

Fluent Bit is a Fast and Lightweight Telemetry Agent for Logs, Metrics, and Traces for Linux, macOS, Windows, and BSD family operating systems. It has been made with a strong focus on performance to allow the collection and processing of telemetry data from different sources without complexity.

\n

It was originally created by Eduardo Silva and is now an independent project.

\n
\n

To learn about Fluent Bit, the Open Source telemetry agent that Calyptia maintains, check out their docs.

\n

The Problem

\n

One of the best things about Fluent Bit is that we provide native packages (RPMs and DEBs) for a myriad of supported targets (various Linux, macOS and Windows), however to do this is also one of the hardest things to support due to the complexity of building and testing across all these targets.

\n

When PRs are provided we would like to ensure they function across the targets but doing so can take a very long time (hours) and consume a lot of resources (that must be paid for). This means that these long running jobs are only done via exception (manually labelling a PR or on full builds for releases) leading to issues only discovered when a full build & test is done, e.g. during the release process so blocking the release until it is fixed.

\n

The long build time problem came to a head when we discovered we could no longer build for Amazon Linux 2023 (AL2023) because the build time exceeded the 6 hour limit for a single job on Github. We had to disable the AL2023 target for releases which means users cannot then update to the latest release leading to missing features or security problems: See the issue here

\n

In addition to challenges in the OSS, there are also challenges on the commercial side. Here, we are seeing issues with extended build times for ARM64 targets because our CI is based on Github Actions and currently only AMD64 (also called x86-64 or x64) runners are provided for builds. This slows down development and can mean bugs are not caught as early as possible.

\n

Why not use self-hosted runners?

\n

One way to speed up builds is to provide self-hosted ARM64 runners.

\n

Unfortunately, runners pose security implications, particularly for public repositories. In fact, Github recommends against using self-hosted runners: About self-hosted runners - GitHub Docs

\n

In addition to security concerns, there are also infrastructure implications for using self-hosted runners. We have to provide the infrastructure around deploying and managing the self-hosted runners, installing an agent, configuring it for jobs, etc. From a perspective of OSS we want anything we do to be simple and easy for maintenance purposes.

\n

Any change we make needs to be compatible with downstream forks as well. We do not want to break builds for existing users, particularly for those who are contributors as well to the open source project. Therefore we need a solution that does not impact them.

\n

There are various tools that can help with managing self-hosted runners, https://jonico.github.io/awesome-runners/ provides a good curated list. I performed an evaluation of some of the recommended tools but the solution would be non-trivial and require some effort to maintain.

\n

Our considerations

\n

We have the following high level goals in a rough priority order:

\n\n

The solution

\n

At Kubecon EU 2023 I met up with Alex Ellis from Actuated (and of OpenFaaS fame) in-person and we wanted to put Alex and his technology to the test, to see if the Actuated technology could fix the problems we see with our build process.

\n

To understand what Actuated is then it is best to refer to their documentation with this specific blog post being a good overview of why we considered adopting it. We're not the only CNCF project that Alex's team was able to help. He describes how he helped Parca and Network Service Mesh to slash their build teams by using native Arm hardware.

\n

A quick TLDR; though would be that Actuated provides an agent you install which then automatically creates ephemeral VMs on the host for each build job. Actuated seemed to tick the various boxes (see the considerations above) we had for it but never trust a vendor until you’ve tried it yourself!

\n

Quote from Alex:

\n
\n

\"Actuated aims to give teams the closest possible experience to managed runners, but with native arm support flat rate billing, and secure VM-level isolation. Since Calyptia adopted actuated, we’ve also shipped an SSH debug experience (like you’d find with CircleCI) and detailed reports and insights on usage across repos, users and organisations.\"

\n
\n

To use Actuated, you have to provision a machine with the Actuated agent, which is trivial and well documented: https://docs.actuated.com/install-agent/.

\n

We deployed an Ampere Altra Q80 server with 256GB of RAM and 80 cores ARM64 machine via Equinix (Equinix donates resources to the CNCF which we use for Fluent Bit so this satisfies the cost side of things) and installed the Actuated agent on it per the Actuated docs.

\n

The update required to start using Actuated in OSS Fluent Bit is a one-liner. (Thanks in part to my excellent work refactoring the CI workflows, or so I like to think. You can see the actual PR here for the change: https://github.com/fluent/fluent-bit/pull/7527.)

\n

The following is the code required to start using Actuated:

\n
-    runs-on: ubuntu-latest\n+    runs-on: ${{ (contains(matrix.distro, 'arm' ) & 'actuated-arm64') || 'ubuntu-latest' }}\n
\n

For most people, the change will be much simpler:

\n
-    runs-on: ubuntu-latest\n+    runs-on: actuated\n
\n

In Github Actions parlance, the code above translates to “if we are doing an ARM build, then use the Actuated runner; otherwise, use the default Github Hosted (AMD64) Ubuntu runner”.

\n

In the real code, I added an extra check so that we only use Actuated runners for the official source repo which means any forks will also carry on running as before on the Github Hosted runner.

\n

With this very simple change, all the ARM64 builds that used to take hours to complete now finish in minutes. In addition, we can actually build the AL2023 ARM64 target to satisfy those users too. A simple change gave us a massive boost to performance and also provided a missing target.

\n

To demonstrate this is not specific to Equinix hosts or in some fashion difficult to manage in heterogeneous infrastructure (e.g. various hosts/VMs from different providers), we also replicated this for all our commercial offerings using a bare-metal Hetzner host. The process was identical: install the agent and make the runs-on code change as above to use Actuated. Massive improvements in build time were seen again as expected.

\n

The usage of bare-metal (or cloud) hosts providers is invisible and only a choice of which provider you want to put the agent on. In our case we have a mixed set up with no difference in usage or maintenance.

\n

Challenges building containers

\n

The native package (RPM/DEB) building described above was quite simple to integrate via the existing workflows we had.

\n

Building the native packages is done via a process that runs a target-specific container for each of the builds, e.g. we run a CentOS container to build for that target. This allows a complete build to be run on any Linux-compatible machine with a container runtime either in CI or locally. For ARM builds, we were using QEMU emulation for ARM builds hence the slowdown as this has to emulate instructions between architectures.

\n

Container builds are the primary commercial area for improvement as we provide a SAAS solution running on K8S. Container builds were also a trickier proposition for OSS as we were using a single job to build all architectures using the docker/build-push-action. The builds were incredibly slow for ARM and also atomic, which means if you received a transient issue in one of the architecture builds, you would have to repeat the whole lot.

\n

As an example: https://github.com/fluent/fluent-bit/blob/master/.github/workflows/call-build-images.yaml

\n
      - name: Build the production images\n        id: build_push\n        uses: docker/build-push-action@v4\n        with:\n          file: ./dockerfiles/Dockerfile\n          context: .\n          tags: ${{ steps.meta.outputs.tags }}\n          labels: ${{ steps.meta.outputs.labels }}\n          platforms: linux/amd64, linux/arm64, linux/arm/v7\n          target: production\n          # Must be disabled to provide legacy format images from the registry\n          provenance: false\n          push: true\n          load: false\n          build-args: |\n            FLB_NIGHTLY_BUILD=${{ inputs.unstable }}\n            RELEASE_VERSION=${{ inputs.version }}\n
\n

The build step above is a bit more complex to tease out into separate components: we need to run single architecture builds for each target then provide a multi-arch manifest that links them together.

\n

We reached out to Alex on a good way to modify this to work within a split build per architecture approach. The Actuated team has been very responsive on these types of questions along with proactive monitoring of our build queue and runners.

\n

Within Calyptia we have followed the approach Docker provided here and suggested by the Actuated team: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners

\n

Based on what we learned, we recommend the following process is followed:

\n

Build each architecture and push by digest in a set of parallel matrix jobs.\nCapture the output digest of each build.\nCreate the multi-arch manifest made up of each digest we have pushed in step 1 using the artefact from step 2.

\n

This approach provides two key benefits. First, it allows us to run on dedicated runners per-arch. Second, if a job fails we only need to repeat the single job, instead of having to rebuild all architectures.

\n

The new approach reduced the time for the release process for the Calyptia Core K8S Operator from more than an hour to minutes. Additionally, because we can do this so quickly, we now build all architectures for every change rather than just on release. This helps developers who are running ARM locally for development as they have containers always available.

\n

The example time speed up for the Calyptia Core K8S operator process was replicated across all the other components. A very good bang for your buck!

\n

For us, the actuated subscription fee has been of great value. Initially we tested the waters on the Basic Plan, but soon upgraded when we saw more areas where we could use it. The cost for us has been offset against a massive improvement in CI time and development time plus reducing the infrastructure costs of managing the self-hosted runners.

\n

Lessons learned

\n

The package updates were seamless really, however we did encounter some issues with the ecosystem (not with actuated), when refactoring and updating our container builds. The issues with the container builds are covered below to help anyone else with the same problems.

\n

Provenance is now enabled by default

\n

We were using v3 of Docker’s docker/build-push-action, but they made a breaking change which caused us a headache. They changed the default in v4 to create the various extra artifacts for provenance (e.g. SBOMs) which did have a few extra side effects both at the time and even now.

\n

If you do not disable this then it will push manifest lists rather than images so you will subsequently get an error message when you try to create a manifest list of another manifest list.

\n

Separately this also causes issues for older docker clients or organisations that need the legacy Docker schema format from a registry: using it means only OCI format schemas are pushed. This impacted both OSS and our commercial offerings: https://github.com/fluent/fluent-bit/issues/7748.

\n

It meant people on older OS’s or with requirements on only consuming Docker schema (e.g. maybe an internal mirror registry only supports that) could not pull the images.

\n

Invalid timestamps for gcr.io with manifests

\n

A funny problem found with our cloud-run deployments for Calyptia Core SAAS offering was that pushing the manifests to (Google Container Registry) gcr.io meant they ended up with a zero-epoch timestamp. This messed up some internal automation for us when we tried to get the latest version.

\n

To resolve this we just switched back to doing a single architecture build as we do not need multi-arch manifests for cloud-run. Internally we still have multi-arch images in ghcr.io for internal use anyway, this is purely the promotion to gcr.io.

\n

Manifests cannot use sub-paths

\n

This was a fun one: when specifying images to make up your manifest they must be in the same registry of course!

\n

Now, we tend to use sub-paths a lot to handle specific use cases for ghcr.io but unfortunately you cannot use them when trying to construct a manifest.

\n

OK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/product:tag-amd64\nNOK: ghcr.io/calyptia/internal/product:tag --> ghcr.io/calyptia/internal/amd64/product:tag

\n

As with all good failures, the tooling let me make a broken manifest at build time but unfortunately trying to pull it meant a failure at runtime.

\n

Actuated container registry mirror

\n

All Github hosted runners provide default credentials to authenticate with docker.io for pulling public images. When running on a self-hosted runner you need to authenticate for this otherwise you will hit rate limits and builds may fail as they cannot download required base images.

\n

Actuated provide a registry mirror and Github Action to simplify this so make sure you set it up: https://docs.actuated.com/tasks/registry-mirror/

\n

As part of this, ensure it is set up for anything that uses images (e.g. we run integration tests on KIND that failed as the cluster could not download its images) and that it is done after any buildx config as it creates a dedicated buildx builder for the mirror usage.

\n

Actuated support

\n

The Actuated team helped us in two ways: the first was that we were able to enable Arm builds for our OSS projects and our commercial products, when they timed out with hosted runners. The second way was where our costs were getting out of hand on GitHub’s larger hosted runners: Actuated not only reduced the build time, but the billing model is flat-rate, meaning our costs are now fixed, rather than growing.

\n

As we made suggestions or collaborated with the Actuated team, they updated the documentation, including our suggestions on smoothing out the onboarding of new build servers and new features for the CLI.

\n

The more improvements we’ve made, the more we’ve seen. Next on our list is getting the runtime of a Go release down from 26 minutes by bringing it over to actuated.

\n

Conclusion

\n

Alex Ellis: We've learned a lot working with Patrick and Calyptia and are pleased to see that they were able to save money, whilst getting much quicker, and safer Open Source and commercial builds.

\n

We value getting feedback and suggestions from customers, and Patrick continues to provide plenty of them.

\n

If you'd like to learn more about actuated, reach out to speak to our team by clicking \"Sign-up\" and filling out the form. We'll be in touch to arrange a call.

","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"rollup":true,"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/case-study-bring-your-own-bare-metal-to-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/case-study-bring-your-own-bare-metal-to-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/case-study-bring-your-own-bare-metal-to-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/case-study-bring-your-own-bare-metal-to-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/cncf-arm-march-update.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/cncf-arm-march-update.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/cncf-arm-march-update.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/cncf-arm-march-update.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/custom-sizes-bpf-kvm.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/custom-sizes-bpf-kvm.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/custom-sizes-bpf-kvm.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/custom-sizes-bpf-kvm.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/develop-a-great-go-cli.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/develop-a-great-go-cli.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/develop-a-great-go-cli.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/develop-a-great-go-cli.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/faster-nix-builds.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/faster-nix-builds.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/faster-nix-builds.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/faster-nix-builds.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/faster-self-hosted-cache.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/faster-self-hosted-cache.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/faster-self-hosted-cache.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/faster-self-hosted-cache.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/firecracker-container-lab.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/firecracker-container-lab.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/firecracker-container-lab.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/firecracker-container-lab.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/github-actions-usage-cli.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/github-actions-usage-cli.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/github-actions-usage-cli.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/github-actions-usage-cli.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/gpus-for-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/gpus-for-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/gpus-for-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/gpus-for-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/how-secure-are-containers-vs-microvms-ci.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/how-secure-are-containers-vs-microvms-ci.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/how-secure-are-containers-vs-microvms-ci.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/how-secure-are-containers-vs-microvms-ci.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/how-to-run-multi-arch-builds-natively.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/how-to-run-multi-arch-builds-natively.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/how-to-run-multi-arch-builds-natively.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/how-to-run-multi-arch-builds-natively.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/is-the-self-hosted-runner-safe-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/is-the-self-hosted-runner-safe-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/is-the-self-hosted-runner-safe-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/is-the-self-hosted-runner-safe-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/kvm-in-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/kvm-in-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/kvm-in-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/kvm-in-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/local-caching-for-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/local-caching-for-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/local-caching-for-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/local-caching-for-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/managing-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/managing-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/managing-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/managing-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/millions-of-cncf-minutes.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/millions-of-cncf-minutes.json similarity index 98% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/millions-of-cncf-minutes.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/millions-of-cncf-minutes.json index 9312caf..6bfe99b 100644 --- a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/millions-of-cncf-minutes.json +++ b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/millions-of-cncf-minutes.json @@ -1 +1 @@ -{"pageProps":{"post":{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","contentHtml":"

In this post I'll cover:

\n\n

What's Actuated?

\n
\n

Actuated is the only solution that gives a managed experience for self-hosted runners, on your own hardware through immutable microVMs.

\n
\n

The immutability is key to both security and reliability. Each build is run in its own ephemeral and immutable microVM, meaning side effects cannot be left behind by prior builds. In addition, our team manages reliability for your servers and the integration with GitHub, for minimal disruption during GitHub outages.

\n

Why did the CNCF want Actuated for its projects?

\n

We've been working with Ampere and Equinix Metal to provide CI via GitHub Actions for Cloud Native Computing (CNCF) projects. Ampere manufacture Arm-based CPUs with a focus on efficiency and high core density. Equinix Metal provide access to the Ampere Altra in their datacenters around the world.

\n

Last December, we met with Chris Aniszczyck - CTO Linux Foundation/CNCF, Ed Vielmetti - Open Source Manager Equinix, Dave Neary - Director of Developer Relations at Ampere and myself to discuss the program and what impact it was having so far.

\n

Watch the recap on YouTube: The Ampere Developer Impact: CNCF Pilot Discussion

\n

Past articles on the blog include:

\n\n

Before and after the program

\n

Before we started the program, CNCF projects could be divided into three buckets:

\n
    \n
  1. They were running Arm-based CI jobs on static servers that they managed
    2. \n
\n

In this case, etcd for instance had a team of half a dozen maintainers who were responsible for setting up, maintaining, and upgrading statically provisioned CI servers for the project. This was a significant overhead for the project maintainers, and the servers were often underutilized. The risk of side-effects being left behind between builds also posed a serious supply chain risk since etcd is consumed in virtually every Kubernetes deployment.

\n
    \n
  1. They were running Arm-based CI using emulation (QEMU)
    2. \n
\n

QEMU can be combined with Docker's buildx for a quick and convenient way to build container images for x86 and Arm architectures. In the best case, it's a small change and may add a few minutes of extra overhead. In the worst case, we saw that jobs that ran in ~ 5 minutes, took over 6 hours to complete using QEMU and hosted runners. A prime example was fluentd, read their case-study here: Scaling ARM builds with Actuated

\n
    \n
  1. They had no Arm CI at all
    2. \n
\n

In the third case, we saw projects like OpenTelemetry which had no support for Arm at all, but demand from their community to bring it up to on par with x86 builds. The need to self-manage insecure CI servers meant that Arm was a blocker for them.

\n

After the program

\n

After the program was live, teams who had been maintaining their own servers got to remove lengthy documentation on server configuration and maintenance, and relied on our team to manage a pool of servers used for scheduling microVMs.

\n

As demand grew, we saw OpenTelemetry and etcd starve the shared pool of resources through very high usage patterns. This is a classic and known problem called \"Tragedy of the Commons\" - when a shared resource is overused by a subset of users, it can lead to a degradation of service for all users. To combat the problem, we added code to provision self-destructing servers for a period of 24-48 hours as need arose, and prevented the higher usage projects from running on at least on of the permanent servers through scheduling rules. One other issue we saw with OpenTelemetry in particular was that the various Go proxies that offer up Go modules appeared to be rejecting requests when too many jobs were running concurrently. As a workaround, we added a private Go proxy for them into the private network space where the CNCF servers run, this also massively reduced the bandwidth costs for the shared infrastructure.

\n

Teams like fluent moved from flakey builds that couldn't finish in 6 hours, to builds that finished in 5-10 minutes. This meant they could expand on their suite of tests.

\n

Where teams such as Cilium, Falco, or OpenTelemetry had no Arm CI support, we saw them quickly ramp up to running thousands of builds per month.

\n

Here's a quote from Federico Di Pierro, Senior Open Source Engineer @ Sysdig and maintainer of Falco:

\n
\n

Falco really needed arm64 GitHub runners to elevate its support for the architecture and enlarge its userbase.\nActuated was the perfect solution for us because it was easy to leverage and relieved any burden for the maintainers. This way, we as maintainers, can focus on what really matters for the project, instead of fighting with maintaining and deploying self-hosted infrastructure.\nNow we are building, testing and releasing artifacts for arm64 leveraging Actuated for many of our projects, and it works flawlessly.\nSupport from Alex's team is always on point, and new kernel features are coming through super quickly!

\n
\n

Akihiro Suda, Software Engineer at NTT Corp, and maintainer of several open source projects including: runc, containerd and lima had this to say:

\n
\n

Huge thanks to Actuated for enabling us to run ARM64 tests without any mess.\nIt is very important for the runc project to run the tests on ARM64, as runc depends on several architecture-dependent components such as seccomp and criu.\nIt is also so nice that the Arm instance specification can be adjusted in a single line in the GitHub Actions workflow file.

\n
\n

Wei Fu, a maintainer for containerd said:

\n
\n

The containerd project was able to test each pull request for the Linux arm64 platform with the support of Actuated.\nIt's a significant step for containerd to mark the Linux arm64 platform as a top-tier supported platform, similar to amd64, since containerd has been widely used in the Arm world.

\n

Thanks to Actuated, we, the containerd community, were able to test container features (like mount-idmapping) on the new kernel without significant maintenance overhead for the test infrastructure.\nWith Actuated, we can focus on open-source deployment to cover more use case scenarios.

\n
\n

Maintainers have direct access to discuss issues and improvements with us via a private Slack community. One of the things we've done in addition to adding burst capacity to the pool, was to provide a tool to help teams right-size VMs for jobs and to add support for eBPF technologies like BTF in the Kernel.

\n

Numbers at a glance

\n

In our last update, 3 months ago, we'd run just under 400k build minutes for the CNCF. That number has now increased to 1.52M minutes, which is a ~ 300x increase in demand in a short period of time.

\n

Here's a breakdown of the top 9 projects by total minutes run, bearing in mind that this only includes jobs that ran to completion, there are thousands of minutes which ran, but were cancelled mid-way or by automation.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
RankOrganisationTotal minsTotal JobsFirst job
1open-telemetry593726400932024-02-15
2etcd-io372080213472023-10-24
3cri-o163927111312023-11-27
4falcosecurity138469132742023-12-06
5fluent89856106582023-06-07
6containerd87007111922023-12-02
7cilium7340662522023-10-31
8opencontainers37164582023-12-15
9argoproj187122024-01-30
(all)(Total)1520464116217
\n

Most organisations build for several projects or repositories. In the case of etcd, the numbers also include the boltdb project, and for cilium, tetragon, and the Go bindings for ebpf are also included. Open Telemetry is mainly focused around the collectors and SDKs.

\n

runc which is within the opencontainers organisation is technically an Open Container Initiative (OCI) project under the LinuxFoundation, rather than a CNCF project, but we gave them access since it is a key dependency for containerd and cri-o.

\n

What's next?

\n

With the exception of Argo, all of the projects are now relatively heavy users of the platform, with demand growing month on month, as you can see from the uptick from 389k minutes in March to a record high of 1.52 million minutes by the end of May of the same year. In the case of Argo, if you're a contributor or have done previous open source enablement, perhaps you could help them expand their Arm support via a series of Pull Requests to enable unit/e2e tests to run on Arm64?

\n

We're continuing to improve the platform to support users during peak demand, outages on GitHub, and to provide a reliable way for CNCF projects to run their CI on real Arm hardware, at full speed.

\n

For instance, last month we just released a new 6.1 Kernel for the Ampere Altra, which means projects like Cilium and Falco can make use of new eBPF features introduced in recent Kernel versions, and will bring support for newer Kernels as the Firecracker team make them available. The runc and container teams also benefit from the newer Kernel and have been able to enable further tests for (Checkpoint/Restore In Userspace) CRIU and User namespaces for containerd.

\n

You can watch the interview I mentioned earlier with Chris, Ed, Dave and myself on YouTube:

\n\n

We could help you too

\n

Actuated can manage x86 and Arm64 servers for GitHub Actions and self-managed GitLab CI. If you'd like to speak to us about how we can speed up your jobs, reduce your maintenance efforts and lower your CI costs, reach out via this page.

","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"}},"__N_SSG":true} \ No newline at end of file +{"pageProps":{"post":{"slug":"millions-of-cncf-minutes","fileName":"2024-05-30-millions-of-cncf-minutes.md","contentHtml":"

In this post I'll cover:

\n\n

What's Actuated?

\n
\n

Actuated is the only solution that gives a managed experience for self-hosted runners, on your own hardware through immutable microVMs.

\n
\n

The immutability is key to both security and reliability. Each build is run in its own ephemeral and immutable microVM, meaning side effects cannot be left behind by prior builds. In addition, our team manages reliability for your servers and the integration with GitHub, for minimal disruption during GitHub outages.

\n

Why did the CNCF want Actuated for its projects?

\n

We've been working with Ampere and Equinix Metal to provide CI via GitHub Actions for Cloud Native Computing (CNCF) projects. Ampere manufacture Arm-based CPUs with a focus on efficiency and high core density. Equinix Metal provide access to the Ampere Altra in their datacenters around the world.

\n

Last December, we met with Chris Aniszczyck - CTO Linux Foundation/CNCF, Ed Vielmetti - Open Source Manager Equinix, Dave Neary - Director of Developer Relations at Ampere and myself to discuss the program and what impact it was having so far.

\n

Watch the recap on YouTube: The Ampere Developer Impact: CNCF Pilot Discussion

\n

Past articles on the blog include:

\n\n

Before and after the program

\n

Before we started the program, CNCF projects could be divided into three buckets:

\n
    \n
  1. They were running Arm-based CI jobs on static servers that they managed
    2. \n
\n

In this case, etcd for instance had a team of half a dozen maintainers who were responsible for setting up, maintaining, and upgrading statically provisioned CI servers for the project. This was a significant overhead for the project maintainers, and the servers were often underutilized. The risk of side-effects being left behind between builds also posed a serious supply chain risk since etcd is consumed in virtually every Kubernetes deployment.

\n
    \n
  1. They were running Arm-based CI using emulation (QEMU)
    2. \n
\n

QEMU can be combined with Docker's buildx for a quick and convenient way to build container images for x86 and Arm architectures. In the best case, it's a small change and may add a few minutes of extra overhead. In the worst case, we saw that jobs that ran in ~ 5 minutes, took over 6 hours to complete using QEMU and hosted runners. A prime example was fluentd, read their case-study here: Scaling ARM builds with Actuated

\n
    \n
  1. They had no Arm CI at all
    2. \n
\n

In the third case, we saw projects like OpenTelemetry which had no support for Arm at all, but demand from their community to bring it up to on par with x86 builds. The need to self-manage insecure CI servers meant that Arm was a blocker for them.

\n

After the program

\n

After the program was live, teams who had been maintaining their own servers got to remove lengthy documentation on server configuration and maintenance, and relied on our team to manage a pool of servers used for scheduling microVMs.

\n

As demand grew, we saw OpenTelemetry and etcd starve the shared pool of resources through very high usage patterns. This is a classic and known problem called \"Tragedy of the Commons\" - when a shared resource is overused by a subset of users, it can lead to a degradation of service for all users. To combat the problem, we added code to provision self-destructing servers for a period of 24-48 hours as need arose, and prevented the higher usage projects from running on at least on of the permanent servers through scheduling rules. One other issue we saw with OpenTelemetry in particular was that the various Go proxies that offer up Go modules appeared to be rejecting requests when too many jobs were running concurrently. As a workaround, we added a private Go proxy for them into the private network space where the CNCF servers run, this also massively reduced the bandwidth costs for the shared infrastructure.

\n

Teams like fluent moved from flakey builds that couldn't finish in 6 hours, to builds that finished in 5-10 minutes. This meant they could expand on their suite of tests.

\n

Where teams such as Cilium, Falco, or OpenTelemetry had no Arm CI support, we saw them quickly ramp up to running thousands of builds per month.

\n

Here's a quote from Federico Di Pierro, Senior Open Source Engineer @ Sysdig and maintainer of Falco:

\n
\n

Falco really needed arm64 GitHub runners to elevate its support for the architecture and enlarge its userbase.\nActuated was the perfect solution for us because it was easy to leverage and relieved any burden for the maintainers. This way, we as maintainers, can focus on what really matters for the project, instead of fighting with maintaining and deploying self-hosted infrastructure.\nNow we are building, testing and releasing artifacts for arm64 leveraging Actuated for many of our projects, and it works flawlessly.\nSupport from Alex's team is always on point, and new kernel features are coming through super quickly!

\n
\n

Akihiro Suda, Software Engineer at NTT Corp, and maintainer of several open source projects including: runc, containerd and lima had this to say:

\n
\n

Huge thanks to Actuated for enabling us to run ARM64 tests without any mess.\nIt is very important for the runc project to run the tests on ARM64, as runc depends on several architecture-dependent components such as seccomp and criu.\nIt is also so nice that the Arm instance specification can be adjusted in a single line in the GitHub Actions workflow file.

\n
\n

Wei Fu, a maintainer for containerd said:

\n
\n

The containerd project was able to test each pull request for the Linux arm64 platform with the support of Actuated.\nIt's a significant step for containerd to mark the Linux arm64 platform as a top-tier supported platform, similar to amd64, since containerd has been widely used in the Arm world.

\n

Thanks to Actuated, we, the containerd community, were able to test container features (like mount-idmapping) on the new kernel without significant maintenance overhead for the test infrastructure.\nWith Actuated, we can focus on open-source deployment to cover more use case scenarios.

\n
\n

Maintainers have direct access to discuss issues and improvements with us via a private Slack community. One of the things we've done in addition to adding burst capacity to the pool, was to provide a tool to help teams right-size VMs for jobs and to add support for eBPF technologies like BTF in the Kernel.

\n

Numbers at a glance

\n

In our last update, 3 months ago, we'd run just under 400k build minutes for the CNCF. That number has now increased to 1.52M minutes, which is a ~ 300x increase in demand in a short period of time.

\n

Here's a breakdown of the top 9 projects by total minutes run, bearing in mind that this only includes jobs that ran to completion, there are thousands of minutes which ran, but were cancelled mid-way or by automation.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
RankOrganisationTotal minsTotal JobsFirst job
1open-telemetry593726400932024-02-15
2etcd-io372080213472023-10-24
3cri-o163927111312023-11-27
4falcosecurity138469132742023-12-06
5fluent89856106582023-06-07
6containerd87007111922023-12-02
7cilium7340662522023-10-31
8opencontainers37164582023-12-15
9argoproj187122024-01-30
(all)(Total)1520464116217
\n

Most organisations build for several projects or repositories. In the case of etcd, the numbers also include the boltdb project, and for cilium, tetragon, and the Go bindings for ebpf are also included. Open Telemetry is mainly focused around the collectors and SDKs.

\n

runc which is within the opencontainers organisation is technically an Open Container Initiative (OCI) project under the LinuxFoundation, rather than a CNCF project, but we gave them access since it is a key dependency for containerd and cri-o.

\n

What's next?

\n

With the exception of Argo, all of the projects are now relatively heavy users of the platform, with demand growing month on month, as you can see from the uptick from 389k minutes in March to a record high of 1.52 million minutes by the end of May of the same year. In the case of Argo, if you're a contributor or have done previous open source enablement, perhaps you could help them expand their Arm support via a series of Pull Requests to enable unit/e2e tests to run on Arm64?

\n

We're continuing to improve the platform to support users during peak demand, outages on GitHub, and to provide a reliable way for CNCF projects to run their CI on real Arm hardware, at full speed.

\n

For instance, last month we just released a new 6.1 Kernel for the Ampere Altra, which means projects like Cilium and Falco can make use of new eBPF features introduced in recent Kernel versions, and will bring support for newer Kernels as the Firecracker team make them available. The runc and container teams also benefit from the newer Kernel and have been able to enable further tests for (Checkpoint/Restore In Userspace) CRIU and User namespaces for containerd.

\n

You can watch the interview I mentioned earlier with Chris, Ed, Dave and myself on YouTube:

\n\n

We could help you too

\n

Actuated can manage x86 and Arm64 servers for GitHub Actions and self-managed GitLab CI. If you'd like to speak to us about how we can speed up your jobs, reduce your maintenance efforts and lower your CI costs, reach out via this page.

","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"rollup":true,"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/multi-arch-docker-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/multi-arch-docker-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/multi-arch-docker-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/multi-arch-docker-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/native-arm64-for-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/native-arm64-for-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/native-arm64-for-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/native-arm64-for-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/oidc-proxy-for-openfaas.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/oidc-proxy-for-openfaas.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/oidc-proxy-for-openfaas.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/oidc-proxy-for-openfaas.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/ollama-in-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/ollama-in-github-actions.json similarity index 98% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/ollama-in-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/ollama-in-github-actions.json index 16be8e7..fee84c2 100644 --- a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/ollama-in-github-actions.json +++ b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/ollama-in-github-actions.json @@ -1 +1 @@ -{"pageProps":{"post":{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","contentHtml":"

That means you can run real end to end tests in CI with the same models you may use in dev and production. And if you use OpenAI or AWS SageMaker extensively, you could perhaps swap out what can be a very expensive API endpoint for your CI or testing environments to save money.

\n

If you'd like to learn more about how and why you'd want access to GPUs in CI, read my past update: Accelerate GitHub Actions with dedicated GPUs.

\n

We'll first cover what ollama is, why it's so popular, how to get it, what kinds of fun things you can do with it, then how to access it from actuated using a real GPU.

\n

\"ollama

\n
\n

ollama can now run in CI with isolated GPU acceleration using actuated

\n
\n

What's ollama?

\n

ollama is an open source project that aims to do for AI models, what Docker did for Linux containers. Whilst Docker created a user experience to share and run containers using container images in the Open Container Initiative (OCI) format, ollama bundles well-known AI models and makes it easy to run them without having to think about Python versions or Nvidia CUDA libraries.

\n

The project packages and runs various models, but seems to take its name from Meta's popular llama2 model, which whilst not released under an open source license, allows for a generous amount of free usage for most types of users.

\n

The ollama project can be run directly on a Linux, MacOS or Windows host, or within a container. There's a server component, and a CLI that acts as a client to pre-trained models. The main use-case today is that of inference - exercising the model with input data. A more recent feature means that you can create embeddings, if you pull a model that supports them.

\n

On Linux, ollama can be installed using a utility script:

\n
curl -fsSL https://ollama.com/install.sh | sh\n
\n

This provides the ollama CLI command.

\n

A quick tour of ollama

\n

After the initial installation, you can start a server:

\n
ollama serve\n
\n

By default, its REST API will listen on port 11434 on 127.0.0.1.

\n

You can find the reference for ollama's REST API here: API endpoints - which includes things like: creating a chat completion, pulling a model, or generating embeddings.

\n

You can then browse available models on the official website, which resembles the Docker Hub. This set currently includes: gemma (built upon Google's DeepMind), mistral (an LLM), codellama (for generating Code), phi (from Microsoft research), vicuna (for chat, based upon llama2), llava (a vision encoder), and many more.

\n

Most models will download with a default parameter size that's small enough to run on most CPUs or GPUs, but if you need to access it, there are larger models for higher accuracy.

\n

For instance, the llama2 model by Meta will default to the 7b model which needs around 8GB of RAM.

\n
# Pull the default model size:\n\nollama pull llama2\n\n# Override the parameter size\n\nollama pull llama2:13b\n
\n

Once you have a model, you can then either \"run\" it, where you'll be able to ask it questions and interact with it like you would with ChatGPT, or you can send it API requests from your own applications using REST and HTTP.

\n

For an interactive prompt, give no parameters:

\n
ollama run llama2\n
\n

To get an immediate response for use in i.e. scripts:

\n
ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n
\n

And you can use the REST API via curl, or your own codebase:

\n
curl -s http://localhost:11434/api/generate -d '{\n    \"model\": \"llama2\",\n    \"stream\": false,\n    \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n}' | jq\n
\n

We are just scratching the surface with what ollama can do, with a focus on testing and pulling pre-built models, but you can also create and share models using a Modelfile, which is another homage to the Docker experience by the ollama developers.

\n

Access ollama from Python code

\n

Here's how to access the API via Python, the stream parameter will emit JSON progressively when set to True, block until done if set to False. With Node.js, Python, Java, C#, etc the code will be very similar, but using your own preferred HTTP client. For Golang (Go) users, ollama founder Jeffrey Morgan maintains a higher-level Go SDK.

\n
import requests\nimport json\n\nurl = \"http://localhost:11434/api/generate\"\npayload = {\n    \"model\": \"llama2\",\n    \"stream\": False,\n    \"prompt\": \"What are the risks of running privileged Docker containers for CI workloads?\"\n}\nheaders = {\n    \"Content-Type\": \"application/json\"\n}\n\nresponse = requests.post(url, data=json.dumps(payload), headers=headers)\n\n# Parse the JSON response\nresponse_json = response.json()\n\n# Pretty print the JSON response\nprint(json.dumps(response_json, indent=4))\n
\n

When you're constructing a request by API, make sure you include any tags in the model name, if you've used one. I.e. \"model\": \"llama2:13b\".

\n

I hear from so many organisations who have gone to lengths to get SOC2 compliance, doing CVE scanning, or who are running Open Policy Agent or Kyverno to enforce strict Pod admission policies in Kubernetes, but then are happy to run their CI in Pods in privileged mode. So I asked the model why that may not be a smart idea. You can run the sample for yourself or see the response here. We also go into detail in the actuated FAQ, the security situation around self-hosted runners and containers is the main reason we built the solution.

\n

Putting it together for a GitHub Action

\n

The following GitHub Action will run on for customers who are enrolled for GPU support for actuated. If you'd like to gain access, contact us via the form on the Pricing page.

\n

The self-actuated/nvidia-run installs either the consumer or datacenter driver for Nvidia, depending on what you have in your system. This only takes about 30 seconds and could be cached if you like. The ollama models could also be cached using a local S3 bucket.

\n

Then, we simply run the equivalent bash commands from the previous section to:

\n\n
name: ollama-e2e\n\non:\n    workflow_dispatch:\n\njobs:\n    ollama-e2e:\n        name: ollama-e2e\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - uses: self-actuated/nvidia-run@master\n        - name: Install Ollama\n          run: |\n            curl -fsSL https://ollama.com/install.sh | sudo -E sh\n        - name: Start serving\n          run: |\n              # Run the background, there is no way to daemonise at the moment\n              ollama serve &\n\n              # A short pause is required before the HTTP port is opened\n              sleep 5\n\n              # This endpoint blocks until ready\n              time curl -i http://localhost:11434\n\n        - name: Pull llama2\n          run: |\n              ollama pull llama2\n\n        - name: Invoke via the CLI\n          run: |\n              ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n\n        - name: Invoke via API\n          run: |\n            curl -s http://localhost:11434/api/generate -d '{\n              \"model\": \"llama2\",\n              \"stream\": false,\n              \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n            }' | jq\n
\n

There is no built-in way to daemonise the ollama server, so for now we run it in the background using bash. The readiness endpoint can then be accessed which blocks until the server has completed its initialisation.

\n

Interactive access with SSH

\n

By modifying your CI job, you can drop into a remote SSH session and run interactive commands at any point in the workflow.

\n

That's how I came up with the commands for the Nvidia driver installation, and for the various ollama commands I shared.

\n

Find out more about SSH for GitHub Actions in the actuated docs.

\n

\"Pulling

\n
\n

Pulling one of the larger llama2 models interactively in an SSH session, directly to the runner VM

\n
\n

Wrapping up

\n

Within a very short period of time ollama helped us pull a popular AI model that can be used for chat and completions. We were then able to take what we learned and run it on a GPU at an accelerated speed and accuracy by using actuated's new GPU support for GitHub Actions and GitLab CI. Most hosted CI systems provide a relatively small amount of disk space for jobs, with actuated you can customise this and that may be important if you're going to be downloading large AI models. You can also easily customise the amount of RAM and vCPU using the runs-on label to any combination you need.

\n

ollama isn't the only way to find, download and run AI models, just like Docker wasn't the only way to download and install Nginx or Postgresql, but it provides a useful and convenient interface for those of us who are still learning about AI, and are not as concerned with the internal workings of the models.

\n

Over on the OpenFaaS blog, in the tutorial Stream OpenAI responses from functions using Server Sent Events, we covered how to stream a response from a model to a function, and then back to a user. There, we used the llama-api open source project, which is a single-purpose HTTP API for simulating llama2.

\n

One of the benefits of ollama is the detailed range of examples in the docs, and the ability to run other models that may include computer vision such as with the LLaVA: Large Language and Vision Assistant model or generating code with Code Llama.

\n

Right now, many of us are running and tuning models in development, some of us are using OpenAI's API or self-hosted models in production, but there's very little talk about doing thorough end to end testing or exercising models in CI. That's where actuated can help.

\n

Feel free to reach out for early access, or to see if we can help your team with your CI needs.

","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"}},"__N_SSG":true} \ No newline at end of file +{"pageProps":{"post":{"slug":"ollama-in-github-actions","fileName":"2024-03-25-ollama-in-github-actions.md","contentHtml":"

That means you can run real end to end tests in CI with the same models you may use in dev and production. And if you use OpenAI or AWS SageMaker extensively, you could perhaps swap out what can be a very expensive API endpoint for your CI or testing environments to save money.

\n

If you'd like to learn more about how and why you'd want access to GPUs in CI, read my past update: Accelerate GitHub Actions with dedicated GPUs.

\n

We'll first cover what ollama is, why it's so popular, how to get it, what kinds of fun things you can do with it, then how to access it from actuated using a real GPU.

\n

\"ollama

\n
\n

ollama can now run in CI with isolated GPU acceleration using actuated

\n
\n

What's ollama?

\n

ollama is an open source project that aims to do for AI models, what Docker did for Linux containers. Whilst Docker created a user experience to share and run containers using container images in the Open Container Initiative (OCI) format, ollama bundles well-known AI models and makes it easy to run them without having to think about Python versions or Nvidia CUDA libraries.

\n

The project packages and runs various models, but seems to take its name from Meta's popular llama2 model, which whilst not released under an open source license, allows for a generous amount of free usage for most types of users.

\n

The ollama project can be run directly on a Linux, MacOS or Windows host, or within a container. There's a server component, and a CLI that acts as a client to pre-trained models. The main use-case today is that of inference - exercising the model with input data. A more recent feature means that you can create embeddings, if you pull a model that supports them.

\n

On Linux, ollama can be installed using a utility script:

\n
curl -fsSL https://ollama.com/install.sh | sh\n
\n

This provides the ollama CLI command.

\n

A quick tour of ollama

\n

After the initial installation, you can start a server:

\n
ollama serve\n
\n

By default, its REST API will listen on port 11434 on 127.0.0.1.

\n

You can find the reference for ollama's REST API here: API endpoints - which includes things like: creating a chat completion, pulling a model, or generating embeddings.

\n

You can then browse available models on the official website, which resembles the Docker Hub. This set currently includes: gemma (built upon Google's DeepMind), mistral (an LLM), codellama (for generating Code), phi (from Microsoft research), vicuna (for chat, based upon llama2), llava (a vision encoder), and many more.

\n

Most models will download with a default parameter size that's small enough to run on most CPUs or GPUs, but if you need to access it, there are larger models for higher accuracy.

\n

For instance, the llama2 model by Meta will default to the 7b model which needs around 8GB of RAM.

\n
# Pull the default model size:\n\nollama pull llama2\n\n# Override the parameter size\n\nollama pull llama2:13b\n
\n

Once you have a model, you can then either \"run\" it, where you'll be able to ask it questions and interact with it like you would with ChatGPT, or you can send it API requests from your own applications using REST and HTTP.

\n

For an interactive prompt, give no parameters:

\n
ollama run llama2\n
\n

To get an immediate response for use in i.e. scripts:

\n
ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n
\n

And you can use the REST API via curl, or your own codebase:

\n
curl -s http://localhost:11434/api/generate -d '{\n    \"model\": \"llama2\",\n    \"stream\": false,\n    \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n}' | jq\n
\n

We are just scratching the surface with what ollama can do, with a focus on testing and pulling pre-built models, but you can also create and share models using a Modelfile, which is another homage to the Docker experience by the ollama developers.

\n

Access ollama from Python code

\n

Here's how to access the API via Python, the stream parameter will emit JSON progressively when set to True, block until done if set to False. With Node.js, Python, Java, C#, etc the code will be very similar, but using your own preferred HTTP client. For Golang (Go) users, ollama founder Jeffrey Morgan maintains a higher-level Go SDK.

\n
import requests\nimport json\n\nurl = \"http://localhost:11434/api/generate\"\npayload = {\n    \"model\": \"llama2\",\n    \"stream\": False,\n    \"prompt\": \"What are the risks of running privileged Docker containers for CI workloads?\"\n}\nheaders = {\n    \"Content-Type\": \"application/json\"\n}\n\nresponse = requests.post(url, data=json.dumps(payload), headers=headers)\n\n# Parse the JSON response\nresponse_json = response.json()\n\n# Pretty print the JSON response\nprint(json.dumps(response_json, indent=4))\n
\n

When you're constructing a request by API, make sure you include any tags in the model name, if you've used one. I.e. \"model\": \"llama2:13b\".

\n

I hear from so many organisations who have gone to lengths to get SOC2 compliance, doing CVE scanning, or who are running Open Policy Agent or Kyverno to enforce strict Pod admission policies in Kubernetes, but then are happy to run their CI in Pods in privileged mode. So I asked the model why that may not be a smart idea. You can run the sample for yourself or see the response here. We also go into detail in the actuated FAQ, the security situation around self-hosted runners and containers is the main reason we built the solution.

\n

Putting it together for a GitHub Action

\n

The following GitHub Action will run on for customers who are enrolled for GPU support for actuated. If you'd like to gain access, contact us via the form on the Pricing page.

\n

The self-actuated/nvidia-run installs either the consumer or datacenter driver for Nvidia, depending on what you have in your system. This only takes about 30 seconds and could be cached if you like. The ollama models could also be cached using a local S3 bucket.

\n

Then, we simply run the equivalent bash commands from the previous section to:

\n\n
name: ollama-e2e\n\non:\n    workflow_dispatch:\n\njobs:\n    ollama-e2e:\n        name: ollama-e2e\n        runs-on: [actuated-8cpu-16gb, gpu]\n        steps:\n        - uses: actions/checkout@v1\n        - uses: self-actuated/nvidia-run@master\n        - name: Install Ollama\n          run: |\n            curl -fsSL https://ollama.com/install.sh | sudo -E sh\n        - name: Start serving\n          run: |\n              # Run the background, there is no way to daemonise at the moment\n              ollama serve &\n\n              # A short pause is required before the HTTP port is opened\n              sleep 5\n\n              # This endpoint blocks until ready\n              time curl -i http://localhost:11434\n\n        - name: Pull llama2\n          run: |\n              ollama pull llama2\n\n        - name: Invoke via the CLI\n          run: |\n              ollama run llama2 \"What are the pros of MicroVMs for continous integrations, especially if Docker is the alternative?\"\n\n        - name: Invoke via API\n          run: |\n            curl -s http://localhost:11434/api/generate -d '{\n              \"model\": \"llama2\",\n              \"stream\": false,\n              \"prompt\":\"What are the risks of running privileged Docker containers for CI workloads?\"\n            }' | jq\n
\n

There is no built-in way to daemonise the ollama server, so for now we run it in the background using bash. The readiness endpoint can then be accessed which blocks until the server has completed its initialisation.

\n

Interactive access with SSH

\n

By modifying your CI job, you can drop into a remote SSH session and run interactive commands at any point in the workflow.

\n

That's how I came up with the commands for the Nvidia driver installation, and for the various ollama commands I shared.

\n

Find out more about SSH for GitHub Actions in the actuated docs.

\n

\"Pulling

\n
\n

Pulling one of the larger llama2 models interactively in an SSH session, directly to the runner VM

\n
\n

Wrapping up

\n

Within a very short period of time ollama helped us pull a popular AI model that can be used for chat and completions. We were then able to take what we learned and run it on a GPU at an accelerated speed and accuracy by using actuated's new GPU support for GitHub Actions and GitLab CI. Most hosted CI systems provide a relatively small amount of disk space for jobs, with actuated you can customise this and that may be important if you're going to be downloading large AI models. You can also easily customise the amount of RAM and vCPU using the runs-on label to any combination you need.

\n

ollama isn't the only way to find, download and run AI models, just like Docker wasn't the only way to download and install Nginx or Postgresql, but it provides a useful and convenient interface for those of us who are still learning about AI, and are not as concerned with the internal workings of the models.

\n

Over on the OpenFaaS blog, in the tutorial Stream OpenAI responses from functions using Server Sent Events, we covered how to stream a response from a model to a function, and then back to a user. There, we used the llama-api open source project, which is a single-purpose HTTP API for simulating llama2.

\n

One of the benefits of ollama is the detailed range of examples in the docs, and the ability to run other models that may include computer vision such as with the LLaVA: Large Language and Vision Assistant model or generating code with Code Llama.

\n

Right now, many of us are running and tuning models in development, some of us are using OpenAI's API or self-hosted models in production, but there's very little talk about doing thorough end to end testing or exercising models in CI. That's where actuated can help.

\n

Feel free to reach out for early access, or to see if we can help your team with your CI needs.

","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"rollup":true,"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"}},"__N_SSG":true} \ No newline at end of file diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/right-sizing-vms-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/right-sizing-vms-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/right-sizing-vms-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/right-sizing-vms-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/sbom-in-github-actions.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/sbom-in-github-actions.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/sbom-in-github-actions.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/sbom-in-github-actions.json diff --git a/_next/data/fmY28ZvLLPLCazi-OKnDH/blog/secure-microvm-ci-gitlab.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/blog/secure-microvm-ci-gitlab.json similarity index 100% rename from _next/data/fmY28ZvLLPLCazi-OKnDH/blog/secure-microvm-ci-gitlab.json rename to _next/data/y73HVNpk3Hj6pdIAHkKda/blog/secure-microvm-ci-gitlab.json diff --git a/_next/data/y73HVNpk3Hj6pdIAHkKda/index.json b/_next/data/y73HVNpk3Hj6pdIAHkKda/index.json new file mode 100644 index 0000000..e8a7870 --- /dev/null +++ b/_next/data/y73HVNpk3Hj6pdIAHkKda/index.json @@ -0,0 +1 @@ +{"pageProps":{"rollupPosts":[{"slug":"millions-of-cncf-minutes","title":"On Running Millions of Arm CI Minutes for the CNCF","description":"We've now run over 1.5 million minutes of CI time for various CNCF projects on Ampere hardware. Here's what we've learned.","tags":["cncf","enablement","arm"],"rollup":true,"author_img":"alex","image":"/images/2024-05-cncf-millions/background.png","date":"2024-05-30"},{"slug":"ollama-in-github-actions","title":"Run AI models with ollama in CI with GitHub Actions","description":"With the new GPU support for actuated, we've been able to run models like llama2 from ollama in CI on consumer and datacenter grade Nvidia cards.","tags":["ai","ollama","ml","localmodels","githubactions","openai","llama","machinelearning"],"rollup":true,"author_img":"alex","image":"/images/2024-04-ollama-in-ci/background.png","date":"2024-04-25"},{"slug":"calyptia-case-study-arm","title":"How Calyptia fixed its Arm builds whilst saving money","description":"Learn how Calyptia fixed its failing Arm builds for open-source Fluent Bit and accelerated our commercial development by adopting Actuated and bare-metal runners.","tags":["images","packer","qemu","kvm"],"rollup":true,"author_img":"patrick-stephens","image":"/images/2023-08-calyptia-casestudy/background.png","date":"2023-08-11"}]},"__N_SSG":true} \ No newline at end of file diff --git a/_next/static/chunks/pages/index-1a777de527b4f542.js b/_next/static/chunks/pages/index-1a777de527b4f542.js deleted file mode 100644 index 168c65e..0000000 --- a/_next/static/chunks/pages/index-1a777de527b4f542.js +++ /dev/null @@ -1 +0,0 @@ -(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[405],{5557:function(e,t,a){(window.__NEXT_P=window.__NEXT_P||[]).push(["/",function(){return a(3260)}])},3260:function(e,t,a){"use strict";a.r(t),a.d(t,{default:function(){return Y}});var s,n,l=a(5893),i=a(1664),r=a.n(i),o=a(7294),d=a(2984),c=a(2351),m=a(3784),u=a(9946),x=a(1363),h=a(4103),g=a(6567),p=a(4157),f=a(5466),b=a(3781),y=((s=y||{})[s.Open=0]="Open",s[s.Closed=1]="Closed",s),w=((n=w||{})[n.ToggleDisclosure=0]="ToggleDisclosure",n[n.CloseDisclosure=1]="CloseDisclosure",n[n.SetButtonId=2]="SetButtonId",n[n.SetPanelId=3]="SetPanelId",n[n.LinkPanel=4]="LinkPanel",n[n.UnlinkPanel=5]="UnlinkPanel",n);let j={0:e=>({...e,disclosureState:(0,d.E)(e.disclosureState,{0:1,1:0})}),1:e=>1===e.disclosureState?e:{...e,disclosureState:1},4:e=>!0===e.linkedPanel?e:{...e,linkedPanel:!0},5:e=>!1===e.linkedPanel?e:{...e,linkedPanel:!1},2:(e,t)=>e.buttonId===t.buttonId?e:{...e,buttonId:t.buttonId},3:(e,t)=>e.panelId===t.panelId?e:{...e,panelId:t.panelId}},v=(0,o.createContext)(null);function N(e){let t=(0,o.useContext)(v);if(null===t){let a=Error(`<${e} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(a,N),a}return t}v.displayName="DisclosureContext";let k=(0,o.createContext)(null);k.displayName="DisclosureAPIContext";let C=(0,o.createContext)(null);function A(e,t){return(0,d.E)(t.type,j,e,t)}C.displayName="DisclosurePanelContext";let S=o.Fragment,I=(0,c.yV)(function(e,t){let{defaultOpen:a=!1,...s}=e,n=(0,o.useRef)(null),l=(0,m.T)(t,(0,m.h)(e=>{n.current=e},void 0===e.as||e.as===o.Fragment)),i=(0,o.useRef)(null),r=(0,o.useRef)(null),u=(0,o.useReducer)(A,{disclosureState:a?0:1,linkedPanel:!1,buttonRef:r,panelRef:i,buttonId:null,panelId:null}),[{disclosureState:x,buttonId:h},p]=u,y=(0,b.z)(e=>{p({type:1});let t=(0,f.r)(n);if(!t||!h)return;let a=e?e instanceof HTMLElement?e:e.current instanceof HTMLElement?e.current:t.getElementById(h):t.getElementById(h);null==a||a.focus()}),w=(0,o.useMemo)(()=>({close:y}),[y]),j=(0,o.useMemo)(()=>({open:0===x,close:y}),[x,y]);return o.createElement(v.Provider,{value:u},o.createElement(k.Provider,{value:w},o.createElement(g.up,{value:(0,d.E)(x,{0:g.ZM.Open,1:g.ZM.Closed})},(0,c.sY)({ourProps:{ref:l},theirProps:s,slot:j,defaultTag:S,name:"Disclosure"}))))}),E=(0,c.yV)(function(e,t){let a=(0,u.M)(),{id:s=`headlessui-disclosure-button-${a}`,...n}=e,[l,i]=N("Disclosure.Button"),r=(0,o.useContext)(C),d=null!==r&&r===l.panelId,g=(0,o.useRef)(null),f=(0,m.T)(g,t,d?null:l.buttonRef);(0,o.useEffect)(()=>{if(!d)return i({type:2,buttonId:s}),()=>{i({type:2,buttonId:null})}},[s,i,d]);let y=(0,b.z)(e=>{var t;if(d){if(1===l.disclosureState)return;switch(e.key){case x.R.Space:case x.R.Enter:e.preventDefault(),e.stopPropagation(),i({type:0}),null==(t=l.buttonRef.current)||t.focus()}}else switch(e.key){case x.R.Space:case x.R.Enter:e.preventDefault(),e.stopPropagation(),i({type:0})}}),w=(0,b.z)(e=>{e.key===x.R.Space&&e.preventDefault()}),j=(0,b.z)(t=>{var a;(0,h.P)(t.currentTarget)||e.disabled||(d?(i({type:0}),null==(a=l.buttonRef.current)||a.focus()):i({type:0}))}),v=(0,o.useMemo)(()=>({open:0===l.disclosureState}),[l]),k=(0,p.f)(e,g),A=d?{ref:f,type:k,onKeyDown:y,onClick:j}:{ref:f,id:s,type:k,"aria-expanded":e.disabled?void 0:0===l.disclosureState,"aria-controls":l.linkedPanel?l.panelId:void 0,onKeyDown:y,onKeyUp:w,onClick:j};return(0,c.sY)({ourProps:A,theirProps:n,slot:v,defaultTag:"button",name:"Disclosure.Button"})}),D=c.AN.RenderStrategy|c.AN.Static,M=Object.assign(I,{Button:E,Panel:(0,c.yV)(function(e,t){let a=(0,u.M)(),{id:s=`headlessui-disclosure-panel-${a}`,...n}=e,[l,i]=N("Disclosure.Panel"),{close:r}=function e(t){let a=(0,o.useContext)(k);if(null===a){let s=Error(`<${t} /> is missing a parent component.`);throw Error.captureStackTrace&&Error.captureStackTrace(s,e),s}return a}("Disclosure.Panel"),d=(0,m.T)(t,l.panelRef,e=>{i({type:e?4:5})});(0,o.useEffect)(()=>(i({type:3,panelId:s}),()=>{i({type:3,panelId:null})}),[s,i]);let x=(0,g.oJ)(),h=null!==x?x===g.ZM.Open:0===l.disclosureState,p=(0,o.useMemo)(()=>({open:0===l.disclosureState,close:r}),[l,r]);return o.createElement(C.Provider,{value:l.panelId},(0,c.sY)({ourProps:{ref:d,id:s},theirProps:n,slot:p,defaultTag:"div",features:D,visible:h,name:"Disclosure.Panel"}))})}),P=o.forwardRef(function({title:e,titleId:t,...a},s){return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:1.5,stroke:"currentColor","aria-hidden":"true",ref:s,"aria-labelledby":t},a),e?o.createElement("title",{id:t},e):null,o.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19.5 8.25l-7.5 7.5-7.5-7.5"}))});var H=[{question:"What is Firecracker? Why is it better than a container?",answer:"Generally, any use of containers for CI means bypassing security, so that any job can take over a host or the cluster. Actuated uses Firecracker, an open-source project built by AWS to fully isolate every job with an immutable microVM.",link:"https://www.youtube.com/watch?v=CYCsa5e2vqg",action:"Watch our Firecracker webinar"},{question:"Can we talk to you before signing up for a plan?",answer:"Just fill out the form for a call with our founder. If you think our solution is a good fit, you can be up and running on the same day.",link:"https://docs.google.com/forms/d/e/1FAIpQLScA12IGyVFrZtSAp2Oj24OdaSMloqARSwoxx3AZbQbs0wpGww/viewform",action:"Schedule a call"},{question:"How much does it cost? What is the right plan for our team?",answer:"We are offering unmetered billing with a flat-rate monthly subscription. You can use as many minutes as you like."},{question:"What kind of support do you offer?",answer:"For the pilot, every customer is invited to a Slack channel for collaboration and support. We have operational experience with GitHub Actions, Docker and Kubernetes and we're making time to help you tune your builds up to get the most out of them."},{question:"What kinds of servers do I need?",answer:"You can use your own physical servers, nested virtualisation with cloud VMs or rent instances paid by the hour.",link:"https://docs.actuated.com/provision-server/",action:"Explore server options"},{question:"Doesn't GitHub already offer faster runners?",answer:"GitHub are in a beta phase for larger runners for their Team and Enterprises plans, these have an increased cost vs. standard runners and there is no Arm support. With actuated you get much faster speeds at a flat rate cost, with usage insights across your team and organisation."},{question:"What are the differences vs. Actions Runtime Controller?",answer:"actions-runtime-controller compromises the security of a Kubernetes cluster by using privileged containers or by mounting the Docker socket - both mean that code in a CI job can take over the host - or potentially the cluster.",link:"/blog/blazing-fast-ci-with-microvms"},{question:"How much faster is an Arm build than using hosted runners?",answer:"In our testing of the open source Parca project, we got the build time down from 33 minutes to 1 minute 26s simply by changing to an Arm runner instead of using QEMU. For Network Service Mesh, Dasch Swiss and Calyptia - their builds couldn't complete within 6 hours, all complete in 4-20 minutes with actuated.",link:"/blog/native-arm64-for-github-actions",action:"Read a case study"},{question:"How do I launch jobs in parallel?",answer:"Have a look at the examples in our docs for matrix builds. Each job within the workflow will be launched in a separate, parallel VM.",link:"https://docs.actuated.com/"},{question:"How mature is actuated?",answer:"Actuated is built on battle tested technology that's run in production at huge scale by Amazon Web Services (AWS) and GitHub. Our solution has launched over 25k VMs for customers already, without issue."},{question:"Is GitLab supported?",answer:"We have a tech preview for self-hosted GitLab.",link:"/blog/secure-microvm-ci-gitlab",action:"Read the announcement"},{question:"Where can I find detailed information about actuated?",answer:"We cover most common questions in much more detail over in the FAQ in the docs.",link:"https://docs.actuated.com/faq",action:"FAQ"}];function R(){return(0,l.jsx)("div",{className:"bg-gray-50",children:(0,l.jsx)("div",{className:"mx-auto max-w-7xl py-12 px-4 sm:py-16 sm:px-6 lg:px-8",children:(0,l.jsxs)("div",{className:"mx-auto max-w-3xl divide-y-2 divide-gray-200",children:[(0,l.jsx)("h2",{className:"text-center text-3xl font-bold tracking-tight text-gray-900 sm:text-4xl",children:"Frequently asked questions"}),(0,l.jsx)("dl",{className:"mt-6 space-y-6 divide-y divide-gray-200",children:H.map(function(e){return(0,l.jsx)(M,{as:"div",className:"pt-6",children:function(t){var a=t.open;return(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)("dt",{className:"text-lg",children:(0,l.jsxs)(M.Button,{className:"flex w-full items-start justify-between text-left text-gray-400",children:[(0,l.jsx)("span",{className:"font-medium text-gray-900",children:e.question}),(0,l.jsx)("span",{className:"ml-6 flex h-7 items-center",children:(0,l.jsx)(P,{className:function(){for(var e=arguments.length,t=Array(e),a=0;a