Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache: Add support for explicitly choosing compression, and zero compression #1877

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
add 'None' compression type and add tests
planetmarshall committed Dec 6, 2024
commit 85a278dd2ffa7740a2beaf72f045f1c4f1ad3b30
102 changes: 102 additions & 0 deletions packages/cache/__tests__/tar.test.ts
Original file line number Diff line number Diff line change
@@ -46,6 +46,40 @@ afterAll(async () => {
await jest.requireActual('@actions/io').rmRF(getTempDir())
})

test('extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')

const archivePath = IS_WINDOWS
? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar'
const workspace = process.env['GITHUB_WORKSPACE']
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath

await tar.extractTar(archivePath, CompressionMethod.None)

expect(mkdirMock).toHaveBeenCalledWith(workspace)
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'-xf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P',
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.join(' '),
undefined,
{
cwd: undefined,
env: expect.objectContaining(defaultEnv)
}
)
})

test('zstd extract tar', async () => {
const mkdirMock = jest.spyOn(io, 'mkdirP')
const execMock = jest.spyOn(exec, 'exec')
@@ -201,6 +235,45 @@ test('gzip extract GNU tar on windows with GNUtar in path', async () => {
}
})

test('create tar', async () => {
const execMock = jest.spyOn(exec, 'exec')

const archiveFolder = getTempDir()
const workspace = process.env['GITHUB_WORKSPACE']
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]

await fs.promises.mkdir(archiveFolder, {recursive: true})

await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.None)

const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath

expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'--posix',
'-cf',
IS_WINDOWS ? CacheFilename.None.replace(/\\/g, '/') : CacheFilename.None,
'--exclude',
IS_WINDOWS ? CacheFilename.None.replace(/\\/g, '/') : CacheFilename.None,
'-P',
'-C',
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
'--files-from',
ManifestFilename
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.join(' '),
undefined, // args
{
cwd: archiveFolder,
env: expect.objectContaining(defaultEnv)
}
)
})

test('zstd create tar', async () => {
const execMock = jest.spyOn(exec, 'exec')

@@ -345,6 +418,35 @@ test('gzip create tar', async () => {
)
})

test('list tar', async () => {
const execMock = jest.spyOn(exec, 'exec')

const archivePath = IS_WINDOWS
? `${process.env['windir']}\\fakepath\\cache.tar`
: 'cache.tar'

await tar.listTar(archivePath, CompressionMethod.None)

const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
expect(execMock).toHaveBeenCalledTimes(1)
expect(execMock).toHaveBeenCalledWith(
[
`"${tarPath}"`,
'-tf',
IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath,
'-P'
]
.concat(IS_WINDOWS ? ['--force-local'] : [])
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
.join(' '),
undefined,
{
cwd: undefined,
env: expect.objectContaining(defaultEnv)
}
)
})

test('zstd list tar', async () => {
const execMock = jest.spyOn(exec, 'exec')

12 changes: 9 additions & 3 deletions packages/cache/src/internal/cacheUtils.ts
Original file line number Diff line number Diff line change
@@ -112,9 +112,15 @@ export async function getCompressionMethod(): Promise<CompressionMethod> {
}

export function getCacheFileName(compressionMethod: CompressionMethod): string {
return compressionMethod === CompressionMethod.Gzip
? CacheFilename.Gzip
: CacheFilename.Zstd
switch (compressionMethod) {
default:
case CompressionMethod.Gzip:
return CacheFilename.Gzip
case CompressionMethod.Zstd:
return CacheFilename.Zstd
case CompressionMethod.None:
return CacheFilename.None
}
}

export async function getGnuTarPathOnWindows(): Promise<string> {
6 changes: 4 additions & 2 deletions packages/cache/src/internal/constants.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
export enum CacheFilename {
Gzip = 'cache.tgz',
Zstd = 'cache.tzst'
Zstd = 'cache.tzst',
None = 'cache.tar'
}

export enum CompressionMethod {
Gzip = 'gzip',
// Long range mode was added to zstd in v1.3.2.
// This enum is for earlier version of zstd that does not have --long support
ZstdWithoutLong = 'zstd-without-long',
Zstd = 'zstd'
Zstd = 'zstd',
None = 'none'
}

export enum ArchiveToolType {
17 changes: 14 additions & 3 deletions packages/cache/src/internal/tar.ts
Original file line number Diff line number Diff line change
@@ -5,11 +5,11 @@ import * as path from 'path'
import * as utils from './cacheUtils'
import {ArchiveTool} from './contracts'
import {
ArchiveToolType,
CompressionMethod,
ManifestFilename,
SystemTarPathOnWindows,
ArchiveToolType,
TarFilename,
ManifestFilename
TarFilename
} from './constants'

const IS_WINDOWS = process.platform === 'win32'
@@ -65,6 +65,7 @@ async function getTarArgs(
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
compressionMethod !== CompressionMethod.None &&
IS_WINDOWS

// Method specific args
@@ -139,10 +140,16 @@ async function getCommands(
type,
archivePath
)

if (compressionMethod === CompressionMethod.None) {
return [tarArgs.join(' ')]
}

const compressionArgs =
type !== 'create'
? await getDecompressionProgram(tarPath, compressionMethod, archivePath)
: await getCompressionProgram(tarPath, compressionMethod)

const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
@@ -178,6 +185,7 @@ async function getDecompressionProgram(
const BSD_TAR_ZSTD =
tarPath.type === ArchiveToolType.BSD &&
compressionMethod !== CompressionMethod.Gzip &&
compressionMethod !== CompressionMethod.None &&
IS_WINDOWS
switch (compressionMethod) {
case CompressionMethod.Zstd:
@@ -199,7 +207,10 @@ async function getDecompressionProgram(
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
]
: ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']
case CompressionMethod.None:
return []
default:
case CompressionMethod.Gzip:
return ['-z']
}
}