Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add experimental offline mode #183

Merged
merged 28 commits into from
Aug 9, 2023
Merged

feat: add experimental offline mode #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
04816fe
feat: add experimental local and offline modes
G-Rath Mar 2, 2023
c2944f8
fix: offline implies comparing locally, and comparing locally implies…
G-Rath Jun 27, 2023
1ec628d
refactor: remove unused `WorkingDirectory` field
G-Rath Jun 27, 2023
b76f4d4
feat: make database storage location customizable and default to user…
G-Rath Jun 29, 2023
42096b6
feat: remove support for `If-None-Match` and `If-Modified-Since`
G-Rath Jun 29, 2023
a60ee00
feat: store database as a plain zip file rather than JSON
G-Rath Jun 29, 2023
4e99005
feat: only re-download databases if their contents have changed
G-Rath Jun 30, 2023
71c286b
perf: use `cachedregexp`
G-Rath Jul 3, 2023
b8d5b8d
fix: ensure custom user agent has a value before using
G-Rath Jul 3, 2023
ad2cdf9
test: add cli cases
G-Rath Jul 25, 2023
54bb2f8
fix: improve error handling when databases cannot be found
G-Rath Jul 26, 2023
ca90763
refactor: use constants
G-Rath Jul 29, 2023
fffee83
refactor: don't use dedicated types to avoid breaking change
G-Rath Jul 29, 2023
979454a
fix: ensure that events are stored
G-Rath Jul 29, 2023
7bc2890
feat: add hidden local db path option
G-Rath Jul 29, 2023
b1f54f5
fix: check status code of response when there is no existing local db
G-Rath Jul 29, 2023
e106799
feat: mention when a local database has been loaded
G-Rath Jul 29, 2023
8acbbe6
refactor: remove local package name normalization
G-Rath Jul 29, 2023
843a852
refactor: make method private
G-Rath Aug 1, 2023
9742e90
fix: invert condition for checking if temp directory has already been…
G-Rath Aug 1, 2023
989dcbb
fix: consider env based-path as explicitly provided paths
G-Rath Aug 1, 2023
a3ebead
refactor: improve `setupLocalDBDirectory` implementation
G-Rath Aug 1, 2023
7fdbc16
refactor: invert condition to be based on implicit path rather than e…
G-Rath Aug 1, 2023
6f56008
chore: add comments to struct properties
G-Rath Aug 1, 2023
7f39e13
chore: explain why packages without versions are always considered vu…
G-Rath Aug 1, 2023
e29fe7a
fix: ensure body is closed regardless of response status code
G-Rath Aug 1, 2023
5698e16
refactor: rename `offline.Check` to `offline.MakeRequest`
G-Rath Aug 7, 2023
6d813c2
refactor: rename `offline` to `local`
G-Rath Aug 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion cmd/osv-scanner/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,19 @@ func run(args []string, stdout, stderr io.Writer) int {
Usage: "also scan files that would be ignored by .gitignore",
Value: false,
},
&cli.BoolFlag{
Name: "experimental-local-db",
Usage: "checks for vulnerabilities using local databases",
},
&cli.BoolFlag{
Name: "experimental-offline",
Usage: "checks for vulnerabilities using local databases that are already cached",
},
&cli.StringFlag{
Name: "experimental-local-db-path",
Usage: "sets the path that local databases should be stored",
Hidden: true,
},
},
ArgsUsage: "[directory1 directory2...]",
Action: func(context *cli.Context) error {
Expand Down Expand Up @@ -149,7 +162,10 @@ func run(args []string, stdout, stderr io.Writer) int {
ConfigOverridePath: context.String("config"),
DirectoryPaths: context.Args().Slice(),
ExperimentalScannerActions: osvscanner.ExperimentalScannerActions{
CallAnalysis: context.Bool("experimental-call-analysis"),
LocalDBPath: context.String("experimental-local-db-path"),
CallAnalysis: context.Bool("experimental-call-analysis"),
CompareLocally: context.Bool("experimental-local-db"),
CompareOffline: context.Bool("experimental-offline"),
},
}, r)

Expand Down
233 changes: 233 additions & 0 deletions cmd/osv-scanner/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,19 @@ import (
"github.com/go-git/go-git/v5"
)

func createTestDir(t *testing.T) (string, func()) {
t.Helper()

p, err := os.MkdirTemp("", "osv-scanner-test-*")
if err != nil {
t.Fatalf("could not create test directory: %v", err)
}

return p, func() {
_ = os.RemoveAll(p)
}
}

func dedent(t *testing.T, str string) string {
t.Helper()

Expand Down Expand Up @@ -610,6 +623,226 @@ func TestRun_LockfileWithExplicitParseAs(t *testing.T) {
}
}

func TestRun_LocalDatabases(t *testing.T) {
t.Parallel()

tests := []cliTestCase{
// one specific supported lockfile
{
name: "",
args: []string{"", "--experimental-local-db", "./fixtures/locks-many/composer.lock"},
wantExitCode: 0,
wantStdout: `
Scanning dir ./fixtures/locks-many/composer.lock
Scanned %%/fixtures/locks-many/composer.lock file and found 1 package
Loaded Packagist local db from %%/osv-scanner/Packagist/all.zip
No vulnerabilities found
`,
wantStderr: "",
},
// one specific supported sbom with vulns
{
name: "",
args: []string{"", "--experimental-local-db", "--config=./fixtures/osv-scanner-empty-config.toml", "./fixtures/sbom-insecure/postgres-stretch.cdx.xml"},
wantExitCode: 1,
wantStdout: `
Scanning dir ./fixtures/sbom-insecure/postgres-stretch.cdx.xml
Scanned %%/fixtures/sbom-insecure/postgres-stretch.cdx.xml as CycloneDX SBOM and found 136 packages
Loaded Debian local db from %%/osv-scanner/Debian/all.zip
Loaded Go local db from %%/osv-scanner/Go/all.zip
Loaded OSS-Fuzz local db from %%/osv-scanner/OSS-Fuzz/all.zip
+-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+
| OSV URL | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE |
+-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+
| https://osv.dev/GHSA-f3fp-gc8g-vw66 | 5.9 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/GHSA-g2j6-57v7-gm8c | 6.1 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/GHSA-m8cg-xc2p-r3fc | 2.5 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/GHSA-v95c-p5hm-xq8f | 6 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/GHSA-vpvm-3wq2-2wvm | 7 | Go | github.com/opencontainers/runc | v1.0.1 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
| https://osv.dev/GHSA-p782-xgp4-8hr8 | 5.3 | Go | golang.org/x/sys | v0.0.0-20210817142637-7d9622a276b7 | fixtures/sbom-insecure/postgres-stretch.cdx.xml |
+-------------------------------------+------+-----------+--------------------------------+------------------------------------+-------------------------------------------------+
`,
wantStderr: "",
},
// one specific unsupported lockfile
{
name: "",
args: []string{"", "--experimental-local-db", "./fixtures/locks-many/not-a-lockfile.toml"},
wantExitCode: 128,
wantStdout: `
Scanning dir ./fixtures/locks-many/not-a-lockfile.toml
`,
wantStderr: `
No package sources found, --help for usage information.
`,
},
// all supported lockfiles in the directory should be checked
{
name: "",
args: []string{"", "--experimental-local-db", "./fixtures/locks-many"},
wantExitCode: 0,
wantStdout: `
Scanning dir ./fixtures/locks-many
Scanned %%/fixtures/locks-many/Gemfile.lock file and found 1 package
Scanned %%/fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found 15 packages
Scanned %%/fixtures/locks-many/composer.lock file and found 1 package
Scanned %%/fixtures/locks-many/package-lock.json file and found 1 package
Scanned %%/fixtures/locks-many/yarn.lock file and found 1 package
Loaded RubyGems local db from %%/osv-scanner/RubyGems/all.zip
Loaded Alpine local db from %%/osv-scanner/Alpine/all.zip
Loaded Packagist local db from %%/osv-scanner/Packagist/all.zip
Loaded npm local db from %%/osv-scanner/npm/all.zip
Loaded filter from: %%/fixtures/locks-many/osv-scanner.toml
GHSA-whgm-jr23-g3j9 has been filtered out because: Test manifest file
Filtered 1 vulnerability from output
No vulnerabilities found
`,
wantStderr: "",
},
// all supported lockfiles in the directory should be checked
{
name: "",
args: []string{"", "--experimental-local-db", "./fixtures/locks-many-with-invalid"},
wantExitCode: 127,
wantStdout: `
Scanning dir ./fixtures/locks-many-with-invalid
Scanned %%/fixtures/locks-many-with-invalid/Gemfile.lock file and found 1 package
Scanned %%/fixtures/locks-many-with-invalid/yarn.lock file and found 1 package
Loaded RubyGems local db from %%/osv-scanner/RubyGems/all.zip
Loaded npm local db from %%/osv-scanner/npm/all.zip
`,
wantStderr: `
Attempted to scan lockfile but failed: %%/fixtures/locks-many-with-invalid/composer.lock
`,
},
// only the files in the given directories are checked by default (no recursion)
{
name: "",
args: []string{"", "--experimental-local-db", "./fixtures/locks-one-with-nested"},
wantExitCode: 0,
wantStdout: `
Scanning dir ./fixtures/locks-one-with-nested
Scanned %%/fixtures/locks-one-with-nested/yarn.lock file and found 1 package
Loaded npm local db from %%/osv-scanner/npm/all.zip
No vulnerabilities found
`,
wantStderr: "",
},
// nested directories are checked when `--recursive` is passed
{
name: "",
args: []string{"", "--experimental-local-db", "--recursive", "./fixtures/locks-one-with-nested"},
wantExitCode: 0,
wantStdout: `
Scanning dir ./fixtures/locks-one-with-nested
Scanned %%/fixtures/locks-one-with-nested/nested/composer.lock file and found 1 package
Scanned %%/fixtures/locks-one-with-nested/yarn.lock file and found 1 package
Loaded Packagist local db from %%/osv-scanner/Packagist/all.zip
Loaded npm local db from %%/osv-scanner/npm/all.zip
No vulnerabilities found
`,
wantStderr: "",
},
// .gitignored files
{
name: "",
args: []string{"", "--experimental-local-db", "--recursive", "./fixtures/locks-gitignore"},
wantExitCode: 0,
wantStdout: `
Scanning dir ./fixtures/locks-gitignore
Scanned %%/fixtures/locks-gitignore/Gemfile.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/subdir/yarn.lock file and found 1 package
Loaded RubyGems local db from %%/osv-scanner/RubyGems/all.zip
Loaded npm local db from %%/osv-scanner/npm/all.zip
No vulnerabilities found
`,
wantStderr: "",
},
// ignoring .gitignore
{
name: "",
args: []string{"", "--experimental-local-db", "--recursive", "--no-ignore", "./fixtures/locks-gitignore"},
wantExitCode: 0,
wantStdout: `
Scanning dir ./fixtures/locks-gitignore
Scanned %%/fixtures/locks-gitignore/Gemfile.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/composer.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/ignored/Gemfile.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/ignored/yarn.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/subdir/Gemfile.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/subdir/composer.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/subdir/yarn.lock file and found 1 package
Scanned %%/fixtures/locks-gitignore/yarn.lock file and found 1 package
Loaded RubyGems local db from %%/osv-scanner/RubyGems/all.zip
Loaded Packagist local db from %%/osv-scanner/Packagist/all.zip
Loaded npm local db from %%/osv-scanner/npm/all.zip
No vulnerabilities found
`,
wantStderr: "",
},
// output with json
{
name: "",
args: []string{"", "--experimental-local-db", "--json", "./fixtures/locks-many/composer.lock"},
wantExitCode: 0,
wantStdout: `
{
"results": []
}
`,
wantStderr: `
Scanning dir ./fixtures/locks-many/composer.lock
Scanned %%/fixtures/locks-many/composer.lock file and found 1 package
Loaded Packagist local db from %%/osv-scanner/Packagist/all.zip
`,
},
{
name: "",
args: []string{"", "--experimental-local-db", "--format", "json", "./fixtures/locks-many/composer.lock"},
wantExitCode: 0,
wantStdout: `
{
"results": []
}
`,
wantStderr: `
Scanning dir ./fixtures/locks-many/composer.lock
Scanned %%/fixtures/locks-many/composer.lock file and found 1 package
Loaded Packagist local db from %%/osv-scanner/Packagist/all.zip
`,
},
// output format: markdown table
{
name: "",
args: []string{"", "--experimental-local-db", "--format", "markdown", "./fixtures/locks-many/composer.lock"},
wantExitCode: 0,
wantStdout: `
Scanning dir ./fixtures/locks-many/composer.lock
Scanned %%/fixtures/locks-many/composer.lock file and found 1 package
Loaded Packagist local db from %%/osv-scanner/Packagist/all.zip
No vulnerabilities found
`,
wantStderr: "",
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
G-Rath marked this conversation as resolved.
Show resolved Hide resolved

testDir, cleanupTestDir := createTestDir(t)
defer cleanupTestDir()

old := tt.args

tt.args = []string{"", "--experimental-local-db-path", testDir}
tt.args = append(tt.args, old[1:]...)

testCli(t, tt)
})
}
}

func TestMain(m *testing.M) {
// ensure a git repository doesn't already exist in the fixtures directory,
// in case we didn't get a chance to clean-up properly in the last run
Expand Down
Loading