fetcher: new implementation using unlinked files

apk/scanner_test.go

@@ -2,21 +2,19 @@ package apk
 
 import (
 	"context"
-	"net/http"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/quay/zlog"
 
 	"github.com/quay/claircore"
-	"github.com/quay/claircore/test/fetch"
+	"github.com/quay/claircore/test"
 )
 
 func TestScan(t *testing.T) {
-	hash, err := claircore.ParseDigest("sha256:89d9c30c1d48bac627e5c6cb0d1ed1eec28e7dbdfbcc04712e4c79c0f83faf17")
-	if err != nil {
-		t.Fatal(err)
-	}
+	t.Parallel()
+	ctx := zlog.Test(context.Background(), t)
+	// TODO(hank) Turn into a fixture.
 	want := []*claircore.Package{
 		{
 			Name:           "musl",
@@ -146,22 +144,13 @@ func TestScan(t *testing.T) {
 		},
 	}
 
-	ctx := zlog.Test(context.Background(), t)
-	l := &claircore.Layer{
-		Hash: hash,
-	}
-
-	n, err := fetch.Layer(ctx, t, http.DefaultClient, "docker.io", "library/alpine", hash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer n.Close()
-
-	if err := l.SetLocal(n.Name()); err != nil {
-		t.Error(err)
-	}
+	l := test.RealizeLayer(ctx, t, test.LayerRef{
+		Registry: "docker.io",
+		Name:     "library/alpine",
+		Digest:   "sha256:89d9c30c1d48bac627e5c6cb0d1ed1eec28e7dbdfbcc04712e4c79c0f83faf17",
+	})
+	var s Scanner
 
-	s := &Scanner{}
 	got, err := s.Scan(ctx, l)
 	if err != nil {
 		t.Fatal(err)

dpkg/distroless_scanner_test.go

@@ -2,16 +2,17 @@ package dpkg
 
 import (
 	"context"
-	"net/http"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/quay/claircore"
-	"github.com/quay/claircore/test/fetch"
 	"github.com/quay/zlog"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/test"
 )
 
 func TestDistrolessLayer(t *testing.T) {
+	ctx := zlog.Test(context.Background(), t)
 	want := []*claircore.Package{
 		{
 			Name:           "base-files",
@@ -41,25 +42,15 @@ func TestDistrolessLayer(t *testing.T) {
 			RepositoryHint: "",
 		},
 	}
+	l := test.RealizeLayer(ctx, t, test.LayerRef{
+		Registry: "gcr.io",
+		Name:     "distroless/static-debian11",
+		Digest:   `sha256:8fdb1fc20e240e9cae976518305db9f9486caa155fd5fc53e7b3a3285fe8a990`,
+	})
+	var s DistrolessScanner
 
 	t.Parallel()
-	ctx := zlog.Test(context.Background(), t)
-	hash := claircore.MustParseDigest(`sha256:8fdb1fc20e240e9cae976518305db9f9486caa155fd5fc53e7b3a3285fe8a990`)
-	l := claircore.Layer{
-		Hash: hash,
-	}
-	n, err := fetch.Layer(ctx, t, http.DefaultClient, "gcr.io", "distroless/static-debian11", hash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer n.Close()
-
-	if err := l.SetLocal(n.Name()); err != nil {
-		t.Error(err)
-	}
-
-	s := new(DistrolessScanner)
-	ps, err := s.Scan(ctx, &l)
+	ps, err := s.Scan(ctx, l)
 	if err != nil {
 		t.Error(err)
 	}

dpkg/scanner_test.go

@@ -4,28 +4,21 @@ import (
 	"archive/tar"
 	"bufio"
 	"context"
-	"errors"
 	"io"
-	"net/http"
 	"net/textproto"
 	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/quay/zlog"
 
 	"github.com/quay/claircore"
-	"github.com/quay/claircore/test/fetch"
-	"github.com/quay/claircore/test/integration"
+	"github.com/quay/claircore/test"
 )
 
 func TestScanner(t *testing.T) {
 	t.Parallel()
-	hash, err := claircore.ParseDigest("sha256:35c102085707f703de2d9eaad8752d6fe1b8f02b5d2149f1d8357c9cc7fb7d0a")
-	if err != nil {
-		t.Fatal(err)
-	}
+	// TODO(hank) Cook up a manifest format for dpkg ala `test/rpmtest.Manifest`
 	want := []*claircore.Package{
 		{
 			Name:           "fdisk",
@@ -803,21 +796,13 @@ func TestScanner(t *testing.T) {
 		},
 	}
 	ctx := zlog.Test(context.Background(), t)
-	l := &claircore.Layer{
-		Hash: hash,
-	}
-
-	n, err := fetch.Layer(ctx, t, http.DefaultClient, "docker.io", "library/ubuntu", hash)
-	if err != nil {
-		t.Error(err)
-	}
-	defer n.Close()
-
-	if err := l.SetLocal(n.Name()); err != nil {
-		t.Error(err)
-	}
+	l := test.RealizeLayer(ctx, t, test.LayerRef{
+		Registry: "docker.io",
+		Name:     "library/ubuntu",
+		Digest:   "sha256:35c102085707f703de2d9eaad8752d6fe1b8f02b5d2149f1d8357c9cc7fb7d0a",
+	})
+	var s Scanner
 
-	s := &Scanner{}
 	got, err := s.Scan(ctx, l)
 	if err != nil {
 		t.Fatal(err)
@@ -830,25 +815,12 @@ func TestScanner(t *testing.T) {
 func TestAbsolutePaths(t *testing.T) {
 	t.Parallel()
 	ctx := zlog.Test(context.Background(), t)
-	hash, err := claircore.ParseDigest("sha256:3c9020349340788076971d5ea638b71e35233fd8e149e269d8eebfa17960c03f")
-	if err != nil {
-		t.Fatal(err)
-	}
-	l := &claircore.Layer{
-		Hash: hash,
-	}
-
-	n, err := fetch.Layer(ctx, t, http.DefaultClient, "gcr.io", "vmwarecloudadvocacy/acmeshop-user", hash)
-	if err != nil {
-		t.Error(err)
-	}
-	defer n.Close()
-
-	if err := l.SetLocal(n.Name()); err != nil {
-		t.Error(err)
-	}
-
-	s := &Scanner{}
+	l := test.RealizeLayer(ctx, t, test.LayerRef{
+		Registry: "gcr.io",
+		Name:     "vmwarecloudadvocacy/acmeshop-user",
+		Digest:   "sha256:3c9020349340788076971d5ea638b71e35233fd8e149e269d8eebfa17960c03f",
+	})
+	var s Scanner
 	got, err := s.Scan(ctx, l)
 	if err != nil {
 		t.Fatal(err)
@@ -861,21 +833,21 @@ func TestAbsolutePaths(t *testing.T) {
 
 func TestExtraMetadata(t *testing.T) {
 	t.Parallel()
-	layerfile := filepath.Join(integration.PackageCacheDir(t), `extrametadata.layer`)
-	l := claircore.Layer{
-		Hash: claircore.MustParseDigest(`sha256:25fd87072f39aaebd1ee24dca825e61d9f5a0f87966c01551d31a4d8d79d37d8`),
-		URI:  "file:///dev/null",
-	}
+	mod := test.Modtime(t, "scanner_test.go")
+	layerfile := test.GenerateFixture(t, `extrametadata.layer`, mod, extraMetadataSetup)
 	ctx := zlog.Test(context.Background(), t)
+	var l claircore.Layer
+	var s Scanner
 
-	// Set up the crafted layer
-	extraMetadataSetup(t, layerfile)
-	l.SetLocal(layerfile)
-	if t.Failed() {
-		return
+	f, err := os.Open(layerfile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	if err := l.Init(ctx, &test.AnyDescription, f); err != nil {
+		t.Error(err)
 	}
 
-	s := new(Scanner)
 	ps, err := s.Scan(ctx, &l)
 	if err != nil {
 		t.Error(err)
@@ -886,47 +858,7 @@ func TestExtraMetadata(t *testing.T) {
 }
 
 // ExtraMetadataSetup is a helper to craft a layer that trips PROJQUAY-1308.
-func extraMetadataSetup(t *testing.T, layer string) {
-	t.Helper()
-
-	fi, err := os.Stat(layer)
-	if err != nil {
-		t.Log(err)
-	}
-	switch {
-	case err == nil:
-		// If everything looks okay, check if this test has been touched. If so,
-		// remove the layer and recurse, so that it's re-created.
-		tf, err := os.Stat(`scanner_test.go`)
-		if err != nil {
-			t.Log(err)
-		}
-		if !fi.ModTime().After(tf.ModTime()) {
-			t.Log("recreating layer")
-			if err := os.Remove(layer); err != nil {
-				t.Log(err)
-			}
-			extraMetadataSetup(t, layer)
-		}
-		return
-	case errors.Is(err, os.ErrNotExist): // OK
-		os.Mkdir(`testdata`, 0o755)
-	default:
-		t.Error(err)
-		return
-	}
-
-	// If we're here, time to create the layer.
-	t.Log("creating layer")
-	f, err := os.Create(layer)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer func() {
-		if err := f.Close(); err != nil {
-			t.Error(err)
-		}
-	}()
+func extraMetadataSetup(t testing.TB, f *os.File) {
 	w := tar.NewWriter(f)
 	defer func() {
 		if err := w.Close(); err != nil {

go.mod

@@ -23,8 +23,11 @@ require (
 	github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319
 	github.com/rs/zerolog v1.30.0
 	github.com/ulikunitz/xz v0.5.11
+	go.opentelemetry.io/otel v1.18.0
+	go.opentelemetry.io/otel/trace v1.18.0
 	golang.org/x/crypto v0.14.0
 	golang.org/x/sync v0.4.0
+	golang.org/x/sys v0.13.0
 	golang.org/x/text v0.13.0
 	golang.org/x/time v0.3.0
 	golang.org/x/tools v0.12.0
@@ -36,6 +39,8 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
@@ -51,9 +56,8 @@ require (
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	go.opentelemetry.io/otel v1.11.0 // indirect
+	go.opentelemetry.io/otel/metric v1.18.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect

go.sum

@@ -24,6 +24,11 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -184,15 +189,19 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
-go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk=
-go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk=
+go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs=
+go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI=
+go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ=
+go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k=
+go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10=
+go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=