diff --git a/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt b/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt index 454285b100ce8..081e56479729c 100644 --- a/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt @@ -26,7 +26,7 @@ import com.github.ajalt.clikt.parameters.options.required import com.github.ajalt.clikt.parameters.types.file import org.ossreviewtoolkit.helper.utils.readOrtResult -import org.ossreviewtoolkit.scanner.storages.FileBasedStorage +import org.ossreviewtoolkit.scanner.storages.PackageBasedFileStorage import org.ossreviewtoolkit.utils.common.expandTilde import org.ossreviewtoolkit.utils.ort.storage.LocalFileStorage @@ -51,7 +51,7 @@ internal class ImportScanResultsCommand : CliktCommand( override fun run() { val ortResult = readOrtResult(ortFile) - val scanResultsStorage = FileBasedStorage(LocalFileStorage(scanResultsStorageDir)) + val scanResultsStorage = PackageBasedFileStorage(LocalFileStorage(scanResultsStorageDir)) val ids = ortResult.getProjects().map { it.id } + ortResult.getPackages().map { it.metadata.id } ids.forEach { id -> diff --git a/helper-cli/src/main/kotlin/commands/packageconfig/CreateCommand.kt b/helper-cli/src/main/kotlin/commands/packageconfig/CreateCommand.kt index b7db5664414e9..4b9697f86a6e8 100644 --- a/helper-cli/src/main/kotlin/commands/packageconfig/CreateCommand.kt +++ b/helper-cli/src/main/kotlin/commands/packageconfig/CreateCommand.kt @@ -43,7 +43,7 @@ import org.ossreviewtoolkit.model.config.PackageConfiguration import org.ossreviewtoolkit.model.config.VcsMatcher import org.ossreviewtoolkit.model.licenses.LicenseClassifications import org.ossreviewtoolkit.model.readValue -import org.ossreviewtoolkit.scanner.storages.FileBasedStorage +import org.ossreviewtoolkit.scanner.storages.PackageBasedFileStorage import org.ossreviewtoolkit.utils.common.expandTilde import org.ossreviewtoolkit.utils.common.safeMkdirs import org.ossreviewtoolkit.utils.ort.storage.LocalFileStorage @@ -119,7 +119,7 @@ internal class CreateCommand : CliktCommand( override fun run() { outputDir.safeMkdirs() - val scanResultsStorage = FileBasedStorage(LocalFileStorage(scanResultsStorageDir)) + val scanResultsStorage = PackageBasedFileStorage(LocalFileStorage(scanResultsStorageDir)) val scanResults = scanResultsStorage.read(Package.EMPTY.copy(id = packageId)).getOrThrow().run { listOfNotNull( find { it.provenance is RepositoryProvenance }, diff --git a/model/src/main/resources/reference.yml b/model/src/main/resources/reference.yml index f6c5ae1ec42b1..2779ad4a979e1 100644 --- a/model/src/main/resources/reference.yml +++ b/model/src/main/resources/reference.yml @@ -191,7 +191,6 @@ ort: sslcert: /defaultdir/postgresql.crt sslkey: /defaultdir/postgresql.pk8 sslrootcert: /defaultdir/root.crt - parallelTransactions: 5 createMissingArchives: false @@ -217,7 +216,6 @@ ort: sslcert: /defaultdir/postgresql.crt sslkey: /defaultdir/postgresql.pk8 sslrootcert: /defaultdir/root.crt - parallelTransactions: 5 config: # A map from scanner plugin types to the plugin configuration. diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index b031caac5f11e..4dba0345d6ff8 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -30,7 +30,7 @@ import org.ossreviewtoolkit.model.config.PluginConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext -import org.ossreviewtoolkit.scanner.ScanResultsStorage +import org.ossreviewtoolkit.scanner.ScanStorage import org.ossreviewtoolkit.scanner.ScannerMatcher import org.ossreviewtoolkit.scanner.ScannerWrapperConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory @@ -52,7 +52,7 @@ import org.semver4j.Semver * configuration [options][PluginConfiguration.options]: * * * **"commandLine":** Command line options that modify the result. These are added to the [ScannerDetails] when - * looking up results from the [ScanResultsStorage]. Defaults to [ScanCodeConfig.DEFAULT_COMMAND_LINE_OPTIONS]. + * looking up results from a [ScanStorage]. Defaults to [ScanCodeConfig.DEFAULT_COMMAND_LINE_OPTIONS]. * * **"commandLineNonConfig":** Command line options that do not modify the result and should therefore not be * considered in [configuration], like "--processes". Defaults to * [ScanCodeConfig.DEFAULT_COMMAND_LINE_NON_CONFIG_OPTIONS]. diff --git a/scanner/src/funTest/kotlin/storages/AbstractPackageBasedScanStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/AbstractPackageBasedScanStorageFunTest.kt new file mode 100644 index 0000000000000..60047d6996e23 --- /dev/null +++ b/scanner/src/funTest/kotlin/storages/AbstractPackageBasedScanStorageFunTest.kt @@ -0,0 +1,318 @@ +/* + * Copyright (C) 2017 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.scanner.storages + +import io.kotest.core.listeners.TestListener +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.containExactly +import io.kotest.matchers.collections.containExactlyInAnyOrder +import io.kotest.matchers.result.shouldBeFailure +import io.kotest.matchers.result.shouldBeSuccess +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe + +import java.time.Duration +import java.time.Instant + +import org.ossreviewtoolkit.model.ArtifactProvenance +import org.ossreviewtoolkit.model.Hash +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.Issue +import org.ossreviewtoolkit.model.LicenseFinding +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.RepositoryProvenance +import org.ossreviewtoolkit.model.ScanResult +import org.ossreviewtoolkit.model.ScanSummary +import org.ossreviewtoolkit.model.ScannerDetails +import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.UnknownProvenance +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.scanner.ScannerMatcher + +import org.semver4j.Semver + +private val DUMMY_TEXT_LOCATION = TextLocation("fakepath", 13, 21) + +abstract class AbstractPackageBasedScanStorageFunTest(vararg listeners: TestListener) : WordSpec() { + private val id = Identifier("type", "namespace", "name1", "version") + + private val sourceArtifact = RemoteArtifact("url1", Hash.create("0123456789abcdef0123456789abcdef01234567")) + + private val vcs = VcsInfo(VcsType.forName("type"), "url1", "revision", "path") + private val vcsWithoutRevision = VcsInfo(VcsType.forName("type"), "url", "") + + private val pkg = Package.EMPTY.copy( + id = id, + sourceArtifact = sourceArtifact, + vcs = VcsInfo.EMPTY, + vcsProcessed = vcs + ) + + private val pkgWithoutRevision = pkg.copy(vcs = vcsWithoutRevision, vcsProcessed = vcsWithoutRevision.normalize()) + + private val provenanceEmpty = UnknownProvenance + private val provenanceWithoutRevision = RepositoryProvenance( + vcsInfo = pkgWithoutRevision.vcsProcessed, + resolvedRevision = "resolvedRevision" + ) + private val provenanceWithSourceArtifact = ArtifactProvenance(sourceArtifact = sourceArtifact) + private val provenanceWithVcsInfo = RepositoryProvenance(vcsInfo = vcs, resolvedRevision = "resolvedRevision") + + private val scannerDetails1 = ScannerDetails("name 1", "1.0.0", "config 1") + private val scannerDetails2 = ScannerDetails("name 2", "2.0.0", "config 2") + private val scannerDetailsCompatibleVersion1 = ScannerDetails("name 1", "1.0.1", "config 1") + private val scannerDetailsCompatibleVersion2 = ScannerDetails("name 1", "1.0.1-alpha.1", "config 1") + private val scannerDetailsIncompatibleVersion = ScannerDetails("name 1", "1.1.0", "config 1") + + private val scannerMatcherForDetails1 = ScannerMatcher.create(scannerDetails1) + + private val scanSummaryWithFiles = ScanSummary.EMPTY.copy( + startTime = Instant.EPOCH + Duration.ofMinutes(1), + endTime = Instant.EPOCH + Duration.ofMinutes(2), + licenseFindings = setOf( + LicenseFinding("license-1.1", DUMMY_TEXT_LOCATION), + LicenseFinding("license-1.2", DUMMY_TEXT_LOCATION) + ), + issues = listOf( + Issue(source = "source-1", message = "error-1"), + Issue(source = "source-2", message = "error-2") + ) + ) + + private lateinit var storage: AbstractPackageBasedScanStorage + + abstract fun createStorage(): AbstractPackageBasedScanStorage + + init { + register(*listeners) + + beforeEach { + storage = createStorage() + } + + "Adding a scan result" should { + "succeed for a valid scan result" { + val scanResult = ScanResult(provenanceWithSourceArtifact, scannerDetails1, scanSummaryWithFiles) + + val addResult = storage.add(id, scanResult) + val readResult = storage.read(pkg) + + addResult.shouldBeSuccess() + readResult.shouldBeSuccess { + it should containExactly(scanResult) + } + } + + "fail if provenance information is missing" { + val scanResult = ScanResult(provenanceEmpty, scannerDetails1, scanSummaryWithFiles) + + val addResult = storage.add(id, scanResult) + val readResult = storage.read(pkg) + + addResult.shouldBeFailure { + it.message shouldBe "Not storing scan result for '${id.toCoordinates()}' because no provenance " + + "information is available." + } + + readResult.shouldBeSuccess { + it should beEmpty() + } + } + + "not store a result for the same scanner and provenance twice" { + val summary1 = scanSummaryWithFiles + val summary2 = scanSummaryWithFiles.copy( + startTime = scanSummaryWithFiles.startTime.plusSeconds(10) + ) + + val scanResult1 = ScanResult(provenanceWithSourceArtifact, scannerDetails1, summary1) + val scanResult2 = ScanResult(provenanceWithSourceArtifact, scannerDetails1, summary2) + + val addResult1 = storage.add(id, scanResult1) + val addResult2 = storage.add(id, scanResult2) + + addResult1.shouldBeSuccess() + addResult2.shouldBeFailure() + + val readResult = storage.read(pkg) + readResult.shouldBeSuccess { + it should containExactly(scanResult1) + } + } + } + + "Reading a scan result" should { + "find all scan results for an id" { + val scanResult1 = ScanResult(provenanceWithSourceArtifact, scannerDetails1, scanSummaryWithFiles) + val scanResult2 = ScanResult(provenanceWithSourceArtifact, scannerDetails2, scanSummaryWithFiles) + + storage.add(id, scanResult1).shouldBeSuccess() + storage.add(id, scanResult2).shouldBeSuccess() + val readResult = storage.read(pkg) + + readResult.shouldBeSuccess { + it should containExactlyInAnyOrder(scanResult1, scanResult2) + } + } + + "find all scan results for a specific scanner" { + val scanResult1 = ScanResult(provenanceWithSourceArtifact, scannerDetails1, scanSummaryWithFiles) + val scanResult2 = ScanResult(provenanceWithVcsInfo, scannerDetails1, scanSummaryWithFiles) + val scanResult3 = ScanResult(provenanceWithSourceArtifact, scannerDetails2, scanSummaryWithFiles) + + storage.add(id, scanResult1).shouldBeSuccess() + storage.add(id, scanResult2).shouldBeSuccess() + storage.add(id, scanResult3).shouldBeSuccess() + val readResult = storage.read(pkg, scannerMatcherForDetails1) + + readResult.shouldBeSuccess { + it should containExactlyInAnyOrder(scanResult1, scanResult2) + } + } + + "find all scan results for scanners with names matching a pattern" { + val detailsCompatibleOtherScanner = scannerDetails1.copy(name = "name 2") + val detailsIncompatibleOtherScanner = scannerDetails1.copy(name = "other Scanner name") + val scanResult1 = ScanResult(provenanceWithSourceArtifact, scannerDetails1, scanSummaryWithFiles) + val scanResult2 = + ScanResult(provenanceWithSourceArtifact, detailsCompatibleOtherScanner, scanSummaryWithFiles) + val scanResult3 = + ScanResult(provenanceWithSourceArtifact, detailsIncompatibleOtherScanner, scanSummaryWithFiles) + val matcher = scannerMatcherForDetails1.copy(regScannerName = "name.+") + + storage.add(id, scanResult1).shouldBeSuccess() + storage.add(id, scanResult2).shouldBeSuccess() + storage.add(id, scanResult3).shouldBeSuccess() + val readResult = storage.read(pkg, matcher) + + readResult.shouldBeSuccess { + it should containExactlyInAnyOrder(scanResult1, scanResult2) + } + } + + "find all scan results for compatible scanners" { + val scanResult = ScanResult(provenanceWithSourceArtifact, scannerDetails1, scanSummaryWithFiles) + val scanResultCompatible1 = + ScanResult(provenanceWithSourceArtifact, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) + val scanResultCompatible2 = + ScanResult(provenanceWithSourceArtifact, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) + val scanResultIncompatible = + ScanResult(provenanceWithSourceArtifact, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) + + storage.add(id, scanResult).shouldBeSuccess() + storage.add(id, scanResultCompatible1).shouldBeSuccess() + storage.add(id, scanResultCompatible2).shouldBeSuccess() + storage.add(id, scanResultIncompatible).shouldBeSuccess() + val readResult = storage.read(pkg, scannerMatcherForDetails1) + + readResult.shouldBeSuccess { + it should containExactlyInAnyOrder(scanResult, scanResultCompatible1, scanResultCompatible2) + } + } + + "find all scan results for a scanner in a version range" { + val scanResult = ScanResult(provenanceWithSourceArtifact, scannerDetails1, scanSummaryWithFiles) + val scanResultCompatible1 = + ScanResult(provenanceWithSourceArtifact, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) + val scanResultCompatible2 = + ScanResult(provenanceWithSourceArtifact, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) + val scanResultIncompatible = + ScanResult(provenanceWithSourceArtifact, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) + val matcher = scannerMatcherForDetails1.copy(maxVersion = Semver("1.5.0")) + + storage.add(id, scanResult).shouldBeSuccess() + storage.add(id, scanResultCompatible1).shouldBeSuccess() + storage.add(id, scanResultCompatible2).shouldBeSuccess() + storage.add(id, scanResultIncompatible).shouldBeSuccess() + val readResult = storage.read(pkg, matcher) + + readResult.shouldBeSuccess { + it should containExactlyInAnyOrder( + scanResult, + scanResultCompatible1, + scanResultCompatible2, + scanResultIncompatible + ) + } + } + + "find only packages with matching provenance" { + val scanResultSourceArtifactMatching = + ScanResult(provenanceWithSourceArtifact, scannerDetails1, scanSummaryWithFiles) + val scanResultVcsMatching = ScanResult(provenanceWithVcsInfo, scannerDetails1, scanSummaryWithFiles) + val provenanceSourceArtifactNonMatching = provenanceWithSourceArtifact.copy( + sourceArtifact = sourceArtifact.copy( + hash = Hash.create("0000000000000000000000000000000000000000") + ) + ) + val scanResultSourceArtifactNonMatching = + ScanResult(provenanceSourceArtifactNonMatching, scannerDetails1, scanSummaryWithFiles) + val provenanceVcsNonMatching = provenanceWithVcsInfo.copy( + vcsInfo = vcs.copy(revision = "revision2"), + resolvedRevision = "resolvedRevision2" + ) + val scanResultVcsNonMatching = + ScanResult(provenanceVcsNonMatching, scannerDetails1, scanSummaryWithFiles) + + storage.add(id, scanResultSourceArtifactMatching).shouldBeSuccess() + storage.add(id, scanResultVcsMatching).shouldBeSuccess() + storage.add(id, scanResultSourceArtifactNonMatching).shouldBeSuccess() + storage.add(id, scanResultVcsNonMatching).shouldBeSuccess() + val readResult = storage.read(pkg, scannerMatcherForDetails1) + + readResult.shouldBeSuccess { + it should containExactlyInAnyOrder(scanResultSourceArtifactMatching, scanResultVcsMatching) + } + } + + "find a scan result if the revision was resolved from a version" { + val scanResult = ScanResult(provenanceWithoutRevision, scannerDetails1, scanSummaryWithFiles) + + storage.add(id, scanResult).shouldBeSuccess() + val readResult = storage.read(pkgWithoutRevision, scannerMatcherForDetails1) + + readResult.shouldBeSuccess { + it should containExactly(scanResult) + } + } + + "not find a scan result if vcs matches (but not vcsProcessed)" { + val pkg = Package.EMPTY.copy( + id = id, + sourceArtifact = RemoteArtifact.EMPTY, + vcs = vcs, + vcsProcessed = VcsInfo.EMPTY + ) + val scanResult = ScanResult(provenanceWithVcsInfo, scannerDetails1, scanSummaryWithFiles) + + storage.add(id, scanResult).shouldBeSuccess() + + val readResult = storage.read(pkg, scannerMatcherForDetails1) + + readResult.shouldBeSuccess { + it should beEmpty() + } + } + } + } +} diff --git a/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedScanStorageFunTest.kt similarity index 98% rename from scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedStorageFunTest.kt rename to scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedScanStorageFunTest.kt index cb4b4f4767cdb..3a964ee967900 100644 --- a/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/AbstractProvenanceBasedScanStorageFunTest.kt @@ -45,7 +45,7 @@ import org.ossreviewtoolkit.scanner.ProvenanceBasedScanStorage import org.ossreviewtoolkit.scanner.ScanStorageException import org.ossreviewtoolkit.scanner.ScannerMatcher -abstract class AbstractProvenanceBasedStorageFunTest(vararg listeners: TestListener) : WordSpec() { +abstract class AbstractProvenanceBasedScanStorageFunTest(vararg listeners: TestListener) : WordSpec() { private lateinit var storage: ProvenanceBasedScanStorage protected abstract fun createStorage(): ProvenanceBasedScanStorage diff --git a/scanner/src/funTest/kotlin/storages/AbstractStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/AbstractStorageFunTest.kt deleted file mode 100644 index 34a8c84db1c60..0000000000000 --- a/scanner/src/funTest/kotlin/storages/AbstractStorageFunTest.kt +++ /dev/null @@ -1,551 +0,0 @@ -/* - * Copyright (C) 2017 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.scanner.storages - -import io.kotest.core.listeners.TestListener -import io.kotest.core.spec.style.WordSpec -import io.kotest.matchers.collections.beEmpty -import io.kotest.matchers.collections.containExactly -import io.kotest.matchers.collections.containExactlyInAnyOrder -import io.kotest.matchers.result.shouldBeFailure -import io.kotest.matchers.result.shouldBeSuccess -import io.kotest.matchers.should -import io.kotest.matchers.shouldBe - -import java.time.Duration -import java.time.Instant - -import org.ossreviewtoolkit.model.ArtifactProvenance -import org.ossreviewtoolkit.model.Hash -import org.ossreviewtoolkit.model.Identifier -import org.ossreviewtoolkit.model.Issue -import org.ossreviewtoolkit.model.LicenseFinding -import org.ossreviewtoolkit.model.Package -import org.ossreviewtoolkit.model.RemoteArtifact -import org.ossreviewtoolkit.model.RepositoryProvenance -import org.ossreviewtoolkit.model.ScanResult -import org.ossreviewtoolkit.model.ScanSummary -import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.TextLocation -import org.ossreviewtoolkit.model.UnknownProvenance -import org.ossreviewtoolkit.model.VcsInfo -import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.scanner.ScanResultsStorage -import org.ossreviewtoolkit.scanner.ScannerMatcher - -import org.semver4j.Semver - -private val DUMMY_TEXT_LOCATION = TextLocation("fakepath", 13, 21) - -abstract class AbstractStorageFunTest(vararg listeners: TestListener) : WordSpec() { - private val id1 = Identifier("type", "namespace", "name1", "version") - private val id2 = Identifier("type", "namespace", "name2", "version") - - private val sourceArtifact1 = RemoteArtifact("url1", Hash.create("0123456789abcdef0123456789abcdef01234567")) - private val sourceArtifact2 = RemoteArtifact("url2", Hash.create("0123456789abcdef0123456789abcdef01234567")) - - private val vcs1 = VcsInfo(VcsType.forName("type"), "url1", "revision", "path") - private val vcs2 = VcsInfo(VcsType.forName("type"), "url2", "revision", "path") - private val vcsWithoutRevision = VcsInfo(VcsType.forName("type"), "url", "") - - private val pkg1 = Package.EMPTY.copy( - id = id1, - sourceArtifact = sourceArtifact1, - vcs = VcsInfo.EMPTY, - vcsProcessed = vcs1 - ) - - private val pkg2 = Package.EMPTY.copy( - id = id2, - sourceArtifact = sourceArtifact2, - vcs = VcsInfo.EMPTY, - vcsProcessed = vcs2 - ) - - private val pkgWithoutRevision = pkg1.copy(vcs = vcsWithoutRevision, vcsProcessed = vcsWithoutRevision.normalize()) - - private val provenanceEmpty = UnknownProvenance - private val provenanceWithoutRevision = RepositoryProvenance( - vcsInfo = pkgWithoutRevision.vcsProcessed, - resolvedRevision = "resolvedRevision" - ) - private val provenanceWithSourceArtifact1 = ArtifactProvenance(sourceArtifact = sourceArtifact1) - private val provenanceWithSourceArtifact2 = ArtifactProvenance(sourceArtifact = sourceArtifact2) - private val provenanceWithVcsInfo1 = RepositoryProvenance(vcsInfo = vcs1, resolvedRevision = "resolvedRevision") - private val provenanceWithVcsInfo2 = RepositoryProvenance(vcsInfo = vcs2, resolvedRevision = "resolvedRevision") - - private val scannerDetails1 = ScannerDetails("name 1", "1.0.0", "config 1") - private val scannerDetails2 = ScannerDetails("name 2", "2.0.0", "config 2") - private val scannerDetailsCompatibleVersion1 = ScannerDetails("name 1", "1.0.1", "config 1") - private val scannerDetailsCompatibleVersion2 = ScannerDetails("name 1", "1.0.1-alpha.1", "config 1") - private val scannerDetailsIncompatibleVersion = ScannerDetails("name 1", "1.1.0", "config 1") - - private val scannerMatcherForDetails1 = ScannerMatcher.create(scannerDetails1) - - private val scanSummaryWithFiles = ScanSummary.EMPTY.copy( - startTime = Instant.EPOCH + Duration.ofMinutes(1), - endTime = Instant.EPOCH + Duration.ofMinutes(2), - licenseFindings = setOf( - LicenseFinding("license-1.1", DUMMY_TEXT_LOCATION), - LicenseFinding("license-1.2", DUMMY_TEXT_LOCATION) - ), - issues = listOf( - Issue(source = "source-1", message = "error-1"), - Issue(source = "source-2", message = "error-2") - ) - ) - - private lateinit var storage: ScanResultsStorage - - abstract fun createStorage(): ScanResultsStorage - - init { - register(*listeners) - - beforeEach { - storage = createStorage() - } - - "Adding a scan result" should { - "succeed for a valid scan result" { - val scanResult = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - - val addResult = storage.add(id1, scanResult) - val readResult = storage.read(pkg1) - - addResult.shouldBeSuccess() - readResult.shouldBeSuccess { - it should containExactly(scanResult) - } - } - - "fail if provenance information is missing" { - val scanResult = ScanResult(provenanceEmpty, scannerDetails1, scanSummaryWithFiles) - - val addResult = storage.add(id1, scanResult) - val readResult = storage.read(pkg1) - - addResult.shouldBeFailure { - it.message shouldBe "Not storing scan result for '${id1.toCoordinates()}' because no provenance " + - "information is available." - } - - readResult.shouldBeSuccess { - it should beEmpty() - } - } - - "not store a result for the same scanner and provenance twice" { - val summary1 = scanSummaryWithFiles - val summary2 = scanSummaryWithFiles.copy( - startTime = scanSummaryWithFiles.startTime.plusSeconds(10) - ) - - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, summary1) - val scanResult2 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, summary2) - - val addResult1 = storage.add(id1, scanResult1) - val addResult2 = storage.add(id1, scanResult2) - - addResult1.shouldBeSuccess() - addResult2.shouldBeFailure() - - val readResult = storage.read(pkg1) - readResult.shouldBeSuccess { - it should containExactly(scanResult1) - } - } - } - - "Reading a scan result" should { - "find all scan results for an id" { - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResult2 = ScanResult(provenanceWithSourceArtifact1, scannerDetails2, scanSummaryWithFiles) - - storage.add(id1, scanResult1).shouldBeSuccess() - storage.add(id1, scanResult2).shouldBeSuccess() - val readResult = storage.read(pkg1) - - readResult.shouldBeSuccess { - it should containExactlyInAnyOrder(scanResult1, scanResult2) - } - } - - "find all scan results for a specific scanner" { - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResult2 = ScanResult(provenanceWithVcsInfo1, scannerDetails1, scanSummaryWithFiles) - val scanResult3 = ScanResult(provenanceWithSourceArtifact1, scannerDetails2, scanSummaryWithFiles) - - storage.add(id1, scanResult1).shouldBeSuccess() - storage.add(id1, scanResult2).shouldBeSuccess() - storage.add(id1, scanResult3).shouldBeSuccess() - val readResult = storage.read(pkg1, scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it should containExactlyInAnyOrder(scanResult1, scanResult2) - } - } - - "find all scan results for scanners with names matching a pattern" { - val detailsCompatibleOtherScanner = scannerDetails1.copy(name = "name 2") - val detailsIncompatibleOtherScanner = scannerDetails1.copy(name = "other Scanner name") - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResult2 = - ScanResult(provenanceWithSourceArtifact1, detailsCompatibleOtherScanner, scanSummaryWithFiles) - val scanResult3 = - ScanResult(provenanceWithSourceArtifact1, detailsIncompatibleOtherScanner, scanSummaryWithFiles) - val matcher = scannerMatcherForDetails1.copy(regScannerName = "name.+") - - storage.add(id1, scanResult1).shouldBeSuccess() - storage.add(id1, scanResult2).shouldBeSuccess() - storage.add(id1, scanResult3).shouldBeSuccess() - val readResult = storage.read(pkg1, matcher) - - readResult.shouldBeSuccess { - it should containExactlyInAnyOrder(scanResult1, scanResult2) - } - } - - "find all scan results for compatible scanners" { - val scanResult = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResultCompatible1 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) - val scanResultCompatible2 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) - val scanResultIncompatible = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) - - storage.add(id1, scanResult).shouldBeSuccess() - storage.add(id1, scanResultCompatible1).shouldBeSuccess() - storage.add(id1, scanResultCompatible2).shouldBeSuccess() - storage.add(id1, scanResultIncompatible).shouldBeSuccess() - val readResult = storage.read(pkg1, scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it should containExactlyInAnyOrder(scanResult, scanResultCompatible1, scanResultCompatible2) - } - } - - "find all scan results for a scanner in a version range" { - val scanResult = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResultCompatible1 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) - val scanResultCompatible2 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) - val scanResultIncompatible = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) - val matcher = scannerMatcherForDetails1.copy(maxVersion = Semver("1.5.0")) - - storage.add(id1, scanResult).shouldBeSuccess() - storage.add(id1, scanResultCompatible1).shouldBeSuccess() - storage.add(id1, scanResultCompatible2).shouldBeSuccess() - storage.add(id1, scanResultIncompatible).shouldBeSuccess() - val readResult = storage.read(pkg1, matcher) - - readResult.shouldBeSuccess { - it should containExactlyInAnyOrder( - scanResult, - scanResultCompatible1, - scanResultCompatible2, - scanResultIncompatible - ) - } - } - - "find only packages with matching provenance" { - val scanResultSourceArtifactMatching = - ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResultVcsMatching = ScanResult(provenanceWithVcsInfo1, scannerDetails1, scanSummaryWithFiles) - val provenanceSourceArtifactNonMatching = provenanceWithSourceArtifact1.copy( - sourceArtifact = sourceArtifact1.copy( - hash = Hash.create("0000000000000000000000000000000000000000") - ) - ) - val scanResultSourceArtifactNonMatching = - ScanResult(provenanceSourceArtifactNonMatching, scannerDetails1, scanSummaryWithFiles) - val provenanceVcsNonMatching = provenanceWithVcsInfo1.copy( - vcsInfo = vcs1.copy(revision = "revision2"), - resolvedRevision = "resolvedRevision2" - ) - val scanResultVcsNonMatching = - ScanResult(provenanceVcsNonMatching, scannerDetails1, scanSummaryWithFiles) - - storage.add(id1, scanResultSourceArtifactMatching).shouldBeSuccess() - storage.add(id1, scanResultVcsMatching).shouldBeSuccess() - storage.add(id1, scanResultSourceArtifactNonMatching).shouldBeSuccess() - storage.add(id1, scanResultVcsNonMatching).shouldBeSuccess() - val readResult = storage.read(pkg1, scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it should containExactlyInAnyOrder(scanResultSourceArtifactMatching, scanResultVcsMatching) - } - } - - "find a scan result if the revision was resolved from a version" { - val scanResult = ScanResult(provenanceWithoutRevision, scannerDetails1, scanSummaryWithFiles) - - storage.add(id1, scanResult).shouldBeSuccess() - val readResult = storage.read(pkgWithoutRevision, scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it should containExactly(scanResult) - } - } - - "not find a scan result if vcs matches (but not vcsProcessed)" { - val pkg = Package.EMPTY.copy( - id = id1, - sourceArtifact = RemoteArtifact.EMPTY, - vcs = vcs1, - vcsProcessed = VcsInfo.EMPTY - ) - val scanResult = ScanResult(provenanceWithVcsInfo1, scannerDetails1, scanSummaryWithFiles) - - storage.add(id1, scanResult).shouldBeSuccess() - - val readResult = storage.read(pkg, scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it should beEmpty() - } - } - } - - "Reading scan results for multiple packages" should { - "find all scan results for a specific scanner" { - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResult2 = ScanResult(provenanceWithVcsInfo1, scannerDetails1, scanSummaryWithFiles) - val scanResult3 = ScanResult(provenanceWithSourceArtifact1, scannerDetails2, scanSummaryWithFiles) - val scanResult4 = ScanResult(provenanceWithSourceArtifact2, scannerDetails1, scanSummaryWithFiles) - val scanResult5 = ScanResult(provenanceWithVcsInfo2, scannerDetails1, scanSummaryWithFiles) - val scanResult6 = ScanResult(provenanceWithSourceArtifact2, scannerDetails2, scanSummaryWithFiles) - - storage.add(id1, scanResult1).shouldBeSuccess() - storage.add(id1, scanResult2).shouldBeSuccess() - storage.add(id1, scanResult3).shouldBeSuccess() - storage.add(id2, scanResult4).shouldBeSuccess() - storage.add(id2, scanResult5).shouldBeSuccess() - storage.add(id2, scanResult6).shouldBeSuccess() - val readResult = storage.read(listOf(pkg1, pkg2), scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it.keys shouldBe setOf(id1, id2) - it[id1] should containExactlyInAnyOrder(scanResult1, scanResult2) - it[id2] should containExactlyInAnyOrder(scanResult4, scanResult5) - } - } - - "find all scan results for scanners with names matching a pattern" { - val detailsCompatibleOtherScanner = scannerDetails1.copy(name = "name 2") - val detailsIncompatibleOtherScanner = scannerDetails1.copy(name = "other Scanner name") - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResult2 = - ScanResult(provenanceWithSourceArtifact1, detailsCompatibleOtherScanner, scanSummaryWithFiles) - val scanResult3 = - ScanResult(provenanceWithSourceArtifact1, detailsIncompatibleOtherScanner, scanSummaryWithFiles) - val scanResult4 = ScanResult(provenanceWithSourceArtifact2, scannerDetails1, scanSummaryWithFiles) - val scanResult5 = - ScanResult(provenanceWithSourceArtifact2, detailsCompatibleOtherScanner, scanSummaryWithFiles) - val scanResult6 = - ScanResult(provenanceWithSourceArtifact2, detailsIncompatibleOtherScanner, scanSummaryWithFiles) - val matcher = scannerMatcherForDetails1.copy(regScannerName = "name.+") - - storage.add(id1, scanResult1).shouldBeSuccess() - storage.add(id1, scanResult2).shouldBeSuccess() - storage.add(id1, scanResult3).shouldBeSuccess() - storage.add(id2, scanResult4).shouldBeSuccess() - storage.add(id2, scanResult5).shouldBeSuccess() - storage.add(id2, scanResult6).shouldBeSuccess() - val readResult = storage.read(listOf(pkg1, pkg2), matcher) - - readResult.shouldBeSuccess { - it.keys shouldBe setOf(id1, id2) - it[id1] should containExactlyInAnyOrder(scanResult1, scanResult2) - it[id2] should containExactlyInAnyOrder(scanResult4, scanResult5) - } - } - - "find all scan results for compatible scanners" { - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResult1Compatible1 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) - val scanResult1Compatible2 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) - val scanResult1Incompatible = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) - - val scanResult2 = ScanResult(provenanceWithSourceArtifact2, scannerDetails1, scanSummaryWithFiles) - val scanResult2Compatible1 = - ScanResult(provenanceWithSourceArtifact2, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) - val scanResult2Compatible2 = - ScanResult(provenanceWithSourceArtifact2, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) - val scanResult2Incompatible = - ScanResult(provenanceWithSourceArtifact2, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) - - storage.add(id1, scanResult1).shouldBeSuccess() - storage.add(id1, scanResult1Compatible1).shouldBeSuccess() - storage.add(id1, scanResult1Compatible2).shouldBeSuccess() - storage.add(id1, scanResult1Incompatible).shouldBeSuccess() - - storage.add(id2, scanResult2).shouldBeSuccess() - storage.add(id2, scanResult2Compatible1).shouldBeSuccess() - storage.add(id2, scanResult2Compatible2).shouldBeSuccess() - storage.add(id2, scanResult2Incompatible).shouldBeSuccess() - - val readResult = storage.read(listOf(pkg1, pkg2), scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it.keys shouldBe setOf(id1, id2) - it[id1] should containExactlyInAnyOrder( - scanResult1, - scanResult1Compatible1, - scanResult1Compatible2 - ) - it[id2] should containExactlyInAnyOrder( - scanResult2, - scanResult2Compatible1, - scanResult2Compatible2 - ) - } - } - - "find all scan results for a scanner in a version range" { - val scanResult1 = ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResult1Compatible1 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) - val scanResult1Compatible2 = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) - val scanResult1Incompatible = - ScanResult(provenanceWithSourceArtifact1, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) - - val scanResult2 = ScanResult(provenanceWithSourceArtifact2, scannerDetails1, scanSummaryWithFiles) - val scanResult2Compatible1 = - ScanResult(provenanceWithSourceArtifact2, scannerDetailsCompatibleVersion1, scanSummaryWithFiles) - val scanResult2Compatible2 = - ScanResult(provenanceWithSourceArtifact2, scannerDetailsCompatibleVersion2, scanSummaryWithFiles) - val scanResult2Incompatible = - ScanResult(provenanceWithSourceArtifact2, scannerDetailsIncompatibleVersion, scanSummaryWithFiles) - - val matcher = scannerMatcherForDetails1.copy(maxVersion = Semver("1.5.0")) - - storage.add(id1, scanResult1).shouldBeSuccess() - storage.add(id1, scanResult1Compatible1).shouldBeSuccess() - storage.add(id1, scanResult1Compatible2).shouldBeSuccess() - storage.add(id1, scanResult1Incompatible).shouldBeSuccess() - - storage.add(id2, scanResult2).shouldBeSuccess() - storage.add(id2, scanResult2Compatible1).shouldBeSuccess() - storage.add(id2, scanResult2Compatible2).shouldBeSuccess() - storage.add(id2, scanResult2Incompatible).shouldBeSuccess() - - val readResult = storage.read(listOf(pkg1, pkg2), matcher) - - readResult.shouldBeSuccess { - it.keys shouldBe setOf(id1, id2) - it[id1] should containExactlyInAnyOrder( - scanResult1, - scanResult1Compatible1, - scanResult1Compatible2, - scanResult1Incompatible - ) - it[id2] should containExactlyInAnyOrder( - scanResult2, - scanResult2Compatible1, - scanResult2Compatible2, - scanResult2Incompatible - ) - } - } - - "find only packages with matching provenance" { - val scanResultSourceArtifactMatching1 = - ScanResult(provenanceWithSourceArtifact1, scannerDetails1, scanSummaryWithFiles) - val scanResultVcsMatching1 = ScanResult(provenanceWithVcsInfo1, scannerDetails1, scanSummaryWithFiles) - val provenanceSourceArtifactNonMatching1 = provenanceWithSourceArtifact1.copy( - sourceArtifact = sourceArtifact1.copy( - hash = Hash.create("0000000000000000000000000000000000000000") - ) - ) - val scanResultSourceArtifactNonMatching1 = - ScanResult(provenanceSourceArtifactNonMatching1, scannerDetails1, scanSummaryWithFiles) - val provenanceVcsNonMatching1 = provenanceWithVcsInfo1.copy( - vcsInfo = vcs1.copy(revision = "revision2"), - resolvedRevision = "resolvedRevision2" - ) - val scanResultVcsNonMatching1 = - ScanResult(provenanceVcsNonMatching1, scannerDetails1, scanSummaryWithFiles) - - val scanResultSourceArtifactMatching2 = - ScanResult(provenanceWithSourceArtifact2, scannerDetails1, scanSummaryWithFiles) - val scanResultVcsMatching2 = ScanResult(provenanceWithVcsInfo2, scannerDetails1, scanSummaryWithFiles) - val provenanceSourceArtifactNonMatching2 = provenanceWithSourceArtifact2.copy( - sourceArtifact = sourceArtifact2.copy( - hash = Hash.create("0000000000000000000000000000000000000000") - ) - ) - val scanResultSourceArtifactNonMatching2 = - ScanResult(provenanceSourceArtifactNonMatching2, scannerDetails1, scanSummaryWithFiles) - val provenanceVcsNonMatching2 = provenanceWithVcsInfo2.copy( - vcsInfo = vcs2.copy(revision = "revision2"), - resolvedRevision = "resolvedRevision2" - ) - val scanResultVcsNonMatching2 = - ScanResult(provenanceVcsNonMatching2, scannerDetails1, scanSummaryWithFiles) - - storage.add(id1, scanResultSourceArtifactMatching1).shouldBeSuccess() - storage.add(id1, scanResultVcsMatching1).shouldBeSuccess() - storage.add(id1, scanResultSourceArtifactNonMatching1).shouldBeSuccess() - storage.add(id1, scanResultVcsNonMatching1).shouldBeSuccess() - - storage.add(id2, scanResultSourceArtifactMatching2).shouldBeSuccess() - storage.add(id2, scanResultVcsMatching2).shouldBeSuccess() - storage.add(id2, scanResultSourceArtifactNonMatching2).shouldBeSuccess() - storage.add(id2, scanResultVcsNonMatching2).shouldBeSuccess() - - val readResult = storage.read(listOf(pkg1, pkg2), scannerMatcherForDetails1) - - readResult.shouldBeSuccess { - it.keys shouldBe setOf(id1, id2) - it[id1] should containExactlyInAnyOrder( - scanResultSourceArtifactMatching1, - scanResultVcsMatching1 - ) - it[id2] should containExactlyInAnyOrder( - scanResultSourceArtifactMatching2, - scanResultVcsMatching2 - ) - } - } - - "find a scan result if the revision was resolved from a version" { - val scanResult = ScanResult(provenanceWithoutRevision, scannerDetails1, scanSummaryWithFiles) - - val addResult = storage.add(id1, scanResult) - val readResult = storage.read(listOf(pkgWithoutRevision), scannerMatcherForDetails1) - - addResult.shouldBeSuccess() - readResult.shouldBeSuccess { - it.keys shouldBe setOf(id1) - it[id1] should containExactly(scanResult) - } - } - } - } -} diff --git a/scanner/src/funTest/kotlin/storages/FileBasedStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/PackageBasedFileStorageFunTest.kt similarity index 84% rename from scanner/src/funTest/kotlin/storages/FileBasedStorageFunTest.kt rename to scanner/src/funTest/kotlin/storages/PackageBasedFileStorageFunTest.kt index 175a9b18c7a83..35b097bc113b2 100644 --- a/scanner/src/funTest/kotlin/storages/FileBasedStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/PackageBasedFileStorageFunTest.kt @@ -23,6 +23,6 @@ import io.kotest.engine.spec.tempdir import org.ossreviewtoolkit.utils.ort.storage.LocalFileStorage -class FileBasedStorageFunTest : AbstractStorageFunTest() { - override fun createStorage() = FileBasedStorage(LocalFileStorage(tempdir())) +class PackageBasedFileStorageFunTest : AbstractPackageBasedScanStorageFunTest() { + override fun createStorage() = PackageBasedFileStorage(LocalFileStorage(tempdir())) } diff --git a/scanner/src/funTest/kotlin/storages/PostgresStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/PackageBasedPostgresStorageFunTest.kt similarity index 81% rename from scanner/src/funTest/kotlin/storages/PostgresStorageFunTest.kt rename to scanner/src/funTest/kotlin/storages/PackageBasedPostgresStorageFunTest.kt index 07b581b4f4734..ef71291581f43 100644 --- a/scanner/src/funTest/kotlin/storages/PostgresStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/PackageBasedPostgresStorageFunTest.kt @@ -23,6 +23,6 @@ import org.ossreviewtoolkit.utils.test.PostgresListener private val postgresListener = PostgresListener() -class PostgresStorageFunTest : AbstractStorageFunTest(postgresListener) { - override fun createStorage() = PostgresStorage(dataSource = postgresListener.dataSource, parallelTransactions = 5) +class PackageBasedPostgresStorageFunTest : AbstractPackageBasedScanStorageFunTest(postgresListener) { + override fun createStorage() = PackageBasedPostgresStorage(dataSource = postgresListener.dataSource) } diff --git a/scanner/src/funTest/kotlin/storages/ProvenanceBasedFileStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/ProvenanceBasedFileStorageFunTest.kt index b5de56ece21b2..13eceabb47887 100644 --- a/scanner/src/funTest/kotlin/storages/ProvenanceBasedFileStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/ProvenanceBasedFileStorageFunTest.kt @@ -23,6 +23,6 @@ import io.kotest.engine.spec.tempdir import org.ossreviewtoolkit.utils.ort.storage.LocalFileStorage -class ProvenanceBasedFileStorageFunTest : AbstractProvenanceBasedStorageFunTest() { +class ProvenanceBasedFileStorageFunTest : AbstractProvenanceBasedScanStorageFunTest() { override fun createStorage() = ProvenanceBasedFileStorage(LocalFileStorage(tempdir())) } diff --git a/scanner/src/funTest/kotlin/storages/ProvenanceBasedPostgresStorageFunTest.kt b/scanner/src/funTest/kotlin/storages/ProvenanceBasedPostgresStorageFunTest.kt index 465f8e06bcf41..d4b9a135ae699 100644 --- a/scanner/src/funTest/kotlin/storages/ProvenanceBasedPostgresStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/storages/ProvenanceBasedPostgresStorageFunTest.kt @@ -23,6 +23,6 @@ import org.ossreviewtoolkit.utils.test.PostgresListener private val postgresListener = PostgresListener() -class ProvenanceBasedPostgresStorageFunTest : AbstractProvenanceBasedStorageFunTest(postgresListener) { +class ProvenanceBasedPostgresStorageFunTest : AbstractProvenanceBasedScanStorageFunTest(postgresListener) { override fun createStorage() = ProvenanceBasedPostgresStorage(postgresListener.dataSource) } diff --git a/scanner/src/main/kotlin/ScanResultsStorage.kt b/scanner/src/main/kotlin/ScanResultsStorage.kt deleted file mode 100644 index 8bb659e4c1bd3..0000000000000 --- a/scanner/src/main/kotlin/ScanResultsStorage.kt +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (C) 2017 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.scanner - -import kotlin.time.measureTimedValue - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.runBlocking - -import org.apache.logging.log4j.kotlin.logger - -import org.ossreviewtoolkit.model.Identifier -import org.ossreviewtoolkit.model.Package -import org.ossreviewtoolkit.model.ScanResult -import org.ossreviewtoolkit.model.UnknownProvenance -import org.ossreviewtoolkit.scanner.provenance.NestedProvenance -import org.ossreviewtoolkit.scanner.provenance.NestedProvenanceScanResult - -/** - * The abstract class that storage backends for scan results need to implement. - */ -abstract class ScanResultsStorage : PackageBasedScanStorage { - /** - * A companion object that allow to configure the globally used storage backend. - */ - companion object { - /** - * A successful [Result] with an empty list of [ScanResult]s. - */ - val EMPTY_RESULT = Result.success>(emptyList()) - } - - /** - * The name to refer to this storage implementation. - */ - open val name: String = javaClass.simpleName - - /** - * Return all [ScanResult]s contained in this [ScanResultsStorage] corresponding to the [package][pkg] wrapped in a - * [Result]. - */ - fun read(pkg: Package): Result> { - val (result, duration) = measureTimedValue { readInternal(pkg) } - - result.onSuccess { results -> - logger.info { - "Read ${results.size} scan result(s) for '${pkg.id.toCoordinates()}' from ${javaClass.simpleName} in " + - "$duration." - } - } - - return result - } - - /** - * Return all [ScanResult]s contained in this [ScanResultsStorage] corresponding to the given [package][pkg] that - * are [compatible][ScannerMatcher.matches] with the provided [scannerMatcher] wrapped in a [Result]. Also, - * [Package.sourceArtifact], [Package.vcs], and [Package.vcsProcessed] are used to check if the scan result matches - * the expected source code location. That check is important to find the correct results when different revisions - * of a package using the same version name are used (e.g. multiple scans of a "1.0-SNAPSHOT" version during - * development). - */ - fun read(pkg: Package, scannerMatcher: ScannerMatcher): Result> { - val (result, duration) = measureTimedValue { readInternal(pkg, scannerMatcher) } - - result.onSuccess { results -> - logger.info { - "Read ${results.size} matching scan result(s) for '${pkg.id.toCoordinates()}' from " + - "${javaClass.simpleName} in $duration." - } - } - - return result - } - - /** - * Return all [ScanResult]s contained in this [ScanResultsStorage] corresponding to the given [packages] that - * are [compatible][ScannerMatcher.matches] with the provided [scannerMatcher] wrapped in a [Result]. Also, - * [Package.sourceArtifact], [Package.vcs], and [Package.vcsProcessed] are used to check if the scan result matches - * the expected source code location. That check is important to find the correct results when different revisions - * of a package using the same version name are used (e.g. multiple scans of a "1.0-SNAPSHOT" version during - * development). - */ - fun read(packages: Collection, scannerMatcher: ScannerMatcher): Result>> { - val (result, duration) = measureTimedValue { readInternal(packages, scannerMatcher) } - - result.onSuccess { results -> - logger.info { - val count = results.values.sumOf { it.size } - "Read $count matching scan result(s) from ${javaClass.simpleName} in $duration." - } - } - - return result - } - - /** - * Add the given [scanResult] to the stored [ScanResult]s for the scanned [Package] with the provided [id]. - * Depending on the storage implementation this might first read any existing [ScanResult]s and write the new - * [ScanResult]s to the storage again, implicitly deleting the original storage entry by overwriting it. - * Return a [Result] describing whether the operation was successful. - */ - fun add(id: Identifier, scanResult: ScanResult): Result { - // Do not store scan results without provenance information, because they cannot be assigned to the revision of - // the package source code later. - if (scanResult.provenance is UnknownProvenance) { - val message = - "Not storing scan result for '${id.toCoordinates()}' because no provenance information is available." - logger.info { message } - - return Result.failure(ScanStorageException(message)) - } - - val (result, duration) = measureTimedValue { addInternal(id, scanResult) } - - logger.info { "Added scan result for '${id.toCoordinates()}' to ${javaClass.simpleName} in $duration." } - - return result - } - - /** - * Internal version of [read]. - */ - protected abstract fun readInternal(pkg: Package): Result> - - /** - * Internal version of [read]. Implementations may want to override this function if they can filter for the wanted - * [scannerMatcher] in a more efficient way than this default implementation. - */ - protected open fun readInternal(pkg: Package, scannerMatcher: ScannerMatcher): Result> = - readInternal(pkg).map { results -> - if (results.isEmpty()) { - results - } else { - val (matchingProvenance, nonMatchingProvenance) = results.partition { it.provenance.matches(pkg) } - - if (matchingProvenance.isEmpty()) { - logger.debug { - "No stored scan results found for $pkg. The following entries with non-matching provenance " + - "have been ignored: ${nonMatchingProvenance.map { it.provenance }}" - } - - matchingProvenance - } else { - val (matchingCriteria, nonMatchingCriteria) = matchingProvenance.partition { - scannerMatcher.matches(it.scanner) - } - - if (matchingCriteria.isEmpty()) { - logger.debug { - "No stored scan results for '${pkg.id.toCoordinates()}' match $scannerMatcher. The " + - "following entries with non-matching criteria have been ignored: " + - nonMatchingCriteria.map { it.scanner } - } - } - - matchingCriteria - } - } - } - - /** - * Internal version of [read]. The default implementation uses [Dispatchers.IO] to run requests for individual - * packages in parallel. Implementations may want to override this function if they can filter for the wanted - * [scannerMatcher] or fetch results for multiple packages in a more efficient way. - */ - protected open fun readInternal( - packages: Collection, - scannerMatcher: ScannerMatcher - ): Result>> { - val results = runBlocking(Dispatchers.IO) { - packages.map { async { it.id to readInternal(it, scannerMatcher) } }.awaitAll() - }.associate { it } - - val successfulResults = results.mapNotNull { (id, scanResults) -> - scanResults.getOrNull()?.let { id to it } - }.toMap() - - return if (successfulResults.isEmpty()) { - Result.failure(ScanStorageException("Could not read any scan results from ${javaClass.simpleName}.")) - } else { - Result.success(successfulResults) - } - } - - /** - * Internal version of [add] that skips common sanity checks. - */ - protected abstract fun addInternal(id: Identifier, scanResult: ScanResult): Result - - override fun read(pkg: Package, nestedProvenance: NestedProvenance): List = - read(pkg).toNestedProvenanceScanResult(nestedProvenance) - - override fun read( - pkg: Package, - nestedProvenance: NestedProvenance, - scannerMatcher: ScannerMatcher - ): List = read(pkg, scannerMatcher).toNestedProvenanceScanResult(nestedProvenance) - - private fun Result>.toNestedProvenanceScanResult(nestedProvenance: NestedProvenance) = - map { scanResults -> - scanResults.filter { it.provenance == nestedProvenance.root } - .map { it.toNestedProvenanceScanResult(nestedProvenance) } - }.getOrThrow() - - override fun write(pkg: Package, nestedProvenanceScanResult: NestedProvenanceScanResult) { - nestedProvenanceScanResult.merge().forEach { scanResult -> - add(pkg.id, scanResult).getOrThrow() - } - } -} diff --git a/scanner/src/main/kotlin/ScanStorage.kt b/scanner/src/main/kotlin/ScanStorage.kt index 3e63d618b5b4d..23d3c221caea9 100644 --- a/scanner/src/main/kotlin/ScanStorage.kt +++ b/scanner/src/main/kotlin/ScanStorage.kt @@ -39,16 +39,9 @@ sealed interface ScanStorageReader */ interface PackageBasedScanStorageReader : ScanStorageReader { /** - * Read all [ScanResult]s for the provided [package][pkg]. The package scan results are converted to a - * [NestedProvenanceScanResult] using the provided [nestedProvenance]. - * - * Throws a [ScanStorageException] if an error occurs while reading from the storage. - */ - fun read(pkg: Package, nestedProvenance: NestedProvenance): List - - /** - * Read all [ScanResult]s for the provided [package][pkg] matching the [provenance][KnownProvenance.matches] and the - * [scannerMatcher]. The package scan results are converted to a [NestedProvenanceScanResult] using the provided + * Read all [ScanResult]s for the provided [package][pkg]. The results have to match the + * [provenance][KnownProvenance.matches] of the package and can optionally be filtered by the provided + * [scannerMatcher]. The results are converted to a [NestedProvenanceScanResult] using the provided * [nestedProvenance]. * * Throws a [ScanStorageException] if an error occurs while reading from the storage. @@ -56,7 +49,7 @@ interface PackageBasedScanStorageReader : ScanStorageReader { fun read( pkg: Package, nestedProvenance: NestedProvenance, - scannerMatcher: ScannerMatcher + scannerMatcher: ScannerMatcher? = null ): List } @@ -68,19 +61,13 @@ interface ProvenanceBasedScanStorageReader : ScanStorageReader { * Read all [ScanResult]s for the provided [provenance]. If the [provenance] is an [ArtifactProvenance], the URL and * the hash value must match. If the [provenance] is a [RepositoryProvenance], the VCS type and URL, and the * resolved revision must match. The VCS revision is ignored, because the resolved revision already defines what was - * scanned. + * scanned. Scan results can optionally be filtered by the provided [scannerMatcher]. * * A [ScanStorageException] is thrown if: * * An error occurs while reading from the storage. * * The [provenance] is a [RepositoryProvenance] with a non-empty VCS path. */ - fun read(provenance: KnownProvenance): List - - /** - * Like [read], but also filters by the provided [scannerMatcher]. - */ - fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher): List = - read(provenance).filter { scannerMatcher.matches(it.scanner) } + fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher? = null): List } /** diff --git a/scanner/src/main/kotlin/ScanStorages.kt b/scanner/src/main/kotlin/ScanStorages.kt index 08f9c9d39f40a..51b7f685a8d22 100644 --- a/scanner/src/main/kotlin/ScanStorages.kt +++ b/scanner/src/main/kotlin/ScanStorages.kt @@ -37,8 +37,8 @@ import org.ossreviewtoolkit.scanner.provenance.ResolvedArtifactProvenance import org.ossreviewtoolkit.scanner.provenance.ResolvedRepositoryProvenance import org.ossreviewtoolkit.scanner.provenance.UnresolvedPackageProvenance import org.ossreviewtoolkit.scanner.storages.ClearlyDefinedStorage -import org.ossreviewtoolkit.scanner.storages.FileBasedStorage -import org.ossreviewtoolkit.scanner.storages.PostgresStorage +import org.ossreviewtoolkit.scanner.storages.PackageBasedFileStorage +import org.ossreviewtoolkit.scanner.storages.PackageBasedPostgresStorage import org.ossreviewtoolkit.scanner.storages.ProvenanceBasedFileStorage import org.ossreviewtoolkit.scanner.storages.ProvenanceBasedPostgresStorage import org.ossreviewtoolkit.scanner.storages.Sw360Storage @@ -132,15 +132,14 @@ private fun createStorage(config: ScanStorageConfiguration): ScanStorage = private fun createFileBasedStorage(config: FileBasedStorageConfiguration) = when (config.type) { - StorageType.PACKAGE_BASED -> FileBasedStorage(config.backend.createFileStorage()) + StorageType.PACKAGE_BASED -> PackageBasedFileStorage(config.backend.createFileStorage()) StorageType.PROVENANCE_BASED -> ProvenanceBasedFileStorage(config.backend.createFileStorage()) } private fun createPostgresStorage(config: PostgresStorageConfiguration) = when (config.type) { - StorageType.PACKAGE_BASED -> PostgresStorage( - DatabaseUtils.createHikariDataSource(config = config.connection, applicationNameSuffix = TOOL_NAME), - config.connection.parallelTransactions + StorageType.PACKAGE_BASED -> PackageBasedPostgresStorage( + DatabaseUtils.createHikariDataSource(config = config.connection, applicationNameSuffix = TOOL_NAME) ) StorageType.PROVENANCE_BASED -> ProvenanceBasedPostgresStorage( DatabaseUtils.createHikariDataSource(config = config.connection, applicationNameSuffix = TOOL_NAME) diff --git a/scanner/src/main/kotlin/ScannerMatcher.kt b/scanner/src/main/kotlin/ScannerMatcher.kt index 74a46a3f4bfa8..6676b1a5a4ef3 100644 --- a/scanner/src/main/kotlin/ScannerMatcher.kt +++ b/scanner/src/main/kotlin/ScannerMatcher.kt @@ -31,14 +31,14 @@ import org.semver4j.Semver * so that it can be used as a replacement for a result produced by an actual scanner. A scanner implementation * creates a [ScannerMatcher] with its exact properties. Users can override some or all of these properties to * state the criteria under which results from a storage are acceptable even if they deviate from the exact - * properties of the scanner. That way it can be configured for instance, that results produced by an older + * properties of the scanner. That way it can be configured, for instance, that results produced by an older * version of the scanner can be used. */ data class ScannerMatcher( /** * Criterion to match the scanner name. This string is interpreted as a regular expression. In the most basic * form, it can be an exact scanner name, but by using features of regular expressions, a more advanced - * matching can be achieved. So it is possible for instance to select multiple scanners using an alternative ('|') + * matching can be achieved. So it is possible, for instance, to select multiple scanners using an alternative ('|') * expression or an arbitrary one using a wildcard ('.*'). */ val regScannerName: String, diff --git a/scanner/src/main/kotlin/storages/AbstractPackageBasedScanStorage.kt b/scanner/src/main/kotlin/storages/AbstractPackageBasedScanStorage.kt new file mode 100644 index 0000000000000..92c18c341a2e5 --- /dev/null +++ b/scanner/src/main/kotlin/storages/AbstractPackageBasedScanStorage.kt @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2017 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.scanner.storages + +import kotlin.time.measureTimedValue + +import org.apache.logging.log4j.kotlin.logger + +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.ScanResult +import org.ossreviewtoolkit.model.UnknownProvenance +import org.ossreviewtoolkit.scanner.PackageBasedScanStorage +import org.ossreviewtoolkit.scanner.ScanStorageException +import org.ossreviewtoolkit.scanner.ScannerMatcher +import org.ossreviewtoolkit.scanner.provenance.NestedProvenance +import org.ossreviewtoolkit.scanner.provenance.NestedProvenanceScanResult +import org.ossreviewtoolkit.scanner.toNestedProvenanceScanResult + +/** + * The abstract class that storage backends for scan results need to implement. + */ +abstract class AbstractPackageBasedScanStorage : PackageBasedScanStorage { + /** + * The name to refer to this storage implementation. + */ + open val name: String = javaClass.simpleName + + /** + * Read all [ScanResult]s for the provided [package][pkg]. The results have to match the + * [provenance][KnownProvenance.matches] of the package and can optionally be filtered by the provided + * [scannerMatcher]. + */ + fun read(pkg: Package, scannerMatcher: ScannerMatcher? = null): Result> { + val (result, duration) = measureTimedValue { + readInternal(pkg, scannerMatcher).map { results -> + results.filter { scannerMatcher?.matches(it.scanner) != false } + } + } + + result.onSuccess { results -> + logger.info { + "Read ${results.size} matching scan result(s) for '${pkg.id.toCoordinates()}' from " + + "${javaClass.simpleName} in $duration." + } + } + + return result + } + + /** + * Add the given [scanResult] to the stored [ScanResult]s for the scanned [Package] with the provided [id]. + * Depending on the storage implementation this might first read any existing [ScanResult]s and write the new + * [ScanResult]s to the storage again, implicitly deleting the original storage entry by overwriting it. + * Return a [Result] describing whether the operation was successful. + */ + fun add(id: Identifier, scanResult: ScanResult): Result { + // Do not store scan results without provenance information, because they cannot be assigned to the revision of + // the package source code later. + if (scanResult.provenance is UnknownProvenance) { + val message = + "Not storing scan result for '${id.toCoordinates()}' because no provenance information is available." + logger.info { message } + + return Result.failure(ScanStorageException(message)) + } + + val (result, duration) = measureTimedValue { addInternal(id, scanResult) } + + logger.info { "Added scan result for '${id.toCoordinates()}' to ${javaClass.simpleName} in $duration." } + + return result + } + + /** + * Internal version of [read]. Implementations do not have to filter results using the provided [scannerMatcher] as + * this is done by the public [read] function. They can use the [scannerMatcher] if they can implement the filtering + * more efficiently, for example, as part of a database query. + */ + abstract fun readInternal(pkg: Package, scannerMatcher: ScannerMatcher? = null): Result> + + /** + * Internal version of [add] that skips common sanity checks. + */ + protected abstract fun addInternal(id: Identifier, scanResult: ScanResult): Result + + override fun read( + pkg: Package, + nestedProvenance: NestedProvenance, + scannerMatcher: ScannerMatcher? + ): List = read(pkg, scannerMatcher).toNestedProvenanceScanResult(nestedProvenance) + + private fun Result>.toNestedProvenanceScanResult(nestedProvenance: NestedProvenance) = + map { scanResults -> + scanResults.filter { it.provenance == nestedProvenance.root } + .map { it.toNestedProvenanceScanResult(nestedProvenance) } + }.getOrThrow() + + override fun write(pkg: Package, nestedProvenanceScanResult: NestedProvenanceScanResult) { + nestedProvenanceScanResult.merge().forEach { scanResult -> + add(pkg.id, scanResult).getOrThrow() + } + } +} diff --git a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt index 6f8268c247348..e87c1d52465c2 100644 --- a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt +++ b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt @@ -50,8 +50,8 @@ import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.utils.toClearlyDefinedCoordinates import org.ossreviewtoolkit.model.utils.toClearlyDefinedSourceLocation import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper -import org.ossreviewtoolkit.scanner.ScanResultsStorage import org.ossreviewtoolkit.scanner.ScanStorageException +import org.ossreviewtoolkit.scanner.ScannerMatcher import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.scanner.storages.utils.getScanCodeDetails import org.ossreviewtoolkit.utils.common.AlphaNumericComparator @@ -72,7 +72,7 @@ class ClearlyDefinedStorage( /** The configuration for this storage implementation. */ config: ClearlyDefinedStorageConfiguration, client: OkHttpClient? = null -) : ScanResultsStorage() { +) : AbstractPackageBasedScanStorage() { constructor(serverUrl: String, client: OkHttpClient? = null) : this( ClearlyDefinedStorageConfiguration(serverUrl), client ) @@ -82,7 +82,7 @@ class ClearlyDefinedStorage( ClearlyDefinedService.create(config.serverUrl, client ?: OkHttpClientHelper.buildClient()) } - override fun readInternal(pkg: Package): Result> = + override fun readInternal(pkg: Package, scannerMatcher: ScannerMatcher?): Result> = runBlocking(Dispatchers.IO) { readFromClearlyDefined(pkg) } override fun addInternal(id: Identifier, scanResult: ScanResult): Result = diff --git a/scanner/src/main/kotlin/storages/FileBasedStorage.kt b/scanner/src/main/kotlin/storages/PackageBasedFileStorage.kt similarity index 81% rename from scanner/src/main/kotlin/storages/FileBasedStorage.kt rename to scanner/src/main/kotlin/storages/PackageBasedFileStorage.kt index ae17b6e582199..fe3deba447137 100644 --- a/scanner/src/main/kotlin/storages/FileBasedStorage.kt +++ b/scanner/src/main/kotlin/storages/PackageBasedFileStorage.kt @@ -31,8 +31,9 @@ import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.yamlMapper -import org.ossreviewtoolkit.scanner.ScanResultsStorage +import org.ossreviewtoolkit.scanner.PackageBasedScanStorage import org.ossreviewtoolkit.scanner.ScanStorageException +import org.ossreviewtoolkit.scanner.ScannerMatcher import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.ort.showStackTrace import org.ossreviewtoolkit.utils.ort.storage.FileStorage @@ -40,18 +41,18 @@ import org.ossreviewtoolkit.utils.ort.storage.FileStorage const val SCAN_RESULTS_FILE_NAME = "scan-results.yml" /** - * A [ScanResultsStorage] using a [FileStorage] as backend. Scan results are serialized using [YAML][yamlMapper]. + * A [PackageBasedScanStorage] using a [FileStorage] as backend. Scan results are serialized using [YAML][yamlMapper]. */ -class FileBasedStorage( +class PackageBasedFileStorage( /** * The [FileStorage] to use for storing scan results. */ val backend: FileStorage -) : ScanResultsStorage() { +) : AbstractPackageBasedScanStorage() { override val name = "${javaClass.simpleName} with ${backend.javaClass.simpleName} backend" - override fun readInternal(pkg: Package): Result> { - val path = storagePath(pkg.id) + private fun readForId(id: Identifier): Result> { + val path = storagePath(id) return runCatching { backend.read(path).use { input -> @@ -59,9 +60,9 @@ class FileBasedStorage( } }.recoverCatching { // If the file cannot be found it means no scan results have been stored, yet. - if (it is FileNotFoundException) return EMPTY_RESULT + if (it is FileNotFoundException) return Result.success(emptyList()) - val message = "Could not read scan results for '${pkg.id.toCoordinates()}' from path '$path': " + + val message = "Could not read scan results for '${id.toCoordinates()}' from path '$path': " + it.collectMessages() logger.info { message } @@ -70,10 +71,11 @@ class FileBasedStorage( } } + override fun readInternal(pkg: Package, scannerMatcher: ScannerMatcher?): Result> = + readForId(pkg.id).map { results -> results.filter { it.provenance.matches(pkg) } } + override fun addInternal(id: Identifier, scanResult: ScanResult): Result { - // Note: The file-based `read()`-implementation does not require the full `Package` information for reading - // existing results, so it is fine to use an empty package here. - val existingScanResults = read(Package.EMPTY.copy(id = id)).getOrDefault(emptyList()) + val existingScanResults = readForId(id).getOrDefault(emptyList()) if (existingScanResults.any { it.scanner == scanResult.scanner && it.provenance == scanResult.provenance }) { val message = "Did not store scan result for '${id.toCoordinates()}' because a scan result for the same " + diff --git a/scanner/src/main/kotlin/storages/PostgresStorage.kt b/scanner/src/main/kotlin/storages/PackageBasedPostgresStorage.kt similarity index 58% rename from scanner/src/main/kotlin/storages/PostgresStorage.kt rename to scanner/src/main/kotlin/storages/PackageBasedPostgresStorage.kt index 4fcabe0f32258..9538a4e0a2e96 100644 --- a/scanner/src/main/kotlin/storages/PostgresStorage.kt +++ b/scanner/src/main/kotlin/storages/PackageBasedPostgresStorage.kt @@ -25,11 +25,6 @@ import java.sql.SQLException import javax.sql.DataSource -import kotlin.math.max - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runBlocking - import org.apache.logging.log4j.kotlin.logger import org.jetbrains.exposed.sql.Database @@ -45,11 +40,9 @@ import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.utils.DatabaseUtils.checkDatabaseEncoding import org.ossreviewtoolkit.model.utils.DatabaseUtils.tableExists import org.ossreviewtoolkit.model.utils.DatabaseUtils.transaction -import org.ossreviewtoolkit.model.utils.DatabaseUtils.transactionAsync import org.ossreviewtoolkit.model.utils.arrayParam import org.ossreviewtoolkit.model.utils.rawParam import org.ossreviewtoolkit.model.utils.tilde -import org.ossreviewtoolkit.scanner.ScanResultsStorage import org.ossreviewtoolkit.scanner.ScanStorageException import org.ossreviewtoolkit.scanner.ScannerMatcher import org.ossreviewtoolkit.scanner.storages.utils.ScanResultDao @@ -62,17 +55,12 @@ private val TABLE_NAME = ScanResults.tableName /** * The Postgres storage back-end. */ -class PostgresStorage( +class PackageBasedPostgresStorage( /** * The JDBC data source to obtain database connections. */ - private val dataSource: Lazy, - - /** - * The number of parallel storage transactions. - */ - private val parallelTransactions: Int -) : ScanResultsStorage() { + private val dataSource: Lazy +) : AbstractPackageBasedScanStorage() { companion object { /** Expression to reference the scanner version as an array. */ private const val VERSION_ARRAY = @@ -131,42 +119,32 @@ class PostgresStorage( """.trimIndent() ) - override fun readInternal(pkg: Package): Result> = - runCatching { - database.transaction { - ScanResultDao.find { ScanResults.identifier eq pkg.id.toCoordinates() }.map { it.scanResult } - } - }.onFailure { - if (it is JsonProcessingException || it is SQLException) { - it.showStackTrace() + override fun readInternal(pkg: Package, scannerMatcher: ScannerMatcher?): Result> { + val minVersionArray = scannerMatcher?.minVersion?.run { intArrayOf(major, minor, patch) } + val maxVersionArray = scannerMatcher?.maxVersion?.run { intArrayOf(major, minor, patch) } - val message = "Could not read scan results for '${pkg.id.toCoordinates()}' from database: " + - it.collectMessages() + return runCatching { + database.transaction { + ScanResultDao.find { + var expression = (ScanResults.identifier eq pkg.id.toCoordinates()) - logger.info { message } + scannerMatcher?.regScannerName?.let { + expression = expression and (rawParam("scan_result->'scanner'->>'name'") tilde it) + } - return Result.failure(ScanStorageException(message)) - } - } + minVersionArray?.let { + expression = expression and (rawParam(VERSION_EXPRESSION) greaterEq arrayParam(it)) + } - override fun readInternal(pkg: Package, scannerMatcher: ScannerMatcher): Result> { - val minVersionArray = with(scannerMatcher.minVersion) { intArrayOf(major, minor, patch) } - val maxVersionArray = with(scannerMatcher.maxVersion) { intArrayOf(major, minor, patch) } + maxVersionArray?.let { + expression = expression and (rawParam(VERSION_EXPRESSION) less arrayParam(it)) + } - return runCatching { - database.transaction { - ScanResultDao.find { - (ScanResults.identifier eq pkg.id.toCoordinates()) and - (rawParam("scan_result->'scanner'->>'name'") tilde scannerMatcher.regScannerName) and - (rawParam(VERSION_EXPRESSION) greaterEq arrayParam(minVersionArray)) and - (rawParam(VERSION_EXPRESSION) less arrayParam(maxVersionArray)) + expression }.map { it.scanResult } // TODO: Currently the query only accounts for the scanner criteria. Ideally also the provenance // should be checked in the query to reduce the downloaded data. .filter { it.provenance.matches(pkg) } - // The scanner compatibility is already checked in the query, but filter here again to be on the - // safe side. - .filter { scannerMatcher.matches(it.scanner) } } }.onFailure { if (it is JsonProcessingException || it is SQLException) { @@ -182,56 +160,6 @@ class PostgresStorage( } } - override fun readInternal( - packages: Collection, - scannerMatcher: ScannerMatcher - ): Result>> { - if (packages.isEmpty()) return Result.success(emptyMap()) - - val minVersionArray = with(scannerMatcher.minVersion) { intArrayOf(major, minor, patch) } - val maxVersionArray = with(scannerMatcher.maxVersion) { intArrayOf(major, minor, patch) } - - return runCatching { - runBlocking(Dispatchers.IO) { - packages.chunked(max(packages.size / parallelTransactions, 1)).map { chunk -> - database.transactionAsync { - @Suppress("MaxLineLength") - ScanResultDao.find { - (ScanResults.identifier inList chunk.map { it.id.toCoordinates() }) and - (rawParam("scan_result->'scanner'->>'name'") tilde scannerMatcher.regScannerName) and - (rawParam(VERSION_EXPRESSION) greaterEq arrayParam(minVersionArray)) and - (rawParam(VERSION_EXPRESSION) less arrayParam(maxVersionArray)) - }.map { it.identifier to it.scanResult } - } - }.flatMap { it.await() } - .groupBy { it.first } - .mapValues { (_, results) -> results.map { it.second } } - .mapValues { (id, results) -> - val pkg = packages.single { it.id == id } - - results - // TODO: Currently the query only accounts for the scanner criteria. Ideally also the - // provenance should be checked in the query to reduce the downloaded data. - .filter { it.provenance.matches(pkg) } - // The scanner compatibility is already checked in the query, but filter here again to be on - // the safe side. - .filter { scannerMatcher.matches(it.scanner) } - } - } - }.onFailure { - if (it is JsonProcessingException || it is SQLException) { - it.showStackTrace() - - val message = "Could not read scan results with $scannerMatcher from database: " + - it.collectMessages() - - logger.info { message } - - return Result.failure(ScanStorageException(message)) - } - } - } - override fun addInternal(id: Identifier, scanResult: ScanResult): Result { logger.info { "Storing scan result for '${id.toCoordinates()}' in storage." } diff --git a/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt b/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt index ba44f33d3450f..77faa20bace52 100644 --- a/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt +++ b/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt @@ -34,6 +34,7 @@ import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.yamlMapper import org.ossreviewtoolkit.scanner.ProvenanceBasedScanStorage import org.ossreviewtoolkit.scanner.ScanStorageException +import org.ossreviewtoolkit.scanner.ScannerMatcher import org.ossreviewtoolkit.scanner.utils.requireEmptyVcsPath import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.fileSystemEncode @@ -41,7 +42,7 @@ import org.ossreviewtoolkit.utils.ort.showStackTrace import org.ossreviewtoolkit.utils.ort.storage.FileStorage class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceBasedScanStorage { - override fun read(provenance: KnownProvenance): List { + override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher?): List { requireEmptyVcsPath(provenance) val path = storagePath(provenance) @@ -52,7 +53,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB // Use the provided provenance for the result instead of building it from the stored values, because // in the case of a RepositoryRevision only the resolved revision matters. it.copy(provenance = provenance) - } + }.filter { scannerMatcher?.matches(it.scanner) != false } } }.getOrElse { when (it) { diff --git a/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt b/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt index 56db42a54a954..50c6403a03039 100644 --- a/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt +++ b/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt @@ -45,6 +45,7 @@ import org.ossreviewtoolkit.model.utils.DatabaseUtils.tableExists import org.ossreviewtoolkit.model.utils.DatabaseUtils.transaction import org.ossreviewtoolkit.scanner.ProvenanceBasedScanStorage import org.ossreviewtoolkit.scanner.ScanStorageException +import org.ossreviewtoolkit.scanner.ScannerMatcher import org.ossreviewtoolkit.scanner.storages.utils.jsonb import org.ossreviewtoolkit.scanner.utils.requireEmptyVcsPath import org.ossreviewtoolkit.utils.common.collectMessages @@ -77,7 +78,7 @@ class ProvenanceBasedPostgresStorage( } } - override fun read(provenance: KnownProvenance): List { + override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher?): List { requireEmptyVcsPath(provenance) try { @@ -104,16 +105,18 @@ class ProvenanceBasedPostgresStorage( // Use the provided provenance for the result instead of building it from the stored values, because in // the case of a RepositoryRevision only the resolved revision matters, therefore the VcsInfo.revision // is not stored in the database. - query.map { - ScanResult( - provenance = provenance, - scanner = ScannerDetails( - name = it[table.scannerName], - version = it[table.scannerVersion], - configuration = it[table.scannerConfiguration] - ), - summary = it[table.scanSummary] + query.mapNotNull { + val details = ScannerDetails( + name = it[table.scannerName], + version = it[table.scannerVersion], + configuration = it[table.scannerConfiguration] ) + + if (scannerMatcher?.matches(details) != false) { + ScanResult(provenance, details, it[table.scanSummary]) + } else { + null + } } } } catch (e: SQLException) { diff --git a/scanner/src/main/kotlin/storages/Sw360Storage.kt b/scanner/src/main/kotlin/storages/Sw360Storage.kt index 68338f8066334..94bd191d95763 100644 --- a/scanner/src/main/kotlin/storages/Sw360Storage.kt +++ b/scanner/src/main/kotlin/storages/Sw360Storage.kt @@ -43,8 +43,8 @@ import org.ossreviewtoolkit.model.config.Sw360StorageConfiguration import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.readValue import org.ossreviewtoolkit.model.writeValue -import org.ossreviewtoolkit.scanner.ScanResultsStorage import org.ossreviewtoolkit.scanner.ScanStorageException +import org.ossreviewtoolkit.scanner.ScannerMatcher import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.safeDeleteRecursively import org.ossreviewtoolkit.utils.ort.createOrtTempDir @@ -55,7 +55,7 @@ import org.ossreviewtoolkit.utils.ort.createOrtTempDir */ class Sw360Storage( configuration: Sw360StorageConfiguration -) : ScanResultsStorage() { +) : AbstractPackageBasedScanStorage() { companion object { val JSON_MAPPER: ObjectMapper = jsonMapper.copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) @@ -86,7 +86,7 @@ class Sw360Storage( private val connectionFactory = createConnection(configuration) private val releaseClient = connectionFactory.releaseAdapter - override fun readInternal(pkg: Package): Result> { + override fun readInternal(pkg: Package, scannerMatcher: ScannerMatcher?): Result> { val tempScanResultFile = createTempFileForUpload(pkg.id) val result = runCatching { diff --git a/scanner/src/test/kotlin/ScannerTest.kt b/scanner/src/test/kotlin/ScannerTest.kt index 10a477846ca0d..89c8becd9bca0 100644 --- a/scanner/src/test/kotlin/ScannerTest.kt +++ b/scanner/src/test/kotlin/ScannerTest.kt @@ -356,7 +356,7 @@ class ScannerTest : WordSpec({ val pkgWithArtifact = Package.new(name = "artifact").withValidSourceArtifact() val scannerWrapper = spyk(FakePackageScannerWrapper()) val reader = spyk(FakePackageBasedStorageReader(scannerWrapper.details)) { - every { read(any(), any()) } returns emptyList() + every { read(any(), any(), any()) } returns emptyList() } val scanner = createScanner( @@ -376,7 +376,7 @@ class ScannerTest : WordSpec({ ) verify(exactly = 1) { - reader.read(pkgWithArtifact, any()) + reader.read(pkgWithArtifact, any(), any()) scannerWrapper.scanPackage(any(), createContext().copy(coveredPackages = listOf(pkgWithArtifact))) } } @@ -433,7 +433,7 @@ class ScannerTest : WordSpec({ } val reader = spyk(FakePackageBasedStorageReader(scannerWrapper.details)) { - every { read(pkgCompletelyScanned, any()) } returns listOf(nestedScanResultCompletelyScanned) + every { read(pkgCompletelyScanned, any(), any()) } returns listOf(nestedScanResultCompletelyScanned) } val scanner = createScanner( @@ -496,7 +496,7 @@ class ScannerTest : WordSpec({ val scannerWrapper = spyk(FakeProvenanceScannerWrapper()) val reader = spyk(FakeProvenanceBasedStorageReader(scannerWrapper.details)) { - every { read(any()) } returns emptyList() + every { read(any(), any()) } returns emptyList() } val scanner = createScanner( @@ -516,7 +516,7 @@ class ScannerTest : WordSpec({ ) verify(exactly = 1) { - reader.read(pkgWithArtifact.artifactProvenance()) + reader.read(pkgWithArtifact.artifactProvenance(), any()) scannerWrapper.scanProvenance(pkgWithArtifact.artifactProvenance(), any()) } } @@ -574,7 +574,7 @@ class ScannerTest : WordSpec({ } val reader = spyk(FakeProvenanceBasedStorageReader(scannerWrapper.details)) { - every { read(unscannedSubRepository) } returns emptyList() + every { read(unscannedSubRepository, any()) } returns emptyList() } val scanner = createScanner( @@ -654,7 +654,7 @@ class ScannerTest : WordSpec({ ) val reader = spyk(FakeProvenanceBasedStorageReader(scannerWrapper.details)) { - every { read(any()) } returns listOf(scanResult) + every { read(any(), any()) } returns listOf(scanResult) } val scanner = createScanner( @@ -823,7 +823,6 @@ class ScannerTest : WordSpec({ scanner.scan(setOf(pkgWithArtifact), createContext()) verify(exactly = 1) { - reader.read(any()) reader.read(any(), any()) } } @@ -986,21 +985,16 @@ private class FakeNestedProvenanceResolver : NestedProvenanceResolver { * with a single license finding for the provided [scannerDetails]. */ private class FakePackageBasedStorageReader(val scannerDetails: ScannerDetails) : PackageBasedScanStorageReader { - override fun read(pkg: Package, nestedProvenance: NestedProvenance): List = - listOf(createStoredNestedScanResult(nestedProvenance.root, scannerDetails)) - override fun read( pkg: Package, nestedProvenance: NestedProvenance, - scannerMatcher: ScannerMatcher - ): List = read(pkg, nestedProvenance) + scannerMatcher: ScannerMatcher? + ): List = listOf(createStoredNestedScanResult(nestedProvenance.root, scannerDetails)) } private class FakeProvenanceBasedStorageReader(val scannerDetails: ScannerDetails) : ProvenanceBasedScanStorageReader { - override fun read(provenance: KnownProvenance): List = + override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher?): List = listOf(createStoredScanResult(provenance, scannerDetails)) - - override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher): List = read(provenance) } private class FakePackageBasedStorageWriter : PackageBasedScanStorageWriter {