diff --git a/analyzer/src/main/kotlin/PackageCurationProvider.kt b/analyzer/src/main/kotlin/PackageCurationProvider.kt index bc9c0256a20cd..9ff51fa5977af 100644 --- a/analyzer/src/main/kotlin/PackageCurationProvider.kt +++ b/analyzer/src/main/kotlin/PackageCurationProvider.kt @@ -21,6 +21,35 @@ package org.ossreviewtoolkit.analyzer import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.PackageCuration +import org.ossreviewtoolkit.model.config.PackageCurationProviderConfiguration +import org.ossreviewtoolkit.utils.common.ConfigurablePluginFactory +import org.ossreviewtoolkit.utils.common.NamedPlugin + +/** + * The extension point for [PackageCurationProvider]s. + */ +interface PackageCurationProviderFactory : ConfigurablePluginFactory { + companion object { + val ALL = NamedPlugin.getAll>() + + fun create(configurations: List) = + // Reverse the list so that curations from providers with higher priority are applied later and can + // overwrite curations from providers with lower priority. + configurations.filter { it.enabled }.map { ALL.getValue(it.name).create(it.config) }.asReversed() + } + + override fun create(config: Map): PackageCurationProvider = create(parseConfig(config)) + + /** + * Create a new [PackageCurationProvider] with [config]. + */ + fun create(config: CONFIG): PackageCurationProvider + + /** + * Parse the [config] map into an object. + */ + fun parseConfig(config: Map): CONFIG +} /** * A provider for [PackageCuration]s. diff --git a/analyzer/src/main/kotlin/curation/ClearlyDefinedPackageCurationProvider.kt b/analyzer/src/main/kotlin/curation/ClearlyDefinedPackageCurationProvider.kt index 77a550d8439bf..48986ac980513 100644 --- a/analyzer/src/main/kotlin/curation/ClearlyDefinedPackageCurationProvider.kt +++ b/analyzer/src/main/kotlin/curation/ClearlyDefinedPackageCurationProvider.kt @@ -28,6 +28,7 @@ import okhttp3.OkHttpClient import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.analyzer.PackageCurationProvider +import org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory import org.ossreviewtoolkit.clients.clearlydefined.ClearlyDefinedService import org.ossreviewtoolkit.clients.clearlydefined.ClearlyDefinedService.Server import org.ossreviewtoolkit.clients.clearlydefined.ComponentType @@ -50,6 +51,24 @@ import org.ossreviewtoolkit.utils.spdx.toSpdx import retrofit2.HttpException +class ClearlyDefinedPackageCurationProviderConfig( + /** + * The URL of the ClearlyDefined server to use. If null, uses the [production server][Server.PRODUCTION.apiUrl]. + */ + val serverUrl: String? = null +) + +class ClearlyDefinedPackageCurationProviderFactory : + PackageCurationProviderFactory { + override val name = "ClearlyDefined" + + override fun create(config: ClearlyDefinedPackageCurationProviderConfig) = + ClearlyDefinedPackageCurationProvider(serverUrl = config.serverUrl) + + override fun parseConfig(config: Map) = + ClearlyDefinedPackageCurationProviderConfig(serverUrl = config["serverUrl"]) +} + /** * A provider for curated package metadata from the [ClearlyDefined](https://clearlydefined.io/) service. */ diff --git a/analyzer/src/main/kotlin/curation/FilePackageCurationProvider.kt b/analyzer/src/main/kotlin/curation/FilePackageCurationProvider.kt index b40093b1c9fb1..63633706276ed 100644 --- a/analyzer/src/main/kotlin/curation/FilePackageCurationProvider.kt +++ b/analyzer/src/main/kotlin/curation/FilePackageCurationProvider.kt @@ -25,25 +25,74 @@ import java.io.IOException import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.analyzer.PackageCurationProvider +import org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory import org.ossreviewtoolkit.model.FileFormat import org.ossreviewtoolkit.model.PackageCuration import org.ossreviewtoolkit.model.readValue import org.ossreviewtoolkit.utils.common.getDuplicates +import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_DIRNAME +import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_FILENAME +import org.ossreviewtoolkit.utils.ort.ortConfigDirectory + +class FilePackageCurationProviderConfig( + /** + * The path of the package curation file or directory. + */ + val path: String +) + +class FilePackageCurationProviderFactory : PackageCurationProviderFactory { + override val name = "File" + + override fun create(config: FilePackageCurationProviderConfig) = FilePackageCurationProvider(config) + + override fun parseConfig(config: Map) = + FilePackageCurationProviderConfig(path = config.getValue("path")) +} + +class DefaultFilePackageCurationProviderFactory : PackageCurationProviderFactory { + override val name = "DefaultFile" + + override fun create(config: Unit) = + ortConfigDirectory.resolve(ORT_PACKAGE_CURATIONS_FILENAME).let { curationsFile -> + when { + curationsFile.isFile -> FilePackageCurationProvider(curationsFile) + else -> PackageCurationProvider.EMPTY + } + } + + override fun parseConfig(config: Map) = Unit +} + +class DefaultDirPackageCurationProviderFactory : PackageCurationProviderFactory { + override val name = "DefaultDir" + + override fun create(config: Unit) = + ortConfigDirectory.resolve(ORT_PACKAGE_CURATIONS_DIRNAME).let { curationsDir -> + when { + curationsDir.isDirectory -> FilePackageCurationProvider(curationsDir) + else -> PackageCurationProvider.EMPTY + } + } + + override fun parseConfig(config: Map) = Unit +} /** * A [PackageCurationProvider] that loads [PackageCuration]s from all given curation files. Supports all file formats * specified in [FileFormat]. */ class FilePackageCurationProvider( - curationFiles: Collection + curationFiles: List ) : SimplePackageCurationProvider(readCurationFiles(curationFiles)) { constructor(curationFile: File) : this(listOf(curationFile)) + constructor(config: FilePackageCurationProviderConfig) : this(File(config.path)) companion object : Logging { fun from(file: File? = null, dir: File? = null): FilePackageCurationProvider { val curationFiles = mutableListOf() file?.takeIf { it.isFile }?.let { curationFiles += it } - dir?.let { curationFiles += FileFormat.findFilesWithKnownExtensions(it) } + dir?.takeIf { it.isDirectory }?.let { curationFiles += FileFormat.findFilesWithKnownExtensions(it) } return FilePackageCurationProvider(curationFiles) } diff --git a/analyzer/src/main/kotlin/curation/OrtConfigPackageCurationProvider.kt b/analyzer/src/main/kotlin/curation/OrtConfigPackageCurationProvider.kt index 4df2376f96b32..b433954caef5f 100644 --- a/analyzer/src/main/kotlin/curation/OrtConfigPackageCurationProvider.kt +++ b/analyzer/src/main/kotlin/curation/OrtConfigPackageCurationProvider.kt @@ -25,6 +25,7 @@ import java.io.IOException import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.analyzer.PackageCurationProvider +import org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory import org.ossreviewtoolkit.downloader.vcs.Git import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.PackageCuration @@ -38,6 +39,14 @@ import org.ossreviewtoolkit.utils.ort.ortDataDirectory private const val ORT_CONFIG_REPOSITORY_BRANCH = "main" private const val ORT_CONFIG_REPOSITORY_URL = "https://github.com/oss-review-toolkit/ort-config.git" +class OrtConfigPackageCurationProviderFactory : PackageCurationProviderFactory { + override val name = "OrtConfig" + + override fun create(config: Unit) = OrtConfigPackageCurationProvider() + + override fun parseConfig(config: Map) = Unit +} + /** * A [PackageCurationProvider] that provides [PackageCuration]s loaded from the * [ort-config repository](https://github.com/oss-review-toolkit/ort-config). diff --git a/analyzer/src/main/kotlin/curation/Sw360PackageCurationProvider.kt b/analyzer/src/main/kotlin/curation/Sw360PackageCurationProvider.kt index 8b038f620e5a3..4fc429cc15f8c 100644 --- a/analyzer/src/main/kotlin/curation/Sw360PackageCurationProvider.kt +++ b/analyzer/src/main/kotlin/curation/Sw360PackageCurationProvider.kt @@ -33,6 +33,7 @@ import org.eclipse.sw360.http.HttpClientFactoryImpl import org.eclipse.sw360.http.config.HttpClientConfig import org.ossreviewtoolkit.analyzer.PackageCurationProvider +import org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory import org.ossreviewtoolkit.model.Hash import org.ossreviewtoolkit.model.HashAlgorithm import org.ossreviewtoolkit.model.Identifier @@ -45,6 +46,23 @@ import org.ossreviewtoolkit.model.orEmpty import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor import org.ossreviewtoolkit.utils.spdx.SpdxExpression +class Sw360PackageCurationProviderFactory : PackageCurationProviderFactory { + override val name = "SW360" + + override fun create(config: Sw360StorageConfiguration) = Sw360PackageCurationProvider(config) + + override fun parseConfig(config: Map) = + Sw360StorageConfiguration( + restUrl = config.getValue("restUrl"), + authUrl = config.getValue("authUrl"), + username = config.getValue("username"), + password = config["password"].orEmpty(), + clientId = config.getValue("clientId"), + clientPassword = config["clientPassword"].orEmpty(), + token = config["token"].orEmpty() + ) +} + /** * A [PackageCurationProvider] for curated package metadata from the configured SW360 instance using the REST API. */ diff --git a/analyzer/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory b/analyzer/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory new file mode 100644 index 0000000000000..ef39f2610dba4 --- /dev/null +++ b/analyzer/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory @@ -0,0 +1,6 @@ +org.ossreviewtoolkit.analyzer.curation.ClearlyDefinedPackageCurationProviderFactory +org.ossreviewtoolkit.analyzer.curation.DefaultDirPackageCurationProviderFactory +org.ossreviewtoolkit.analyzer.curation.DefaultFilePackageCurationProviderFactory +org.ossreviewtoolkit.analyzer.curation.FilePackageCurationProviderFactory +org.ossreviewtoolkit.analyzer.curation.OrtConfigPackageCurationProviderFactory +org.ossreviewtoolkit.analyzer.curation.Sw360PackageCurationProviderFactory diff --git a/cli/src/funTest/kotlin/OrtMainFunTest.kt b/cli/src/funTest/kotlin/OrtMainFunTest.kt index b72989e47bd5c..a1dced9d279b4 100644 --- a/cli/src/funTest/kotlin/OrtMainFunTest.kt +++ b/cli/src/funTest/kotlin/OrtMainFunTest.kt @@ -38,6 +38,7 @@ import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.model.OrtResult import org.ossreviewtoolkit.model.config.OrtConfiguration import org.ossreviewtoolkit.model.config.OrtConfigurationWrapper +import org.ossreviewtoolkit.model.config.PackageCurationProviderConfiguration import org.ossreviewtoolkit.model.readValue import org.ossreviewtoolkit.model.writeValue import org.ossreviewtoolkit.utils.common.EnvironmentVariableFilter @@ -63,7 +64,18 @@ class OrtMainFunTest : StringSpec() { override suspend fun beforeSpec(spec: Spec) { configFile = createSpecTempFile(suffix = ".yml") - configFile.writeValue(OrtConfigurationWrapper(OrtConfiguration())) + configFile.writeValue( + OrtConfigurationWrapper( + OrtConfiguration( + packageCurationProviders = listOf( + PackageCurationProviderConfiguration( + name = "File", + config = mapOf("path" to projectDir.resolve("gradle/curations.yml").path) + ) + ) + ) + ) + ) } override suspend fun beforeTest(testCase: TestCase) { @@ -186,29 +198,6 @@ class OrtMainFunTest : StringSpec() { } "Analyzer creates correct output" { - val expectedResult = patchExpectedResult( - projectDir.resolve("gradle-all-dependencies-expected-result.yml"), - url = vcsUrl, - revision = vcsRevision, - urlProcessed = normalizeVcsUrl(vcsUrl) - ) - - @Suppress("IgnoredReturnValue") - runMain( - "-c", configFile.path, - "-P", "ort.analyzer.enabledPackageManagers=Gradle", - "analyze", - "-i", projectDir.resolve("gradle").absolutePath, - "-o", outputDir.path - ) - - val analyzerResult = outputDir.resolve("analyzer-result.yml").readValue() - val resolvedResult = analyzerResult.withResolvedScopes() - - patchActualResult(resolvedResult, patchStartAndEndTime = true) shouldBe expectedResult - } - - "Package curation data file is applied correctly" { val expectedResult = patchExpectedResult( projectDir.resolve("gradle-all-dependencies-expected-result-with-curations.yml"), url = vcsUrl, @@ -222,8 +211,7 @@ class OrtMainFunTest : StringSpec() { "-P", "ort.analyzer.enabledPackageManagers=Gradle", "analyze", "-i", projectDir.resolve("gradle").absolutePath, - "-o", outputDir.path, - "--package-curations-file", projectDir.resolve("gradle/curations.yml").toString() + "-o", outputDir.path ) val analyzerResult = outputDir.resolve("analyzer-result.yml").readValue() diff --git a/cli/src/main/kotlin/commands/AnalyzerCommand.kt b/cli/src/main/kotlin/commands/AnalyzerCommand.kt index 0849d821c890c..0b257b5ad4bb1 100644 --- a/cli/src/main/kotlin/commands/AnalyzerCommand.kt +++ b/cli/src/main/kotlin/commands/AnalyzerCommand.kt @@ -28,7 +28,6 @@ import com.github.ajalt.clikt.parameters.options.convert import com.github.ajalt.clikt.parameters.options.default import com.github.ajalt.clikt.parameters.options.defaultLazy import com.github.ajalt.clikt.parameters.options.deprecated -import com.github.ajalt.clikt.parameters.options.flag import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.options.required import com.github.ajalt.clikt.parameters.options.split @@ -40,14 +39,11 @@ import java.time.Duration import kotlin.time.toKotlinDuration import org.ossreviewtoolkit.analyzer.Analyzer +import org.ossreviewtoolkit.analyzer.PackageCurationProviderFactory import org.ossreviewtoolkit.analyzer.PackageManager import org.ossreviewtoolkit.analyzer.PackageManagerFactory -import org.ossreviewtoolkit.analyzer.curation.ClearlyDefinedPackageCurationProvider import org.ossreviewtoolkit.analyzer.curation.CompositePackageCurationProvider -import org.ossreviewtoolkit.analyzer.curation.FilePackageCurationProvider -import org.ossreviewtoolkit.analyzer.curation.OrtConfigPackageCurationProvider import org.ossreviewtoolkit.analyzer.curation.SimplePackageCurationProvider -import org.ossreviewtoolkit.analyzer.curation.Sw360PackageCurationProvider import org.ossreviewtoolkit.cli.OrtCommand import org.ossreviewtoolkit.cli.utils.SeverityStats import org.ossreviewtoolkit.cli.utils.configurationGroup @@ -64,8 +60,6 @@ import org.ossreviewtoolkit.model.utils.DefaultResolutionProvider import org.ossreviewtoolkit.model.utils.mergeLabels import org.ossreviewtoolkit.utils.common.expandTilde import org.ossreviewtoolkit.utils.common.safeMkdirs -import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_DIRNAME -import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_FILENAME import org.ossreviewtoolkit.utils.ort.ORT_REPO_CONFIG_FILENAME import org.ossreviewtoolkit.utils.ort.ORT_RESOLUTIONS_FILENAME import org.ossreviewtoolkit.utils.ort.ortConfigDirectory @@ -98,24 +92,6 @@ class AnalyzerCommand : OrtCommand( help = "The list of output formats to be used for the ORT result file(s)." ).enum().split(",").default(listOf(FileFormat.YAML)).outputGroup() - private val packageCurationsFile by option( - "--package-curations-file", - help = "A file containing package curation data." - ).convert { it.expandTilde() } - .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) - .convert { it.absoluteFile.normalize() } - .default(ortConfigDirectory.resolve(ORT_PACKAGE_CURATIONS_FILENAME)) - .configurationGroup() - - private val packageCurationsDir by option( - "--package-curations-dir", - help = "A directory containing package curation data." - ).convert { it.expandTilde() } - .file(mustExist = true, canBeFile = false, canBeDir = true, mustBeWritable = false, mustBeReadable = true) - .convert { it.absoluteFile.normalize() } - .default(ortConfigDirectory.resolve(ORT_PACKAGE_CURATIONS_DIRNAME)) - .configurationGroup() - private val repositoryConfigurationFile by option( "--repository-configuration-file", help = "A file containing the repository configuration. If set, overrides any repository configuration " + @@ -135,21 +111,6 @@ class AnalyzerCommand : OrtCommand( .default(ortConfigDirectory.resolve(ORT_RESOLUTIONS_FILENAME)) .configurationGroup() - private val useClearlyDefinedCurations by option( - "--clearly-defined-curations", - help = "Whether to fall back to package curation data from the ClearlyDefine service or not." - ).flag() - - private val useOrtCurations by option( - "--ort-curations", - help = "Whether to fall back to package curation data from the ort-config repository or not." - ).flag() - - private val useSw360Curations by option( - "--sw360-curations", - help = "Whether to fall back to package curation data from the SW360 service or not." - ).flag() - private val labels by option( "--label", "-l", help = "Set a label in the ORT result, overwriting any existing label of the same name. Can be used multiple " + @@ -199,8 +160,6 @@ class AnalyzerCommand : OrtCommand( } val configurationFiles = listOf( - packageCurationsFile, - packageCurationsDir, repositoryConfigurationFile, resolutionsFile ) @@ -232,15 +191,7 @@ class AnalyzerCommand : OrtCommand( val analyzer = Analyzer(analyzerConfiguration, labels) val curationProviders = buildList { - if (useClearlyDefinedCurations) add(ClearlyDefinedPackageCurationProvider()) - - if (useSw360Curations) { - ortConfig.analyzer.sw360Configuration?.let { add(Sw360PackageCurationProvider(it)) } - } - - if (useOrtCurations) add(OrtConfigPackageCurationProvider()) - - add(FilePackageCurationProvider.from(packageCurationsFile, packageCurationsDir)) + addAll(PackageCurationProviderFactory.create(ortConfig.packageCurationProviders)) val repositoryPackageCurations = repositoryConfiguration.curations.packages diff --git a/model/src/main/kotlin/config/AnalyzerConfiguration.kt b/model/src/main/kotlin/config/AnalyzerConfiguration.kt index 3e3e35393ff7f..c90ea288d6258 100644 --- a/model/src/main/kotlin/config/AnalyzerConfiguration.kt +++ b/model/src/main/kotlin/config/AnalyzerConfiguration.kt @@ -48,12 +48,7 @@ data class AnalyzerConfiguration( * Package manager specific configurations. The key needs to match the name of the package manager class, e.g. * "NuGet" for the NuGet package manager. */ - val packageManagers: Map? = null, - - /** - * Configuration of the SW360 package curation provider. - */ - val sw360Configuration: Sw360StorageConfiguration? = null + val packageManagers: Map? = null ) { /** * A copy of [packageManagers] with case-insensitive keys. @@ -110,8 +105,7 @@ data class AnalyzerConfiguration( allowDynamicVersions = other.allowDynamicVersions, enabledPackageManagers = other.enabledPackageManagers ?: enabledPackageManagers, disabledPackageManagers = other.disabledPackageManagers ?: disabledPackageManagers, - packageManagers = mergedPackageManagers, - sw360Configuration = other.sw360Configuration ?: sw360Configuration + packageManagers = mergedPackageManagers ) } } diff --git a/model/src/main/kotlin/config/OrtConfiguration.kt b/model/src/main/kotlin/config/OrtConfiguration.kt index a79e0ca04c64d..4c4c3b8e81ebc 100644 --- a/model/src/main/kotlin/config/OrtConfiguration.kt +++ b/model/src/main/kotlin/config/OrtConfiguration.kt @@ -30,6 +30,8 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.utils.common.EnvironmentVariableFilter +import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_DIRNAME +import org.ossreviewtoolkit.utils.ort.ORT_PACKAGE_CURATIONS_FILENAME /** * The configuration model for all ORT components. @@ -63,7 +65,7 @@ data class OrtConfiguration( /** * Enable the usage of project-local package curations from the [RepositoryConfiguration]. If set to true, apply * package curations from a local .ort.yml file before applying those specified via the command line i.e. curations - * from the.ort.yml take precedence. + * from the .ort.yml take precedence. */ val enableRepositoryPackageCurations: Boolean = false, @@ -77,6 +79,17 @@ data class OrtConfiguration( */ val licenseFilePatterns: LicenseFilePatterns = LicenseFilePatterns.DEFAULT, + /** + * The package curation providers to use. Defaults to providers for the default configuration locations + * [ORT_PACKAGE_CURATIONS_FILENAME] and [ORT_PACKAGE_CURATIONS_DIRNAME] are added. The order of this list defines + * the priority of the providers: Providers that appear earlier in the list can overwrite curations for the same + * package from providers that appear later in the list. + */ + val packageCurationProviders: List = listOf( + PackageCurationProviderConfiguration("DefaultDir"), + PackageCurationProviderConfiguration("DefaultFile") + ), + /** * The threshold from which on issues count as severe. */ diff --git a/model/src/main/kotlin/config/PackageCurationProviderConfiguration.kt b/model/src/main/kotlin/config/PackageCurationProviderConfiguration.kt new file mode 100644 index 0000000000000..9fa598bbe82f0 --- /dev/null +++ b/model/src/main/kotlin/config/PackageCurationProviderConfiguration.kt @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.config + +import org.ossreviewtoolkit.utils.common.NamedPlugin + +data class PackageCurationProviderConfiguration( + /** + * The [name][NamedPlugin.name] of the package curation provider. + */ + val name: String, + + /** + * Whether this curation provider is enabled. + */ + val enabled: Boolean = true, + + /** + * The configuration of the package curation provider. See the specific implementation for available configuration + * options. + */ + val config: Map = emptyMap() +) diff --git a/model/src/main/resources/reference.yml b/model/src/main/resources/reference.yml index a2fb6babae93b..8887b09f6bdc4 100644 --- a/model/src/main/resources/reference.yml +++ b/model/src/main/resources/reference.yml @@ -19,26 +19,50 @@ # exclusive, so this file is only used to show all options and to validate the configuration. ort: + allowedProcessEnvironmentVariableNames: + - PASSPORT + - USER_HOME + deniedProcessEnvironmentVariablesSubstrings: + - PASS + - SECRET + - TOKEN + - USER + + enableRepositoryPackageConfigurations: true + enableRepositoryPackageCurations: true + licenseFilePatterns: licenseFilenames: ['license*'] patentFilenames: [patents] rootLicenseFilenames: ['readme*'] + packageCurationProviders: + - name: DefaultFile + - name: DefaultDir + - name: File + config: + path: '/some-path/curations.yml' + - name: File + config: + path: '/some-path/curations-dir' + - name: OrtConfig + enabled: '${USE_ORT_CONFIG_CURATIONS:-true}' + - name: ClearlyDefined + config: + serverUrl: 'https://api.clearlydefined.io' + - name: SW360 + config: + restUrl: 'https://your-sw360-rest-url' + authUrl: 'https://your-authentication-url' + username: username + password: password + clientId: clientId + clientPassword: clientPassword + token: token + severeIssueThreshold: ERROR severeRuleViolationThreshold: ERROR - enableRepositoryPackageCurations: true - enableRepositoryPackageConfigurations: true - - deniedProcessEnvironmentVariablesSubstrings: - - PASS - - SECRET - - TOKEN - - USER - allowedProcessEnvironmentVariableNames: - - PASSPORT - - USER_HOME - analyzer: allowDynamicVersions: true @@ -67,15 +91,6 @@ ort: # intercept requests to the registry. disableRegistryCertificateVerification: false - sw360Configuration: - restUrl: 'https://your-sw360-rest-url' - authUrl: 'https://your-authentication-url' - username: username - password: password - clientId: clientId - clientPassword: clientPassword - token: token - advisor: nexusIq: serverUrl: 'https://rest-api-url-of-your-nexus-iq-server' diff --git a/model/src/test/kotlin/config/AnalyzerConfigurationTest.kt b/model/src/test/kotlin/config/AnalyzerConfigurationTest.kt index 6a236af31431c..b09475a2e4193 100644 --- a/model/src/test/kotlin/config/AnalyzerConfigurationTest.kt +++ b/model/src/test/kotlin/config/AnalyzerConfigurationTest.kt @@ -50,36 +50,31 @@ class AnalyzerConfigurationTest : WordSpec({ val self = AnalyzerConfiguration( allowDynamicVersions = false, enabledPackageManagers = listOf("Gradle"), - disabledPackageManagers = listOf("NPM"), - sw360Configuration = sw360Config1 + disabledPackageManagers = listOf("NPM") ) val other = AnalyzerConfiguration( allowDynamicVersions = true, enabledPackageManagers = listOf("Maven"), - disabledPackageManagers = listOf("SBT"), - sw360Configuration = sw360Config2 + disabledPackageManagers = listOf("SBT") ) with(self.merge(other)) { allowDynamicVersions shouldBe true enabledPackageManagers should containExactly("Maven") disabledPackageManagers should containExactly("SBT") - sw360Configuration shouldBe sw360Config2 } } "keep values which are null in other" { val self = AnalyzerConfiguration( enabledPackageManagers = listOf("Gradle"), - disabledPackageManagers = listOf("NPM"), - sw360Configuration = sw360Config1 + disabledPackageManagers = listOf("NPM") ) val other = AnalyzerConfiguration( enabledPackageManagers = null, - disabledPackageManagers = null, - sw360Configuration = null + disabledPackageManagers = null ) self.merge(other) shouldBe self @@ -132,17 +127,3 @@ class AnalyzerConfigurationTest : WordSpec({ } } }) - -private val sw360Config1 = Sw360StorageConfiguration( - restUrl = "url1", - authUrl = "auth1", - username = "user1", - clientId = "client1" -) - -private val sw360Config2 = Sw360StorageConfiguration( - restUrl = "url2", - authUrl = "auth2", - username = "user2", - clientId = "client2" -) diff --git a/model/src/test/kotlin/config/OrtConfigurationTest.kt b/model/src/test/kotlin/config/OrtConfigurationTest.kt index a6ad98fa909c5..e60a366934f50 100644 --- a/model/src/test/kotlin/config/OrtConfigurationTest.kt +++ b/model/src/test/kotlin/config/OrtConfigurationTest.kt @@ -37,6 +37,7 @@ import io.kotest.matchers.types.shouldBeInstanceOf import java.io.File import java.lang.IllegalArgumentException +import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.SourceCodeOrigin import org.ossreviewtoolkit.utils.common.EnvironmentVariableFilter import org.ossreviewtoolkit.utils.test.createTestTempFile @@ -48,14 +49,59 @@ class OrtConfigurationTest : WordSpec({ val refConfig = File("src/main/resources/$REFERENCE_CONFIG_FILENAME") val ortConfig = OrtConfiguration.load(file = refConfig) + ortConfig.allowedProcessEnvironmentVariableNames should containExactlyInAnyOrder("PASSPORT", "USER_HOME") ortConfig.deniedProcessEnvironmentVariablesSubstrings should containExactlyInAnyOrder( "PASS", "SECRET", "TOKEN", "USER" ) - ortConfig.allowedProcessEnvironmentVariableNames should containExactlyInAnyOrder("PASSPORT", "USER_HOME") + + ortConfig.enableRepositoryPackageConfigurations shouldBe true + ortConfig.enableRepositoryPackageCurations shouldBe true + + with(ortConfig.licenseFilePatterns) { + licenseFilenames shouldContainExactly listOf("license*") + patentFilenames shouldContainExactly listOf("patents") + rootLicenseFilenames shouldContainExactly listOf("readme*") + } + + ortConfig.packageCurationProviders should containExactly( + PackageCurationProviderConfiguration(name = "DefaultFile"), + PackageCurationProviderConfiguration(name = "DefaultDir"), + PackageCurationProviderConfiguration( + name = "File", + config = mapOf("path" to "/some-path/curations.yml") + ), + PackageCurationProviderConfiguration( + name = "File", + config = mapOf("path" to "/some-path/curations-dir") + ), + PackageCurationProviderConfiguration(name = "OrtConfig", enabled = true), + PackageCurationProviderConfiguration( + name = "ClearlyDefined", + config = mapOf("serverUrl" to "https://api.clearlydefined.io") + ), + PackageCurationProviderConfiguration( + name = "SW360", + config = mapOf( + "restUrl" to "https://your-sw360-rest-url", + "authUrl" to "https://your-authentication-url", + "username" to "username", + "password" to "password", + "clientId" to "clientId", + "clientPassword" to "clientPassword", + "token" to "token" + ) + ), + ) + + ortConfig.severeIssueThreshold shouldBe Severity.ERROR + ortConfig.severeRuleViolationThreshold shouldBe Severity.ERROR with(ortConfig.analyzer) { allowDynamicVersions shouldBe true + enabledPackageManagers shouldContainExactlyInAnyOrder listOf("DotNet", "Gradle") + disabledPackageManagers shouldContainExactlyInAnyOrder listOf("Maven", "NPM") + packageManagers shouldNotBeNull { get("Gradle") shouldNotBeNull { mustRunAfter should containExactly("NPM") @@ -67,31 +113,47 @@ class OrtConfigurationTest : WordSpec({ } } + get("Yarn2") shouldNotBeNull { + options shouldNotBeNull { + this shouldContainExactly mapOf("disableRegistryCertificateVerification" to "false") + } + } + getPackageManagerConfiguration("gradle") shouldNotBeNull { this shouldBe get("Gradle") } } - - sw360Configuration shouldNotBeNull { - restUrl shouldBe "https://your-sw360-rest-url" - authUrl shouldBe "https://your-authentication-url" - username shouldBe "username" - password shouldBe "password" - clientId shouldBe "clientId" - clientPassword shouldBe "clientPassword" - token shouldBe "token" - } } with(ortConfig.advisor) { nexusIq shouldNotBeNull { serverUrl shouldBe "https://rest-api-url-of-your-nexus-iq-server" + browseUrl shouldBe "https://web-browsing-url-of-your-nexus-iq-server" username shouldBe "username" password shouldBe "password" } vulnerableCode shouldNotBeNull { serverUrl shouldBe "http://localhost:8000" + apiKey shouldBe "0123456789012345678901234567890123456789" + } + + gitHubDefects shouldNotBeNull { + token shouldBe "githubAccessToken" + labelFilter shouldContainExactlyInAnyOrder listOf( + "!duplicate", + "!enhancement", + "!invalid", + "!question", + "!documentation", + "*" + ) + maxNumberOfIssuesPerRepository shouldBe 50 + parallelRequests shouldBe 5 + } + + osv shouldNotBeNull { + serverUrl shouldBe "https://api.osv.dev" } options shouldNotBeNull { @@ -99,13 +161,18 @@ class OrtConfigurationTest : WordSpec({ } } - ortConfig.downloader shouldNotBeNull { + with(ortConfig.downloader) { + allowMovingRevisions shouldBe true includedLicenseCategories should containExactly("category-a", "category-b") sourceCodeOrigins should containExactly(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT) } with(ortConfig.scanner) { + skipConcluded shouldBe true + archive shouldNotBeNull { + enabled shouldBe true + fileStorage shouldNotBeNull { httpFileStorage should beNull() localFileStorage shouldNotBeNull { @@ -129,22 +196,71 @@ class OrtConfigurationTest : WordSpec({ } } + createMissingArchives shouldBe false + + detectedLicenseMapping shouldContainExactly mapOf( + "BSD (Three Clause License)" to "BSD-3-clause", + "LicenseRef-scancode-generic-cla" to "NOASSERTION" + ) + + options shouldNotBeNull { + get("ScanCode") shouldNotBeNull { + this shouldContainExactly mapOf( + "commandLine" to "--copyright --license --info --strip-root --timeout 300", + "parseLicenseExpressions" to "true", + "minVersion" to "3.2.1-rc2", + "maxVersion" to "32.0.0" + ) + } + + get("FossId") shouldNotBeNull { + val urlMapping = "https://my-repo.example.org(?.*) -> " + + "ssh://my-mapped-repo.example.org\${repoPath}" + + this shouldContainExactly mapOf( + "serverUrl" to "https://fossid.example.com/instance/", + "user" to "user", + "apiKey" to "XYZ", + "namingProjectPattern" to "\$Var1_\$Var2", + "namingScanPattern" to "\$Var1_#projectBaseCode_\$Var3", + "namingVariableVar1" to "myOrg", + "namingVariableVar2" to "myTeam", + "namingVariableVar3" to "myUnit", + "waitForResult" to "false", + "keepFailedScans" to "false", + "deltaScans" to "true", + "deltaScanLimit" to "10", + "detectLicenseDeclarations" to "true", + "detectCopyrightStatements" to "true", + "timeout" to "60", + "urlMappingExample" to urlMapping + ) + } + } + storages shouldNotBeNull { keys shouldContainExactlyInAnyOrder setOf( "local", "http", "clearlyDefined", "postgres", "sw360Configuration" ) + + val localStorage = this["local"] + localStorage.shouldBeInstanceOf() + localStorage.backend.localFileStorage shouldNotBeNull { + directory shouldBe File("~/.ort/scanner/results") + compression shouldBe false + } + val httpStorage = this["http"] httpStorage.shouldBeInstanceOf() httpStorage.backend.httpFileStorage shouldNotBeNull { url shouldBe "https://your-http-server" + query shouldBe "?username=user&password=123" headers should containExactlyEntries("key1" to "value1", "key2" to "value2") } - val localStorage = this["local"] - localStorage.shouldBeInstanceOf() - localStorage.backend.localFileStorage shouldNotBeNull { - directory shouldBe File("~/.ort/scanner/results") - } + val cdStorage = this["clearlyDefined"] + cdStorage.shouldBeInstanceOf() + cdStorage.serverUrl shouldBe "https://api.clearlydefined.io" val postgresStorage = this["postgres"] postgresStorage.shouldBeInstanceOf() @@ -158,10 +274,7 @@ class OrtConfigurationTest : WordSpec({ sslkey shouldBe "/defaultdir/postgresql.pk8" sslrootcert shouldBe "/defaultdir/root.crt" } - - val cdStorage = this["clearlyDefined"] - cdStorage.shouldBeInstanceOf() - cdStorage.serverUrl shouldBe "https://api.clearlydefined.io" + postgresStorage.type shouldBe StorageType.PROVENANCE_BASED val sw360Storage = this["sw360Configuration"] sw360Storage.shouldBeInstanceOf() @@ -174,13 +287,6 @@ class OrtConfigurationTest : WordSpec({ sw360Storage.token shouldBe "token" } - options shouldNotBeNull { - val fossIdOptions = getValue("FossId") - val mapping = "https://my-repo.example.org(?.*) -> " + - "ssh://my-mapped-repo.example.org\${repoPath}" - fossIdOptions["urlMappingExample"] shouldBe mapping - } - storageReaders shouldContainExactly listOf("local", "postgres", "http", "clearlyDefined") storageWriters shouldContainExactly listOf("postgres") @@ -191,6 +297,7 @@ class OrtConfigurationTest : WordSpec({ httpFileStorage should beNull() localFileStorage shouldNotBeNull { directory shouldBe File("~/.ort/scanner/provenance") + compression shouldBe false } } @@ -209,6 +316,20 @@ class OrtConfigurationTest : WordSpec({ } } + with(ortConfig.reporter) { + options shouldNotBeNull { + keys shouldContainExactlyInAnyOrder setOf("FossId") + + get("FossId") shouldNotBeNull { + this shouldContainExactly mapOf( + "serverUrl" to "https://fossid.example.com/instance/", + "user" to "user", + "apiKey" to "XYZ" + ) + } + } + } + with(ortConfig.notifier) { mail shouldNotBeNull { hostName shouldBe "localhost" @@ -225,15 +346,6 @@ class OrtConfigurationTest : WordSpec({ password shouldBe "password" } } - - with(ortConfig.licenseFilePatterns) { - licenseFilenames shouldContainExactly listOf("license*") - patentFilenames shouldContainExactly listOf("patents") - rootLicenseFilenames shouldContainExactly listOf("readme*") - } - - ortConfig.enableRepositoryPackageCurations shouldBe true - ortConfig.enableRepositoryPackageConfigurations shouldBe true } "correctly prioritize the sources" { diff --git a/utils/common/src/main/kotlin/PluginManager.kt b/utils/common/src/main/kotlin/PluginManager.kt index 4932508e6b188..c670410052c44 100644 --- a/utils/common/src/main/kotlin/PluginManager.kt +++ b/utils/common/src/main/kotlin/PluginManager.kt @@ -47,3 +47,14 @@ interface NamedPlugin { */ val name: String } + +/** + * An interface to be implemented by plugin factories. Plugin factories are required if a plugin needs configuration + * on initialization and can therefore not be created directly by the [ServiceLoader]. + */ +interface ConfigurablePluginFactory : NamedPlugin { + /** + * Create a new instance of [PLUGIN] from [config]. + */ + fun create(config: Map): PLUGIN +}