diff --git a/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt b/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt index dcf2cfb6d5456..64d6ac544f33b 100644 --- a/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt +++ b/plugins/package-curation-providers/clearly-defined/src/main/kotlin/ClearlyDefinedPackageCurationProvider.kt @@ -47,8 +47,8 @@ import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.ort.OkHttpClientHelper import org.ossreviewtoolkit.utils.ort.showStackTrace -import org.ossreviewtoolkit.utils.spdx.SpdxExpression -import org.ossreviewtoolkit.utils.spdx.toSpdx +import org.ossreviewtoolkit.utils.spdx.SpdxExpression.Strictness +import org.ossreviewtoolkit.utils.spdx.toSpdxOrNull import retrofit2.HttpException @@ -149,12 +149,9 @@ class ClearlyDefinedPackageCurationProvider( filteredCurations.forEach inner@{ (coordinates, curation) -> val pkgId = coordinatesToIds[coordinates] ?: return@inner - val declaredLicenseParsed = curation.licensed?.declared?.let { declaredLicense -> - // Only take curations of good quality (i.e. those not using deprecated identifiers) and in - // particular none that contain "OTHER" as a license, also see - // https://github.com/clearlydefined/curated-data/issues/7836. - runCatching { declaredLicense.toSpdx(SpdxExpression.Strictness.ALLOW_CURRENT) }.getOrNull() - } + // Only take curations of good quality (i.e. those not using deprecated identifiers) and in particular none + // that contain "OTHER" as a license, also see https://github.com/clearlydefined/curated-data/issues/7836. + val declaredLicenseParsed = curation.licensed?.declared?.toSpdxOrNull(Strictness.ALLOW_CURRENT) val sourceLocation = curation.described?.sourceLocation?.toArtifactOrVcs() diff --git a/utils/ort/src/main/kotlin/DeclaredLicenseProcessor.kt b/utils/ort/src/main/kotlin/DeclaredLicenseProcessor.kt index bcbba8a6e0097..be64fe341df4f 100644 --- a/utils/ort/src/main/kotlin/DeclaredLicenseProcessor.kt +++ b/utils/ort/src/main/kotlin/DeclaredLicenseProcessor.kt @@ -24,17 +24,14 @@ import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonPropertyOrder import com.fasterxml.jackson.databind.annotation.JsonSerialize -import org.apache.logging.log4j.kotlin.logger - import org.ossreviewtoolkit.utils.common.StringSortedSetConverter -import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.unquote import org.ossreviewtoolkit.utils.spdx.SpdxCompoundExpression import org.ossreviewtoolkit.utils.spdx.SpdxConstants import org.ossreviewtoolkit.utils.spdx.SpdxDeclaredLicenseMapping import org.ossreviewtoolkit.utils.spdx.SpdxExpression import org.ossreviewtoolkit.utils.spdx.SpdxOperator -import org.ossreviewtoolkit.utils.spdx.toSpdx +import org.ossreviewtoolkit.utils.spdx.toSpdxOrNull object DeclaredLicenseProcessor { private val urlPrefixesToRemove = listOf( @@ -92,9 +89,9 @@ object DeclaredLicenseProcessor { ?: SpdxDeclaredLicenseMapping.map(strippedLicense) ?: SpdxDeclaredLicenseMapping.map(strippedLicense.unquote()) ?: SpdxDeclaredLicenseMapping.map(strippedLicense.removePrefix(SpdxConstants.TAG).trim()) - ?: parseLicense(strippedLicense) - return mappedLicense?.normalize()?.takeIf { it.isValid() || it.toString() == SpdxConstants.NONE } + val processedLicense = mappedLicense ?: strippedLicense.toSpdxOrNull() + return processedLicense?.normalize()?.takeIf { it.isValid() || it.toString() == SpdxConstants.NONE } } /** @@ -133,13 +130,6 @@ object DeclaredLicenseProcessor { return ProcessedDeclaredLicense(spdxExpression, mapped, unmapped) } - - private fun parseLicense(declaredLicense: String) = - runCatching { - declaredLicense.toSpdx() - }.onFailure { - logger.debug { "Could not parse declared license '$declaredLicense': ${it.collectMessages()}" } - }.getOrNull() } data class ProcessedDeclaredLicense( diff --git a/utils/spdx/src/main/kotlin/Extensions.kt b/utils/spdx/src/main/kotlin/Extensions.kt index 022dacc2c32ee..a85d23c9036d5 100644 --- a/utils/spdx/src/main/kotlin/Extensions.kt +++ b/utils/spdx/src/main/kotlin/Extensions.kt @@ -17,10 +17,19 @@ * License-Filename: LICENSE */ +@file:Suppress("TooManyFunctions") + package org.ossreviewtoolkit.utils.spdx +import java.lang.invoke.MethodHandles + +import org.apache.logging.log4j.kotlin.loggerOf + +import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.spdx.SpdxExpression.Strictness +private val logger = loggerOf(MethodHandles.lookup().lookupClass()) + /** * Create an [SpdxExpression] by concatenating [this][SpdxLicense] and [other] using [SpdxOperator.AND]. */ @@ -79,12 +88,23 @@ fun String.isSpdxExpressionOrNotPresent(strictness: Strictness = Strictness.ALLO SpdxConstants.isNotPresent(this) || isSpdxExpression(strictness) /** - * Parses the string as an [SpdxExpression] of the given [strictness] and returns the result on success, or throws an + * Parse this string as an [SpdxExpression] of the given [strictness] and return the result on success, or throw an * [SpdxException] if the string cannot be parsed. */ fun String.toSpdx(strictness: Strictness = Strictness.ALLOW_ANY): SpdxExpression = SpdxExpression.parse(this, strictness) +/** + * Parse this string as an [SpdxExpression] of the given [strictness] and return the result on success, or null if this + * string cannot be parsed. + */ +fun String.toSpdxOrNull(strictness: Strictness = Strictness.ALLOW_ANY): SpdxExpression? = + runCatching { + toSpdx(strictness) + }.onFailure { + logger.debug { "Could not parse '$this' as an SPDX license: ${it.collectMessages()}" } + }.getOrNull() + /** * Convert a [String] to an SPDX "idstring" (like license IDs, package IDs, etc.) which may only contain letters, * numbers, ".", and / or "-". If [allowPlusSuffix] is enabled, a "+" (as used in license IDs) is kept as the suffix. diff --git a/utils/spdx/src/main/kotlin/SpdxExpression.kt b/utils/spdx/src/main/kotlin/SpdxExpression.kt index 778a7e9e1f0ab..341a6ecc20048 100644 --- a/utils/spdx/src/main/kotlin/SpdxExpression.kt +++ b/utils/spdx/src/main/kotlin/SpdxExpression.kt @@ -107,10 +107,9 @@ sealed class SpdxExpression { } /** - * Normalize all license IDs using a mapping containing common misspellings of license IDs. If [mapDeprecated] is - * `true`, also deprecated IDs are mapped to their current counterparts. The result of this function is not - * guaranteed to contain only valid IDs. Use [validate] to check the returned [SpdxExpression] for validity - * afterwards. + * Normalize all license IDs using [SpdxSimpleLicenseMapping]. If [mapDeprecated] is `true`, also deprecated IDs are + * mapped to their current counterparts. The result of this function is not guaranteed to contain only valid IDs. + * Use [validate] or [isValid] to check the returned [SpdxExpression] for validity afterwards. */ abstract fun normalize(mapDeprecated: Boolean = true): SpdxExpression diff --git a/utils/spdx/src/main/kotlin/SpdxSimpleLicenseMapping.kt b/utils/spdx/src/main/kotlin/SpdxSimpleLicenseMapping.kt index bc5a45f3cb795..e8a35c72e1eff 100644 --- a/utils/spdx/src/main/kotlin/SpdxSimpleLicenseMapping.kt +++ b/utils/spdx/src/main/kotlin/SpdxSimpleLicenseMapping.kt @@ -23,9 +23,8 @@ import com.fasterxml.jackson.module.kotlin.readValue /** * A mapping from simple license names to valid SPDX license IDs. This mapping only contains license strings which *can* - * be parsed by [SpdxExpression.parse] but have a corresponding valid SPDX license ID that should be used instead. When - * mapping a name without any indication of a version to an ID with a version, the most commonly used version at the - * time of writing is used. See [SpdxDeclaredLicenseMapping] for a mapping of unparsable license strings. + * be parsed by [SpdxExpression.parse] but have a corresponding valid SPDX license ID that should be used instead. See + * [SpdxDeclaredLicenseMapping] for a mapping of unparsable license strings. */ object SpdxSimpleLicenseMapping { /** diff --git a/utils/spdx/src/main/resources/declared-license-mapping.yml b/utils/spdx/src/main/resources/declared-license-mapping.yml index 352bc05755939..c62bf57579653 100644 --- a/utils/spdx/src/main/resources/declared-license-mapping.yml +++ b/utils/spdx/src/main/resources/declared-license-mapping.yml @@ -15,6 +15,10 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE +# A mapping from license strings collected from the declared licenses of Open Source packages to SPDX expressions. This +# mapping only contains license strings which can *not* be parsed by [SpdxExpression.parse], for example because the +# license names contain white spaces. See [SpdxSimpleLicenseMapping] for a mapping of varied license names. + # Sort the entries below via IntelliJ's "Edit" -> "Sort Lines". # Map a declared license string to "NONE" in order to discard it. --- diff --git a/utils/spdx/src/main/resources/simple-license-mapping.yml b/utils/spdx/src/main/resources/simple-license-mapping.yml index bac7dfd47e5e2..15bcc1d6a5931 100644 --- a/utils/spdx/src/main/resources/simple-license-mapping.yml +++ b/utils/spdx/src/main/resources/simple-license-mapping.yml @@ -15,6 +15,10 @@ # SPDX-License-Identifier: Apache-2.0 # License-Filename: LICENSE +# A mapping from simple license names to valid SPDX license IDs. This mapping only contains license strings which *can* +# be parsed by [SpdxExpression.parse] but have a corresponding valid SPDX license ID that should be used instead. See +# [SpdxDeclaredLicenseMapping] for a mapping of unparsable license strings. + # Sort the entries below via IntelliJ's "Edit" -> "Sort Lines". ---