diff --git a/model/src/main/kotlin/config/PackageConfiguration.kt b/model/src/main/kotlin/config/PackageConfiguration.kt index 5f9137e3069e1..e6393e4a4678d 100644 --- a/model/src/main/kotlin/config/PackageConfiguration.kt +++ b/model/src/main/kotlin/config/PackageConfiguration.kt @@ -73,6 +73,12 @@ data class PackageConfiguration( @JsonInclude(JsonInclude.Include.NON_EMPTY) val pathExcludes: List = emptyList(), + /** + * Path includes. + */ + @JsonInclude(JsonInclude.Include.NON_EMPTY) + val pathIncludes: List = emptyList(), + /** * License finding curations. */ diff --git a/model/src/main/kotlin/licenses/DefaultLicenseInfoProvider.kt b/model/src/main/kotlin/licenses/DefaultLicenseInfoProvider.kt index 8125fb0d1d267..6d6530ceac6a5 100644 --- a/model/src/main/kotlin/licenses/DefaultLicenseInfoProvider.kt +++ b/model/src/main/kotlin/licenses/DefaultLicenseInfoProvider.kt @@ -27,6 +27,7 @@ import org.ossreviewtoolkit.model.OrtResult import org.ossreviewtoolkit.model.Provenance import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.PathInclude import org.ossreviewtoolkit.model.utils.filterByVcsPath import org.ossreviewtoolkit.utils.ort.ProcessedDeclaredLicense @@ -92,6 +93,7 @@ class DefaultLicenseInfoProvider(val ortResult: OrtResult) : LicenseInfoProvider copyrights = it.summary.copyrightFindings, licenseFindingCurations = config.licenseFindingCurations, pathExcludes = config.pathExcludes, + pathIncludes = config.pathIncludes, relativeFindingsPath = config.relativeFindingsPath ) } @@ -104,12 +106,14 @@ class DefaultLicenseInfoProvider(val ortResult: OrtResult) : LicenseInfoProvider Configuration( ortResult.repository.config.curations.licenseFindings, ortResult.repository.config.excludes.paths, + ortResult.repository.config.includes.paths, ortResult.repository.getRelativePath(project.vcsProcessed).orEmpty() ) } ?: ortResult.getPackageConfigurations(id, provenance).let { packageConfigurations -> Configuration( packageConfigurations.flatMap { it.licenseFindingCurations }, packageConfigurations.flatMap { it.pathExcludes }, + packageConfigurations.flatMap { it.pathIncludes }, "" ) } @@ -118,5 +122,6 @@ class DefaultLicenseInfoProvider(val ortResult: OrtResult) : LicenseInfoProvider private data class Configuration( val licenseFindingCurations: List, val pathExcludes: List, + val pathIncludes: List, val relativeFindingsPath: String ) diff --git a/model/src/main/kotlin/licenses/LicenseInfo.kt b/model/src/main/kotlin/licenses/LicenseInfo.kt index 8b9510003743a..c9316bb4b0908 100644 --- a/model/src/main/kotlin/licenses/LicenseInfo.kt +++ b/model/src/main/kotlin/licenses/LicenseInfo.kt @@ -27,6 +27,7 @@ import org.ossreviewtoolkit.model.Provenance import org.ossreviewtoolkit.model.Repository import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.PathInclude import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.utils.ort.ProcessedDeclaredLicense import org.ossreviewtoolkit.utils.spdx.SpdxExpression @@ -136,6 +137,11 @@ data class Findings( */ val pathExcludes: List, + /** + * The list of all path includes that apply to this [provenance]. + */ + val pathIncludes: List, + /** * The root path of the locations of the [licenses] and [copyrights] relative to the paths used in the * [licenseFindingCurations] and [pathExcludes]. An empty string, if all refer to the same root path. diff --git a/model/src/main/kotlin/licenses/LicenseInfoResolver.kt b/model/src/main/kotlin/licenses/LicenseInfoResolver.kt index 228de03dd3941..a97a9b72f3783 100644 --- a/model/src/main/kotlin/licenses/LicenseInfoResolver.kt +++ b/model/src/main/kotlin/licenses/LicenseInfoResolver.kt @@ -32,6 +32,7 @@ import org.ossreviewtoolkit.model.UnknownProvenance import org.ossreviewtoolkit.model.config.CopyrightGarbage import org.ossreviewtoolkit.model.config.LicenseFilePatterns import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.PathInclude import org.ossreviewtoolkit.model.utils.FileArchiver import org.ossreviewtoolkit.model.utils.FindingCurationMatcher import org.ossreviewtoolkit.model.utils.FindingsMatcher @@ -123,9 +124,16 @@ class LicenseInfoResolver( ).mapNotNull { curationResult -> val licenseFinding = curationResult.curatedFinding ?: return@mapNotNull null - licenseFinding.license to findings.pathExcludes.any { pathExclude -> + val isExcludedByPathIncludes = licenseFinding.location.isExcludedByPathIncludes( + findings.pathIncludes, + findings.relativeFindingsPath + ) + + val isMatchedByPathExcludes = findings.pathExcludes.any { pathExclude -> pathExclude.matches(licenseFinding.location.prependedPath(findings.relativeFindingsPath)) } + + licenseFinding.license to (isMatchedByPathExcludes || isExcludedByPathIncludes) } }.groupBy(keySelector = { it.first }, valueTransform = { it.second }).mapValues { (_, excluded) -> excluded.all { it } @@ -190,6 +198,7 @@ class LicenseInfoResolver( val resolvedCopyrightFindings = resolveCopyrights( copyrightFindings, findings.pathExcludes, + findings.pathIncludes, findings.relativeFindingsPath ) @@ -204,12 +213,18 @@ class LicenseInfoResolver( it.matches(licenseFinding.location.prependedPath(findings.relativeFindingsPath)) } + val isExcludedByPathIncludes = licenseFinding.location.isExcludedByPathIncludes( + findings.pathIncludes, + findings.relativeFindingsPath + ) + licenseFinding.license.decompose().forEach { singleLicense -> resolvedLocations.getOrPut(singleLicense) { mutableSetOf() } += ResolvedLicenseLocation( findings.provenance, licenseFinding.location, appliedCuration = appliedCuration, matchingPathExcludes = matchingPathExcludes, + isExcludedByPathIncludes = isExcludedByPathIncludes, copyrights = resolvedCopyrightFindings ) } @@ -218,6 +233,7 @@ class LicenseInfoResolver( unmatchedCopyrights.getOrPut(findings.provenance) { mutableSetOf() } += resolveCopyrights( copyrightFindings = matchResult.unmatchedCopyrights, pathExcludes = findings.pathExcludes, + pathIncludes = findings.pathIncludes, relativeFindingsPath = findings.relativeFindingsPath ) } @@ -228,6 +244,7 @@ class LicenseInfoResolver( private fun resolveCopyrights( copyrightFindings: Set, pathExcludes: List, + pathIncludes: List, relativeFindingsPath: String ): Set = copyrightFindings.mapTo(mutableSetOf()) { finding -> @@ -235,7 +252,17 @@ class LicenseInfoResolver( it.matches(finding.location.prependedPath(relativeFindingsPath)) } - ResolvedCopyrightFinding(finding.statement, finding.location, matchingPathExcludes) + val isExcludedByPathIncludes = finding.location.isExcludedByPathIncludes( + pathIncludes, + relativeFindingsPath + ) + + ResolvedCopyrightFinding( + finding.statement, + finding.location, + matchingPathExcludes, + isExcludedByPathIncludes + ) } private fun createLicenseFileInfo(id: Identifier): ResolvedLicenseFileInfo { @@ -290,6 +317,7 @@ class LicenseInfoResolver( location = UNDEFINED_TEXT_LOCATION, appliedCuration = null, matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false, copyrights = authors.mapTo(mutableSetOf()) { author -> val statement = "Copyright (C) $author".takeUnless { author.contains("Copyright", ignoreCase = true) @@ -298,10 +326,23 @@ class LicenseInfoResolver( ResolvedCopyrightFinding( statement = statement, location = UNDEFINED_TEXT_LOCATION, - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ) } ) + + /** + * Return true if the [TextLocation] is excluded because some path includes are defined and none of them matches + * this [TextLocation] prepended with the [relativeFindingsPath]. + */ + private fun TextLocation.isExcludedByPathIncludes(pathIncludes: List, relativeFindingsPath: String) = + when { + pathIncludes.isEmpty() -> false + else -> pathIncludes.none { + it.matches(prependedPath(relativeFindingsPath)) + } + } } private class ResolvedLicenseBuilder(val license: SpdxSingleLicenseExpression) { diff --git a/model/src/main/kotlin/licenses/ResolvedCopyrightFinding.kt b/model/src/main/kotlin/licenses/ResolvedCopyrightFinding.kt index 31bed47e63c51..6e4b58e612ee6 100644 --- a/model/src/main/kotlin/licenses/ResolvedCopyrightFinding.kt +++ b/model/src/main/kotlin/licenses/ResolvedCopyrightFinding.kt @@ -39,5 +39,10 @@ data class ResolvedCopyrightFinding( /** * All [PathExclude]s matching this [location]. */ - val matchingPathExcludes: List + val matchingPathExcludes: List, + + /** + * If true, some includes are defines and are not matching this [location]. + */ + val isExcludedByPathIncludes: Boolean ) diff --git a/model/src/main/kotlin/licenses/ResolvedLicense.kt b/model/src/main/kotlin/licenses/ResolvedLicense.kt index e3d653edad5ef..2d44a1251ac1c 100644 --- a/model/src/main/kotlin/licenses/ResolvedLicense.kt +++ b/model/src/main/kotlin/licenses/ResolvedLicense.kt @@ -60,7 +60,9 @@ data class ResolvedLicense( * True, if this license was [detected][LicenseSource.DETECTED] and all [locations] have matching path excludes. */ val isDetectedExcluded by lazy { - LicenseSource.DETECTED in sources && locations.all { it.matchingPathExcludes.isNotEmpty() } + LicenseSource.DETECTED in sources && locations.all { + it.matchingPathExcludes.isNotEmpty() || it.isExcludedByPathIncludes + } } init { @@ -95,13 +97,16 @@ data class ResolvedLicense( /** * Filter all excluded copyrights. Copyrights which have - * [matching path excludes][ResolvedCopyrightFinding.matchingPathExcludes] are removed. + * [matching path excludes][ResolvedCopyrightFinding.matchingPathExcludes] or for which [no include are defined] + * [ResolvedCopyrightFinding.isExcludedByPathIncludes] are removed. */ fun filterExcludedCopyrights(): ResolvedLicense = copy( locations = locations.mapTo(mutableSetOf()) { location -> location.copy( - copyrights = location.copyrights.filterTo(mutableSetOf()) { it.matchingPathExcludes.isEmpty() } + copyrights = location.copyrights.filterTo(mutableSetOf()) { + it.matchingPathExcludes.isEmpty() || it.isExcludedByPathIncludes + } ) } ) diff --git a/model/src/main/kotlin/licenses/ResolvedLicenseLocation.kt b/model/src/main/kotlin/licenses/ResolvedLicenseLocation.kt index 52f4c4799c133..f72fca63c97df 100644 --- a/model/src/main/kotlin/licenses/ResolvedLicenseLocation.kt +++ b/model/src/main/kotlin/licenses/ResolvedLicenseLocation.kt @@ -48,6 +48,11 @@ data class ResolvedLicenseLocation( */ val matchingPathExcludes: List, + /** + * If true, some includes are defines and are not matching this [location]. + */ + val isExcludedByPathIncludes: Boolean, + /** * All copyright findings associated to this license location, excluding copyright garbage. */ diff --git a/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt b/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt index aca740de41388..e4ed82cd973f4 100644 --- a/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt +++ b/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt @@ -53,6 +53,8 @@ import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.LicenseFindingCurationReason import org.ossreviewtoolkit.model.config.PathExclude import org.ossreviewtoolkit.model.config.PathExcludeReason +import org.ossreviewtoolkit.model.config.PathInclude +import org.ossreviewtoolkit.model.config.PathIncludeReason import org.ossreviewtoolkit.model.declaredLicenses import org.ossreviewtoolkit.model.utils.FileArchiver import org.ossreviewtoolkit.model.utils.FileProvenanceFileStorage @@ -68,6 +70,9 @@ import org.ossreviewtoolkit.utils.test.createDefault import org.ossreviewtoolkit.utils.test.transformingCollectionEmptyMatcher import org.ossreviewtoolkit.utils.test.transformingCollectionMatcher +const val COPYRIGHT = "(c) 2010 Holder" + +@Suppress("LargeClass") class LicenseInfoResolverTest : WordSpec({ val pkgId = Identifier("Gradle:org.ossreviewtoolkit:ort:1.0.0") val vcsInfo = VcsInfo( @@ -122,6 +127,7 @@ class LicenseInfoResolverTest : WordSpec({ ), licenseFindingCurations = emptyList(), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) @@ -144,7 +150,8 @@ class LicenseInfoResolverTest : WordSpec({ ResolvedCopyrightFinding( statement = "Copyright Apache-2.0", location = TextLocation("LICENSE", 1), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ) ) ) @@ -162,7 +169,8 @@ class LicenseInfoResolverTest : WordSpec({ ResolvedCopyrightFinding( statement = "Copyright MIT", location = TextLocation("LICENSE", 31), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ) ) ) @@ -218,6 +226,7 @@ class LicenseInfoResolverTest : WordSpec({ ), licenseFindingCurations = emptyList(), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) @@ -251,12 +260,13 @@ class LicenseInfoResolverTest : WordSpec({ ).toFindingsSet(), copyrights = setOf( CopyrightFinding("(c) 2009 Holder", TextLocation("LICENSE", 1)), - CopyrightFinding("(c) 2010 Holder", TextLocation("LICENSE", 2)), + CopyrightFinding(COPYRIGHT, TextLocation("LICENSE", 2)), CopyrightFinding("(c) 2011 Holder", TextLocation("LICENSE", 50)), CopyrightFinding("(c) 2012 Holder", TextLocation("LICENSE", 51)) ), licenseFindingCurations = emptyList(), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) @@ -270,7 +280,7 @@ class LicenseInfoResolverTest : WordSpec({ result should containCopyrightStatementsForLicenseExactly( "Apache-2.0", "(c) 2009 Holder", - "(c) 2010 Holder" + COPYRIGHT ) result should containCopyrightStatementsForLicenseExactly( "MIT", @@ -311,6 +321,7 @@ class LicenseInfoResolverTest : WordSpec({ ), licenseFindingCurations = emptyList(), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) @@ -367,11 +378,12 @@ class LicenseInfoResolverTest : WordSpec({ ) ).toFindingsSet(), copyrights = setOf( - CopyrightFinding("(c) 2010 Holder", TextLocation("LICENSE", 1)), - CopyrightFinding("(c) 2010 Holder", TextLocation("a/b", 1)) + CopyrightFinding(COPYRIGHT, TextLocation("LICENSE", 1)), + CopyrightFinding(COPYRIGHT, TextLocation("a/b", 1)) ), licenseFindingCurations = emptyList(), pathExcludes = listOf(vcsPathExclude), + pathIncludes = emptyList(), relativeFindingsPath = "" ), Findings( @@ -383,11 +395,12 @@ class LicenseInfoResolverTest : WordSpec({ ) ).toFindingsSet(), copyrights = setOf( - CopyrightFinding("(c) 2010 Holder", TextLocation("LICENSE", 1)), - CopyrightFinding("(c) 2010 Holder", TextLocation("a/b", 1)) + CopyrightFinding(COPYRIGHT, TextLocation("LICENSE", 1)), + CopyrightFinding(COPYRIGHT, TextLocation("a/b", 1)) ), licenseFindingCurations = emptyList(), pathExcludes = listOf(sourceArtifactPathExclude), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) @@ -412,16 +425,16 @@ class LicenseInfoResolverTest : WordSpec({ ) should beEmpty() result.pathExcludesForCopyright( - "(c) 2010 Holder", provenance, TextLocation("LICENSE", 1) + COPYRIGHT, provenance, TextLocation("LICENSE", 1) ) should beEmpty() result.pathExcludesForCopyright( - "(c) 2010 Holder", provenance, TextLocation("a/b", 1) + COPYRIGHT, provenance, TextLocation("a/b", 1) ) should containExactly(vcsPathExclude) result.pathExcludesForCopyright( - "(c) 2010 Holder", sourceArtifactProvenance, TextLocation("LICENSE", 1) + COPYRIGHT, sourceArtifactProvenance, TextLocation("LICENSE", 1) ) should containExactly(sourceArtifactPathExclude) result.pathExcludesForCopyright( - "(c) 2010 Holder", sourceArtifactProvenance, TextLocation("a/b", 1) + COPYRIGHT, sourceArtifactProvenance, TextLocation("a/b", 1) ) should beEmpty() result.licenses.flatMap { resolvedLicense -> @@ -432,6 +445,142 @@ class LicenseInfoResolverTest : WordSpec({ ) } + "apply path includes" { + val sourceArtifact = RemoteArtifact( + url = "https://example.com/", + hash = Hash.NONE + ) + val sourceArtifactProvenance = ArtifactProvenance( + sourceArtifact = sourceArtifact + ) + val sourceArtifactPathInclude = PathInclude( + pattern = "LICENSE", + reason = PathIncludeReason.SOURCE_OF + ) + val vcsPathInclude = PathInclude( + pattern = "a/b", + reason = PathIncludeReason.SOURCE_OF + ) + + val licenseInfos = listOf( + createLicenseInfo( + id = pkgId, + detectedLicenses = listOf( + Findings( + provenance = provenance, + licenses = mapOf( + "Apache-2.0" to listOf( + TextLocation("LICENSE", 1), + TextLocation("a/b", 1) + ), + "MIT" to listOf( + TextLocation("c/d", 4) + ) + ).toFindingsSet(), + copyrights = setOf( + CopyrightFinding(COPYRIGHT, TextLocation("LICENSE", 1)), + CopyrightFinding(COPYRIGHT, TextLocation("a/b", 1)) + ), + licenseFindingCurations = emptyList(), + pathExcludes = emptyList(), + pathIncludes = listOf(vcsPathInclude), + relativeFindingsPath = "" + ), + Findings( + provenance = sourceArtifactProvenance, + licenses = mapOf( + "Apache-2.0" to listOf( + TextLocation("LICENSE", 1), + TextLocation("a/b", 1) + ), + "MIT" to listOf( + TextLocation("c/d", 4) + ) + ).toFindingsSet(), + copyrights = setOf( + CopyrightFinding(COPYRIGHT, TextLocation("LICENSE", 1)), + CopyrightFinding(COPYRIGHT, TextLocation("a/b", 1)) + ), + licenseFindingCurations = emptyList(), + pathExcludes = emptyList(), + pathIncludes = listOf(sourceArtifactPathInclude), + relativeFindingsPath = "" + ) + ) + ) + ) + + val resolver = createResolver(licenseInfos) + + val result = resolver.resolveLicenseInfo(pkgId) + + result.isExcludedByPathIncludesForLicense( + "MIT", + provenance, + TextLocation("c/d", 4) + ) shouldBe true + result.isExcludedByPathIncludesForLicense( + "MIT", + sourceArtifactProvenance, + TextLocation("c/d", 4) + ) shouldBe true + + result.isExcludedByPathIncludesForLicense( + "Apache-2.0", + provenance, + TextLocation("LICENSE", 1) + ) shouldBe true + + result.isExcludedByPathIncludesForLicense( + "Apache-2.0", + provenance, + TextLocation("a/b", 1) + ) shouldBe false + + result.isExcludedByPathIncludesForLicense( + "Apache-2.0", + sourceArtifactProvenance, + TextLocation("LICENSE", 1) + ) shouldBe false + + result.isExcludedByPathIncludesForLicense( + "Apache-2.0", + sourceArtifactProvenance, + TextLocation("a/b", 1) + ) shouldBe true + + result.isExcludedByPathIncludesForCopyright( + COPYRIGHT, + provenance, + TextLocation("LICENSE", 1) + ) shouldBe true + + result.isExcludedByPathIncludesForCopyright( + COPYRIGHT, + provenance, + TextLocation("a/b", 1) + ) shouldBe false + + result.isExcludedByPathIncludesForCopyright( + COPYRIGHT, + sourceArtifactProvenance, + TextLocation("LICENSE", 1) + ) shouldBe false + + result.isExcludedByPathIncludesForCopyright( + COPYRIGHT, + sourceArtifactProvenance, + TextLocation("a/b", 1) + ) shouldBe true + + result.licenses.flatMap { resolvedLicense -> + resolvedLicense.originalExpressions.filter { it.source == LicenseSource.DETECTED } + } should containExactlyInAnyOrder( + ResolvedOriginalExpression("Apache-2.0".toSpdx(), LicenseSource.DETECTED, false), + ResolvedOriginalExpression("MIT".toSpdx(), LicenseSource.DETECTED, true) + ) + } + "apply license finding curations" { val curation = LicenseFindingCuration( path = "LICENSE", @@ -456,6 +605,7 @@ class LicenseInfoResolverTest : WordSpec({ ), licenseFindingCurations = listOf(curation), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) @@ -476,7 +626,8 @@ class LicenseInfoResolverTest : WordSpec({ ResolvedCopyrightFinding( statement = "(c) 2010 Holder 1", location = TextLocation("LICENSE", 1), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ) ) ) @@ -517,6 +668,7 @@ class LicenseInfoResolverTest : WordSpec({ ), licenseFindingCurations = emptyList(), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ), @@ -651,6 +803,7 @@ class LicenseInfoResolverTest : WordSpec({ ), licenseFindingCurations = emptyList(), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) @@ -686,7 +839,8 @@ class LicenseInfoResolverTest : WordSpec({ ResolvedCopyrightFinding( statement = "Copyright 2020 Holder", location = TextLocation("LICENSE", 1), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ) ) ) @@ -825,6 +979,7 @@ private fun containLocationForLicense( location: TextLocation, appliedCuration: LicenseFindingCuration? = null, matchingPathExcludes: List = emptyList(), + isExcludedByPathIncludes: Boolean = false, copyrights: Set = emptySet() ): Matcher?> = neverNullMatcher { value -> @@ -834,6 +989,7 @@ private fun containLocationForLicense( location, appliedCuration, matchingPathExcludes, + isExcludedByPathIncludes, copyrights ) @@ -861,3 +1017,21 @@ private fun ResolvedLicenseInfo.pathExcludesForCopyright( .find { it.statement == copyright && it.location == location } ?.matchingPathExcludes ?.toSet().orEmpty() + +private fun ResolvedLicenseInfo.isExcludedByPathIncludesForLicense( + license: String, + provenance: Provenance, + location: TextLocation +) = find { it.license == SpdxSingleLicenseExpression.parse(license) } + ?.locations + ?.find { it.provenance == provenance && it.location == location } + ?.isExcludedByPathIncludes + +private fun ResolvedLicenseInfo.isExcludedByPathIncludesForCopyright( + copyright: String, + provenance: Provenance, + location: TextLocation +) = flatMap { license -> license.locations.filter { it.provenance == provenance } } + .flatMap { it.copyrights } + .find { it.statement == copyright && it.location == location } + ?.isExcludedByPathIncludes diff --git a/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt b/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt index da6fbe1c46187..030d6e1b29d96 100644 --- a/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt +++ b/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt @@ -206,6 +206,7 @@ private val RESOLVED_LICENSE_INFO: ResolvedLicenseInfo by lazy { location = TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), appliedCuration = null, matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false, copyrights = emptySet() ) ) @@ -223,6 +224,7 @@ private val RESOLVED_LICENSE_INFO: ResolvedLicenseInfo by lazy { location = TextLocation("LICENCE", TextLocation.UNKNOWN_LINE), appliedCuration = null, matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false, copyrights = emptySet() ) ) @@ -309,6 +311,7 @@ private val COMPUTATION_HEAVY_RESOLVED_LICENSE_INFO: ResolvedLicenseInfo by lazy copyrights = emptySet(), licenseFindingCurations = emptyList(), pathExcludes = emptyList(), + pathIncludes = emptyList(), relativeFindingsPath = "" ) ) diff --git a/model/src/test/kotlin/licenses/ResolvedLicenseTest.kt b/model/src/test/kotlin/licenses/ResolvedLicenseTest.kt index 0527fe0898b82..5585dcd73998a 100644 --- a/model/src/test/kotlin/licenses/ResolvedLicenseTest.kt +++ b/model/src/test/kotlin/licenses/ResolvedLicenseTest.kt @@ -37,7 +37,8 @@ class ResolvedLicenseTest : WordSpec({ path = "/path/to/file/A", line = 2 ), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ), ResolvedCopyrightFinding( statement = "Copyright (C) 2022 The ORT Project Authors", @@ -45,7 +46,8 @@ class ResolvedLicenseTest : WordSpec({ path = "/path/to/file/B", line = 2 ), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ) ) @@ -66,7 +68,8 @@ class ResolvedLicenseTest : WordSpec({ path = "/path/to/file/A", line = 2 ), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ), ResolvedCopyrightFinding( // Note the "." at the end. @@ -75,7 +78,8 @@ class ResolvedLicenseTest : WordSpec({ path = "/path/to/file/B", line = 2 ), - matchingPathExcludes = emptyList() + matchingPathExcludes = emptyList(), + isExcludedByPathIncludes = false ) ) diff --git a/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedFinding.kt b/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedFinding.kt index afb110852ab11..9a595c59be676 100644 --- a/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedFinding.kt +++ b/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedFinding.kt @@ -38,5 +38,9 @@ data class EvaluatedFinding( val endLine: Int, val scanResult: EvaluatedScanResult, @JsonInclude(JsonInclude.Include.NON_EMPTY) - val pathExcludes: List + val pathExcludes: List, + + // If the finding is excluded by the presence of a [PathInclude], this property is true. + @JsonInclude(JsonInclude.Include.NON_EMPTY) + val isExcludedByPathIncludes: Boolean ) diff --git a/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModel.kt b/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModel.kt index 800e4ecb21429..447db2b837961 100644 --- a/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModel.kt +++ b/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModel.kt @@ -39,6 +39,7 @@ import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.config.IssueResolution import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.PathInclude import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.config.RuleViolationResolution import org.ossreviewtoolkit.model.config.ScopeExclude @@ -91,6 +92,7 @@ import org.ossreviewtoolkit.reporter.Statistics */ data class EvaluatedModel( val pathExcludes: List, + val pathIncludes: List, val scopeExcludes: List, val copyrights: List, val licenses: List, diff --git a/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModelMapper.kt b/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModelMapper.kt index 6574ad2be739f..758009479fb8c 100644 --- a/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModelMapper.kt +++ b/plugins/reporters/evaluated-model/src/main/kotlin/EvaluatedModelMapper.kt @@ -39,6 +39,7 @@ import org.ossreviewtoolkit.model.config.Excludes import org.ossreviewtoolkit.model.config.IssueResolution import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.PathInclude import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.config.Resolutions import org.ossreviewtoolkit.model.config.RuleViolationResolution @@ -59,7 +60,7 @@ import org.ossreviewtoolkit.utils.spdx.calculatePackageVerificationCode /** * Maps the [reporter input][input] to an [EvaluatedModel]. */ -@Suppress("TooManyFunctions") +@Suppress("LargeClass", "TooManyFunctions") internal class EvaluatedModelMapper(private val input: ReporterInput) { private val packages = mutableMapOf() private val paths = mutableListOf() @@ -71,6 +72,7 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { private val issues = mutableListOf() private val issueResolutions = mutableListOf() private val pathExcludes = mutableListOf() + private val pathIncludes = mutableListOf() private val scopeExcludes = mutableListOf() private val ruleViolations = mutableListOf() private val ruleViolationResolutions = mutableListOf() @@ -84,6 +86,7 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { var id: Identifier, var isExcluded: Boolean, val pathExcludes: MutableList = mutableListOf(), + val pathIncludes: MutableList = mutableListOf(), val scopeExcludes: MutableList = mutableListOf() ) @@ -134,6 +137,7 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { return EvaluatedModel( pathExcludes = pathExcludes, + pathIncludes = pathIncludes, scopeExcludes = scopeExcludes, issueResolutions = issueResolutions, issues = issues, @@ -169,14 +173,17 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { input.ortResult.getProjects().forEach { project -> val pathExcludes = input.ortResult.getExcludes().findPathExcludes(project, input.ortResult) + val pathIncludes = input.ortResult.getIncludes().findPathIncludes(project, input.ortResult) val dependencies = input.ortResult.dependencyNavigator.projectDependencies(project) - if (pathExcludes.isEmpty()) { + if (pathExcludes.isEmpty() && pathIncludes.isEmpty()) { val info = packageExcludeInfo.getValue(project.id) if (info.isExcluded) { info.isExcluded = false info.pathExcludes.clear() info.scopeExcludes.clear() } + + info.pathIncludes.clear() } else { dependencies.forEach { id -> val info = packageExcludeInfo.getOrPut(id) { PackageExcludeInfo(id, true) } @@ -184,6 +191,8 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { if (info.isExcluded) { info.pathExcludes += pathExcludes } + + info.pathIncludes += pathIncludes } } @@ -238,6 +247,14 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { .flatMap { it.pathExcludes } } + private fun getPathIncludes(id: Identifier, provenance: Provenance): List = + if (input.ortResult.isProject(id)) { + input.ortResult.getIncludes().paths + } else { + input.ortResult.getPackageConfigurations(id, provenance) + .flatMap { it.pathIncludes } + } + private fun addProject(project: Project) { val scanResults = mutableListOf() val detectedLicenses = mutableSetOf() @@ -246,7 +263,9 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { val issues = mutableListOf() val applicablePathExcludes = input.ortResult.getExcludes().findPathExcludes(project, input.ortResult) + val applicablePathIncludes = input.ortResult.getIncludes().findPathIncludes(project, input.ortResult) val evaluatedPathExcludes = pathExcludes.addIfRequired(applicablePathExcludes) + pathIncludes.addIfRequired(applicablePathIncludes) val evaluatedPackage = EvaluatedPackage( id = project.id, @@ -287,7 +306,8 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { findings.filter { it.type == EvaluatedFindingType.LICENSE }.mapNotNullTo(detectedLicenses) { it.license } val includedDetectedLicenses = findings.filter { - it.type == EvaluatedFindingType.LICENSE && it.pathExcludes.isEmpty() + val isIncluded = pathIncludes.isEmpty() || pathIncludes.any { include -> include.matches(it.path) } + it.type == EvaluatedFindingType.LICENSE && it.pathExcludes.isEmpty() && isIncluded }.mapNotNullTo(mutableSetOf()) { it.license } detectedExcludedLicenses += detectedLicenses - includedDetectedLicenses @@ -660,6 +680,7 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { findings: MutableList ) { val pathExcludes = getPathExcludes(id, scanResult.provenance) + val pathIncludes = getPathIncludes(id, scanResult.provenance) val licenseFindingCurations = getLicenseFindingCurations(id, scanResult.provenance) // Sort the curated findings here to avoid the need to sort in the web-app each time it is loaded. val curatedFindings = curationsMatcher.applyAll(scanResult.summary.licenseFindings, licenseFindingCurations) @@ -679,6 +700,16 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { val evaluatedPathExcludes = pathExcludes .filter { it.matches(copyrightFinding.location.getRelativePathToRoot(id)) } .let { this@EvaluatedModelMapper.pathExcludes.addIfRequired(it) } + pathIncludes.let { this@EvaluatedModelMapper.pathIncludes.addIfRequired(it) } + + val evaluatedPathIncludes = if (pathIncludes.isEmpty() || pathIncludes.any { include -> + include.matches(copyrightFinding.location.getRelativePathToRoot(id)) + } + ) { + emptyList() + } else { + pathIncludes + } findings += EvaluatedFinding( type = EvaluatedFindingType.COPYRIGHT, @@ -688,7 +719,8 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { startLine = copyrightFinding.location.startLine, endLine = copyrightFinding.location.endLine, scanResult = evaluatedScanResult, - pathExcludes = evaluatedPathExcludes + pathExcludes = evaluatedPathExcludes, + isExcludedByPathIncludes = evaluatedPathExcludes.isEmpty() && evaluatedPathIncludes.isNotEmpty() ) } @@ -699,6 +731,18 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { .filter { it.matches(licenseFinding.location.getRelativePathToRoot(id)) } .let { this@EvaluatedModelMapper.pathExcludes.addIfRequired(it) } + pathIncludes.let { this@EvaluatedModelMapper.pathIncludes.addIfRequired(it) } + + val evaluatedPathIncludes = if (pathIncludes.isEmpty() || pathIncludes.any { + include -> + include.matches(licenseFinding.location.getRelativePathToRoot(id)) + } + ) { + emptyList() + } else { + pathIncludes + } + findings += EvaluatedFinding( type = EvaluatedFindingType.LICENSE, license = actualLicense, @@ -707,7 +751,8 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { startLine = licenseFinding.location.startLine, endLine = licenseFinding.location.endLine, scanResult = evaluatedScanResult, - pathExcludes = evaluatedPathExcludes + pathExcludes = evaluatedPathExcludes, + isExcludedByPathIncludes = evaluatedPathExcludes.isEmpty() && evaluatedPathIncludes.isNotEmpty() ) } } @@ -772,6 +817,12 @@ internal class EvaluatedModelMapper(private val input: ReporterInput) { ) } - return copy(config = config.copy(excludes = excludes, resolutions = resolutions)) + val includes = with(config.includes) { + copy( + paths = paths.map { pathIncludes.addIfRequired(it) } + ) + } + + return copy(config = config.copy(excludes = excludes, includes = includes, resolutions = resolutions)) } } diff --git a/plugins/reporters/web-app-template/src/components/PackageFindingsTable.jsx b/plugins/reporters/web-app-template/src/components/PackageFindingsTable.jsx index 843b62eab3925..417a55dc0d78f 100644 --- a/plugins/reporters/web-app-template/src/components/PackageFindingsTable.jsx +++ b/plugins/reporters/web-app-template/src/components/PackageFindingsTable.jsx @@ -52,6 +52,7 @@ const PackageFindingsTable = ({ webAppPackage }) => { key: finding.key, path: finding.path, pathExcludes: finding.pathExcludes, + isExcludedByPathIncludes: finding.isExcludedByPathIncludes, pathExcludeReasonsText: Array.from(finding.pathExcludeReasons).join(', '), startLine: finding.startLine, value: finding.value @@ -124,7 +125,7 @@ const PackageFindingsTable = ({ webAppPackage }) => { @@ -138,11 +139,19 @@ const PackageFindingsTable = ({ webAppPackage }) => { }); expandable = { - expandedRowRender: (webAppFinding) => ( - - ), + expandedRowRender: (webAppFinding) => { + if (webAppFinding.isExcludedByPathIncludes) { + return () + } else { + return () + } + + }, expandIcon: (obj) => { const { expanded, onExpand, record } = obj; diff --git a/plugins/reporters/web-app-template/src/components/PathExcludesTable.jsx b/plugins/reporters/web-app-template/src/components/PathExcludesTable.jsx index 0779ef41de2d3..fed3f4aeebc90 100644 --- a/plugins/reporters/web-app-template/src/components/PathExcludesTable.jsx +++ b/plugins/reporters/web-app-template/src/components/PathExcludesTable.jsx @@ -20,12 +20,18 @@ import { Table } from 'antd'; // Generates the HTML to display webAppPathExclude(s) as a table -const PathExcludesTable = ({ excludes }) => { +const PathExcludesTable = ({ excludes, isIncludes = false }) => { const columns = [ { dataIndex: 'reason', key: 'reason', - title: 'Reason' + title: 'Reason', + + render: (text) => ( + <> + {text} {isIncludes && (includes)} + + ) }, { dataIndex: 'pattern', diff --git a/plugins/reporters/web-app-template/src/models/WebAppFinding.js b/plugins/reporters/web-app-template/src/models/WebAppFinding.js index eeb23a33ba08e..c1aa3d6130f53 100644 --- a/plugins/reporters/web-app-template/src/models/WebAppFinding.js +++ b/plugins/reporters/web-app-template/src/models/WebAppFinding.js @@ -32,9 +32,12 @@ class WebAppFinding { #pathExcludes; + #isExcludedByPathIncludes; + #pathExcludeIndexes = new Set(); #pathExcludeReasons; + #pathIncludeReasons; #startLine; @@ -63,11 +66,18 @@ class WebAppFinding { } const pathExcludes = obj.path_excludes || obj.pathExcludes; + if (Array.isArray(pathExcludes) && pathExcludes.length > 0) { this.#pathExcludeIndexes = new Set(pathExcludes); this.#isExcluded = true; } + const isExcludedByPathIncludes = obj.isExcludedByPathIncludes || obj.is_excluded_by_path_includes; + if (isExcludedByPathIncludes) { + this.#isExcludedByPathIncludes = true; + this.#isExcluded = true; + } + if (Number.isInteger(obj.start_line) || Number.isInteger(obj.startLine)) { this.#startLine = obj.start_line || obj.startLine; } @@ -155,6 +165,10 @@ class WebAppFinding { return this.#pathExcludeReasons; } + get isExcludedByPathIncludes() { + return this.#isExcludedByPathIncludes; + } + get startLine() { return this.#startLine; } diff --git a/plugins/reporters/web-app-template/src/models/WebAppOrtResult.js b/plugins/reporters/web-app-template/src/models/WebAppOrtResult.js index a7ee925549c15..f15fe47933e6c 100644 --- a/plugins/reporters/web-app-template/src/models/WebAppOrtResult.js +++ b/plugins/reporters/web-app-template/src/models/WebAppOrtResult.js @@ -80,6 +80,10 @@ class WebAppOrtResult { #pathExcludes = []; + #pathIncludes = []; + + #pathIncludeReasons; + #paths = []; #projects = []; @@ -180,6 +184,14 @@ class WebAppOrtResult { } } + if (obj.path_includes || obj.pathIncludes) { + const pathIncludes = obj.path_includes || obj.pathIncludes; + + for (let i = 0, len = pathIncludes.length; i < len; i++) { + this.#pathIncludes.push(new WebAppPathExclude(pathIncludes[i])); + } + } + if (obj.paths) { const { paths } = obj; for (let i = 0, len = paths.length; i < len; i++) { @@ -447,6 +459,22 @@ class WebAppOrtResult { return this.#pathExcludes; } + get pathIncludes() { + return this.#pathIncludes; + } + + get pathIncludeReasons() { + if (!this.#pathIncludeReasons) { + if (Array.isArray(this.#pathIncludes) && this.#pathIncludes.length > 0) { + this.#pathIncludeReasons = new Set(this.#pathIncludes.map((value) => value.reason)) + } else { + this.#pathIncludeReasons = new Set(); + } + } + + return this.#pathIncludeReasons; + } + get paths() { return this.#paths; } diff --git a/plugins/reporters/web-app-template/src/models/WebAppPackage.js b/plugins/reporters/web-app-template/src/models/WebAppPackage.js index 45f9dbb5294f6..6877a83e0a065 100644 --- a/plugins/reporters/web-app-template/src/models/WebAppPackage.js +++ b/plugins/reporters/web-app-template/src/models/WebAppPackage.js @@ -411,6 +411,10 @@ class WebAppPackage { return new Set([...pathExcludeReasons, ...scopeExcludeReasons]); } + get pathIncludeReasons() { + return this.#webAppOrtResult.pathIncludeReasons + } + get excludedFindings() { if (!this.#excludedFindings) { this.#excludedFindings = []; @@ -477,6 +481,10 @@ class WebAppPackage { return this.#pathExcludes; } + get pathIncludes() { + return this.#webAppOrtResult.pathIncludes; + } + get pathExcludeIndexes() { return this.#pathExcludeIndexes; } diff --git a/website/docs/configuration/package-configurations.md b/website/docs/configuration/package-configurations.md index 6b3584cc070f8..1b756763c579c 100644 --- a/website/docs/configuration/package-configurations.md +++ b/website/docs/configuration/package-configurations.md @@ -1,6 +1,6 @@ # Package Configurations -A package configuration file allows you to define path excludes and license finding curations for a specific package (dependency) and provenance. +A package configuration file allows you to define path includes, path excludes and license finding curations for a specific package (dependency) and provenance. Conceptually, the file is similar to [.ort.yml](ort-yml.md), but it is used only for packages included via a package manager as project dependencies, and not for the project's own source code repository to be scanned. ## When To Use @@ -47,12 +47,14 @@ source_code_origin: ARTIFACT id: "NPM::ansi-styles:4.2.1" ``` -## Defining Path Excludes and License Finding Curations +## Defining Path Includes, Path Excludes and License Finding Curations +Path includes define which files and directories are part of the distributed release artifact(s) for a package, for example source code files. +When defining includes, all the non-matching paths are considered as excluded by default. Path excludes define which code is not part of the distributed release artifact(s) for a package, for example, code found in the source repository but only used for building, documenting or testing the code. License finding curations are used to fix incorrect scan results, for example, if a wrong license was detected, or if a finding is a false positive. -The entries for path excludes and license finding curations have the same syntax and semantics as in the `ort.yml` file, see [excluding paths](ort-yml.md#excluding-paths) and [curating license findings](ort-yml.md#curating-project-license-findings) for details. +The entries for path includes, path excludes and license finding curations have the same syntax and semantics as in the `ort.yml` file, see [including paths](ort-yml.md#including-paths), [excluding paths](ort-yml.md#excluding-paths) and [curating license findings](ort-yml.md#curating-project-license-findings) for details. ```yaml id: "Pip::example-package:0.0.1" @@ -61,6 +63,10 @@ path_excludes: - pattern: "docs/**" reason: "DOCUMENTATION_OF" comment: "This directory contains documentation which is not distributed." +path_includes: +- pattern: "src/**" + reason: "SOURCE_OF" + comment: "This directory contains source code of distributed application." license_finding_curations: - path: "src/**/*.cpp" start_lines: "3"