From 0c33e3fc3cc2229425693b44aaa63e7244431ec3 Mon Sep 17 00:00:00 2001 From: hainenber Date: Sun, 29 Sep 2024 19:51:33 +0700 Subject: [PATCH 1/5] Refactor license checker task to work with Apache Rat v0.16.1 Signed-off-by: hainenber --- buildSrc/build.gradle | 2 +- .../precommit/LicenseHeadersTask.groovy | 209 ++++++++++-------- 2 files changed, 115 insertions(+), 96 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index b984ef3800490..e4e701c7adc96 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -108,7 +108,7 @@ dependencies { api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0' api 'com.netflix.nebula:nebula-publishing-plugin:21.0.0' api 'com.netflix.nebula:gradle-info-plugin:12.1.6' - api 'org.apache.rat:apache-rat:0.15' + api 'org.apache.rat:apache-rat:0.16.1' api "commons-io:commons-io:${props.getProperty('commonsio')}" api "net.java.dev.jna:jna:5.14.0" api 'com.github.johnrengelman:shadow:8.1.1' diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy b/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy index b8d0ed2b9c43c..9a82bd20517a9 100644 --- a/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy +++ b/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy @@ -28,35 +28,38 @@ */ package org.opensearch.gradle.precommit -import org.apache.rat.anttasks.Report -import org.apache.rat.anttasks.SubstringLicenseMatcher -import org.apache.rat.license.SimpleLicenseFamily +import org.apache.rat.analysis.HeaderCheckWorker +import org.apache.rat.analysis.matchers.SimpleTextMatcher +import org.apache.rat.anttasks.License +import org.apache.rat.api.MetaData +import org.apache.rat.document.impl.FileDocument +import org.apache.rat.license.ILicense +import org.apache.rat.license.ILicenseFamily +import org.apache.rat.license.ILicenseFamilyBuilder import org.opensearch.gradle.AntTask import org.gradle.api.file.FileCollection import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal import org.gradle.api.tasks.IgnoreEmptyDirectories; -import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.PathSensitive import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.SkipWhenEmpty -import java.nio.file.Files - /** * Checks files for license headers. *

- * This is a port of the apache lucene check */ class LicenseHeadersTask extends AntTask { - @OutputFile - File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat.log') - /** Allowed license families for this project. */ @Input List approvedLicenses = ['Apache', 'Generated', 'SPDX', 'Vendored'] + /** Disallowed license families for this project. BSD4 at the moment */ + @Internal + List disapprovedLicenses = ['Original BSD License (with advertising clause)'] + /** * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the @@ -87,6 +90,50 @@ class LicenseHeadersTask extends AntTask { return project.sourceSets.collect({it.allJava}) } + /** + * Internal class to hold license metadata + */ + private class LicenseSettings { + private String licenseCategory + private String licenseFamilyName + private String pattern + private LicenseSettings(String licenseCategory, String licenseFamilyName, String pattern) { + this.licenseCategory = licenseCategory + this.licenseFamilyName = licenseFamilyName + this.pattern = pattern + } + } + + /** + * Create license matcher from allowed/disallowed license list. + * + * @param licenseSettingsMap A map of license identifier and its associated data (family name, category and pattern) + */ + private Map getLicenseMatchers(Map licenseSettingsMap) { + Map licenseHashMap = new HashMap<>() + + for (Map.Entry entry: licenseSettingsMap.entrySet()) { + SortedSet licenseCtx = new TreeSet() + var licenseCodeName = entry.getKey() + var licenseSetting = entry.getValue() + var licenseFamilyBuilder = new ILicenseFamilyBuilder() + var licenseFamily = licenseFamilyBuilder.setLicenseFamilyCategory(licenseSetting.licenseCategory) + .setLicenseFamilyName(licenseSetting.licenseFamilyName) + .build() + licenseCtx.add(licenseFamily) + + var license = new License() + license.setName(licenseFamily.getFamilyName()) + license.setFamily(licenseFamily.getFamilyCategory()) + license.add(new SimpleTextMatcher(licenseSetting.pattern)) + + var configuredLicense = license.build(licenseCtx) + licenseHashMap.put(licenseCodeName, configuredLicense) + } + + return licenseHashMap + } + /** * Add a new license type. * @@ -105,100 +152,72 @@ class LicenseHeadersTask extends AntTask { @Override protected void runAnt(AntBuilder ant) { - ant.project.addTaskDefinition('ratReport', Report) - ant.project.addDataTypeDefinition('substringMatcher', SubstringLicenseMatcher) - ant.project.addDataTypeDefinition('approvedLicense', SimpleLicenseFamily) - - Files.deleteIfExists(reportFile.toPath()) - - // run rat, going to the file - ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) { - for (FileCollection dirSet : javaFiles) { - for (File dir: dirSet.srcDirs) { - // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main... - if (dir.exists()) { - ant.fileset(dir: dir, excludes: excludes.join(' ')) - } - } - } - - // BSD 4-clause stuff (is disallowed below) - // we keep this here, in case someone adds BSD code for some reason, it should never be allowed. - substringMatcher(licenseFamilyCategory: "BSD4 ", - licenseFamilyName: "Original BSD License (with advertising clause)") { - pattern(substring: "All advertising materials") - } - - // Apache - substringMatcher(licenseFamilyCategory: "AL ", - licenseFamilyName: "Apache") { - // Apache license (ES) - pattern(substring: "Licensed to Elasticsearch under one or more contributor") - } - - // SPDX - substringMatcher(licenseFamilyCategory: "SPDX ", - licenseFamilyName: "SPDX") { - // Apache license (OpenSearch) - pattern(substring: "SPDX-License-Identifier: Apache-2.0") - pattern(substring: "Copyright OpenSearch Contributors.") - } - - // Generated resources - substringMatcher(licenseFamilyCategory: "GEN ", - licenseFamilyName: "Generated") { - // parsers generated by antlr - pattern(substring: "ANTLR GENERATED CODE") - } - - // Vendored Code - substringMatcher(licenseFamilyCategory: "VEN ", - licenseFamilyName: "Vendored") { - pattern(substring: "@notice") - } + Map licenseSettingsHashMap = new HashMap<>() + Map> licenseStats = new HashMap<>() + + // BSD 4-clause stuff (is disallowed below) + // we keep this here, in case someone adds BSD code for some reason, it should never be allowed. + licenseSettingsHashMap.put("BSD4", new LicenseSettings("BSD4", "Original BSD License (with advertising clause)", "All advertising materials")) + // Apache + licenseSettingsHashMap.put("AL", new LicenseSettings("AL", "Apache", "Licensed to Elasticsearch under one or more contributor")) + // SPDX + licenseSettingsHashMap.put("SPDX1", new LicenseSettings("SPDX", "SPDX", "SPDX-License-Identifier: Apache-2.0")) + // SPDX: Apache license (OpenSearch) + licenseSettingsHashMap.put("SPDX2", new LicenseSettings("SPDX", "SPDX", "Copyright OpenSearch Contributors.")) + // Generated resources + licenseSettingsHashMap.put("GEN", new LicenseSettings("GEN", "Generated", "ANTLR GENERATED CODE")) + // Vendored code + licenseSettingsHashMap.put("VEN", new LicenseSettings("VEN", "Vendored", "@noticed")) + + // License types added by the project + for (Map.Entry additional : additionalLicenses.entrySet()) { + String category = additional.getKey().substring(0, 5) + String family = additional.getKey().substring(5) + licenseSettingsHashMap.put(category, new LicenseSettings( + category, + family, + additional.getValue(), + )) + } - // license types added by the project - for (Map.Entry additional : additionalLicenses.entrySet()) { - String category = additional.getKey().substring(0, 5) - String family = additional.getKey().substring(5) - substringMatcher(licenseFamilyCategory: category, - licenseFamilyName: family) { - pattern(substring: additional.getValue()) + Map licenseHashMap = this.getLicenseMatchers(licenseSettingsHashMap); + + for (FileCollection dirSet : javaFiles) { + for (File file: dirSet) { + var ratDoc = new FileDocument(file) + var detectedLicenseFamilyName = null + var detectedLicenseCodename = null + for (Map.Entry entry: licenseHashMap.entrySet()) { + try (Reader reader = ratDoc.reader()) { + var worker = new HeaderCheckWorker(reader, HeaderCheckWorker.DEFAULT_NUMBER_OF_RETAINED_HEADER_LINES, entry.getValue(), ratDoc) + worker.read() + detectedLicenseFamilyName = ratDoc.getMetaData().get(MetaData.RAT_URL_LICENSE_FAMILY_NAME).value + if (!detectedLicenseFamilyName.equals("Unknown license")) { + detectedLicenseCodename = entry.getKey() + break; + } + } + } + var sourceFilePath = file.getCanonicalPath() + if (disapprovedLicenses.contains(detectedLicenseFamilyName)) { + licenseStats.computeIfAbsent('DISAPPROVED', k -> new ArrayList<>()).add("(" + detectedLicenseCodename + ") " + sourceFilePath) + } else if (detectedLicenseFamilyName.equals("Unknown license")) { + licenseStats.computeIfAbsent('MISSING/UNKNOWN', k -> new ArrayList<>()).add(sourceFilePath) } - } - - // approved categories - for (String licenseFamily : approvedLicenses) { - approvedLicense(familyName: licenseFamily) } } // check the license file for any errors, this should be fast. - boolean zeroUnknownLicenses = false - boolean foundProblemsWithFiles = false - reportFile.eachLine('UTF-8') { line -> - if (line.startsWith("0 Unknown Licenses")) { - zeroUnknownLicenses = true - } + boolean hasDisapprovedLicenses = licenseStats.containsKey('DISAPPROVED') && licenseStats.get('DISAPPROVED').size() > 0 + boolean hasUnknownLicenses = licenseStats.containsKey('MISSING/UNKNOWN') && licenseStats.get('MISSING/UNKNOWN').size() > 0 - if (line.startsWith(" !")) { - foundProblemsWithFiles = true - } - } - - if (zeroUnknownLicenses == false || foundProblemsWithFiles) { - // print the unapproved license section, usually its all you need to fix problems. - int sectionNumber = 0 - reportFile.eachLine('UTF-8') { line -> - if (line.startsWith("*******************************")) { - sectionNumber++ - } else { - if (sectionNumber == 2) { - logger.error(line) - } + if (hasDisapprovedLicenses || hasUnknownLicenses) { + for (Map.Entry> entry: licenseStats.entrySet()) { + for (String line: entry.getValue()) { + logger.error(entry.getKey() + " " + line) } } - throw new IllegalStateException("License header problems were found! Full details: " + reportFile.absolutePath) + throw new IllegalStateException("License header problems were found!") } } } From 03d29543971870100dd5ca88f61e0031d0c309a7 Mon Sep 17 00:00:00 2001 From: hainenber Date: Sun, 29 Sep 2024 21:18:07 +0700 Subject: [PATCH 2/5] Accept Apache v2 and Protoc license headers Signed-off-by: hainenber --- .../opensearch/gradle/precommit/LicenseHeadersTask.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy b/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy index 9a82bd20517a9..dde4b4a85b08d 100644 --- a/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy +++ b/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy @@ -155,10 +155,16 @@ class LicenseHeadersTask extends AntTask { Map licenseSettingsHashMap = new HashMap<>() Map> licenseStats = new HashMap<>() + // Acceptable licenses by Apache Rat's defaults + // Apache 2 + licenseSettingsHashMap.put("AL2", new LicenseSettings("AL", "Apache License Version 2.0", "Licensed to the Apache Software Foundation (ASF)")) + // Generate code from Protocol Buffer compiler + licenseSettingsHashMap.put("GENPROTOC", new LicenseSettings("GEN", "Generated", "Generated by the protocol buffer compiler. DO NOT EDIT!")) + // BSD 4-clause stuff (is disallowed below) // we keep this here, in case someone adds BSD code for some reason, it should never be allowed. licenseSettingsHashMap.put("BSD4", new LicenseSettings("BSD4", "Original BSD License (with advertising clause)", "All advertising materials")) - // Apache + // Apache (ES) licenseSettingsHashMap.put("AL", new LicenseSettings("AL", "Apache", "Licensed to Elasticsearch under one or more contributor")) // SPDX licenseSettingsHashMap.put("SPDX1", new LicenseSettings("SPDX", "SPDX", "SPDX-License-Identifier: Apache-2.0")) From d3521ae1182dbf5fb201928c4cd8727ce12a751b Mon Sep 17 00:00:00 2001 From: hainenber Date: Tue, 1 Oct 2024 21:58:19 +0700 Subject: [PATCH 3/5] chore: add CHANGELOG entry Signed-off-by: hainenber --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3df8aec0f376a..0e7c2fb8f41cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) +- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) +- Bump `org.apache.rat:apache-rat'` from 0.15 to 0.16.1 ([#16121](https://github.com/opensearch-project/OpenSearch/pull/16121)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) From 9b2dc224357c3b0e702272355f9819f399638313 Mon Sep 17 00:00:00 2001 From: hainenber Date: Tue, 1 Oct 2024 21:59:27 +0700 Subject: [PATCH 4/5] chore: remove extraneous CHANGELOG entry Signed-off-by: hainenber --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e7c2fb8f41cd..009a342bc0bc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) -- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) - Bump `org.apache.rat:apache-rat'` from 0.15 to 0.16.1 ([#16121](https://github.com/opensearch-project/OpenSearch/pull/16121)) ### Changed From 755a373a28493be526299a81532e0ed41ba026c1 Mon Sep 17 00:00:00 2001 From: hainenber Date: Sun, 6 Oct 2024 16:39:05 +0700 Subject: [PATCH 5/5] feat(build-src): refactor licenseHeader task to work with ReportConfig per recommendation Signed-off-by: hainenber --- .../precommit/LicenseHeadersTask.groovy | 185 ++++++++---------- 1 file changed, 87 insertions(+), 98 deletions(-) diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy b/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy index dde4b4a85b08d..3eae27a3e2896 100644 --- a/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy +++ b/buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy @@ -28,20 +28,24 @@ */ package org.opensearch.gradle.precommit -import org.apache.rat.analysis.HeaderCheckWorker +import org.apache.rat.Defaults +import org.apache.rat.ReportConfiguration +import org.apache.rat.Reporter import org.apache.rat.analysis.matchers.SimpleTextMatcher import org.apache.rat.anttasks.License -import org.apache.rat.api.MetaData -import org.apache.rat.document.impl.FileDocument +import org.apache.rat.anttasks.ResourceCollectionContainer import org.apache.rat.license.ILicense import org.apache.rat.license.ILicenseFamily import org.apache.rat.license.ILicenseFamilyBuilder +import org.apache.rat.utils.DefaultLog +import org.apache.tools.ant.types.resources.FileResource +import org.apache.tools.ant.types.resources.Union +import org.gradle.api.tasks.OutputFile import org.opensearch.gradle.AntTask import org.gradle.api.file.FileCollection import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.Internal -import org.gradle.api.tasks.IgnoreEmptyDirectories; +import org.gradle.api.tasks.IgnoreEmptyDirectories import org.gradle.api.tasks.PathSensitive import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.SkipWhenEmpty @@ -52,14 +56,13 @@ import org.gradle.api.tasks.SkipWhenEmpty */ class LicenseHeadersTask extends AntTask { + @OutputFile + File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat1.log') + /** Allowed license families for this project. */ @Input List approvedLicenses = ['Apache', 'Generated', 'SPDX', 'Vendored'] - /** Disallowed license families for this project. BSD4 at the moment */ - @Internal - List disapprovedLicenses = ['Original BSD License (with advertising clause)'] - /** * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the @@ -90,48 +93,27 @@ class LicenseHeadersTask extends AntTask { return project.sourceSets.collect({it.allJava}) } - /** - * Internal class to hold license metadata - */ - private class LicenseSettings { - private String licenseCategory - private String licenseFamilyName - private String pattern - private LicenseSettings(String licenseCategory, String licenseFamilyName, String pattern) { - this.licenseCategory = licenseCategory - this.licenseFamilyName = licenseFamilyName - this.pattern = pattern - } - } - /** * Create license matcher from allowed/disallowed license list. * * @param licenseSettingsMap A map of license identifier and its associated data (family name, category and pattern) */ - private Map getLicenseMatchers(Map licenseSettingsMap) { - Map licenseHashMap = new HashMap<>() - - for (Map.Entry entry: licenseSettingsMap.entrySet()) { - SortedSet licenseCtx = new TreeSet() - var licenseCodeName = entry.getKey() - var licenseSetting = entry.getValue() - var licenseFamilyBuilder = new ILicenseFamilyBuilder() - var licenseFamily = licenseFamilyBuilder.setLicenseFamilyCategory(licenseSetting.licenseCategory) - .setLicenseFamilyName(licenseSetting.licenseFamilyName) - .build() - licenseCtx.add(licenseFamily) - - var license = new License() - license.setName(licenseFamily.getFamilyName()) - license.setFamily(licenseFamily.getFamilyCategory()) - license.add(new SimpleTextMatcher(licenseSetting.pattern)) - - var configuredLicense = license.build(licenseCtx) - licenseHashMap.put(licenseCodeName, configuredLicense) - } - - return licenseHashMap + private static ILicense generateRatLicense(String licenseCategory, String licenseFamilyName, String pattern) { + SortedSet licenseCtx = new TreeSet() + var licenseFamilyBuilder = new ILicenseFamilyBuilder() + var licenseFamily = licenseFamilyBuilder.setLicenseFamilyCategory(licenseCategory) + .setLicenseFamilyName(licenseFamilyName) + .build() + licenseCtx.add(licenseFamily) + + var license = new License() + license.setName(licenseFamily.getFamilyName()) + license.setFamily(licenseFamily.getFamilyCategory()) + license.add(new SimpleTextMatcher(pattern)) + + var configuredLicense = license.build(licenseCtx) + + return configuredLicense } /** @@ -152,78 +134,85 @@ class LicenseHeadersTask extends AntTask { @Override protected void runAnt(AntBuilder ant) { - Map licenseSettingsHashMap = new HashMap<>() - Map> licenseStats = new HashMap<>() - - // Acceptable licenses by Apache Rat's defaults - // Apache 2 - licenseSettingsHashMap.put("AL2", new LicenseSettings("AL", "Apache License Version 2.0", "Licensed to the Apache Software Foundation (ASF)")) - // Generate code from Protocol Buffer compiler - licenseSettingsHashMap.put("GENPROTOC", new LicenseSettings("GEN", "Generated", "Generated by the protocol buffer compiler. DO NOT EDIT!")) - - // BSD 4-clause stuff (is disallowed below) - // we keep this here, in case someone adds BSD code for some reason, it should never be allowed. - licenseSettingsHashMap.put("BSD4", new LicenseSettings("BSD4", "Original BSD License (with advertising clause)", "All advertising materials")) - // Apache (ES) - licenseSettingsHashMap.put("AL", new LicenseSettings("AL", "Apache", "Licensed to Elasticsearch under one or more contributor")) - // SPDX - licenseSettingsHashMap.put("SPDX1", new LicenseSettings("SPDX", "SPDX", "SPDX-License-Identifier: Apache-2.0")) - // SPDX: Apache license (OpenSearch) - licenseSettingsHashMap.put("SPDX2", new LicenseSettings("SPDX", "SPDX", "Copyright OpenSearch Contributors.")) - // Generated resources - licenseSettingsHashMap.put("GEN", new LicenseSettings("GEN", "Generated", "ANTLR GENERATED CODE")) - // Vendored code - licenseSettingsHashMap.put("VEN", new LicenseSettings("VEN", "Vendored", "@noticed")) + List approvedLicenses = List.of( + // Apache 2 + generateRatLicense("AL2", "Apache License Version 2.0", "Licensed to the Apache Software Foundation (ASF)"), + // Generated code from Protocol Buffer compiler + generateRatLicense("GEN", "Generated", "Generated by the protocol buffer compiler. DO NOT EDIT!"), + // Apache (ES) + generateRatLicense("AL", "Apache", "Licensed to Elasticsearch under one or more contributor"), + // SPDX + generateRatLicense("SPDX", "SPDX", "SPDX-License-Identifier: Apache-2.0"), + // SPDX: Apache license (OpenSearch) + generateRatLicense("SPDX-ES", "SPDX", "Copyright OpenSearch Contributors."), + // Generated resources + generateRatLicense("GEN", "Generated", "ANTLR GENERATED CODE"), + // Vendored code + generateRatLicense("VEN", "Vendored", "@noticed"), + ) + + // Currently Apache Rat doesn't display header for source code file with negative matching + // like BSD4 + // Source: https://github.com/apache/creadur-rat/blob/apache-rat-project-0.16.1/apache-rat-core/src/main/resources/org/apache/rat/plain-rat.xsl#L85-L87) + // Uncomment and integrate the negative matcher for BSD4 once Rat supports + // List disapprovedLicenses = List.of( + // // BSD 4-clause stuff (is disallowed below) + // // we keep this here, in case someone adds BSD code for some reason, it should never be allowed. + // generateRatLicense("BSD4", "Original BSD License (with advertising clause)", "All advertising materials"), + // ) + + ReportConfiguration configuration = new ReportConfiguration(DefaultLog.INSTANCE); + configuration.setOut(reportFile) + configuration.setStyleSheet(Defaults.getPlainStyleSheet()) + configuration.addLicensesIfNotPresent(approvedLicenses) + configuration.addApprovedLicenseCategories(approvedLicenses.stream().map(l -> l.getLicenseFamily().getFamilyCategory()).toList()) // License types added by the project for (Map.Entry additional : additionalLicenses.entrySet()) { String category = additional.getKey().substring(0, 5) String family = additional.getKey().substring(5) - licenseSettingsHashMap.put(category, new LicenseSettings( + configuration.addLicense(generateRatLicense( category, family, additional.getValue(), )) + configuration.addApprovedLicenseCategory(category) } - Map licenseHashMap = this.getLicenseMatchers(licenseSettingsHashMap); - + Union union = new Union() for (FileCollection dirSet : javaFiles) { for (File file: dirSet) { - var ratDoc = new FileDocument(file) - var detectedLicenseFamilyName = null - var detectedLicenseCodename = null - for (Map.Entry entry: licenseHashMap.entrySet()) { - try (Reader reader = ratDoc.reader()) { - var worker = new HeaderCheckWorker(reader, HeaderCheckWorker.DEFAULT_NUMBER_OF_RETAINED_HEADER_LINES, entry.getValue(), ratDoc) - worker.read() - detectedLicenseFamilyName = ratDoc.getMetaData().get(MetaData.RAT_URL_LICENSE_FAMILY_NAME).value - if (!detectedLicenseFamilyName.equals("Unknown license")) { - detectedLicenseCodename = entry.getKey() - break; - } - } - } - var sourceFilePath = file.getCanonicalPath() - if (disapprovedLicenses.contains(detectedLicenseFamilyName)) { - licenseStats.computeIfAbsent('DISAPPROVED', k -> new ArrayList<>()).add("(" + detectedLicenseCodename + ") " + sourceFilePath) - } else if (detectedLicenseFamilyName.equals("Unknown license")) { - licenseStats.computeIfAbsent('MISSING/UNKNOWN', k -> new ArrayList<>()).add(sourceFilePath) - } + union.add(new FileResource(file)) } } + configuration.setReportable(new ResourceCollectionContainer(union)) + Reporter.report(configuration) // check the license file for any errors, this should be fast. - boolean hasDisapprovedLicenses = licenseStats.containsKey('DISAPPROVED') && licenseStats.get('DISAPPROVED').size() > 0 - boolean hasUnknownLicenses = licenseStats.containsKey('MISSING/UNKNOWN') && licenseStats.get('MISSING/UNKNOWN').size() > 0 + boolean zeroUnknownLicenses = false + boolean foundProblemsWithFiles = false + reportFile.eachLine('UTF-8') { line -> + if (line.startsWith("0 Unknown Licenses")) { + zeroUnknownLicenses = true + } + if (line.startsWith(" !")) { + foundProblemsWithFiles = true + } + } - if (hasDisapprovedLicenses || hasUnknownLicenses) { - for (Map.Entry> entry: licenseStats.entrySet()) { - for (String line: entry.getValue()) { - logger.error(entry.getKey() + " " + line) + if (zeroUnknownLicenses == false || foundProblemsWithFiles) { + // print the unapproved license section, usually its all you need to fix problems. + int sectionNumber = 0 + reportFile.eachLine('UTF-8') { line -> + if (line.startsWith("*******************************")) { + sectionNumber++ + } else { + if (sectionNumber == 2) { + logger.error(line) + } } } - throw new IllegalStateException("License header problems were found!") + throw new IllegalStateException("License header problems were found! Full details: " + reportFile.absolutePath) } } }