diff --git a/cli/pom.xml b/cli/pom.xml index 8d87452..c488391 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -12,6 +12,7 @@ UTF-8 UTF-8 UTF-8 + 4.7.7 @@ -27,16 +28,16 @@ com.fasterxml.jackson.module jackson-module-kotlin + + info.picocli + picocli + ${picocli.version} + io.github.ardoco metrics ${revision} - - org.jetbrains.kotlinx - kotlinx-cli-jvm - 0.3.6 - org.slf4j slf4j-simple @@ -81,6 +82,28 @@ + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + kapt + + kapt + + + + + info.picocli + picocli-codegen + ${picocli.version} + + + + + + diff --git a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/App.kt b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/App.kt index b1b0ded..b47f1c6 100644 --- a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/App.kt +++ b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/App.kt @@ -4,19 +4,21 @@ import edu.kit.kastel.mcse.ardoco.metrics.cli.commands.AggregationClassification import edu.kit.kastel.mcse.ardoco.metrics.cli.commands.AggregationRankCommand import edu.kit.kastel.mcse.ardoco.metrics.cli.commands.ClassificationCommand import edu.kit.kastel.mcse.ardoco.metrics.cli.commands.RankCommand -import kotlinx.cli.ArgParser -import kotlinx.cli.ArgType -import kotlinx.cli.ExperimentalCli +import picocli.CommandLine -@OptIn(ExperimentalCli::class) fun main(args: Array) { - val parser = ArgParser("ArDoCo Metrics") - - val outputFileOption = parser.option(ArgType.String, shortName = "o", description = "The output file", fullName = "output") - val classificationCommand = ClassificationCommand(outputFileOption) - val aggregationClassificationCommand = AggregationClassificationCommand(outputFileOption) - val rankCommand = RankCommand(outputFileOption) - val aggregationRankCommand = AggregationRankCommand(outputFileOption) - parser.subcommands(classificationCommand, aggregationClassificationCommand, rankCommand, aggregationRankCommand) - parser.parse(args) + val rootCommand = RootCommand() + CommandLine(rootCommand) + .addSubcommand("classification", ClassificationCommand()) + .addSubcommand("aggCl", AggregationClassificationCommand()) + .addSubcommand("rank", RankCommand()) + .addSubcommand("aggRnk", AggregationRankCommand()) + .execute(*args) } + +@CommandLine.Command( + name = "ArDoCo Metrics", + mixinStandardHelpOptions = true, + description = ["CLI for ArDoCo Metrics"] +) +class RootCommand diff --git a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationClassificationCommand.kt b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationClassificationCommand.kt index aeefa69..937d82b 100644 --- a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationClassificationCommand.kt +++ b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationClassificationCommand.kt @@ -6,28 +6,27 @@ import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.registerKotlinModule import edu.kit.kastel.mcse.ardoco.metrics.ClassificationMetricsCalculator import edu.kit.kastel.mcse.ardoco.metrics.result.SingleClassificationResult -import kotlinx.cli.ArgType -import kotlinx.cli.ExperimentalCli -import kotlinx.cli.SingleNullableOption -import kotlinx.cli.Subcommand -import kotlinx.cli.required -import java.io.File +import picocli.CommandLine.Command +import picocli.CommandLine.Option +import java.util.concurrent.Callable -@OptIn(ExperimentalCli::class) -class AggregationClassificationCommand( - private val outputFileOption: SingleNullableOption -) : Subcommand("aggCl", "Aggregate results of multiple classifications. I.e., Macro Average + WeightedAverage + Micro Average") { - private val directoryWithResultsOption by option( - ArgType.String, - shortName = "d", - description = "The directory with the classification results" - ).required() +@Command( + name = "aggCl", + description = ["Aggregate results of multiple classifications. I.e., Macro Average + WeightedAverage + Micro Average"], + mixinStandardHelpOptions = true +) +class AggregationClassificationCommand : Callable { + @Option(names = ["-d", "--directory"], description = ["The directory with the classification results"], required = true) + lateinit var directoryWithResults: String - override fun execute() { - val directory = File(directoryWithResultsOption) + @Option(names = ["-o", "--output"], description = ["The output file"]) + var outputFile: String? = null + + override fun call(): Int { + val directory = java.io.File(directoryWithResults) if (!directory.isDirectory) { println("The provided path is not a directory") - return + return 1 } val oom = ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).registerKotlinModule() val results: List> = @@ -38,16 +37,15 @@ class AggregationClassificationCommand( } ?: emptyList() if (results.isEmpty()) { println("No classification results found") - return + return 1 } val classificationMetrics = ClassificationMetricsCalculator.Instance val average = classificationMetrics.calculateAverages(results) average.forEach { it.prettyPrint() } - - val output = outputFileOption.value - if (output != null) { - val outputFile = File(output) - oom.writeValue(outputFile, average) + outputFile?.let { + val outputFileObj = java.io.File(it) + oom.writeValue(outputFileObj, average) } + return 0 } } diff --git a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationRankCommand.kt b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationRankCommand.kt index 1ad69ae..68ad305 100644 --- a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationRankCommand.kt +++ b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/AggregationRankCommand.kt @@ -6,28 +6,27 @@ import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.registerKotlinModule import edu.kit.kastel.mcse.ardoco.metrics.RankMetricsCalculator import edu.kit.kastel.mcse.ardoco.metrics.result.SingleRankMetricsResult -import kotlinx.cli.ArgType -import kotlinx.cli.ExperimentalCli -import kotlinx.cli.SingleNullableOption -import kotlinx.cli.Subcommand -import kotlinx.cli.required -import java.io.File +import picocli.CommandLine.Command +import picocli.CommandLine.Option +import java.util.concurrent.Callable -@OptIn(ExperimentalCli::class) -class AggregationRankCommand( - private val outputFileOption: SingleNullableOption -) : Subcommand("aggRnk", "Aggregate results of multiple rank metrics runs. I.e., Macro Average + WeightedAverage") { - private val directoryWithResultsOption by option( - ArgType.String, - shortName = "d", - description = "The directory with the rank results" - ).required() +@Command( + name = "aggRnk", + description = ["Aggregate results of multiple rank metrics runs. I.e., Macro Average + WeightedAverage"], + mixinStandardHelpOptions = true +) +class AggregationRankCommand : Callable { + @Option(names = ["-d", "--directory"], description = ["The directory with the rank results"], required = true) + lateinit var directoryWithResults: String - override fun execute() { - val directory = File(directoryWithResultsOption) + @Option(names = ["-o", "--output"], description = ["The output file"]) + var outputFile: String? = null + + override fun call(): Int { + val directory = java.io.File(directoryWithResults) if (!directory.isDirectory) { println("The provided path is not a directory") - return + return 1 } val oom = ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).registerKotlinModule() val results: List = @@ -38,16 +37,15 @@ class AggregationRankCommand( } ?: emptyList() if (results.isEmpty()) { println("No classification results found") - return + return 1 } val rankMetrics = RankMetricsCalculator.Instance val average = rankMetrics.calculateAverages(results) average.forEach { it.prettyPrint() } - - val output = outputFileOption.value - if (output != null) { - val outputFile = File(output) - oom.writeValue(outputFile, average) + outputFile?.let { + val outputFileObj = java.io.File(it) + oom.writeValue(outputFileObj, average) } + return 0 } } diff --git a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/ClassificationCommand.kt b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/ClassificationCommand.kt index 3852e86..6193185 100644 --- a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/ClassificationCommand.kt +++ b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/ClassificationCommand.kt @@ -3,50 +3,60 @@ package edu.kit.kastel.mcse.ardoco.metrics.cli.commands import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.registerKotlinModule -import edu.kit.kastel.mcse.ardoco.metrics.ClassificationMetricsCalculator -import kotlinx.cli.ArgType -import kotlinx.cli.ExperimentalCli -import kotlinx.cli.SingleNullableOption -import kotlinx.cli.Subcommand -import kotlinx.cli.default -import kotlinx.cli.required -import java.io.File +import picocli.CommandLine.Command +import picocli.CommandLine.Option +import java.util.concurrent.Callable -@OptIn(ExperimentalCli::class) -class ClassificationCommand(private val outputFileOption: SingleNullableOption) : Subcommand("classification", "Calculates classification metrics") { - private val classificationFileOption by option(ArgType.String, shortName = "c", description = "The classification file", fullName = "classification").required() - private val groundTruthFileOption by option(ArgType.String, shortName = "g", description = "The ground truth file", fullName = "ground-truth").required() - private val fileHeaderOption by option(ArgType.Boolean, description = "Whether the files have a header", fullName = "header").default(false) - private val confusionMatrixSumOption by option(ArgType.Int, shortName = "s", description = "The sum of the confusion matrix", fullName = "sum").default(-1) +@Command(name = "classification", description = ["Calculates classification metrics"], mixinStandardHelpOptions = true) +class ClassificationCommand : Callable { + @Option(names = ["-c", "--classification"], description = ["The classification file"], required = true) + lateinit var classificationFile: String - override fun execute() { - println("Calculating classification metrics") - val classificationFile = File(classificationFileOption) - val groundTruthFile = File(groundTruthFileOption) + @Option(names = ["-g", "--ground-truth"], description = ["The ground truth file"], required = true) + lateinit var groundTruthFile: String - if (!classificationFile.exists() || !groundTruthFile.exists()) { - println("Classification file or ground truth file does not exist") - return - } + @Option(names = ["--header"], description = ["Whether the files have a header"]) + var fileHeader: Boolean = false - val classification = classificationFile.readLines().filter { it.isNotBlank() }.drop(if (fileHeaderOption) 1 else 0).toSet() - val groundTruth = groundTruthFile.readLines().filter { it.isNotBlank() }.drop(if (fileHeaderOption) 1 else 0).toSet() + @Option(names = ["-s", "--sum"], description = ["The sum of the confusion matrix"]) + var confusionMatrixSum: Int = -1 - val classificationMetrics = ClassificationMetricsCalculator.Instance + @Option(names = ["-o", "--output"], description = ["The output file"]) + var outputFile: String? = null + override fun call(): Int { + println("Calculating classification metrics") + val classificationFileObj = java.io.File(classificationFile) + val groundTruthFileObj = java.io.File(groundTruthFile) + if (!classificationFileObj.exists() || !groundTruthFileObj.exists()) { + println("Classification file or ground truth file does not exist") + return 1 + } + val classification = + classificationFileObj + .readLines() + .filter { it.isNotBlank() } + .drop(if (fileHeader) 1 else 0) + .toSet() + val groundTruth = + groundTruthFileObj + .readLines() + .filter { it.isNotBlank() } + .drop(if (fileHeader) 1 else 0) + .toSet() + val classificationMetrics = edu.kit.kastel.mcse.ardoco.metrics.ClassificationMetricsCalculator.Instance val result = classificationMetrics.calculateMetrics( classification, groundTruth, - if (confusionMatrixSumOption < 0) null else confusionMatrixSumOption + if (confusionMatrixSum < 0) null else confusionMatrixSum ) result.prettyPrint() - - val output = outputFileOption.value - if (output != null) { - val outputFile = File(output) + outputFile?.let { + val outputFileObj = java.io.File(it) val oom = ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).registerKotlinModule() - oom.writeValue(outputFile, result) + oom.writeValue(outputFileObj, result) } + return 0 } } diff --git a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/RankCommand.kt b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/RankCommand.kt index 76257fe..2bb3107 100644 --- a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/RankCommand.kt +++ b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/RankCommand.kt @@ -4,89 +4,108 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.registerKotlinModule import edu.kit.kastel.mcse.ardoco.metrics.RankMetricsCalculator -import kotlinx.cli.ArgType -import kotlinx.cli.ExperimentalCli -import kotlinx.cli.SingleNullableOption -import kotlinx.cli.Subcommand -import kotlinx.cli.default -import kotlinx.cli.required -import java.io.File +import picocli.CommandLine.Command +import picocli.CommandLine.Option +import java.util.concurrent.Callable -@OptIn(ExperimentalCli::class) -class RankCommand(private val outputFileOption: SingleNullableOption) : Subcommand("rank", "Calculates rank metrics") { - private val rankedListDirectoryOption by option( - ArgType.String, shortName = "r", description = "The directory of the ranked list files", fullName = "ranked-list-directory" - ).required() - private val groundTruthFileOption by option( - ArgType.String, shortName = "g", description = "The ground truth file", fullName = "ground-truth" - ).required() - private val fileHeaderOption by option(ArgType.Boolean, description = "Whether the files have a header", fullName = "header").default(false) - private val rankedRelevanceListDirectoryOption by option( - ArgType.String, - shortName = "rrl", - description = "The directory of the ranked relevance list files", - fullName = "ranked-relevance-list-directory" - ) - private val biggerIsMoreSimilar by option( - ArgType.String, shortName = "b", description = "Whether the relevance scores are more similar if bigger", fullName = "bigger-is-more-similar" - ) +@Command(name = "rank", description = ["Calculates rank metrics"], mixinStandardHelpOptions = true) +class RankCommand : Callable { + @Option(names = ["-r", "--ranked-list-directory"], description = ["The directory of the ranked list files"], required = true) + lateinit var rankedListDirectory: String + @Option(names = ["-g", "--ground-truth"], description = ["The ground truth file"], required = true) + lateinit var groundTruthFile: String - override fun execute() { - println("Calculating rank metrics") - val rankedListDirectory = File(rankedListDirectoryOption) - val groundTruthFile = File(groundTruthFileOption) + @Option(names = ["--header"], description = ["Whether the files have a header"]) + var fileHeader: Boolean = false + + @Option(names = ["--ranked-relevance-list-directory", "-rrl"], description = ["The directory of the ranked relevance list files"]) + var rankedRelevanceListDirectory: String? = null + + @Option(names = ["-b", "--bigger-is-more-similar"], description = ["Whether the relevance scores are more similar if bigger"]) + var biggerIsMoreSimilar: Boolean? = null + + @Option(names = ["-o", "--output"], description = ["The output file"]) + var outputFile: String? = null - if (!rankedListDirectory.exists() || !groundTruthFile.exists()) { + override fun call(): Int { + println("Calculating rank metrics") + val rankedListDirectoryFile = java.io.File(rankedListDirectory) + val groundTruthFileObj = java.io.File(groundTruthFile) + if (!rankedListDirectoryFile.exists() || !groundTruthFileObj.exists()) { println("The directory of the ranked list files or ground truth file does not exist") - return + return 1 } - if (!rankedListDirectory.isDirectory) { + if (!rankedListDirectoryFile.isDirectory) { println("The provided path is not a directory") - return + return 1 } - val rankedResults: List> = rankedListDirectory.listFiles()?.filter { file -> - file.isFile - }?.map { file -> file.readLines().filter { it.isNotBlank() }.drop(if (fileHeaderOption) 1 else 0) } ?: emptyList() + val rankedResults: List> = + rankedListDirectoryFile + .listFiles() + ?.filter { file -> + file.isFile + }?.map { file -> file.readLines().filter { it.isNotBlank() }.drop(if (fileHeader) 1 else 0) } ?: emptyList() if (rankedResults.isEmpty()) { println("No classification results found") - return + return 1 } - val groundTruth = groundTruthFile.readLines().filter { it.isNotBlank() }.drop(if (fileHeaderOption) 1 else 0).toSet() - + val groundTruth = + groundTruthFileObj + .readLines() + .filter { it.isNotBlank() } + .drop(if (fileHeader) 1 else 0) + .toSet() var relevanceBasedInput: RankMetricsCalculator.RelevanceBasedInput? = null - if (rankedRelevanceListDirectoryOption != null) { - val rankedRelevanceListDirectory = File(rankedRelevanceListDirectoryOption!!) - if (!rankedRelevanceListDirectory.exists() || !rankedRelevanceListDirectory.isDirectory) { + if (rankedRelevanceListDirectory != null) { + val rankedRelevanceListDirectoryFile = java.io.File(rankedRelevanceListDirectory!!) + if (!rankedRelevanceListDirectoryFile.exists() || !rankedRelevanceListDirectoryFile.isDirectory) { println("The directory of the ranked relevance list files does not exist or is not a directory") - return + return 1 } - val rankedRelevances = rankedRelevanceListDirectory.listFiles()?.filter { file -> - file.isFile - }?.map { file -> file.readLines().filter { it.isNotBlank() }.map { it.toDouble() }.drop(if (fileHeaderOption) 1 else 0) } ?: emptyList() + val rankedRelevances = + rankedRelevanceListDirectoryFile + .listFiles() + ?.filter { file -> + file.isFile + }?.map { file -> + file + .readLines() + .filter { it.isNotBlank() } + .map { it.toDouble() } + .drop(if (fileHeader) 1 else 0) + } ?: emptyList() if (rankedRelevances.isEmpty()) { println("No relevance scores found") - return + return 1 } if (biggerIsMoreSimilar == null) { - throw IllegalArgumentException("ranked relevances and bigger is more similar can only occur together") + throw IllegalArgumentException("Both 'ranked-relevance-list-directory' and 'bigger-is-more-similar' must be specified together.") } - relevanceBasedInput = if (biggerIsMoreSimilar != null) RankMetricsCalculator.RelevanceBasedInput( - rankedRelevances, { it }, biggerIsMoreSimilar.toBoolean() - ) else null + relevanceBasedInput = + RankMetricsCalculator.RelevanceBasedInput( + rankedRelevances, + { it }, + biggerIsMoreSimilar!! + ) } val rankMetrics = RankMetricsCalculator.Instance - - val result = rankMetrics.calculateMetrics( - rankedResults, groundTruth, relevanceBasedInput - ) + val result = + rankMetrics.calculateMetrics( + rankedResults, + groundTruth, + relevanceBasedInput + ) result.prettyPrint() - - val output = outputFileOption.value - if (output != null) { - val outputFile = File(output) - val oom = ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).registerKotlinModule() - oom.writeValue(outputFile, result) + outputFile?.let { + val outputFileObj = java.io.File(it) + val oom = + ObjectMapper() + .enable( + SerializationFeature.INDENT_OUTPUT + ).registerKotlinModule() + oom.writeValue(outputFileObj, result) } + return 0 } }