diff --git a/INSTALL.md b/INSTALL.md index 5be7cd639..61256cb45 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -106,14 +106,14 @@ You can use the packaged Dockerfile and docker-compose.yml files [here](./docker 4. Copy the RCA framework artifact and the Performance Analyzer plugin JAR into this folder `cp /build/distributions/performance-analyzer-rca-3.0.0.0-SNAPSHOT.zip ./` - `cp /build/distributions/opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip ./` + `cp /build/distributions/opensearch-performance-analyzer-3.0.0.0.zip ./` ### Installation 1. Make sure you're in the `rca-infra` folder. 2. Build and tag the Docker image with our RCA framework. - + `docker build -t opensearch/pa-rca:3.0 .` 3. Spin up a two node cluster as follows: diff --git a/build.gradle b/build.gradle index b942ad690..0db8e4666 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,6 @@ buildscript { dependencies { classpath "org.opensearch.gradle:build-tools:${opensearch_version}" classpath group: 'com.google.guava', name: 'guava', version: '31.1-jre' - classpath 'org.ajoberstar:gradle-git:0.2.3' } } @@ -309,10 +308,12 @@ project.afterEvaluate { repositories { mavenCentral() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } } configurations { includeJars + dockerBuild } tasks.withType(JavaCompile) { @@ -375,6 +376,9 @@ dependencies { testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.1' testImplementation group: 'junit', name: 'junit', version: "${junitVersion}" + + // Required for Docker build + dockerBuild group: 'org.opensearch.plugin', name:'performance-analyzer', version: "${opensearch_build}" } configurations.each { @@ -414,65 +418,34 @@ idea { import groovy.json.JsonSlurper import java.nio.file.Paths -import org.ajoberstar.gradle.git.tasks.GitClone String paDir String dockerBuildDir String dockerArtifactsDir +String file_ext = "zip" -task clonePaGitRepo(type: GitClone) { +task pullPAPlugin(type: Copy) { dockerBuildDir = Paths.get(getProject().getBuildDir().toString(), "docker-build").toString() mkdir dockerBuildDir - if (localPaDir != null && !localPaDir.empty) { - // If you already have the performance Analyzer sources checked out then update the - // localPaDir in gradle.properties with the path. - paDir = localPaDir - } else { - paDir = Paths.get(dockerBuildDir, "pa").toString() - } - - def destination = file(paDir) - uri = gitPaRepo - branch = gitPaBranch - destinationPath = destination - bare = false - enabled = !destination.exists() //to clone only once -} + paDir = Paths.get(dockerBuildDir, "pa") + mkdir paDir -task removeLicenses(type: Delete) { - dependsOn(clonePaGitRepo) - def toDelete = Paths.get(paDir, 'licenses', "performanceanalyzer-rca-${version}.jar.sha1") - delete(toDelete) -} - -task regenerateLicenses(type: Exec) { - dependsOn(removeLicenses) - workingDir(paDir) - commandLine './gradlew', 'updateShas' -} - -task buildPa(type: Exec) { - dependsOn(assemble, publishToMavenLocal, regenerateLicenses) - workingDir paDir - println String.format('pa in dir: (%s) will be built.', paDir) println String.format('opensearch_version: (%s), plugin_version: (%s), snapshot: (%s), qualifier: (%s).', opensearch_version, version, isSnapshot, buildVersionQualifier) - if (buildVersionQualifier == null || buildVersionQualifier == '' || buildVersionQualifier == 'null') { - commandLine './gradlew', 'assemble', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}" - } - else { - commandLine './gradlew', 'assemble', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}", "-Dbuild.version_qualifier=${buildVersionQualifier}" - } - println "PA repo located at '" + paDir + "' will be used." + from(configurations.dockerBuild) + into(paDir) + println String.format('Pulled PA Plugin at: (%s), will be used for Docker Build', Paths.get(paDir, "performance-analyzer-${version}.${file_ext}")) } task copyAllArtifacts(type: Copy) { - dependsOn(buildPa) + dependsOn(assemble, publishToMavenLocal, pullPAPlugin) def projectPathStr = getProjectDir().toPath().toString() def dockerArtifacts = Paths.get(projectPathStr, 'docker').toString() - def rcaArtifacts = Paths.get(projectPathStr, 'build', 'distributions', "performance-analyzer-rca-${version}.zip") - def paArtifacts = Paths.get(paDir, 'build', 'distributions', "opensearch-performance-analyzer-${version}.zip") + def rcaArtifacts = Paths.get(projectPathStr, 'build', 'distributions', "performance-analyzer-rca-${version}.${file_ext}") + def paArtifacts = Paths.get(dockerBuildDir, 'pa', "performance-analyzer-${version}.${file_ext}") dockerArtifactsDir = Paths.get(dockerBuildDir, 'rca-docker') + mkdir dockerArtifactsDir + from(dockerArtifacts) from(rcaArtifacts) from(paArtifacts) @@ -480,9 +453,9 @@ task copyAllArtifacts(type: Copy) { } task copyReaderMetricsFiles(type: Copy) { - dependsOn(buildPa) - copy{ - from ('src/test/resources/reader'){ + dependsOn(pullPAPlugin) + copy { + from ('src/test/resources/reader') { } into 'build/private/test_resources' } @@ -490,7 +463,6 @@ task copyReaderMetricsFiles(type: Copy) { task buildDocker(type: Exec) { dependsOn(copyAllArtifacts) - workingDir(dockerArtifactsDir) commandLine 'docker', 'build', '-t', 'opensearch/pa-rca:3.0', '.', '--build-arg', "JDK_VER=${buildDockerJdkVersion}" } diff --git a/docker/Dockerfile b/docker/Dockerfile index 8d8310165..85e29e437 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -52,11 +52,10 @@ COPY --chown=1000:0 opensearch.yml log4j2.properties config/ COPY --chown=1000:0 performance-analyzer-rca-3.0.0.0-SNAPSHOT.zip config/ -COPY --chown=1000:0 opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip /tmp/ - -RUN opensearch-plugin install --batch file:///tmp/opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip; \ - rm /tmp/opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip +COPY --chown=1000:0 performance-analyzer-3.0.0.0-SNAPSHOT.zip /tmp/ +RUN opensearch-plugin install --batch file:///tmp/performance-analyzer-3.0.0.0-SNAPSHOT.zip; \ + rm /tmp/performance-analyzer-3.0.0.0-SNAPSHOT.zip USER 0 # Set gid to 0 for opensearch and make group permission similar to that of user diff --git a/gradle.properties b/gradle.properties index 9398d1449..755130d6c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,10 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # -localPaDir=../performance-analyzer # Below were added to fix build issue. Refer - https://github.com/diffplug/spotless/issues/834 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ - --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED \ No newline at end of file + --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED diff --git a/src/main/java/org/opensearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java b/src/main/java/org/opensearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java index 220a9737c..82fdbcb8e 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java +++ b/src/main/java/org/opensearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java @@ -43,7 +43,8 @@ public class PerformanceAnalyzerMetrics { public static final String sHeapPath = "heap_metrics"; public static final String sNodesPath = "node_metrics"; public static final String sPendingTasksPath = "pending_tasks"; - public static final String sClusterManagerThrottledTasksPath = "cluster_manager_throttling_metrics"; + public static final String sClusterManagerThrottledTasksPath = + "cluster_manager_throttling_metrics"; public static final String sShardStatePath = "shard_state_metrics"; public static final String sDisksPath = "disk_metrics"; public static final String sTCPPath = "tcp_metrics"; diff --git a/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/summaries/HotNodeSummary.java b/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/summaries/HotNodeSummary.java index 9ad834402..428f82ab1 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/summaries/HotNodeSummary.java +++ b/src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/summaries/HotNodeSummary.java @@ -72,6 +72,7 @@ public void setHotShardSummaryList(List summaryList) { this.hotShardSummaryList = summaryList; } } + public void appendNestedSummary(HotResourceSummary summary) { if (summary != null) { hotResourceSummaryList.add(summary); diff --git a/src/main/java/org/opensearch/performanceanalyzer/rca/store/metric/AggregateMetric.java b/src/main/java/org/opensearch/performanceanalyzer/rca/store/metric/AggregateMetric.java index 9c8f34f23..8b8532969 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/rca/store/metric/AggregateMetric.java +++ b/src/main/java/org/opensearch/performanceanalyzer/rca/store/metric/AggregateMetric.java @@ -115,8 +115,7 @@ public MetricFlowUnit gather(final Queryable queryable) { context, tableName, aggDimension, groupByFieldsList, selectFieldsList); } catch (Exception e) { // TODO: Emit log/stats that gathering failed. - LOG.error("RCA: Caught an exception while getting the DB {}", - e.getMessage()); + LOG.error("RCA: Caught an exception while getting the DB {}", e.getMessage()); return MetricFlowUnit.generic(); } return new MetricFlowUnit(0, result); diff --git a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerAppTest.java b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerAppTest.java index de2f44668..f24b83727 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerAppTest.java +++ b/src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerAppTest.java @@ -39,5 +39,4 @@ public void testStartErrorHandlingThread() throws InterruptedException { errorHandlingThread.interrupt(); Assert.assertTrue(RcaTestHelper.verify(ReaderMetrics.ERROR_HANDLER_THREAD_STOPPED)); } - }