Updating the PA zip creation logic for docker build

INSTALL.md

@@ -106,14 +106,14 @@ You can use the packaged Dockerfile and docker-compose.yml files [here](./docker
 4. Copy the RCA framework artifact and the Performance Analyzer plugin JAR into this folder
 
     `cp <RCA framework root>/build/distributions/performance-analyzer-rca-3.0.0.0-SNAPSHOT.zip ./`
-    `cp <Performance Analyzer plugin root>/build/distributions/opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip ./`
+    `cp <Performance Analyzer plugin root>/build/distributions/opensearch-performance-analyzer-3.0.0.0.zip ./`
 
  ### Installation
 
  1. Make sure you're in the `rca-infra` folder.
 
  2. Build and tag the Docker image with our RCA framework.
-    
+
     `docker build -t opensearch/pa-rca:3.0 .`
 
  3. Spin up a two node cluster as follows:

build.gradle

@@ -35,7 +35,6 @@ buildscript {
     dependencies {
         classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
         classpath group: 'com.google.guava', name: 'guava', version: '31.1-jre'
-        classpath 'org.ajoberstar:gradle-git:0.2.3'
     }
 }
 
@@ -309,10 +308,12 @@ project.afterEvaluate {
 
 repositories {
     mavenCentral()
+    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
 }
 
 configurations {
     includeJars
+    dockerBuild
 }
 
 tasks.withType(JavaCompile) {
@@ -375,6 +376,9 @@ dependencies {
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
     testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.1'
     testImplementation group: 'junit', name: 'junit', version: "${junitVersion}"
+
+    // Required for Docker build
+    dockerBuild group: 'org.opensearch.plugin', name:'performance-analyzer', version: "${opensearch_build}"
 }
 
 configurations.each {
@@ -414,83 +418,51 @@ idea {
 
 import groovy.json.JsonSlurper
 import java.nio.file.Paths
-import org.ajoberstar.gradle.git.tasks.GitClone
 
 String paDir
 String dockerBuildDir
 String dockerArtifactsDir
+String file_ext = "zip"
 
-task clonePaGitRepo(type: GitClone) {
+task pullPAPlugin(type: Copy) {
     dockerBuildDir = Paths.get(getProject().getBuildDir().toString(), "docker-build").toString()
     mkdir dockerBuildDir
-    if (localPaDir != null && !localPaDir.empty) {
-        // If you already have the performance Analyzer sources checked out then update the
-        // localPaDir in gradle.properties with the path.
-        paDir = localPaDir
-    } else {
-        paDir = Paths.get(dockerBuildDir, "pa").toString()
-    }
-
-    def destination = file(paDir)
-    uri = gitPaRepo
-    branch = gitPaBranch
-    destinationPath = destination
-    bare = false
-    enabled = !destination.exists() //to clone only once
-}
+    paDir = Paths.get(dockerBuildDir, "pa")
+    mkdir paDir
 
-task removeLicenses(type: Delete) {
-    dependsOn(clonePaGitRepo)
-    def toDelete = Paths.get(paDir, 'licenses', "performanceanalyzer-rca-${version}.jar.sha1")
-    delete(toDelete)
-}
-
-task regenerateLicenses(type: Exec) {
-    dependsOn(removeLicenses)
-    workingDir(paDir)
-    commandLine './gradlew', 'updateShas'
-}
-
-task buildPa(type: Exec) {
-    dependsOn(assemble, publishToMavenLocal, regenerateLicenses)
-    workingDir paDir
-    println String.format('pa in dir: (%s) will be built.', paDir)
     println String.format('opensearch_version: (%s), plugin_version: (%s), snapshot: (%s), qualifier: (%s).', opensearch_version, version, isSnapshot, buildVersionQualifier)
-    if (buildVersionQualifier == null || buildVersionQualifier == '' || buildVersionQualifier == 'null') {
-        commandLine './gradlew', 'assemble', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}"
-    }
-    else {
-        commandLine './gradlew', 'assemble', "-Dopensearch.version=${opensearch_version}", "-Dbuild.snapshot=${isSnapshot}", "-Dbuild.version_qualifier=${buildVersionQualifier}"
-    }
-    println "PA repo located at '" + paDir + "' will be used."
+    from(configurations.dockerBuild)
+    into(paDir)
+    println String.format('Pulled PA Plugin at: (%s), will be used for Docker Build', Paths.get(paDir, "performance-analyzer-${version}.${file_ext}"))
 }
 
 task copyAllArtifacts(type: Copy) {
-    dependsOn(buildPa)
+    dependsOn(assemble, publishToMavenLocal, pullPAPlugin)
     def projectPathStr = getProjectDir().toPath().toString()
     def dockerArtifacts = Paths.get(projectPathStr, 'docker').toString()
-    def rcaArtifacts = Paths.get(projectPathStr, 'build', 'distributions', "performance-analyzer-rca-${version}.zip")
-    def paArtifacts = Paths.get(paDir, 'build', 'distributions', "opensearch-performance-analyzer-${version}.zip")
+    def rcaArtifacts = Paths.get(projectPathStr, 'build', 'distributions', "performance-analyzer-rca-${version}.${file_ext}")
+    def paArtifacts = Paths.get(dockerBuildDir, 'pa', "performance-analyzer-${version}.${file_ext}")
 
     dockerArtifactsDir = Paths.get(dockerBuildDir, 'rca-docker')
+    mkdir dockerArtifactsDir
+
     from(dockerArtifacts)
     from(rcaArtifacts)
     from(paArtifacts)
     into(dockerArtifactsDir)
 }
 
 task copyReaderMetricsFiles(type: Copy) {
-    dependsOn(buildPa)
-    copy{
-        from ('src/test/resources/reader'){
+    dependsOn(pullPAPlugin)
+    copy {
+        from ('src/test/resources/reader') {
         }
         into 'build/private/test_resources'
     }
 }
 
 task buildDocker(type: Exec) {
     dependsOn(copyAllArtifacts)
-
     workingDir(dockerArtifactsDir)
     commandLine 'docker', 'build', '-t', 'opensearch/pa-rca:3.0', '.', '--build-arg', "JDK_VER=${buildDockerJdkVersion}"
 }

docker/Dockerfile

@@ -52,11 +52,10 @@ COPY --chown=1000:0 opensearch.yml log4j2.properties config/
 
 COPY --chown=1000:0 performance-analyzer-rca-3.0.0.0-SNAPSHOT.zip config/
 
-COPY --chown=1000:0 opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip /tmp/
-
-RUN opensearch-plugin install --batch file:///tmp/opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip; \
-        rm /tmp/opensearch-performance-analyzer-3.0.0.0-SNAPSHOT.zip
+COPY --chown=1000:0 performance-analyzer-3.0.0.0-SNAPSHOT.zip /tmp/
 
+RUN opensearch-plugin install --batch file:///tmp/performance-analyzer-3.0.0.0-SNAPSHOT.zip; \
+        rm /tmp/performance-analyzer-3.0.0.0-SNAPSHOT.zip
 USER 0
 
 # Set gid to 0 for opensearch and make group permission similar to that of user

gradle.properties

@@ -3,10 +3,9 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-localPaDir=../performance-analyzer
 # Below were added to fix build issue. Refer - https://github.com/diffplug/spotless/issues/834
 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
   --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
   --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
   --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
-  --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+  --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED

src/main/java/org/opensearch/performanceanalyzer/metrics/PerformanceAnalyzerMetrics.java

@@ -43,7 +43,8 @@ public class PerformanceAnalyzerMetrics {
     public static final String sHeapPath = "heap_metrics";
     public static final String sNodesPath = "node_metrics";
     public static final String sPendingTasksPath = "pending_tasks";
-    public static final String sClusterManagerThrottledTasksPath = "cluster_manager_throttling_metrics";
+    public static final String sClusterManagerThrottledTasksPath =
+            "cluster_manager_throttling_metrics";
     public static final String sShardStatePath = "shard_state_metrics";
     public static final String sDisksPath = "disk_metrics";
     public static final String sTCPPath = "tcp_metrics";

src/main/java/org/opensearch/performanceanalyzer/rca/framework/api/summaries/HotNodeSummary.java

@@ -72,6 +72,7 @@ public void setHotShardSummaryList(List<HotShardSummary> summaryList) {
             this.hotShardSummaryList = summaryList;
         }
     }
+
     public void appendNestedSummary(HotResourceSummary summary) {
         if (summary != null) {
             hotResourceSummaryList.add(summary);

src/main/java/org/opensearch/performanceanalyzer/rca/store/metric/AggregateMetric.java

@@ -115,8 +115,7 @@ public MetricFlowUnit gather(final Queryable queryable) {
                             context, tableName, aggDimension, groupByFieldsList, selectFieldsList);
         } catch (Exception e) {
             // TODO: Emit log/stats that gathering failed.
-             LOG.error("RCA: Caught an exception while getting the DB {}",
-             e.getMessage());
+            LOG.error("RCA: Caught an exception while getting the DB {}", e.getMessage());
             return MetricFlowUnit.generic();
         }
         return new MetricFlowUnit(0, result);

src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerAppTest.java

@@ -39,5 +39,4 @@ public void testStartErrorHandlingThread() throws InterruptedException {
         errorHandlingThread.interrupt();
         Assert.assertTrue(RcaTestHelper.verify(ReaderMetrics.ERROR_HANDLER_THREAD_STOPPED));
     }
-
 }