Induce artificial latency and fail 30% of uploads

Signed-off-by: Ashish Singh <ssashish@amazon.com>

Author: ashking94

server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java

@@ -37,6 +37,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -72,6 +73,8 @@ public final class RemoteStoreRefreshListener implements ReferenceManager.Refres
     private final AtomicLong refreshSeqNo = new AtomicLong();
     private final Map<String, Long> fileSizeMap = new HashMap<>();
 
+    private final Random random = new Random();
+
     public RemoteStoreRefreshListener(IndexShard indexShard) {
         this.indexShard = indexShard;
         this.storeDirectory = indexShard.store().directory();
@@ -153,6 +156,15 @@ public void afterRefresh(boolean didRefresh) {
                             // Start tracking total uploads started
                             statsTracker.incrementTotalUploadsStarted();
 
+                            // For test purpose
+                            // Step 1 - Injection artificial delay of 1s to 100s
+                            Thread.sleep(random.nextInt(100) + 1);
+
+                            // Step 2 - Fail 30% of requests by throwing exception
+                            if (random.nextInt(100) < 30) {
+                                throw new RuntimeException("Failing upload for test purpose");
+                            }
+
                             // Start the segments upload
                             segmentsUploadStatus = UploadStatus.STARTED;
                             segmentsUploadStatus = uploadNewSegments(localSegmentsPostRefresh, statsTracker, sizeMap);