Skip to content

Commit a81eaed

Browse files
osamahan999osamahan999
authored
Add retrying for listAlbums in GoogleMediaExporter (#1324)
* Add retrying for listALbums in GoogleMediaExporter * fix comment
1 parent d911ff5 commit a81eaed

File tree

3 files changed

+68
-28
lines changed

3 files changed

+68
-28
lines changed

‎extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/FailedToListAlbumsException.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package org.datatransferproject.datatransfer.google.common;
2+
3+
/**
4+
* FailedToListAlbumsException is thrown when we try to call PhotosInterface.listAlbums and are
5+
* unsuccessful.
6+
*/
7+
public class FailedToListAlbumsException extends Exception {
8+
public FailedToListAlbumsException(String message, Exception cause) {
9+
super(message, cause);
10+
}
11+
}

‎extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporter.java‎

Lines changed: 52 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@
3535
import java.util.Optional;
3636
import java.util.UUID;
3737
import javax.annotation.Nullable;
38+
3839
import org.datatransferproject.api.launcher.Monitor;
40+
import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException;
3941
import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
4042
import org.datatransferproject.datatransfer.google.common.GoogleErrorLogger;
4143
import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
@@ -44,7 +46,6 @@
4446
import org.datatransferproject.datatransfer.google.mediaModels.MediaItemSearchResponse;
4547
import org.datatransferproject.datatransfer.google.photos.GooglePhotosInterface;
4648
import org.datatransferproject.spi.cloud.storage.JobStore;
47-
import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore;
4849
import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor;
4950
import org.datatransferproject.spi.transfer.provider.ExportResult;
5051
import org.datatransferproject.spi.transfer.provider.ExportResult.ResultType;
@@ -143,7 +144,7 @@ private static String createCacheKey() {
143144
@Override
144145
public ExportResult<MediaContainerResource> export(
145146
UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation)
146-
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
147+
throws UploadErrorException, FailedToListAlbumsException, InvalidTokenException, PermissionDeniedException, IOException {
147148
if (!exportInformation.isPresent()) {
148149
// Make list of photos contained in albums so they are not exported twice later on
149150
populateContainedMediaList(jobId, authData);
@@ -321,7 +322,7 @@ private ExportResult<MediaContainerResource> exportMediaContainer(
321322
@VisibleForTesting
322323
ExportResult<MediaContainerResource> exportAlbums(
323324
TokensAndUrlAuthData authData, Optional<PaginationData> paginationData, UUID jobId)
324-
throws IOException, InvalidTokenException, PermissionDeniedException {
325+
throws FailedToListAlbumsException {
325326
Optional<String> paginationToken = Optional.empty();
326327
if (paginationData.isPresent()) {
327328
String token = ((StringPaginationToken) paginationData.get()).getToken();
@@ -330,9 +331,7 @@ ExportResult<MediaContainerResource> exportAlbums(
330331
paginationToken = Optional.of(token.substring(ALBUM_TOKEN_PREFIX.length()));
331332
}
332333

333-
AlbumListResponse albumListResponse;
334-
335-
albumListResponse = getOrCreatePhotosInterface(authData).listAlbums(paginationToken);
334+
AlbumListResponse albumListResponse = listAlbums(jobId, authData, paginationToken);
336335

337336
PaginationData nextPageData;
338337
String token = albumListResponse.getNextPageToken();
@@ -406,7 +405,7 @@ ExportResult<MediaContainerResource> exportMedia(
406405

407406
/** Method for storing a list of all photos that are already contained in albums */
408407
void populateContainedMediaList(UUID jobId, TokensAndUrlAuthData authData)
409-
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
408+
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException {
410409
// This method is only called once at the beginning of the transfer, so we can start by
411410
// initializing a new TempMediaData to be store in the job store.
412411
TempMediaData tempMediaData = new TempMediaData(jobId);
@@ -415,25 +414,29 @@ void populateContainedMediaList(UUID jobId, TokensAndUrlAuthData authData)
415414
AlbumListResponse albumListResponse;
416415
MediaItemSearchResponse containedMediaSearchResponse;
417416
do {
418-
albumListResponse =
419-
getOrCreatePhotosInterface(authData).listAlbums(Optional.ofNullable(albumToken));
420-
if (albumListResponse.getAlbums() != null) {
421-
for (GoogleAlbum album : albumListResponse.getAlbums()) {
422-
String albumId = album.getId();
423-
String photoToken = null;
424-
do {
425-
containedMediaSearchResponse =
426-
getOrCreatePhotosInterface(authData)
427-
.listMediaItems(Optional.of(albumId), Optional.ofNullable(photoToken));
428-
if (containedMediaSearchResponse.getMediaItems() != null) {
429-
for (GoogleMediaItem mediaItem : containedMediaSearchResponse.getMediaItems()) {
430-
tempMediaData.addContainedPhotoId(mediaItem.getId());
431-
}
417+
albumListResponse = listAlbums(jobId, authData, Optional.ofNullable(albumToken));
418+
albumToken = albumListResponse.getNextPageToken();
419+
if (albumListResponse.getAlbums() == null) {
420+
continue;
421+
}
422+
423+
for (GoogleAlbum album : albumListResponse.getAlbums()) {
424+
String albumId = album.getId();
425+
String photoToken = null;
426+
427+
do {
428+
containedMediaSearchResponse =
429+
getOrCreatePhotosInterface(authData)
430+
.listMediaItems(Optional.of(albumId), Optional.ofNullable(photoToken));
431+
if (containedMediaSearchResponse.getMediaItems() != null) {
432+
for (GoogleMediaItem mediaItem : containedMediaSearchResponse.getMediaItems()) {
433+
tempMediaData.addContainedPhotoId(mediaItem.getId());
432434
}
433-
photoToken = containedMediaSearchResponse.getNextPageToken();
434-
} while (photoToken != null);
435-
}
435+
}
436+
photoToken = containedMediaSearchResponse.getNextPageToken();
437+
} while (photoToken != null);
436438
}
439+
437440
albumToken = albumListResponse.getNextPageToken();
438441
} while (albumToken != null);
439442

@@ -558,6 +561,31 @@ GoogleMediaItem getGoogleMediaItem(String photoIdempotentId, String photoDataId,
558561
return null;
559562
}
560563

564+
/**
565+
* Tries to call PhotosInterface.listAlbums, and retries on failure. If unsuccessful, throws a
566+
* FailedToListAlbumsException.
567+
*/
568+
private AlbumListResponse listAlbums(UUID jobId, TokensAndUrlAuthData authData, Optional<String> albumToken)
569+
throws FailedToListAlbumsException {
570+
if (retryingExecutor == null || !enableRetrying) {
571+
try {
572+
return getOrCreatePhotosInterface(authData).listAlbums(albumToken);
573+
} catch (IOException | InvalidTokenException | PermissionDeniedException e) {
574+
throw new FailedToListAlbumsException(e.getMessage(), e);
575+
}
576+
}
577+
578+
try {
579+
return retryingExecutor.executeOrThrowException(
580+
format("%s: listAlbums(page=%s)", jobId, albumToken),
581+
format("listAlbums(page=%s)", albumToken),
582+
() -> getOrCreatePhotosInterface(authData).listAlbums(albumToken)
583+
);
584+
} catch (Exception e) {
585+
throw new FailedToListAlbumsException(e.getMessage(), e);
586+
}
587+
}
588+
561589
private synchronized GooglePhotosInterface getOrCreatePhotosInterface(
562590
TokensAndUrlAuthData authData) {
563591
return photosInterface == null ? makePhotosInterface(authData) : photosInterface;

‎extensions/data-transfer/portability-data-transfer-google/src/test/java/org/datatransferproject/datatransfer/google/media/GoogleMediaExporterTest.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import java.util.UUID;
4343
import java.util.stream.Collectors;
4444
import org.datatransferproject.api.launcher.Monitor;
45+
import org.datatransferproject.datatransfer.google.common.FailedToListAlbumsException;
4546
import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
4647
import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
4748
import org.datatransferproject.datatransfer.google.mediaModels.GoogleAlbum;
@@ -145,7 +146,7 @@ public void setup()
145146
}
146147

147148
@Test
148-
public void exportAlbumFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException {
149+
public void exportAlbumFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException, FailedToListAlbumsException {
149150
setUpSingleAlbum();
150151
when(albumListResponse.getNextPageToken()).thenReturn(ALBUM_TOKEN);
151152

@@ -183,7 +184,7 @@ public void exportAlbumFirstSet() throws IOException, InvalidTokenException, Per
183184
}
184185

185186
@Test
186-
public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException, PermissionDeniedException {
187+
public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException, PermissionDeniedException, FailedToListAlbumsException {
187188
setUpSingleAlbum();
188189
when(albumListResponse.getNextPageToken()).thenReturn(null);
189190

@@ -319,7 +320,7 @@ public void exportPhotoSubsequentSet()
319320

320321
@Test
321322
public void populateContainedMediaList()
322-
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
323+
throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException {
323324
// Set up an album with two photos
324325
setUpSingleAlbum();
325326
when(albumListResponse.getNextPageToken()).thenReturn(null);
@@ -397,7 +398,7 @@ public void testGetGoogleMediaItemSucceeds() throws IOException, InvalidTokenExc
397398
}
398399

399400
@Test
400-
public void testExportPhotosContainer_photosRetrying() throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
401+
public void testExportPhotosContainer_photosRetrying() throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException, FailedToListAlbumsException {
401402
String photoIdToFail1 = "photo3";
402403
String photoIdToFail2 = "photo5";
403404

0 commit comments

Comments
 (0)