├── .cz.toml ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── commitlint.yml │ ├── gradle.yml │ └── release.yml ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Documentation ├── Developer.md ├── Glossary.md ├── Integration.md ├── JettyApiSpec.yaml ├── Keys.md ├── Overview.md ├── Publishing.md ├── README.md └── RunningLocally.md ├── LICENSE ├── README.md ├── build.gradle ├── client-rest ├── angular.json ├── build.gradle ├── e2e │ ├── protractor.conf.js │ ├── src │ │ ├── app.e2e-spec.ts │ │ └── app.po.ts │ └── tsconfig.e2e.json ├── package-lock.json ├── package.json ├── proxy.conf.json ├── src │ ├── app │ │ ├── app.component.html │ │ ├── app.component.spec.ts │ │ ├── app.component.ts │ │ ├── app.module.ts │ │ ├── app.routing.ts │ │ ├── data │ │ │ ├── data-types.service.ts │ │ │ ├── data.component.html │ │ │ ├── data.component.ts │ │ │ └── index.ts │ │ ├── progress │ │ │ ├── index.ts │ │ │ ├── progress.component.html │ │ │ ├── progress.component.ts │ │ │ ├── progress.guard.ts │ │ │ └── progress.service.ts │ │ ├── start │ │ │ ├── index.ts │ │ │ ├── start.component.html │ │ │ └── start.component.ts │ │ ├── transfer │ │ │ ├── auth-callback.component.ts │ │ │ ├── create-transfer.component.html │ │ │ ├── create-transfer.component.ts │ │ │ ├── index.ts │ │ │ ├── initiate-transfer.component.html │ │ │ ├── initiate-transfer.component.ts │ │ │ └── transfer.service.ts │ │ ├── transport │ │ │ ├── index.ts │ │ │ └── transport.functions.ts │ │ ├── types │ │ │ ├── create-transfer-job.ts │ │ │ ├── data-types.ts │ │ │ ├── generate-service-auth-data.ts │ │ │ ├── get-reserved-worker.ts │ │ │ ├── index.ts │ │ │ ├── reserve-worker.ts │ │ │ ├── reserved-worker.ts │ │ │ ├── service-auth-data.ts │ │ │ ├── start-transfer-job.ts │ │ │ ├── transfer-job.ts │ │ │ └── transfer-services.ts │ │ └── util │ │ │ ├── arrays.ts │ │ │ └── index.ts │ ├── assets │ │ └── .gitkeep │ ├── browserslist │ ├── environments │ │ ├── .gitignore │ │ └── environment.ts │ ├── favicon.ico │ ├── index.html │ ├── karma.conf.js │ ├── main.ts │ ├── polyfills.ts │ ├── sass │ │ ├── _progress-indicator.scss │ │ └── styles.scss │ ├── test-keys │ │ ├── server.crt │ │ └── server.key │ ├── test.ts │ ├── tsconfig.app.json │ ├── tsconfig.spec.json │ └── tslint.json ├── tsconfig.json └── tslint.json ├── config └── checkstyle │ └── checkstyle.xml ├── distributions ├── README.md ├── api-default │ └── build.gradle ├── common │ └── k8s │ │ ├── README.md │ │ ├── api-deployment.yaml │ │ ├── api-service.yaml │ │ └── transfer-deployment.yaml ├── demo-google-deployment │ ├── README.md │ ├── api │ │ └── build.gradle │ ├── bin │ │ ├── .gitignore │ │ ├── build_and_deploy_static_content.sh │ │ ├── encrypt_and_upload_secrets.sh │ │ ├── iam-policy.json │ │ ├── init_project_vars_example.sh │ │ └── setup_gke_environment.sh │ ├── resources │ │ ├── config │ │ │ ├── api.yaml │ │ │ ├── common.yaml │ │ │ └── environments │ │ │ │ └── qa │ │ │ │ └── index.html │ │ └── log4j.properties │ └── transfer │ │ └── build.gradle ├── demo-server │ ├── .gitignore │ ├── build.gradle │ ├── env.secrets.template │ └── src │ │ ├── config │ │ ├── client │ │ │ └── nginx.conf │ │ └── demo │ │ │ └── supervisord.conf │ │ └── main │ │ └── resources │ │ ├── config │ │ ├── common.yaml │ │ ├── retry │ │ │ └── default.yaml │ │ └── transfer.yaml │ │ ├── demo-selfsigned-keystore.jks │ │ └── log4j.properties └── transfer-default │ └── build.gradle ├── extensions ├── README.md ├── auth │ ├── portability-auth-apple │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── apple │ │ │ │ ├── AppleAuthServiceExtension.java │ │ │ │ ├── AppleOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ ├── META-INF.services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ │ │ └── config │ │ │ └── retry │ │ │ └── default.yaml │ ├── portability-auth-daybook │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── daybook │ │ │ │ ├── DaybookAuthServiceExtension.java │ │ │ │ ├── DaybookOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-deezer │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── deezer │ │ │ │ ├── DeezerAuthServiceExtension.java │ │ │ │ ├── DeezerOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-facebook │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── facebook │ │ │ │ ├── FacebookAuthServiceExtension.java │ │ │ │ ├── FacebookOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-flickr │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── auth │ │ │ │ │ └── flickr │ │ │ │ │ ├── FlickrAuthServiceExtension.java │ │ │ │ │ └── FlickrOAuthConfig.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── auth │ │ │ └── flickr │ │ │ └── FlickrAuthDataGeneratorTest.java │ ├── portability-auth-google │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── google │ │ │ │ ├── BloggerAuthServiceExtension.java │ │ │ │ ├── BloggerOAuthConfig.java │ │ │ │ ├── GoogleAuthServiceExtension.java │ │ │ │ ├── GoogleOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-harness-microsoft │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── auth │ │ │ └── microsoft │ │ │ └── harness │ │ │ ├── AuthTestDriver.java │ │ │ └── TestHelper.java │ ├── portability-auth-imgur │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org.datatransferproject.auth.imgur │ │ │ │ ├── ImgurAuthServiceExtension.java │ │ │ │ ├── ImgurOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-instagram │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── instagram │ │ │ │ ├── InstagramAuthServiceExtension.java │ │ │ │ └── InstagramOAuthConfig.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-koofr │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── koofr │ │ │ │ ├── KoofrAuthServiceExtension.java │ │ │ │ ├── KoofrOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-microsoft │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── microsoft │ │ │ │ ├── MicrosoftAuthServiceExtension.java │ │ │ │ ├── MicrosoftOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-offline-demo │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── offline │ │ │ │ ├── OfflineDemoAuthDataGenerator.java │ │ │ │ └── OfflineDemoAuthServiceExtension.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-rememberthemilk │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── rememberthemilk │ │ │ │ ├── RememberTheMilkAuthDataGenerator.java │ │ │ │ ├── RememberTheMilkAuthServiceExtension.java │ │ │ │ ├── RememberTheMilkSignatureGenerator.java │ │ │ │ └── model │ │ │ │ ├── AuthElement.java │ │ │ │ └── RememberTheMilkResponse.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-smugmug │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── smugmug │ │ │ │ ├── SmugMugAuthServiceExtension.java │ │ │ │ └── SmugMugOAuthConfig.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ ├── portability-auth-spotify │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── auth │ │ │ │ └── spotify │ │ │ │ ├── SpotifyAuthServiceExtension.java │ │ │ │ ├── SpotifyOAuthConfig.java │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension │ └── portability-auth-twitter │ │ ├── build.gradle │ │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── datatransferproject │ │ │ └── auth │ │ │ └── twitter │ │ │ ├── TwitterAuthServiceExtension.java │ │ │ └── TwitterOAuthConfig.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.datatransferproject.spi.api.auth.extension.AuthServiceExtension ├── cloud │ ├── portability-cloud-google │ │ ├── build.gradle │ │ ├── gradle.properties │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── cloud │ │ │ │ │ └── google │ │ │ │ │ ├── GoogleAppCredentialStore.java │ │ │ │ │ ├── GoogleAppSecretDecrypter.java │ │ │ │ │ ├── GoogleCloudExtension.java │ │ │ │ │ ├── GoogleCloudExtensionModule.java │ │ │ │ │ ├── GoogleCloudIdempotentImportExecutor.java │ │ │ │ │ ├── GoogleCloudIdempotentImportExecutorExtension.java │ │ │ │ │ ├── GoogleCloudUtils.java │ │ │ │ │ ├── GoogleCredentialException.java │ │ │ │ │ ├── GoogleDtpInternalMetricExtension.java │ │ │ │ │ ├── GoogleDtpInternalMetricRecorder.java │ │ │ │ │ ├── GoogleJobStore.java │ │ │ │ │ ├── GoogleTempFileStore.java │ │ │ │ │ ├── StackdriverMonitor.java │ │ │ │ │ ├── StackdriverMonitorExtension.java │ │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ ├── org.datatransferproject.api.launcher.MonitorExtension │ │ │ │ ├── org.datatransferproject.spi.cloud.extension.CloudExtension │ │ │ │ ├── org.datatransferproject.spi.service.extension.ServiceExtension │ │ │ │ └── org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutorExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── cloud │ │ │ └── google │ │ │ ├── GoogleCloudIdempotentImportExecutorTest.java │ │ │ ├── GoogleCloudModuleTest.java │ │ │ └── GoogleJobStoreTest.java │ ├── portability-cloud-local │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── cloud │ │ │ │ │ └── local │ │ │ │ │ ├── LocalAppCredentialStore.java │ │ │ │ │ ├── LocalCloudExtension.java │ │ │ │ │ ├── LocalJobStore.java │ │ │ │ │ ├── LocalTempFileStore.java │ │ │ │ │ └── package-info.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.cloud.extension.CloudExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── cloud │ │ │ └── local │ │ │ └── LocalJobStoreTest.java │ └── portability-cloud-microsoft │ │ ├── build.gradle │ │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── datatransferproject │ │ │ └── cloud │ │ │ └── microsoft │ │ │ ├── cosmos │ │ │ ├── AzureDtpInternalMetricExtension.java │ │ │ ├── AzureDtpInternalMetricRecorder.java │ │ │ ├── AzureKeyVaultStore.java │ │ │ ├── AzureTableStore.java │ │ │ ├── DataWrapper.java │ │ │ ├── MicrosoftCloudExtension.java │ │ │ ├── MicrosoftStorageException.java │ │ │ └── TableStoreConfiguration.java │ │ │ └── package-info.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── org.datatransferproject.spi.cloud.extension.CloudExtension │ │ └── org.datatransferproject.spi.service.extension.ServiceExtension ├── config │ └── portability-config-yaml │ │ ├── build.gradle │ │ └── src │ │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── config │ │ │ │ └── yaml │ │ │ │ ├── YamlSettingsExtension.java │ │ │ │ └── package-info.java │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.config.extension.SettingsExtension │ │ └── test │ │ ├── java │ │ └── org │ │ │ └── datatransferproject │ │ │ └── config │ │ │ └── yaml │ │ │ └── YamlSettingsExtensionTest.java │ │ └── resources │ │ ├── api-1.yaml │ │ └── api-2.yaml ├── copier │ └── portability-stack-copier │ │ ├── build.gradle │ │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── datatransferproject │ │ │ └── copier │ │ │ └── stack │ │ │ ├── PortabilityStackInMemoryDataCopier.java │ │ │ ├── PortabilityStackInMemoryDataCopierExtension.java │ │ │ └── package-info.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.datatransferproject.transfer.copier.InMemoryDataCopierExtension ├── data-transfer │ ├── portability-data-transfer-apple │ │ ├── Readme.md │ │ ├── build.gradle │ │ ├── idea.gradle │ │ ├── proto.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── datatransfer │ │ │ │ │ └── apple │ │ │ │ │ ├── AppleBaseInterface.java │ │ │ │ │ ├── AppleInterfaceFactory.java │ │ │ │ │ ├── AppleTransferExtension.java │ │ │ │ │ ├── constants │ │ │ │ │ ├── AppleConstants.java │ │ │ │ │ ├── AppleMusicConstants.java │ │ │ │ │ ├── ApplePhotosConstants.java │ │ │ │ │ ├── AuditKeys.java │ │ │ │ │ └── Headers.java │ │ │ │ │ ├── exceptions │ │ │ │ │ ├── AppleContentException.java │ │ │ │ │ └── AppleHttpCopyException.java │ │ │ │ │ ├── http │ │ │ │ │ └── TokenRefresher.java │ │ │ │ │ ├── music │ │ │ │ │ ├── AppleMusicImporter.java │ │ │ │ │ ├── AppleMusicInterface.java │ │ │ │ │ └── data │ │ │ │ │ │ └── converters │ │ │ │ │ │ └── AppleMusicPlaylistConverter.java │ │ │ │ │ ├── photos │ │ │ │ │ ├── AppleMediaImporter.java │ │ │ │ │ ├── AppleMediaInterface.java │ │ │ │ │ ├── ApplePhotosImporter.java │ │ │ │ │ ├── AppleVideosImporter.java │ │ │ │ │ ├── DownUpResult.java │ │ │ │ │ └── streaming │ │ │ │ │ │ └── StreamingContentClient.java │ │ │ │ │ └── signals │ │ │ │ │ ├── AppleSignalHandler.java │ │ │ │ │ └── AppleSignalInterface.java │ │ │ ├── proto │ │ │ │ ├── ImportMusicPlaylists.proto │ │ │ │ └── ImportPhotos.proto │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── datatransfer │ │ │ │ └── apple │ │ │ │ ├── http │ │ │ │ ├── FakeSynchronousSubscriber.java │ │ │ │ └── TokenRefresherTest.java │ │ │ │ ├── music │ │ │ │ ├── AppleMusicImporterTest.java │ │ │ │ └── data │ │ │ │ │ └── converters │ │ │ │ │ └── AppleMusicPlaylistConverterTest.java │ │ │ │ ├── photos │ │ │ │ ├── AppleImporterTestBase.java │ │ │ │ ├── AppleMediaImporterTest.java │ │ │ │ ├── ApplePhotosImporterTest.java │ │ │ │ ├── AppleVideosImporterTest.java │ │ │ │ ├── StreamingUtils.java │ │ │ │ └── TestConstants.java │ │ │ │ └── signals │ │ │ │ ├── AppleSignalHandlerTest.java │ │ │ │ └── AppleSignalInterfaceTest.java │ │ │ └── resources │ │ │ └── mockito-extensions │ │ │ └── org.mockito.plugins.MockMaker │ ├── portability-data-transfer-backblaze │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── datatransfer │ │ │ │ │ └── backblaze │ │ │ │ │ ├── BackblazeTransferExtension.java │ │ │ │ │ ├── common │ │ │ │ │ ├── BackblazeDataTransferClient.java │ │ │ │ │ ├── BackblazeDataTransferClientFactory.java │ │ │ │ │ ├── BackblazeS3ClientFactory.java │ │ │ │ │ └── BaseBackblazeS3ClientFactory.java │ │ │ │ │ ├── exception │ │ │ │ │ └── BackblazeCredentialsException.java │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── photos │ │ │ │ │ └── BackblazePhotosImporter.java │ │ │ │ │ └── videos │ │ │ │ │ └── BackblazeVideosImporter.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── datatransfer │ │ │ │ └── backblaze │ │ │ │ ├── common │ │ │ │ └── BackblazeDataTransferClientTest.java │ │ │ │ ├── photos │ │ │ │ └── BackblazePhotosImporterTest.java │ │ │ │ └── videos │ │ │ │ └── BackblazeVideosImporterTest.java │ │ │ └── resources │ │ │ └── test.txt │ ├── portability-data-transfer-daybook │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── transfer │ │ │ │ │ └── daybook │ │ │ │ │ ├── DaybookTransferExtension.java │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── photos │ │ │ │ │ └── DaybookPhotosImporter.java │ │ │ │ │ └── social │ │ │ │ │ └── DaybookPostsImporter.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── transfer │ │ │ └── daybook │ │ │ ├── photos │ │ │ └── DaybookPhotosImporterTest.java │ │ │ └── social │ │ │ └── DaybookPostsImporterTest.java │ ├── portability-data-transfer-deezer │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── transfer │ │ │ │ └── deezer │ │ │ │ ├── DeezerApi.java │ │ │ │ ├── DeezerTransferExtension.java │ │ │ │ ├── model │ │ │ │ ├── Album.java │ │ │ │ ├── Artist.java │ │ │ │ ├── Error.java │ │ │ │ ├── InsertResponse.java │ │ │ │ ├── PlaylistDetails.java │ │ │ │ ├── PlaylistSummary.java │ │ │ │ ├── PlaylistsResponse.java │ │ │ │ ├── Track.java │ │ │ │ ├── TrackCollection.java │ │ │ │ └── User.java │ │ │ │ └── playlists │ │ │ │ ├── DeezerPlaylistExporter.java │ │ │ │ └── DeezerPlaylistImporter.java │ │ │ └── resources │ │ │ ├── META-INF │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── config │ │ │ └── deezer.yaml │ ├── portability-data-transfer-facebook │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── transfer │ │ │ │ │ └── facebook │ │ │ │ │ ├── FacebookTransferExtension.java │ │ │ │ │ ├── photos │ │ │ │ │ ├── FacebookPhotosExporter.java │ │ │ │ │ ├── FacebookPhotosInterface.java │ │ │ │ │ └── RestFbFacebookPhotos.java │ │ │ │ │ ├── utils │ │ │ │ │ └── FacebookTransferUtils.java │ │ │ │ │ └── videos │ │ │ │ │ ├── FacebookVideosExporter.java │ │ │ │ │ ├── FacebookVideosImporter.java │ │ │ │ │ ├── FacebookVideosInterface.java │ │ │ │ │ └── RestFbFacebookVideos.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── transfer │ │ │ │ └── facebook │ │ │ │ ├── photos │ │ │ │ └── FacebookPhotosExporterTest.java │ │ │ │ └── videos │ │ │ │ ├── FacebookVideosExporterTest.java │ │ │ │ └── FacebookVideosImporterTest.java │ │ │ └── resources │ │ │ └── test.jpeg │ ├── portability-data-transfer-flickr │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── datatransfer │ │ │ │ │ └── flickr │ │ │ │ │ ├── FlickrTransferExtension.java │ │ │ │ │ ├── media │ │ │ │ │ └── FlickrMediaImporter.java │ │ │ │ │ └── photos │ │ │ │ │ ├── FlickrPhotosExporter.java │ │ │ │ │ ├── FlickrPhotosImporter.java │ │ │ │ │ ├── FlickrTempPhotoData.java │ │ │ │ │ └── FlickrUtils.java │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ │ └── config │ │ │ │ └── flickr.yaml │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── datatransfer │ │ │ └── flickr │ │ │ ├── FlickrTestUtils.java │ │ │ ├── media │ │ │ └── FlickrMediaImporterTest.java │ │ │ └── photos │ │ │ ├── FlickrPhotosExporterTest.java │ │ │ └── FlickrPhotosImporterTest.java │ ├── portability-data-transfer-generic │ │ ├── README.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── datatransfer │ │ │ │ └── generic │ │ │ │ ├── BlobIdToName.java │ │ │ │ ├── BlobbySerializer.java │ │ │ │ ├── CalendarSerializer.java │ │ │ │ ├── ContainerSerializer.java │ │ │ │ ├── GenericFileImporter.java │ │ │ │ ├── GenericImporter.java │ │ │ │ ├── GenericPayload.java │ │ │ │ ├── GenericTransferConstants.java │ │ │ │ ├── GenericTransferExtension.java │ │ │ │ ├── ImportableData.java │ │ │ │ ├── ImportableFileData.java │ │ │ │ ├── MediaSerializer.java │ │ │ │ ├── SocialPostsSerializer.java │ │ │ │ └── auth │ │ │ │ └── OAuthTokenManager.java │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── datatransfer │ │ │ └── generic │ │ │ ├── BlobIdToNameTest.java │ │ │ ├── BlobbySerializerSimulationTest.java │ │ │ ├── BlobbySerializerTest.java │ │ │ ├── CalendarSerializerTest.java │ │ │ ├── GenericFileImporterTest.java │ │ │ ├── GenericImportSerializerTestBase.java │ │ │ ├── GenericImporterTest.java │ │ │ ├── MediaSerializerTest.java │ │ │ ├── SocialPostsSerializerTest.java │ │ │ └── auth │ │ │ └── OAuthTokenManagerTest.java │ ├── portability-data-transfer-google │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── datatransfer │ │ │ │ │ └── google │ │ │ │ │ ├── BloggerTransferExtension.java │ │ │ │ │ ├── GoogleTransferExtension.java │ │ │ │ │ ├── blogger │ │ │ │ │ └── GoogleBloggerImporter.java │ │ │ │ │ ├── calendar │ │ │ │ │ ├── GoogleCalendarExporter.java │ │ │ │ │ └── GoogleCalendarImporter.java │ │ │ │ │ ├── common │ │ │ │ │ ├── FailedToListAlbumsException.java │ │ │ │ │ ├── FailedToListMediaItemsException.java │ │ │ │ │ ├── GoogleCredentialFactory.java │ │ │ │ │ ├── GoogleErrorLogger.java │ │ │ │ │ ├── GooglePhotosImportUtils.java │ │ │ │ │ ├── GoogleStaticObjects.java │ │ │ │ │ ├── InvalidExportedItemException.java │ │ │ │ │ └── gphotos │ │ │ │ │ │ └── GPhotosUpload.java │ │ │ │ │ ├── contacts │ │ │ │ │ ├── GoogleContactsExporter.java │ │ │ │ │ └── GoogleContactsImporter.java │ │ │ │ │ ├── drive │ │ │ │ │ ├── DriveExporter.java │ │ │ │ │ ├── DriveFolderMapping.java │ │ │ │ │ └── DriveImporter.java │ │ │ │ │ ├── gplus │ │ │ │ │ └── GooglePlusExporter.java │ │ │ │ │ ├── mail │ │ │ │ │ ├── GoogleMailExporter.java │ │ │ │ │ └── GoogleMailImporter.java │ │ │ │ │ ├── media │ │ │ │ │ ├── GoogleMediaExporter.java │ │ │ │ │ └── GoogleMediaImporter.java │ │ │ │ │ ├── mediaModels │ │ │ │ │ ├── AlbumListResponse.java │ │ │ │ │ ├── BatchMediaItemResponse.java │ │ │ │ │ ├── GoogleAlbum.java │ │ │ │ │ ├── GoogleMediaItem.java │ │ │ │ │ ├── MediaItemSearchResponse.java │ │ │ │ │ ├── MediaMetadata.java │ │ │ │ │ ├── NewMediaItem.java │ │ │ │ │ ├── NewMediaItemResult.java │ │ │ │ │ ├── NewMediaItemUpload.java │ │ │ │ │ ├── Photo.java │ │ │ │ │ ├── SimpleMediaItem.java │ │ │ │ │ ├── Status.java │ │ │ │ │ └── Video.java │ │ │ │ │ ├── music │ │ │ │ │ ├── GoogleMusicExporter.java │ │ │ │ │ ├── GoogleMusicHttpApi.java │ │ │ │ │ ├── GoogleMusicImporter.java │ │ │ │ │ └── GooglePlaylistInsertionToken.java │ │ │ │ │ ├── musicModels │ │ │ │ │ ├── BatchPlaylistItemRequest.java │ │ │ │ │ ├── BatchPlaylistItemResponse.java │ │ │ │ │ ├── GoogleArtist.java │ │ │ │ │ ├── GooglePlaylist.java │ │ │ │ │ ├── GooglePlaylistItem.java │ │ │ │ │ ├── GoogleRelease.java │ │ │ │ │ ├── GoogleTrack.java │ │ │ │ │ ├── ImportPlaylistItemRequest.java │ │ │ │ │ ├── ImportPlaylistRequest.java │ │ │ │ │ ├── NewPlaylistItemResult.java │ │ │ │ │ ├── PlaylistExportResponse.java │ │ │ │ │ ├── PlaylistItemExportResponse.java │ │ │ │ │ └── Status.java │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── photos │ │ │ │ │ ├── GooglePhotosExporter.java │ │ │ │ │ ├── GooglePhotosImporter.java │ │ │ │ │ ├── GooglePhotosInterface.java │ │ │ │ │ └── PhotoResult.java │ │ │ │ │ ├── tasks │ │ │ │ │ ├── GoogleTasksExporter.java │ │ │ │ │ └── GoogleTasksImporter.java │ │ │ │ │ └── videos │ │ │ │ │ ├── GoogleVideosExporter.java │ │ │ │ │ ├── GoogleVideosImporter.java │ │ │ │ │ ├── GoogleVideosInterface.java │ │ │ │ │ └── VideoResult.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── datatransfer │ │ │ │ └── google │ │ │ │ ├── calendar │ │ │ │ ├── GoogleCalendarExporterTest.java │ │ │ │ └── GoogleCalendarImporterTest.java │ │ │ │ ├── common │ │ │ │ └── GoogleErrorLoggerTest.java │ │ │ │ ├── contacts │ │ │ │ ├── GoogleContactsExportConversionTest.java │ │ │ │ ├── GoogleContactsExporterTest.java │ │ │ │ ├── GoogleContactsImportConversionTest.java │ │ │ │ └── GoogleContactsImporterTest.java │ │ │ │ ├── mail │ │ │ │ ├── GoogleMailExporterTest.java │ │ │ │ └── GoogleMailImporterTest.java │ │ │ │ ├── media │ │ │ │ ├── GoogleMediaExporterTest.java │ │ │ │ └── GoogleMediaImporterTest.java │ │ │ │ ├── mediaModels │ │ │ │ ├── AlbumListResponseTest.java │ │ │ │ ├── BatchMediaItemResponseTest.java │ │ │ │ ├── GoogleAlbumTest.java │ │ │ │ ├── GoogleMediaItemTest.java │ │ │ │ ├── MediaItemSearchResponseTest.java │ │ │ │ ├── MediaMetadataTest.java │ │ │ │ ├── PhotoTest.java │ │ │ │ ├── StatusTest.java │ │ │ │ └── VideoTest.java │ │ │ │ ├── music │ │ │ │ ├── GoogleMusicExporterTest.java │ │ │ │ └── GoogleMusicImporterTest.java │ │ │ │ ├── photos │ │ │ │ ├── GooglePhotosExporterTest.java │ │ │ │ ├── GooglePhotosImporterTest.java │ │ │ │ └── PhotoResultSerializationTest.java │ │ │ │ ├── tasks │ │ │ │ └── GoogleTasksExporterTest.java │ │ │ │ └── videos │ │ │ │ ├── GoogleVideosExporterTest.java │ │ │ │ ├── GoogleVideosImporterTest.java │ │ │ │ └── VideoResultSerializationTest.java │ │ │ └── resources │ │ │ └── mockito-extensions │ │ │ └── org.mockito.plugins.MockMaker │ ├── portability-data-transfer-imgur │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── datatransfer │ │ │ │ │ └── imgur │ │ │ │ │ ├── ImgurTransferExtension.java │ │ │ │ │ └── photos │ │ │ │ │ ├── ImgurPhotosExporter.java │ │ │ │ │ └── ImgurPhotosImporter.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── transfer │ │ │ │ └── imgur │ │ │ │ └── photos │ │ │ │ └── ImgurPhotoExporterTest.java │ │ │ └── resources │ │ │ ├── album_1_images.json │ │ │ ├── albums.json │ │ │ ├── all_images.json │ │ │ ├── page1.json │ │ │ └── page2.json │ ├── portability-data-transfer-instagram │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── transfer │ │ │ │ │ └── instagram │ │ │ │ │ ├── InstagramTransferExtension.java │ │ │ │ │ └── photos │ │ │ │ │ ├── InstagramPhotoExporter.java │ │ │ │ │ └── model │ │ │ │ │ ├── Caption.java │ │ │ │ │ ├── ImageData.java │ │ │ │ │ ├── ImageObject.java │ │ │ │ │ ├── MediaFeedData.java │ │ │ │ │ ├── MediaResponse.java │ │ │ │ │ └── Meta.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── transfer │ │ │ └── instagram │ │ │ └── photos │ │ │ └── InstagramPhotoExporterTest.java │ ├── portability-data-transfer-koofr │ │ ├── README.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── transfer │ │ │ │ │ └── koofr │ │ │ │ │ ├── KoofrTransferExtension.java │ │ │ │ │ ├── KoofrTransmogrificationConfig.java │ │ │ │ │ ├── common │ │ │ │ │ ├── ApiErrorDetails.java │ │ │ │ │ ├── FilesFile.java │ │ │ │ │ ├── FilesListRecursiveItem.java │ │ │ │ │ ├── InputStreamRequestBody.java │ │ │ │ │ ├── KoofrClient.java │ │ │ │ │ ├── KoofrClientFactory.java │ │ │ │ │ ├── KoofrCredentialFactory.java │ │ │ │ │ └── KoofrMediaExport.java │ │ │ │ │ ├── exceptions │ │ │ │ │ └── KoofrClientIOException.java │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── photos │ │ │ │ │ ├── KoofrPhotosExporter.java │ │ │ │ │ └── KoofrPhotosImporter.java │ │ │ │ │ └── videos │ │ │ │ │ ├── KoofrVideosExporter.java │ │ │ │ │ └── KoofrVideosImporter.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── transfer │ │ │ │ └── koofr │ │ │ │ ├── KoofrTransmogrificationConfigTest.java │ │ │ │ ├── common │ │ │ │ ├── Fixtures.java │ │ │ │ └── KoofrClientTest.java │ │ │ │ ├── photos │ │ │ │ ├── KoofrPhotosExporterTest.java │ │ │ │ └── KoofrPhotosImporterTest.java │ │ │ │ └── videos │ │ │ │ ├── KoofrVideosExporterTest.java │ │ │ │ └── KoofrVideosImporterTest.java │ │ │ └── resources │ │ │ ├── blank.jpg │ │ │ └── listrecursive.jsonl │ ├── portability-data-transfer-mastodon │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── transfer │ │ │ │ └── mastodon │ │ │ │ ├── model │ │ │ │ ├── Account.java │ │ │ │ └── Status.java │ │ │ │ └── social │ │ │ │ ├── MastodonActivityExport.java │ │ │ │ ├── MastodonActivityImport.java │ │ │ │ └── MastodonHttpUtilities.java │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── transfer │ │ │ └── mastodon │ │ │ └── contacts │ │ │ └── ManualTest.java │ ├── portability-data-transfer-microsoft │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── transfer │ │ │ │ │ └── microsoft │ │ │ │ │ ├── DataChunk.java │ │ │ │ │ ├── MicrosoftApiResponse.java │ │ │ │ │ ├── MicrosoftTransferExtension.java │ │ │ │ │ ├── MicrosoftTransmogrificationConfig.java │ │ │ │ │ ├── StreamChunker.java │ │ │ │ │ ├── calendar │ │ │ │ │ ├── MicrosoftCalendarExporter.java │ │ │ │ │ └── MicrosoftCalendarImporter.java │ │ │ │ │ ├── common │ │ │ │ │ ├── MicrosoftCredentialFactory.java │ │ │ │ │ └── RequestHelper.java │ │ │ │ │ ├── contacts │ │ │ │ │ ├── MicrosoftContactsExporter.java │ │ │ │ │ └── MicrosoftContactsImporter.java │ │ │ │ │ ├── driveModels │ │ │ │ │ ├── MicrosoftDriveFolder.java │ │ │ │ │ ├── MicrosoftDriveItem.java │ │ │ │ │ ├── MicrosoftDriveItemsResponse.java │ │ │ │ │ ├── MicrosoftFileMetadata.java │ │ │ │ │ ├── MicrosoftPhotoMetadata.java │ │ │ │ │ ├── MicrosoftSpecialFolder.java │ │ │ │ │ └── MicrosoftVideoMetadata.java │ │ │ │ │ ├── media │ │ │ │ │ ├── MicrosoftMediaExporter.java │ │ │ │ │ ├── MicrosoftMediaImporter.java │ │ │ │ │ └── MicrosoftMediaInterface.java │ │ │ │ │ ├── offline │ │ │ │ │ └── MicrosoftOfflineDataExporter.java │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── photos │ │ │ │ │ ├── MicrosoftPhotosExporter.java │ │ │ │ │ ├── MicrosoftPhotosImporter.java │ │ │ │ │ └── MicrosoftPhotosInterface.java │ │ │ │ │ ├── spi │ │ │ │ │ └── types │ │ │ │ │ │ └── MicrosoftOfflineData.java │ │ │ │ │ ├── transformer │ │ │ │ │ ├── TransformConstants.java │ │ │ │ │ ├── TransformResult.java │ │ │ │ │ ├── TransformerContext.java │ │ │ │ │ ├── TransformerService.java │ │ │ │ │ ├── TransformerServiceImpl.java │ │ │ │ │ ├── calendar │ │ │ │ │ │ ├── ToCalendarAttendeeModelTransformer.java │ │ │ │ │ │ ├── ToCalendarEventModelTransformer.java │ │ │ │ │ │ ├── ToCalendarEventTimeTransformer.java │ │ │ │ │ │ ├── ToCalendarModelTransformer.java │ │ │ │ │ │ ├── ToGraphCalendarTransformer.java │ │ │ │ │ │ └── ToGraphEventTransformer.java │ │ │ │ │ ├── common │ │ │ │ │ │ └── TransformerHelper.java │ │ │ │ │ └── contacts │ │ │ │ │ │ ├── ToGraphAddressTransformer.java │ │ │ │ │ │ ├── ToGraphContactTransformer.java │ │ │ │ │ │ ├── ToVCardAddressTransformer.java │ │ │ │ │ │ └── ToVCardTransformer.java │ │ │ │ │ └── types │ │ │ │ │ └── GraphPagination.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── transfer │ │ │ └── microsoft │ │ │ ├── MicrosoftApiResponseTest.java │ │ │ ├── StreamChunkerTest.java │ │ │ ├── helper │ │ │ ├── MockJobStore.java │ │ │ └── TestTransformerContext.java │ │ │ ├── integration │ │ │ ├── LocalExportTestRunner.java │ │ │ ├── LocalImportTestRunner.java │ │ │ ├── MicrosoftCalendarExportTest.java │ │ │ └── MicrosoftCalendarImportTest.java │ │ │ ├── media │ │ │ ├── MicrosoftMediaExporterTest.java │ │ │ └── MicrosoftMediaImporterTest.java │ │ │ ├── photos │ │ │ ├── MicrosoftPhotosExporterTest.java │ │ │ └── MicrosoftPhotosImporterTest.java │ │ │ └── transformer │ │ │ └── calendar │ │ │ ├── ToCalendarAttendeeModelTransformerTest.java │ │ │ ├── ToCalendarEventModelTransformerTest.java │ │ │ ├── ToCalendarEventTimeTransformerTest.java │ │ │ └── ToCalendarModelTransformerTest.java │ ├── portability-data-transfer-offline-demo │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── transfer │ │ │ │ └── offline │ │ │ │ ├── OfflineDemoImporter.java │ │ │ │ └── OfflineDemoTransferExtension.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ ├── portability-data-transfer-rememberthemilk │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── transfer │ │ │ │ │ └── rememberthemilk │ │ │ │ │ ├── RememberTheMilkTransferExtension.java │ │ │ │ │ ├── model │ │ │ │ │ └── tasks │ │ │ │ │ │ ├── Error.java │ │ │ │ │ │ ├── GetListResponse.java │ │ │ │ │ │ ├── GetListsResponse.java │ │ │ │ │ │ ├── ListAddResponse.java │ │ │ │ │ │ ├── ListInfo.java │ │ │ │ │ │ ├── Notes.java │ │ │ │ │ │ ├── RememberTheMilkResponse.java │ │ │ │ │ │ ├── Task.java │ │ │ │ │ │ ├── TaskList.java │ │ │ │ │ │ ├── TaskSeries.java │ │ │ │ │ │ ├── TaskUpdateResponse.java │ │ │ │ │ │ ├── Tasks.java │ │ │ │ │ │ ├── TimelineCreateResponse.java │ │ │ │ │ │ └── Transaction.java │ │ │ │ │ └── tasks │ │ │ │ │ ├── RememberTheMilkService.java │ │ │ │ │ ├── RememberTheMilkSignatureGenerator.java │ │ │ │ │ ├── RememberTheMilkTasksExporter.java │ │ │ │ │ └── RememberTheMilkTasksImporter.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── transfer │ │ │ └── rememberthemilk │ │ │ ├── model │ │ │ └── tasks │ │ │ │ └── ModelTest.java │ │ │ └── tasks │ │ │ └── RememberTheMilkSignatureGeneratorTest.java │ ├── portability-data-transfer-smugmug │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── org │ │ │ │ │ └── datatransferproject │ │ │ │ │ └── transfer │ │ │ │ │ └── smugmug │ │ │ │ │ ├── SmugMugInterface.java │ │ │ │ │ ├── SmugMugTransferExtension.java │ │ │ │ │ ├── SmugMugTransmogrificationConfig.java │ │ │ │ │ ├── media │ │ │ │ │ └── SmugMugMediaImporter.java │ │ │ │ │ └── photos │ │ │ │ │ ├── SmugMugOauthApi.java │ │ │ │ │ ├── SmugMugPhotoTempData.java │ │ │ │ │ ├── SmugMugPhotosExporter.java │ │ │ │ │ ├── SmugMugPhotosImporter.java │ │ │ │ │ └── model │ │ │ │ │ ├── SmugMugAlbum.java │ │ │ │ │ ├── SmugMugAlbumImageResponse.java │ │ │ │ │ ├── SmugMugAlbumInfoResponse.java │ │ │ │ │ ├── SmugMugAlbumResponse.java │ │ │ │ │ ├── SmugMugAlbumsResponse.java │ │ │ │ │ ├── SmugMugImage.java │ │ │ │ │ ├── SmugMugImageUploadResponse.java │ │ │ │ │ ├── SmugMugPageInfo.java │ │ │ │ │ ├── SmugMugResponse.java │ │ │ │ │ ├── SmugMugUrl.java │ │ │ │ │ ├── SmugMugUser.java │ │ │ │ │ └── SmugMugUserResponse.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ │ │ └── test │ │ │ └── java │ │ │ └── org │ │ │ └── datatransferproject │ │ │ └── transfer │ │ │ └── smugmug │ │ │ ├── media │ │ │ └── SmugMugMediaImporterTest.java │ │ │ └── photos │ │ │ ├── SmugMugInterfaceTest.java │ │ │ └── SmugMugPhotosImporterTest.java │ ├── portability-data-transfer-spotify │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── transfer │ │ │ │ └── spotify │ │ │ │ ├── SpotifyTransferExtension.java │ │ │ │ └── playlists │ │ │ │ ├── SpotifyPlaylistExporter.java │ │ │ │ └── SpotifyPlaylistImporter.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension │ └── portability-data-transfer-twitter │ │ ├── Readme.md │ │ ├── build.gradle │ │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── datatransferproject │ │ │ └── transfer │ │ │ └── twitter │ │ │ ├── TwitterApiWrapper.java │ │ │ ├── TwitterPhotosExporter.java │ │ │ ├── TwitterPhotosImporter.java │ │ │ └── TwitterTransferExtension.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.datatransferproject.spi.transfer.extension.TransferExtension ├── security │ ├── portability-security-cleartext │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── org │ │ │ │ └── datatransferproject │ │ │ │ └── security │ │ │ │ └── cleartext │ │ │ │ ├── ClearTextAuthDataDecryptService.java │ │ │ │ ├── ClearTextKeyGenerator.java │ │ │ │ ├── ClearTextPublicKeySerializer.java │ │ │ │ └── ClearTextSecurityExtension.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.datatransferproject.spi.transfer.security.SecurityExtension │ └── portability-security-jwe │ │ ├── build.gradle │ │ └── src │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── datatransferproject │ │ │ └── security │ │ │ └── jwe │ │ │ ├── JWEAuthDataDecryptService.java │ │ │ ├── JWEKeyGenerator.java │ │ │ ├── JWEPublicKeySerializer.java │ │ │ └── JWESecurityExtension.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.datatransferproject.spi.transfer.security.SecurityExtension └── transport │ └── portability-transport-jettyrest │ ├── build.gradle │ └── src │ └── main │ ├── java │ └── org │ │ └── datatransferproject │ │ └── transport │ │ └── jettyrest │ │ ├── JettyRestExtension.java │ │ ├── http │ │ ├── JettyMonitor.java │ │ └── JettyTransport.java │ │ └── rest │ │ ├── DataTypesController.java │ │ ├── JerseyTransportBinder.java │ │ ├── RESTConstants.java │ │ ├── TransferController.java │ │ └── TransferServicesController.java │ └── resources │ └── META-INF │ └── services │ └── org.datatransferproject.spi.service.extension.ServiceExtension ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── libraries ├── README.md ├── auth │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── auth │ │ ├── OAuth1Config.java │ │ ├── OAuth1DataGenerator.java │ │ ├── OAuth1ServiceExtension.java │ │ ├── OAuth2Config.java │ │ ├── OAuth2DataGenerator.java │ │ ├── OAuth2ServiceExtension.java │ │ ├── OAuth2TokenResponse.java │ │ └── OAuthUtils.java ├── config │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── config │ │ ├── ConfigUtils.java │ │ ├── FlagBindingModule.java │ │ └── extension │ │ ├── SettingsExtension.java │ │ └── SettingsExtensionLoader.java └── security │ ├── build.gradle │ └── src │ └── main │ └── java │ └── org │ └── datatransferproject │ └── security │ ├── AesSymmetricKeyGenerator.java │ ├── CryptoTransformation.java │ ├── Decrypter.java │ ├── DecrypterFactory.java │ ├── DecrypterImpl.java │ ├── Encrypter.java │ ├── EncrypterFactory.java │ ├── EncrypterImpl.java │ └── SymmetricKeyGenerator.java ├── package-lock.json ├── portability-api-launcher ├── build.gradle └── src │ └── main │ └── java │ └── org │ └── datatransferproject │ ├── api │ └── launcher │ │ ├── AbstractExtension.java │ │ ├── BootExtension.java │ │ ├── Constants.java │ │ ├── DelegatingExtensionContext.java │ │ ├── DtpInternalMetricRecorder.java │ │ ├── ExtensionContext.java │ │ ├── Flag.java │ │ ├── JobAwareMonitor.java │ │ ├── MetricRecorder.java │ │ ├── Monitor.java │ │ ├── MonitorExtension.java │ │ ├── SystemExtension.java │ │ ├── TypeManager.java │ │ └── Version.java │ └── launcher │ ├── metrics │ ├── LoggingDtpInternalMetricRecorder.java │ └── ServiceAwareMetricRecorder.java │ ├── monitor │ ├── ConsoleMonitor.java │ ├── MonitorLoader.java │ ├── MultiplexMonitor.java │ └── events │ │ └── EventCode.java │ └── types │ └── TypeManagerImpl.java ├── portability-api ├── README.md ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── api │ │ ├── ApiExtensionContext.java │ │ ├── ApiMain.java │ │ ├── ApiServicesModule.java │ │ ├── action │ │ ├── ActionUtils.java │ │ ├── datatype │ │ │ └── DataTypesAction.java │ │ └── transfer │ │ │ ├── CreateTransferJobAction.java │ │ │ ├── GenerateServiceAuthDataAction.java │ │ │ ├── GetReservedWorkerAction.java │ │ │ ├── GetTransferJobAction.java │ │ │ ├── GetTransferServicesAction.java │ │ │ ├── ReserveWorkerAction.java │ │ │ ├── StartTransferJobAction.java │ │ │ └── TransferServicesActionModule.java │ │ ├── auth │ │ └── PortabilityAuthServiceProviderRegistry.java │ │ ├── package-info.java │ │ └── token │ │ └── JWTTokenManager.java │ └── test │ └── java │ └── org │ └── datatransferproject │ └── api │ ├── action │ └── datatype │ │ └── DataTypesActionTest.java │ └── auth │ └── PortabilityAuthServiceExtensionRegistryTest.java ├── portability-bootstrap-vm ├── build.gradle └── src │ └── main │ └── java │ └── org │ └── datatransferproject │ └── bootstrap │ └── vm │ └── SingleVMMain.java ├── portability-spi-api ├── README.md ├── build.gradle └── src │ └── main │ └── java │ └── org │ └── datatransferproject │ ├── api │ └── action │ │ └── Action.java │ └── spi │ └── api │ ├── auth │ ├── AuthDataGenerator.java │ ├── AuthServiceProviderRegistry.java │ └── extension │ │ └── AuthServiceExtension.java │ ├── package-info.java │ ├── token │ └── TokenManager.java │ ├── transport │ ├── DiscardingStreamCounter.java │ ├── FileStreamer.java │ ├── JobFileStream.java │ ├── RemoteFileStreamer.java │ ├── TransportBinder.java │ └── UrlGetStreamer.java │ └── types │ └── AuthFlowConfiguration.java ├── portability-spi-cloud ├── README.md ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── spi │ │ └── cloud │ │ ├── connection │ │ └── ConnectionProvider.java │ │ ├── extension │ │ ├── CloudExtension.java │ │ ├── CloudExtensionLoader.java │ │ └── CloudExtensionModule.java │ │ ├── package-info.java │ │ ├── storage │ │ ├── AppCredentialStore.java │ │ ├── CryptoKeyStore.java │ │ ├── JobStore.java │ │ ├── JobStoreWithValidator.java │ │ └── TemporaryPerJobDataStore.java │ │ └── types │ │ ├── JobAuthorization.java │ │ └── PortabilityJob.java │ └── test │ └── java │ └── org │ └── datatransferproject │ └── spi │ └── cloud │ ├── connection │ └── ConnectionProviderTest.java │ └── types │ └── PortabilityJobTest.java ├── portability-spi-service ├── build.gradle └── src │ └── main │ └── java │ └── org │ └── datatransferproject │ └── spi │ └── service │ └── extension │ └── ServiceExtension.java ├── portability-spi-transfer ├── README.md ├── build.gradle └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── datatransferproject │ │ │ └── spi │ │ │ └── transfer │ │ │ ├── extension │ │ │ └── TransferExtension.java │ │ │ ├── hooks │ │ │ ├── DefaultJobHooks.java │ │ │ ├── JobHooks.java │ │ │ ├── JobHooksExtension.java │ │ │ ├── JobHooksLoader.java │ │ │ └── MultiplexJobHooks.java │ │ │ ├── i18n │ │ │ ├── AbstractMultilingualDictionary.java │ │ │ ├── BaseMultilingualDictionary.java │ │ │ ├── BaseMultilingualString.java │ │ │ └── MultilingualString.java │ │ │ ├── idempotentexecutor │ │ │ ├── CachingExecutor.java │ │ │ ├── IdempotentImportExecutor.java │ │ │ ├── IdempotentImportExecutorExtension.java │ │ │ ├── IdempotentImportExecutorLoader.java │ │ │ ├── ImportFunction.java │ │ │ ├── InMemoryIdempotentImportExecutor.java │ │ │ ├── InMemoryIdempotentImportExecutorExtension.java │ │ │ ├── ItemImportResult.java │ │ │ └── RetryingInMemoryIdempotentImportExecutor.java │ │ │ ├── package-info.java │ │ │ ├── provider │ │ │ ├── ExportResult.java │ │ │ ├── Exporter.java │ │ │ ├── ImportResult.java │ │ │ ├── Importer.java │ │ │ ├── SignalHandler.java │ │ │ ├── SignalRequest.java │ │ │ ├── TransferCompatibilityProvider.java │ │ │ └── converter │ │ │ │ ├── AnyToAnyExporter.java │ │ │ │ ├── AnyToAnyImporter.java │ │ │ │ ├── MediaExporterDecorator.java │ │ │ │ └── MediaImporterDecorator.java │ │ │ ├── security │ │ │ ├── AuthDataDecryptService.java │ │ │ ├── PublicKeySerializer.java │ │ │ ├── SecurityException.java │ │ │ ├── SecurityExtension.java │ │ │ ├── SecurityExtensionLoader.java │ │ │ └── TransferKeyGenerator.java │ │ │ ├── store │ │ │ └── TransferStore.java │ │ │ └── types │ │ │ ├── ContinuationData.java │ │ │ ├── CopyException.java │ │ │ ├── CopyExceptionWithFailureReason.java │ │ │ ├── DestinationMemoryFullException.java │ │ │ ├── DestinationNotFoundException.java │ │ │ ├── FailureReasons.java │ │ │ ├── IdOnlyResource.java │ │ │ ├── InvalidTokenException.java │ │ │ ├── PermissionDeniedException.java │ │ │ ├── SessionInvalidatedException.java │ │ │ ├── TempMediaData.java │ │ │ ├── UnconfirmedUserException.java │ │ │ ├── UploadErrorException.java │ │ │ ├── UpstreamApiUnexpectedResponseException.java │ │ │ ├── UserCheckpointedException.java │ │ │ └── signals │ │ │ └── JobLifeCycle.java │ └── resources │ │ ├── MultilingualStringsBundle.properties │ │ ├── MultilingualStringsBundle_ar.properties │ │ ├── MultilingualStringsBundle_cs.properties │ │ ├── MultilingualStringsBundle_de.properties │ │ ├── MultilingualStringsBundle_el.properties │ │ ├── MultilingualStringsBundle_es.properties │ │ ├── MultilingualStringsBundle_fr.properties │ │ ├── MultilingualStringsBundle_id.properties │ │ ├── MultilingualStringsBundle_it.properties │ │ ├── MultilingualStringsBundle_ja.properties │ │ ├── MultilingualStringsBundle_ms.properties │ │ ├── MultilingualStringsBundle_pt.properties │ │ ├── MultilingualStringsBundle_ru.properties │ │ ├── MultilingualStringsBundle_sk.properties │ │ ├── MultilingualStringsBundle_tl.properties │ │ ├── MultilingualStringsBundle_uk.properties │ │ └── MultilingualStringsBundle_zh.properties │ └── test │ └── java │ └── org │ └── datatransferproject │ └── spi │ └── transfer │ ├── i18n │ └── MultilingualDictionaryTest.java │ ├── idempotentexecutor │ ├── ItemImportResultTest.java │ └── RetryingInMemoryIdempotentImporterExecutorTest.java │ ├── provider │ ├── ExportResultTest.java │ ├── ImportResultTest.java │ ├── TransferCompatibilityProviderTest.java │ └── converter │ │ ├── AnyToAnyExporterTest.java │ │ ├── AnyToAnyImporterTest.java │ │ ├── MediaExporterDecoratorTest.java │ │ └── MediaImporterDecoratorTest.java │ └── types │ └── ContinuationDataTest.java ├── portability-test-utilities ├── build.gradle └── src │ └── main │ └── java │ └── org │ └── datatransferproject │ └── test │ └── types │ ├── FakeIdempotentImportExecutor.java │ └── ObjectMapperFactory.java ├── portability-transfer ├── README.md ├── build.gradle ├── settings.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── transfer │ │ ├── Annotations.java │ │ ├── CallableExporter.java │ │ ├── CallableImporter.java │ │ ├── CallableSizeCalculator.java │ │ ├── CredsTimeoutException.java │ │ ├── JobCancelWatchingService.java │ │ ├── JobMetadata.java │ │ ├── JobPoller.java │ │ ├── JobPollingService.java │ │ ├── JobProcessor.java │ │ ├── Worker.java │ │ ├── WorkerExtensionContext.java │ │ ├── WorkerMain.java │ │ ├── WorkerModule.java │ │ └── copier │ │ ├── InMemoryDataCopier.java │ │ ├── InMemoryDataCopierClassLoader.java │ │ ├── InMemoryDataCopierExtension.java │ │ ├── PortabilityAbstractInMemoryDataCopier.java │ │ └── PortabilityInMemoryDataCopier.java │ └── test │ └── java │ └── org │ └── datatransferproject │ └── transfer │ ├── CallableSizeCalculatorTest.java │ ├── JobPollingServiceTest.java │ ├── JobProcessorTest.java │ ├── WorkerModuleTest.java │ └── copier │ └── PortabilityInMemoryDataCopierTest.java ├── portability-types-client ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── types │ │ └── client │ │ ├── datatype │ │ ├── DataTypes.java │ │ └── GetDataTypes.java │ │ ├── package-info.java │ │ └── transfer │ │ ├── CreateTransferJob.java │ │ ├── GenerateServiceAuthData.java │ │ ├── GetReservedWorker.java │ │ ├── GetTransferJob.java │ │ ├── GetTransferServices.java │ │ ├── ReserveWorker.java │ │ ├── ReservedWorker.java │ │ ├── ServiceAuthData.java │ │ ├── StartTransferJob.java │ │ ├── TransferJob.java │ │ └── TransferServices.java │ └── test │ └── java │ └── org │ └── datatransferproject │ └── types │ └── client │ └── transfer │ ├── CreateTransferJobTest.java │ └── TransferJobTest.java ├── portability-types-common ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── types │ │ └── common │ │ ├── DownloadableFile.java │ │ ├── DownloadableItem.java │ │ ├── ExportInformation.java │ │ ├── Fileable.java │ │ ├── FolderItem.java │ │ ├── ImportableItem.java │ │ ├── IntPaginationToken.java │ │ ├── PaginationData.java │ │ ├── PortabilityCommon.java │ │ ├── PortableType.java │ │ ├── StringPaginationToken.java │ │ └── models │ │ ├── ContainerResource.java │ │ ├── CreativeWork.java │ │ ├── DataModel.java │ │ ├── DataVertical.java │ │ ├── DateRangeContainerResource.java │ │ ├── FavoriteInfo.java │ │ ├── IdOnlyContainerResource.java │ │ ├── ItemResource.java │ │ ├── MediaObject.java │ │ ├── README.md │ │ ├── Thing.java │ │ ├── TransmogrificationConfig.java │ │ ├── blob │ │ ├── BlobbyStorageContainerResource.java │ │ ├── DigitalDocumentWrapper.java │ │ └── DtpDigitalDocument.java │ │ ├── calendar │ │ ├── CalendarAttendeeModel.java │ │ ├── CalendarContainerResource.java │ │ ├── CalendarEventModel.java │ │ ├── CalendarModel.java │ │ ├── Property.java │ │ └── RecurrenceRule.java │ │ ├── contacts │ │ └── ContactsModelWrapper.java │ │ ├── mail │ │ ├── MailContainerModel.java │ │ ├── MailContainerResource.java │ │ └── MailMessageModel.java │ │ ├── media │ │ ├── MediaAlbum.java │ │ └── MediaContainerResource.java │ │ ├── music │ │ ├── MusicContainerResource.java │ │ ├── MusicGroup.java │ │ ├── MusicPlaylist.java │ │ ├── MusicPlaylistItem.java │ │ ├── MusicRecording.java │ │ └── MusicRelease.java │ │ ├── photos │ │ ├── PhotoAlbum.java │ │ ├── PhotoModel.java │ │ └── PhotosContainerResource.java │ │ ├── playlists │ │ ├── CreativeWork.java │ │ ├── MusicAlbum.java │ │ ├── MusicGroup.java │ │ ├── MusicPlaylist.java │ │ ├── MusicRecording.java │ │ └── PlaylistContainerResource.java │ │ ├── social │ │ ├── SocialActivityActor.java │ │ ├── SocialActivityAttachment.java │ │ ├── SocialActivityAttachmentType.java │ │ ├── SocialActivityContainerResource.java │ │ ├── SocialActivityLocation.java │ │ ├── SocialActivityModel.java │ │ └── SocialActivityType.java │ │ ├── tasks │ │ ├── TaskContainerResource.java │ │ ├── TaskListModel.java │ │ └── TaskModel.java │ │ └── videos │ │ ├── VideoAlbum.java │ │ ├── VideoModel.java │ │ └── VideosContainerResource.java │ └── test │ └── java │ └── org │ └── datatransferproject │ └── types │ └── common │ └── models │ ├── DataVerticalTest.java │ ├── DateRangeContainerResourceTest.java │ ├── blob │ └── BlobbyStorageContainerResourceTest.java │ ├── calendar │ └── CalendarContainerResourceTest.java │ ├── mail │ └── MailContainerResourceTest.java │ ├── media │ └── MediaContainerResourceTest.java │ ├── music │ └── MusicContainerResourceTest.java │ ├── photos │ ├── PhotoAlbumTest.java │ └── PhotosContainerResourceTest.java │ ├── social │ └── SocialActivityContainerResourceTest.java │ ├── tasks │ └── TaskContainerResourceTest.java │ └── videos │ └── VideosContainerResourceTest.java ├── portability-types-transfer ├── README.md ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── datatransferproject │ │ └── types │ │ └── transfer │ │ ├── auth │ │ ├── AppCredentials.java │ │ ├── AuthData.java │ │ ├── AuthDataPair.java │ │ ├── CookiesAndUrlAuthData.java │ │ ├── TokenAuthData.java │ │ ├── TokenSecretAuthData.java │ │ └── TokensAndUrlAuthData.java │ │ ├── errors │ │ └── ErrorDetail.java │ │ ├── package-info.java │ │ ├── retry │ │ ├── ExponentialBackoffSkipStrategy.java │ │ ├── ExponentialBackoffStrategy.java │ │ ├── NoRetryStrategy.java │ │ ├── RetryException.java │ │ ├── RetryMapping.java │ │ ├── RetryStrategy.java │ │ ├── RetryStrategyLibrary.java │ │ ├── RetryingCallable.java │ │ ├── SkipRetryStrategy.java │ │ ├── UniformRetrySkipStrategy.java │ │ └── UniformRetryStrategy.java │ │ └── serviceconfig │ │ ├── TransferServiceConfig.java │ │ └── TransferServiceConfigSpecification.java │ └── test │ └── java │ └── org │ └── datatransferproject │ └── types │ └── transfer │ └── auth │ └── AuthDataSerializationTest.java ├── scripts └── docker_push.sh └── settings.gradle /.cz.toml: -------------------------------------------------------------------------------- 1 | [tool.commitizen] 2 | name = "cz_conventional_commits" 3 | tag_format = "v$version" 4 | version_scheme = "semver" 5 | version_provider = "scm" 6 | update_changelog_on_bump = false 7 | changelog_start_rev = "v1.0.0" 8 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior, in case people don't have core.autocrlf set. 2 | * text=auto 3 | 4 | # Explicitly declare text files you want to always be normalized and converted 5 | # to native line endings on checkout. 6 | *.c text 7 | *.h text 8 | *.java text 9 | *.gradle text 10 | *.properties text 11 | *.gradle text 12 | 13 | # Declare files that will always have CRLF line endings on checkout. 14 | *.sln text eol=crlf 15 | 16 | # Denote all files that are truly binary and should not be modified. 17 | *.png binary 18 | *.jpg binary 19 | 20 | package-lock.json -diff -merge 21 | package-lock.json linguist-generated=true 22 | 23 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "npm" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | # Always increase the version requirement 8 | # to match the new version. 9 | versioning-strategy: increase 10 | commit-message: 11 | prefix: "fix" 12 | prefix-development: "build" 13 | include: "scope" 14 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | name: Lint PR Title 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, edited] 6 | 7 | jobs: 8 | commitlint: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: "Install commitizen" 13 | run: pip install --user -U commitizen 14 | - name: "Lint the PR title" 15 | env: 16 | TITLE: ${{ github.event.pull_request.title }} 17 | run: cz check --message "${TITLE}" || { echo "see https://github.com/dtinit/data-transfer-project/blob/master/Documentation/Publishing.md#semantic-versioning-and-conventional-commits" >&2; exit 1; } 18 | -------------------------------------------------------------------------------- /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Java CI with Gradle 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Set up JDK 11 20 | uses: actions/setup-java@v4 21 | with: 22 | java-version: '11' 23 | distribution: 'adopt' 24 | cache: gradle 25 | - name: Grant execute permission for gradlew 26 | run: chmod +x gradlew 27 | - name: Build and Test with Gradle 28 | run: ./gradlew build test 29 | -------------------------------------------------------------------------------- /Documentation/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | This folder contains documentation for contributing to and running the Data Transfer Project. 4 | 5 | * [Overview.md](Overview.md) - a high level summary of the project 6 | 7 | * [Developer.md](Developer.md) - developer setup and core framework development 8 | 9 | * [RunningLocally.md](RunningLocally.md) - a quick start to running DTP locally with our demo Docker image 10 | 11 | * [Integration.md](Integration.md) - integrating a new service, data type, and/or hosting platform 12 | 13 | * [Keys.md](Keys.md) - our philosophy for how DTP interacts with service provider APIs 14 | 15 | * [Glossary.md](Glossary.md) - a glossary of recommended terms to use in DTP UIs 16 | -------------------------------------------------------------------------------- /client-rest/e2e/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // Protractor configuration file, see link for more information 2 | // https://github.com/angular/protractor/blob/master/lib/config.ts 3 | 4 | const { SpecReporter } = require('jasmine-spec-reporter'); 5 | 6 | exports.config = { 7 | allScriptsTimeout: 11000, 8 | specs: [ 9 | './src/**/*.e2e-spec.ts' 10 | ], 11 | capabilities: { 12 | 'browserName': 'chrome', 13 | chromeOptions: { 14 | args: ["--disable-dev-shm-usage", "--headless", "--disable-gpu", "--no-sandbox", "--disable-extensions"] 15 | } 16 | }, 17 | directConnect: true, 18 | baseUrl: 'http://localhost:4200/', 19 | framework: 'jasmine', 20 | jasmineNodeOpts: { 21 | showColors: true, 22 | defaultTimeoutInterval: 30000, 23 | print: function() {} 24 | }, 25 | onPrepare() { 26 | require('ts-node').register({ 27 | project: require('path').join(__dirname, './tsconfig.e2e.json') 28 | }); 29 | jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } })); 30 | } 31 | }; 32 | -------------------------------------------------------------------------------- /client-rest/e2e/src/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { AppPage } from './app.po'; 2 | 3 | describe('workspace-project App', () => { 4 | let page: AppPage; 5 | 6 | beforeEach(() => { 7 | page = new AppPage(); 8 | }); 9 | 10 | it('should display welcome message', () => { 11 | page.navigateTo(); 12 | expect(page.getParagraphText()).toEqual('Data Transfer Project Demo'); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /client-rest/e2e/src/app.po.ts: -------------------------------------------------------------------------------- 1 | import { browser, by, element } from 'protractor'; 2 | 3 | export class AppPage { 4 | navigateTo() { 5 | return browser.get('/'); 6 | } 7 | 8 | getParagraphText() { 9 | return element(by.css('app-root h3')).getText(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /client-rest/e2e/tsconfig.e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "module": "commonjs", 6 | "target": "es5", 7 | "types": [ 8 | "jasmine", 9 | "jasminewd2", 10 | "node" 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /client-rest/proxy.conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "/api/*": { 3 | "target": "https://localhost:8080/", 4 | "secure": false, 5 | "logLevel": "debug" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /client-rest/src/app/app.component.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | -------------------------------------------------------------------------------- /client-rest/src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | import { Component } from "@angular/core"; 2 | 3 | @Component({ 4 | selector: "app-root", 5 | templateUrl: "./app.component.html" 6 | }) 7 | export class AppComponent { 8 | title = "Data Transfer Project Demo"; 9 | } 10 | -------------------------------------------------------------------------------- /client-rest/src/app/app.routing.ts: -------------------------------------------------------------------------------- 1 | import {RouterModule, Routes} from "@angular/router"; 2 | import {StartComponent} from "./start"; 3 | import {DataComponent} from "./data"; 4 | import {ProgressGuard} from "./progress"; 5 | import {AuthCallbackComponent, CreateTransferComponent} from "./transfer"; 6 | import {InitiateTransferComponent} from "./transfer/initiate-transfer.component"; 7 | 8 | const appRoutes: Routes = [ 9 | {path: "", component: StartComponent}, 10 | {path: "data", component: DataComponent, canActivate: [ProgressGuard]}, 11 | {path: "create", component: CreateTransferComponent, canActivate: [ProgressGuard]}, 12 | {path: "initiate", component: InitiateTransferComponent, canActivate: [ProgressGuard]}, 13 | {path: "callback/:service", component: AuthCallbackComponent}, 14 | 15 | // otherwise redirect to start 16 | {path: "**", redirectTo: ""} 17 | ]; 18 | 19 | export const routing = RouterModule.forRoot(appRoutes); 20 | 21 | 22 | -------------------------------------------------------------------------------- /client-rest/src/app/data/data-types.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from "@angular/core"; 2 | import {HttpClient} from "@angular/common/http"; 3 | import {Observable} from "rxjs/index"; 4 | import {environment} from "../../environments/environment"; 5 | import {DataTypes} from "../types"; 6 | 7 | /** 8 | * Returns the list of data types from the API server. 9 | */ 10 | @Injectable() 11 | export class DataTypesService { 12 | 13 | constructor(private http: HttpClient) { 14 | } 15 | 16 | getDataTypes(): Observable { 17 | return this.http.get(`${environment.apiBaseUrl}/api/datatypes`) as Observable; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /client-rest/src/app/data/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./data.component"; 2 | export * from "./data-types.service"; 3 | -------------------------------------------------------------------------------- /client-rest/src/app/progress/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./progress.component"; 2 | export * from "./progress.service"; 3 | export * from "./progress.guard"; 4 | -------------------------------------------------------------------------------- /client-rest/src/app/progress/progress.component.html: -------------------------------------------------------------------------------- 1 |
    2 |
  1. 3 | Start 4 |
    2. 5 |
  2. 6 | Choose Data 7 |
    3. 8 |
  3. 9 | Select Services 10 |
    4. 11 |
  4. 12 | Authentication 13 |
    5. 14 |
  5. 15 | Transfer 16 |
    6. 17 |
18 | 19 | 20 | -------------------------------------------------------------------------------- /client-rest/src/app/progress/progress.guard.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from "@angular/core"; 2 | import {ActivatedRouteSnapshot, CanActivate, RouterStateSnapshot} from "@angular/router"; 3 | import {ProgressService, Step} from "./progress.service"; 4 | 5 | /** 6 | * Disables invalid application transitions. 7 | */ 8 | @Injectable() 9 | export class ProgressGuard implements CanActivate { 10 | 11 | constructor(private progressService: ProgressService) { 12 | } 13 | 14 | canActivate(route: ActivatedRouteSnapshot, state: RouterStateSnapshot) { 15 | if ("/create" == state.url && this.progressService.currentStep() < Step.SERVICES) { 16 | return false; 17 | } else if ("/import" == state.url && this.progressService.currentStep() < Step.AUTHENTICATE_EXPORT) { 18 | return false; 19 | } else if ("/initiate" == state.url && this.progressService.currentStep() < Step.WORKER_RESERVED) { 20 | return false; 21 | } 22 | return true; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /client-rest/src/app/start/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./start.component"; 2 | -------------------------------------------------------------------------------- /client-rest/src/app/start/start.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, OnInit} from "@angular/core"; 2 | import {Router} from "@angular/router"; 3 | import {ProgressService} from "../progress"; 4 | import {environment} from "../../environments/environment"; 5 | 6 | /** 7 | * Displays the initial start step. 8 | */ 9 | @Component({ 10 | templateUrl: "./start.component.html" 11 | }) 12 | export class StartComponent implements OnInit { 13 | 14 | constructor(private progressService: ProgressService, private router: Router) { 15 | } 16 | 17 | ngOnInit() { 18 | if ("cleartext" === environment.encryptionScheme) { 19 | console.warn("Client configured to pass authentication credentials as clear text. This scheme should be used for demo purposes only."); 20 | } 21 | } 22 | 23 | next() { 24 | this.progressService.begin(); 25 | this.router.navigate(["data"]); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /client-rest/src/app/transfer/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./create-transfer.component"; 2 | export * from "./transfer.service"; 3 | export * from "./auth-callback.component"; -------------------------------------------------------------------------------- /client-rest/src/app/transport/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./transport.functions"; -------------------------------------------------------------------------------- /client-rest/src/app/transport/transport.functions.ts: -------------------------------------------------------------------------------- 1 | export function transportError(error: any) { 2 | console.error(error); 3 | alert(`Sorry, something is not right.\n\nCode: ${error.status}\nMessage: ${error.message}`); 4 | } -------------------------------------------------------------------------------- /client-rest/src/app/types/create-transfer-job.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Data required to create a transfer job. 3 | */ 4 | export interface CreateTransferJob { 5 | /** 6 | * Source (export) service. 7 | */ 8 | exportService: string; 9 | 10 | /** 11 | * Target (import) service. 12 | */ 13 | importService: string; 14 | 15 | /** 16 | * The URL for export auth callback. 17 | */ 18 | exportCallbackUrl: string; 19 | 20 | /** 21 | * The URL for import auth callback. 22 | */ 23 | importCallbackUrl: string; 24 | 25 | /** 26 | * The data type to transfer. 27 | */ 28 | dataType: string; 29 | 30 | /** 31 | * The encryptionScheme to use. 32 | */ 33 | encryptionScheme: string; 34 | } 35 | -------------------------------------------------------------------------------- /client-rest/src/app/types/data-types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The data types, e.g. contacts, photos, etc. supported by the current installation. 3 | */ 4 | export interface DataTypes { 5 | 6 | dataTypes: Array; 7 | 8 | } 9 | -------------------------------------------------------------------------------- /client-rest/src/app/types/generate-service-auth-data.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Data required to generate auth data. 3 | */ 4 | export interface GenerateServiceAuthData { 5 | id: string; 6 | authToken: string; 7 | mode: "EXPORT" | "IMPORT"; 8 | callbackUrl: string; 9 | } 10 | -------------------------------------------------------------------------------- /client-rest/src/app/types/get-reserved-worker.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Data required to reserve a worker 3 | */ 4 | export interface GetReservedWorker { 5 | id: string; 6 | } 7 | -------------------------------------------------------------------------------- /client-rest/src/app/types/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./data-types"; 2 | export * from "./transfer-services"; 3 | export * from "./create-transfer-job"; 4 | export * from "./transfer-job"; 5 | export * from "./reserve-worker"; 6 | export * from "./get-reserved-worker"; 7 | export * from "./reserved-worker"; 8 | export * from "./start-transfer-job"; 9 | export * from "./generate-service-auth-data"; 10 | export * from "./service-auth-data"; 11 | -------------------------------------------------------------------------------- /client-rest/src/app/types/reserve-worker.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Data required to reserve a worker for a transfer job. 3 | */ 4 | export interface ReserveWorker { 5 | id: string; 6 | } 7 | -------------------------------------------------------------------------------- /client-rest/src/app/types/reserved-worker.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Data to represent a worker reserved for a transfer job. 3 | */ 4 | export interface ReservedWorker { 5 | publicKey: string; 6 | } 7 | -------------------------------------------------------------------------------- /client-rest/src/app/types/service-auth-data.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Encrypted, generated auth data. 3 | */ 4 | export interface ServiceAuthData { 5 | authData: string; 6 | } 7 | -------------------------------------------------------------------------------- /client-rest/src/app/types/start-transfer-job.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Data required to start a transfer job. 3 | */ 4 | export interface StartTransferJob { 5 | id: string; 6 | 7 | /** 8 | * The auth data pair in the form {exportAuthData: AuthData, importAuthData: AuthData} encrypted according to the scheme in use by the client and server. 9 | */ 10 | encryptedAuthData: string; 11 | } 12 | -------------------------------------------------------------------------------- /client-rest/src/app/types/transfer-job.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A transfer job. 3 | */ 4 | export class TransferJob { 5 | /** 6 | * The unique transfer id, url-encoded. 7 | */ 8 | id: string; 9 | 10 | /** 11 | * The url to redirect to for export auth. 12 | */ 13 | exportUrl: string; 14 | 15 | /** 16 | * The url to redirect to for import auth. 17 | */ 18 | importUrl: string; 19 | } 20 | -------------------------------------------------------------------------------- /client-rest/src/app/types/transfer-services.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Available transfer services for a given data type. 3 | */ 4 | export interface TransferServices { 5 | dataType: string; 6 | exportServices: Array; 7 | importServices: Array; 8 | } 9 | -------------------------------------------------------------------------------- /client-rest/src/app/util/arrays.ts: -------------------------------------------------------------------------------- 1 | export class Arrays { 2 | /** 3 | * Removes an element from the array. 4 | * @param element the element 5 | * @param array the array 6 | * @returns {boolean} true if the element was removed 7 | */ 8 | static removeElement = function (element: any, array: Array) { 9 | let index = array.indexOf(element); 10 | if (index > -1) { 11 | array.splice(index, 1); 12 | return true; 13 | } 14 | return false; 15 | }; 16 | 17 | } -------------------------------------------------------------------------------- /client-rest/src/app/util/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./arrays" -------------------------------------------------------------------------------- /client-rest/src/assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinit/data-transfer-project/3a37025eded8c8e50006de79cb58bfa269543471/client-rest/src/assets/.gitkeep -------------------------------------------------------------------------------- /client-rest/src/browserslist: -------------------------------------------------------------------------------- 1 | # This file is currently used by autoprefixer to adjust CSS to support the below specified browsers 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | # For IE 9-11 support, please uncomment the last line of the file and adjust as needed 5 | > 0.5% 6 | last 2 versions 7 | Firefox ESR 8 | not dead 9 | # IE 9-11 -------------------------------------------------------------------------------- /client-rest/src/environments/.gitignore: -------------------------------------------------------------------------------- 1 | environment.prod.ts 2 | -------------------------------------------------------------------------------- /client-rest/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build ---prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false, 7 | encryptionScheme: "jwe", // supported values are "jwe" and "cleartext" which must correspond to the schemes supported by the server 8 | apiBaseUrl: ((typeof document !== "undefined") && document.baseURI.replace(/\/+$/, "")) || 'https://localhost:3000' 9 | }; 10 | -------------------------------------------------------------------------------- /client-rest/src/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinit/data-transfer-project/3a37025eded8c8e50006de79cb58bfa269543471/client-rest/src/favicon.ico -------------------------------------------------------------------------------- /client-rest/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Data Transfer Project Demo 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /client-rest/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule) 12 | .catch(err => console.log(err)); 13 | -------------------------------------------------------------------------------- /client-rest/src/sass/styles.scss: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css?family=Open+Sans:300,400|Roboto:300,400'); 2 | 3 | $header-font-family: 'Open Sans', Helvetica, Roboto, Arial, sans-serif; 4 | $body-font-family: 'Open Sans', Helvetica, Roboto, Arial, sans-serif; 5 | $header-font-weight: 300; 6 | 7 | @import '~foundation-sites/scss/foundation'; 8 | @include foundation-everything(true); 9 | @import "progress-indicator"; 10 | -------------------------------------------------------------------------------- /client-rest/src/test-keys/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICpDCCAYwCCQClaFSfxCiPLjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls 3 | b2NhbGhvc3QwHhcNMjAwNDI0MTIxOTIxWhcNMjAwNTI0MTIxOTIxWjAUMRIwEAYD 4 | VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDF 5 | RxTli0EkW2naOUo7Sl4aFjYagwDOp8cxUG47agrD6xH++0U05nquezYZXjltnEK5 6 | +fkJIQUEf0UkYZK6LUnnw/q6yh7449iCOqikRJbIR3/EU7zKiUvHX6eiOdVkx/nE 7 | zKrhCzm1u7TrAf6hYwHro3nfxUTv4XBEVOAEeMjhvG5c6O6OYVoT+uMeaQgyQw4V 8 | J41nVpHFVQJ28uIfcLBp+PFr2LVF3CPskd/L4IzxrZxJiogAntvxDkm+Sv/BBqtf 9 | V3BX3FvVb+qXAZ/+JFdeFUlTuiIMg0Je1xypBlflU6fAGnuXhbi/KvpKyXNd+AA2 10 | 9OFsxsbmHEyTG30rN3DXAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADUduGZvrSfg 11 | GUfiUMx+SZ9mvGGbUwGM7kgRM8CPzYUkNmnrOMdGuD8haTQ4f0oUxZQuIQ9kLhDE 12 | G6DEunZDzkI6XyrpPTTbD5tuW3jFgqopGlTmeUmq5hegdWRIpjI9cTRlF9HIE8a9 13 | 3zpk928dJvEi9pdES+Bqzozd9D+NsuDFK5+zQHjAjy5wfeJKnTfkUCeQ3TdgRekX 14 | 4S2MTdIDP/k1rptiysHQiJViairPKfQEoKKsKhsWHSvG/c7KwaX7N6wDjpXTmb4W 15 | tGmtyudTOqAWZ7NOIEqqUCZl6nC9Z6qemUVEhP4lFuw8UnNvCcRy7bDNiEO7JrEj 16 | Cd3zjBmRh9E= 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /client-rest/src/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "module": "es2015", 6 | "types": [] 7 | }, 8 | "exclude": [ 9 | "src/test.ts", 10 | "**/*.spec.ts" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /client-rest/src/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/spec", 5 | "module": "commonjs", 6 | "types": [ 7 | "jasmine", 8 | "node" 9 | ] 10 | }, 11 | "files": [ 12 | "test.ts", 13 | "polyfills.ts" 14 | ], 15 | "include": [ 16 | "**/*.spec.ts", 17 | "**/*.d.ts" 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /client-rest/src/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tslint.json", 3 | "rules": { 4 | "directive-selector": [ 5 | true, 6 | "attribute", 7 | "app", 8 | "camelCase" 9 | ], 10 | "component-selector": [ 11 | true, 12 | "element", 13 | "app", 14 | "kebab-case" 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /client-rest/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "baseUrl": "./", 5 | "outDir": "./dist/out-tsc", 6 | "sourceMap": true, 7 | "declaration": false, 8 | "moduleResolution": "node", 9 | "emitDecoratorMetadata": true, 10 | "experimentalDecorators": true, 11 | "target": "es5", 12 | "typeRoots": [ 13 | "node_modules/@types" 14 | ], 15 | "lib": [ 16 | "es2017", 17 | "dom" 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinit/data-transfer-project/3a37025eded8c8e50006de79cb58bfa269543471/config/checkstyle/checkstyle.xml -------------------------------------------------------------------------------- /distributions/README.md: -------------------------------------------------------------------------------- 1 | # Distributions 2 | 3 | Data Transfer Project is platform agnostic. 4 | 5 | Currently supported deployments are: 6 | - Local from source 7 | - [Local from docker](demo-server) 8 | - [Google (GCP)](demo-google-deployment) 9 | - Microsoft (Azure) [in progress]. 10 | 11 | We welcome contributors to add configuration for additional platforms. 12 | 13 | ## Local development 14 | 15 | For local development (only), secrets may be stored in environment variables. 16 | 17 | ### For running from a jar file 18 | 19 | Set environment variables directly in your OS (e.g. `.bashrc` file, or 20 | `System > Control Panel > Advanced system settings > Environment Variables` on Windows). 21 | 22 | ### For running via docker 23 | Add any secrets to your local `distributions/demo-server/env.secrets` file. A 24 | sample is provided in `distributions/demo-server/env.secrets.template`. 25 | -------------------------------------------------------------------------------- /distributions/common/k8s/api-service.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | apiVersion: v1 17 | kind: Service 18 | # This service is associated with the 'portability' app, so can be thought of as 'portability.api' 19 | metadata: 20 | name: api 21 | spec: 22 | externalTrafficPolicy: Cluster 23 | ports: 24 | - nodePort: 30580 25 | port: 8080 26 | protocol: TCP 27 | targetPort: 8080 28 | selector: 29 | app: portability 30 | sessionAffinity: None 31 | type: NodePort 32 | status: 33 | loadBalancer: {} -------------------------------------------------------------------------------- /distributions/demo-google-deployment/bin/.gitignore: -------------------------------------------------------------------------------- 1 | init_project_vars.sh 2 | postprocess_project.sh 3 | # Credentials used for connecting to GCP services for local testing only. In prod we use creds 4 | # stored in a Kubernetes secret rather than a file the image. (https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform) 5 | service_acct_creds.json -------------------------------------------------------------------------------- /distributions/demo-google-deployment/bin/iam-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "members": [ 5 | "serviceAccount:SERVICE_ACCOUNT" 6 | ], 7 | "role": "roles/editor" 8 | }, 9 | { 10 | "members": [ 11 | "OWNERS" 12 | ], 13 | "role": "roles/owner" 14 | }, 15 | { 16 | "members": [ 17 | "serviceAccount:SERVICE_ACCOUNT" 18 | ], 19 | "role": "roles/cloudkms.cryptoKeyDecrypter" 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /distributions/demo-google-deployment/resources/config/api.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # API settings for Google cloud common to all environments 18 | baseUrl: http://localhost:3000 # default to local 19 | baseApiUrl: http://localhost:8080 # default to local 20 | # Whether the API server should serve requests using HTTPS. False because we 21 | # implement HTTPS at the load balancer in front of our API servers and don't 22 | # need it for communication within our network. 23 | useHttps: false 24 | -------------------------------------------------------------------------------- /distributions/demo-google-deployment/resources/config/common.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Google cloud settings common to all environments 18 | cloud: GOOGLE 19 | -------------------------------------------------------------------------------- /distributions/demo-google-deployment/resources/config/environments/qa/index.html: -------------------------------------------------------------------------------- 1 | PortabilityDemoLoading... -------------------------------------------------------------------------------- /distributions/demo-google-deployment/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # SLF4J logging configuration 18 | 19 | log4j.rootLogger=DEBUG, A1 20 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 21 | log4j.appender.A1.layout=org.datatransferproject.logging.EncryptingLayout 22 | -------------------------------------------------------------------------------- /distributions/demo-server/.gitignore: -------------------------------------------------------------------------------- 1 | **/env.secrets 2 | **/key.json 3 | -------------------------------------------------------------------------------- /distributions/demo-server/src/config/demo/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | loglevel=info 3 | nodaemon=true 4 | pidfile=/var/run/supervisord.pid 5 | user=root 6 | 7 | [supervisorctl] 8 | prompt = mysupervisor 9 | 10 | [program:nginx] 11 | command=/usr/sbin/nginx -g 'daemon off;' 12 | autostart=true 13 | autorestart=true 14 | priority=10 15 | stdout_logfile=/dev/fd/1 16 | stdout_logfile_maxbytes=0 17 | 18 | [program:dtp-java] 19 | command=java -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=*:5005 -jar /app/demo-server-all.jar 20 | autostart=true 21 | autorestart=true 22 | priority=10 23 | stdout_logfile=/dev/fd/1 24 | stdout_logfile_maxbytes=0 25 | -------------------------------------------------------------------------------- /distributions/demo-server/src/main/resources/config/common.yaml: -------------------------------------------------------------------------------- 1 | cloud: LOCAL 2 | environment: LOCAL 3 | -------------------------------------------------------------------------------- /distributions/demo-server/src/main/resources/config/retry/default.yaml: -------------------------------------------------------------------------------- 1 | # YAML formatting of a RetryStrategyLibrary 2 | # Currently defined per-distribution. Later on extensions will be able to 3 | # define their own RetryStrategyLibraries, with custom RetryMappings and 4 | # RetryStrategies. 5 | strategyMappings: 6 | - regexes: 7 | - "status code: 4\\d\\d" 8 | strategy: ! {} 9 | - regexes: 10 | - "simple" 11 | strategy: ! 12 | maxAttempts: 5 13 | intervalMillis: 1000 14 | defaultRetryStrategy: ! 15 | maxAttempts: 5 16 | initialIntervalMillis: 1000 17 | multiplier: 1.5 18 | -------------------------------------------------------------------------------- /distributions/demo-server/src/main/resources/config/transfer.yaml: -------------------------------------------------------------------------------- 1 | # job polling interval in milliseconds 2 | pollInterval: 1000 3 | -------------------------------------------------------------------------------- /distributions/demo-server/src/main/resources/demo-selfsigned-keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinit/data-transfer-project/3a37025eded8c8e50006de79cb58bfa269543471/distributions/demo-server/src/main/resources/demo-selfsigned-keystore.jks -------------------------------------------------------------------------------- /distributions/demo-server/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # SLF4J logging configuration 18 | 19 | log4j.rootLogger=DEBUG, consoleAppender 20 | log4j.appender.consoleAppender=org.apache.log4j.ConsoleAppender 21 | log4j.appender.consoleAppender.layout=org.datatransferproject.logging.EncryptingLayout 22 | -------------------------------------------------------------------------------- /extensions/README.md: -------------------------------------------------------------------------------- 1 | # Extensions 2 | 3 | This folder contains all the extensions for individual service providers 4 | and cloud platforms. 5 | 6 | - **auth** contains authorization libraries used by service providers to 7 | authorize users. 8 | 9 | - **cloud** contains cloud platform specific extensions that allow DTP to 10 | be run on various public clouds. 11 | 12 | - **data-transfer** contains the service provider specific adapters 13 | that translate a service's API into one or more 14 | [DTP data models](../portability-types-transfer/src/main/java/org/datatransferproject/types/transfer/models). 15 | 16 | - **transport** contains implementations of various transport layers that 17 | can be used to communicate with the DTP platform. 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-apple/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | plugins { 18 | id 'maven' 19 | id 'signing' 20 | } 21 | 22 | dependencies { 23 | compile project(':portability-spi-api') 24 | compile project(':portability-spi-cloud') 25 | compile project(':libraries:auth') 26 | } 27 | 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-apple/src/main/java/org/datatransferproject/auth/apple/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.auth.apple; 17 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-apple/src/main/resources/META-INF.services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.apple.AppleAuthServiceExtension -------------------------------------------------------------------------------- /extensions/auth/portability-auth-apple/src/main/resources/config/retry/default.yaml: -------------------------------------------------------------------------------- 1 | # YAML formatting of a RetryStrategyLibrary 2 | # Currently defined per-distribution. Later on extensions will be able to 3 | # define their own RetryStrategyLibraries, with custom RetryMappings and 4 | # RetryStrategies. 5 | strategyMappings: 6 | - regexes: 7 | - "status code: 4\\d\\d" 8 | strategy: ! {} 9 | - regexes: 10 | - "simple" 11 | strategy: ! 12 | maxAttempts: 5 13 | intervalMillis: 1000 14 | defaultRetryStrategy: ! 15 | maxAttempts: 5 16 | initialIntervalMillis: 1000 17 | multiplier: 1.5 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-daybook/src/main/java/org/datatransferproject/auth/daybook/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Daybook services. */ 17 | package org.datatransferproject.auth.daybook; -------------------------------------------------------------------------------- /extensions/auth/portability-auth-daybook/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.daybook.DaybookAuthServiceExtension -------------------------------------------------------------------------------- /extensions/auth/portability-auth-deezer/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-api') 23 | compile project(':portability-spi-cloud') 24 | compile project(':libraries:auth') 25 | } 26 | 27 | 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-deezer/src/main/java/org/datatransferproject/auth/deezer/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Deezer. */ 17 | package org.datatransferproject.auth.deezer; 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-deezer/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.deezer.DeezerAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-facebook/src/main/java/org/datatransferproject/auth/facebook/FacebookAuthServiceExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.auth.facebook; 18 | 19 | import org.datatransferproject.auth.OAuth2ServiceExtension; 20 | 21 | public class FacebookAuthServiceExtension extends OAuth2ServiceExtension { 22 | public FacebookAuthServiceExtension() { super(new FacebookOAuthConfig()); } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-facebook/src/main/java/org/datatransferproject/auth/facebook/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Facebook services. */ 17 | package org.datatransferproject.auth.facebook; 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-facebook/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.facebook.FacebookAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-flickr/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.flickr.FlickrAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-flickr/src/test/java/org/datatransferproject/auth/flickr/FlickrAuthDataGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.auth.flickr; 18 | 19 | class FlickrAuthDataGeneratorTest { 20 | 21 | } 22 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-google/src/main/java/org/datatransferproject/auth/google/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Google services. */ 17 | package org.datatransferproject.auth.google; 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-google/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.google.GoogleAuthServiceExtension 2 | org.datatransferproject.auth.google.BloggerAuthServiceExtension -------------------------------------------------------------------------------- /extensions/auth/portability-auth-harness-microsoft/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':extensions:auth:portability-auth-microsoft') 23 | } 24 | 25 | configurePublication(project) 26 | 27 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-imgur/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | plugins { 18 | id 'maven' 19 | id 'signing' 20 | } 21 | 22 | dependencies { 23 | compile project(':portability-spi-api') 24 | compile project(':portability-spi-cloud') 25 | compile project(':libraries:auth') 26 | } 27 | 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-imgur/src/main/java/org.datatransferproject.auth.imgur/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.auth.imgur; 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-imgur/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.imgur.ImgurAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-instagram/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.instagram.InstagramAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-koofr/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-api') 23 | compile project(':portability-spi-cloud') 24 | compile project(':libraries:auth') 25 | } 26 | 27 | 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-koofr/src/main/java/org/datatransferproject/auth/koofr/KoofrAuthServiceExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.auth.koofr; 17 | 18 | import org.datatransferproject.auth.OAuth2ServiceExtension; 19 | 20 | public class KoofrAuthServiceExtension extends OAuth2ServiceExtension { 21 | 22 | public KoofrAuthServiceExtension() { 23 | super(new KoofrOAuthConfig()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-koofr/src/main/java/org/datatransferproject/auth/koofr/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers for Koofr. */ 17 | package org.datatransferproject.auth.koofr; 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-koofr/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.koofr.KoofrAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-microsoft/src/main/java/org/datatransferproject/auth/microsoft/MicrosoftAuthServiceExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.auth.microsoft; 17 | 18 | import org.datatransferproject.auth.OAuth2ServiceExtension; 19 | 20 | public class MicrosoftAuthServiceExtension extends OAuth2ServiceExtension { 21 | 22 | public MicrosoftAuthServiceExtension() { 23 | super(new MicrosoftOAuthConfig()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-microsoft/src/main/java/org/datatransferproject/auth/microsoft/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Google services. */ 17 | package org.datatransferproject.auth.microsoft; 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-microsoft/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.microsoft.MicrosoftAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-offline-demo/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-api') 23 | compile project(':portability-spi-cloud') 24 | } 25 | 26 | 27 | configurePublication(project) 28 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-offline-demo/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.offline.OfflineDemoAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-rememberthemilk/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.rememberthemilk.RememberTheMilkAuthServiceExtension -------------------------------------------------------------------------------- /extensions/auth/portability-auth-smugmug/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | org.datatransferproject.auth.smugmug.SmugMugAuthServiceExtension -------------------------------------------------------------------------------- /extensions/auth/portability-auth-spotify/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-api') 23 | compile project(':portability-spi-cloud') 24 | compile project(':libraries:auth') 25 | } 26 | 27 | 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-spotify/src/main/java/org/datatransferproject/auth/spotify/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Spotify. */ 17 | package org.datatransferproject.auth.spotify; 18 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-spotify/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.auth.spotify.SpotifyAuthServiceExtension 2 | -------------------------------------------------------------------------------- /extensions/auth/portability-auth-twitter/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-api') 23 | compile project(':portability-spi-cloud') 24 | compile project(':libraries:auth') 25 | 26 | compile 'org.twitter4j:twitter4j-core:4.0.3' 27 | } 28 | 29 | configurePublication(project) -------------------------------------------------------------------------------- /extensions/auth/portability-auth-twitter/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | org.datatransferproject.auth.twitter.TwitterAuthServiceExtension -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-google/gradle.properties: -------------------------------------------------------------------------------- 1 | openCensusVersion=0.20.0 2 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-google/src/main/java/org/datatransferproject/cloud/google/GoogleCredentialException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.cloud.google; 18 | 19 | final class GoogleCredentialException extends Exception { 20 | GoogleCredentialException(String message) { 21 | super(message); 22 | } 23 | 24 | GoogleCredentialException(String message, Exception cause) { 25 | super(message, cause); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-google/src/main/java/org/datatransferproject/cloud/google/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Extensions for running the system on Google Cloud Platform. */ 17 | package org.datatransferproject.cloud.google; 18 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-google/src/main/resources/META-INF/services/org.datatransferproject.api.launcher.MonitorExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.cloud.google.StackdriverMonitorExtension 2 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-google/src/main/resources/META-INF/services/org.datatransferproject.spi.cloud.extension.CloudExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.cloud.google.GoogleCloudExtension 2 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-google/src/main/resources/META-INF/services/org.datatransferproject.spi.service.extension.ServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.cloud.google.GoogleDtpInternalMetricExtension 2 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-google/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutorExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.cloud.google.GoogleCloudIdempotentImportExecutorExtension -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-local/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-cloud') 23 | compile project(':portability-spi-service') 24 | compile project(':portability-spi-transfer') 25 | } 26 | 27 | configurePublication(project) 28 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-local/src/main/java/org/datatransferproject/cloud/local/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Extensions for running the system on Google App Engine. */ 17 | package org.datatransferproject.cloud.local; 18 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-local/src/main/resources/META-INF/services/org.datatransferproject.spi.cloud.extension.CloudExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.cloud.local.LocalCloudExtension -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-microsoft/src/main/java/org/datatransferproject/cloud/microsoft/cosmos/DataWrapper.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.cloud.microsoft.cosmos; 2 | 3 | import com.microsoft.azure.storage.table.TableServiceEntity; 4 | 5 | /** Wraps Job data for serialization to Cosmos DB */ 6 | public class DataWrapper extends TableServiceEntity { 7 | private String serialized; 8 | private String state; 9 | 10 | public DataWrapper(String partitionKey, String rowKey, String state, String serialized) { 11 | this.partitionKey = partitionKey; 12 | this.rowKey = rowKey; 13 | this.state = state; 14 | this.serialized = serialized; 15 | } 16 | 17 | /** Required by Azure Table Store API */ 18 | public DataWrapper() {} 19 | 20 | public String getSerialized() { 21 | return serialized; 22 | } 23 | 24 | public void setSerialized(String serialized) { 25 | this.serialized = serialized; 26 | } 27 | 28 | public String getState() { 29 | return state; 30 | } 31 | 32 | public void setState(String state) { 33 | this.state = state; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-microsoft/src/main/java/org/datatransferproject/cloud/microsoft/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Extensions for running the system on Microsoft Azure. */ 17 | package org.datatransferproject.cloud.microsoft; 18 | -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-microsoft/src/main/resources/META-INF/services/org.datatransferproject.spi.cloud.extension.CloudExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.cloud.microsoft.cosmos.MicrosoftCloudExtension -------------------------------------------------------------------------------- /extensions/cloud/portability-cloud-microsoft/src/main/resources/META-INF/services/org.datatransferproject.spi.service.extension.ServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.cloud.microsoft.cosmos.AzureDtpInternalMetricExtension 2 | -------------------------------------------------------------------------------- /extensions/config/portability-config-yaml/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':libraries:config') 23 | compile project(':portability-api-launcher') 24 | compile project(':portability-types-transfer') 25 | 26 | compile("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}") 27 | } 28 | 29 | configurePublication(project) -------------------------------------------------------------------------------- /extensions/config/portability-config-yaml/src/main/java/org/datatransferproject/config/yaml/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.config.yaml; 18 | -------------------------------------------------------------------------------- /extensions/config/portability-config-yaml/src/main/resources/META-INF/services/org.datatransferproject.config.extension.SettingsExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.config.yaml.YamlSettingsExtension 2 | -------------------------------------------------------------------------------- /extensions/config/portability-config-yaml/src/test/resources/api-1.yaml: -------------------------------------------------------------------------------- 1 | baseUrl: https://localhost:3000 2 | baseApiUrl: https://localhost:8080 3 | -------------------------------------------------------------------------------- /extensions/config/portability-config-yaml/src/test/resources/api-2.yaml: -------------------------------------------------------------------------------- 1 | baseUrl: www.aBaseUrl.com 2 | -------------------------------------------------------------------------------- /extensions/copier/portability-stack-copier/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-transfer') 23 | } 24 | 25 | configurePublication(project) 26 | -------------------------------------------------------------------------------- /extensions/copier/portability-stack-copier/src/main/java/org/datatransferproject/copier/stack/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.copier.stack; 18 | -------------------------------------------------------------------------------- /extensions/copier/portability-stack-copier/src/main/resources/META-INF/services/org.datatransferproject.transfer.copier.InMemoryDataCopierExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.copier.stack.PortabilityStackInMemoryDataCopierExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-apple/idea.gradle: -------------------------------------------------------------------------------- 1 | project.ext.ideaCommandLineArgumentProviders = new ArrayList() 2 | project.ext.ideaCommandLineArguments = new ArrayList() 3 | 4 | idea.workspace.iws { 5 | withXml { ws -> 6 | ws.node 7 | .component.find { it.@name == 'RunManager' } 8 | .configuration.findAll { it.@default && (it.@type == 'Application' || it.@type == 'JUnit') } 9 | .each { conf -> 10 | conf.option.find { it.@name == 'VM_PARAMETERS' }.@value = ideaCommandLineArguments() 11 | } 12 | } 13 | } 14 | 15 | def ideaCommandLineArguments() { 16 | def args = new HashSet(project.ideaCommandLineArguments) 17 | args.addAll(project.ideaCommandLineArgumentProviders.collect { it.asArguments().join(' ') }) 18 | return args.join(' ') 19 | } 20 | 21 | cleanIdea.dependsOn cleanIdeaWorkspace 22 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-apple/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.datatransfer.apple.AppleTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-apple/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeS3ClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.datatransfer.backblaze.common; 18 | 19 | import software.amazon.awssdk.services.s3.S3Client; 20 | 21 | public interface BackblazeS3ClientFactory { 22 | S3Client createS3Client(String accessKey, String secretKey, String region); 23 | } 24 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.datatransfer.backblaze; 18 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-backblaze/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.datatransfer.backblaze.BackblazeTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-backblaze/src/test/resources/test.txt: -------------------------------------------------------------------------------- 1 | Oh, hi there! 2 | 3 | I'm just a test file, hanging around here waiting to be read. 4 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-daybook/src/main/java/org/datatransferproject/transfer/daybook/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.transfer.daybook; -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-daybook/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transfer.daybook.DaybookTransferExtension -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/Readme.md: -------------------------------------------------------------------------------- 1 | # Deezer 2 | This folder contains the extension implementation for the 3 | [SpotDeezerfy](https://www.deezer.com/) API. 4 | 5 | 6 | ## Data Supported 7 | 8 | - Playlists 9 | 10 | ## Current State 11 | 12 | - Mostly working 13 | 14 | The base functionality for import and export work, there are some rough edges still. 15 | 16 | **Known Issues:** 17 | 18 | - Playlist art is not transfered 19 | - Playlist order is not preserved 20 | - Song matching is only on ISRC 21 | 22 | ## Keys & Auth 23 | 24 | https://developers.deezer.com/ 25 | 26 | ## Maintained By 27 | 28 | The Deezer extension was created and maintained by the 29 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 30 | (this includes developers from Google). 31 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/Album.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | /** 4 | * POJO of: https://developers.deezer.com/api/album 5 | */ 6 | public class Album { 7 | private long id; 8 | private String title; 9 | private String upc; 10 | private String link; 11 | 12 | public long getId() { 13 | return id; 14 | } 15 | 16 | public String getTitle() { 17 | return title; 18 | } 19 | 20 | public String getUpc() { 21 | return upc; 22 | } 23 | 24 | public String getLink() { 25 | return link; 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return String.format("Album{id=%s, title=%s}", id, title); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/Artist.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | /** 4 | * POJO of: https://developers.deezer.com/api/artist 5 | */ 6 | public class Artist { 7 | private long id; 8 | private String name; 9 | private String link; 10 | 11 | public long getId() { 12 | return id; 13 | } 14 | 15 | public String getName() { 16 | return name; 17 | } 18 | 19 | public String getLink() { 20 | return link; 21 | } 22 | 23 | @Override 24 | public String toString(){ 25 | return String.format("Artist{id=%s, name=%s, link=%s}", 26 | id, name, link); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/Error.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | public class Error { 4 | private String type; 5 | private String message; 6 | private int code; 7 | 8 | public String getType() { 9 | return type; 10 | } 11 | 12 | public String getMessage() { 13 | return message; 14 | } 15 | 16 | public int getCode() { 17 | return code; 18 | } 19 | 20 | public String toString() { 21 | return String.format("Error{type=%s message=%s code=%s}", 22 | type, message, code); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/InsertResponse.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | import java.io.Serializable; 4 | 5 | public class InsertResponse implements Serializable { 6 | private long id; 7 | private Error error; 8 | 9 | public long getId() { 10 | return id; 11 | } 12 | 13 | public String toString() { 14 | if (error != null) { 15 | return error.toString(); 16 | } 17 | return String.format("InsertResponse{id=%s}", id); 18 | } 19 | 20 | public Error getError() { 21 | return error; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/PlaylistDetails.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | public class PlaylistDetails extends PlaylistSummary { 6 | @JsonProperty("tracks") 7 | private TrackCollection tracks; 8 | 9 | public TrackCollection getTrackCollection() { 10 | return tracks; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/PlaylistsResponse.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | /** 4 | * Response format for user/me/platlists, see: 5 | * https://developers.deezer.com/api/explorer?url=user/me 6 | */ 7 | public class PlaylistsResponse { 8 | private PlaylistSummary[] data; 9 | 10 | public PlaylistSummary[] getData() { 11 | return data; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/TrackCollection.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | 6 | public class TrackCollection { 7 | private Track[] data; 8 | 9 | @JsonCreator 10 | TrackCollection(@JsonProperty("data") Track[] data) { 11 | this.data = data; 12 | } 13 | 14 | public Track[] getTracks() { 15 | return data; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/java/org/datatransferproject/transfer/deezer/model/User.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.deezer.model; 2 | 3 | /** 4 | * POJO of https://developers.deezer.com/api/user 5 | */ 6 | public class User { 7 | private long id; 8 | private String name; 9 | private String tracklist; 10 | 11 | public long getId() { 12 | return id; 13 | } 14 | 15 | public String getName() { 16 | return name; 17 | } 18 | 19 | public String getTracklist() { 20 | return tracklist; 21 | } 22 | 23 | @Override 24 | public String toString() { 25 | return String.format("User{id=%s, name=%s, tracklist=%s}", id, name, tracklist); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | org.datatransferproject.transfer.deezer.DeezerTransferExtension 18 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-deezer/src/main/resources/config/deezer.yaml: -------------------------------------------------------------------------------- 1 | # Global rate limit is 50 requests / 5 seconds (https://developers.deezer.com/api) 2 | # so 50 * 12 = 600 qps globally, assume 100 concurent DTP users 3 | perUserRateLimit: 6 4 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-facebook/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transfer.facebook.FacebookTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-facebook/src/test/resources/test.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinit/data-transfer-project/3a37025eded8c8e50006de79cb58bfa269543471/extensions/data-transfer/portability-data-transfer-facebook/src/test/resources/test.jpeg -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-flickr/Readme.md: -------------------------------------------------------------------------------- 1 | # Flickr 2 | This folder contains the extension implementation for the 3 | [Flickr](https://www.flickr.com) service. 4 | 5 | The implementation is based off of [Flickr4Java](https://github.com/boncey/Flickr4Java). 6 | 7 | ## Data Supported 8 | 9 | - [Photos](src/main/java/org/datatransferproject/datatransfer/flickr/photos) Import & Export 10 | 11 | ## Current State 12 | No known issues, however like much of DTP still needs some in depth testing to ensure corner 13 | cases are working. 14 | 15 | ## Keys & Auth 16 | 17 | Apply for a key at [www.flickr.com/services/apps/create/apply](https://www.flickr.com/services/apps/create/apply) 18 | 19 | Flickr uses OAuth 1.0 for authorization. 20 | 21 | ## Maintained By 22 | 23 | The Flickr extension was created by the 24 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 25 | and is not an official product of Flickr. 26 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-flickr/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-cloud') 23 | compile project(':portability-spi-transfer') 24 | 25 | compile("com.flickr4java:flickr4java:${flickrVersion}") 26 | 27 | testCompile project(":extensions:cloud:portability-cloud-local") 28 | } 29 | 30 | configurePublication(project) -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-flickr/src/main/java/org/datatransferproject/datatransfer/flickr/photos/FlickrTempPhotoData.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.flickr.photos; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeName; 6 | import org.datatransferproject.types.common.models.DataModel; 7 | 8 | @JsonTypeName("org.dataportability:FlickrTempPhotoData") 9 | public class FlickrTempPhotoData extends DataModel { 10 | private final String name; 11 | private final String description; 12 | 13 | @JsonCreator 14 | public FlickrTempPhotoData( 15 | @JsonProperty("name") String name, @JsonProperty("description") String description) { 16 | this.name = name; 17 | this.description = description; 18 | } 19 | 20 | public String getName() { 21 | return name; 22 | } 23 | 24 | public String getDescription() { 25 | return description; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-flickr/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.datatransfer.flickr.FlickrTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-flickr/src/main/resources/config/flickr.yaml: -------------------------------------------------------------------------------- 1 | # Global rate limit is 3600 requests / hour (https://www.flickr.com/services/developer/api/) 2 | # so 1 qps. Limiting it to 1 per user, but in production this should be monitored to make sure 3 | # the global limit is not exceeded. 4 | perUserRateLimit: 1 5 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-generic/src/main/java/org/datatransferproject/datatransfer/generic/BlobIdToName.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.generic; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | import org.datatransferproject.types.common.models.DataModel; 7 | 8 | public class BlobIdToName extends DataModel { 9 | @JsonProperty("idToName") 10 | private Map idToName; 11 | 12 | public BlobIdToName() { 13 | this.idToName = new HashMap<>(); 14 | } 15 | 16 | public String get(String id) { 17 | return idToName.get(id); 18 | } 19 | 20 | public String add(String id, String name) { 21 | return idToName.put(id, name); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-generic/src/main/java/org/datatransferproject/datatransfer/generic/ContainerSerializer.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.generic; 2 | 3 | import java.io.IOException; 4 | import org.datatransferproject.types.common.models.ContainerResource; 5 | 6 | @FunctionalInterface 7 | public interface ContainerSerializer { 8 | public Iterable> apply(C containerResource) throws IOException; 9 | } 10 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-generic/src/main/java/org/datatransferproject/datatransfer/generic/GenericPayload.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.generic; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 6 | 7 | @JsonTypeInfo(use = JsonTypeInfo.Id.NAME) 8 | public class GenericPayload { 9 | @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY) 10 | private final T payload; 11 | 12 | private final String schemaSource; 13 | // TODO: update 14 | private final String apiVersion = "0.1.0"; 15 | 16 | @JsonCreator 17 | public GenericPayload(@JsonProperty T payload, @JsonProperty String schemaSource) { 18 | this.payload = payload; 19 | this.schemaSource = schemaSource; 20 | } 21 | 22 | public T getPayload() { 23 | return payload; 24 | } 25 | 26 | public String getApiVersion() { 27 | return apiVersion; 28 | } 29 | 30 | public String getSchemaSource() { 31 | return schemaSource; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-generic/src/main/java/org/datatransferproject/datatransfer/generic/GenericTransferConstants.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.generic; 2 | 3 | final class GenericTransferConstants { 4 | public static final String SCHEMA_SOURCE_BASE = 5 | "https://github.com/dtinit/data-transfer-project/blob/master"; 6 | } 7 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-generic/src/main/java/org/datatransferproject/datatransfer/generic/ImportableData.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.generic; 2 | 3 | public class ImportableData { 4 | /** JSON serializable data to POST */ 5 | private GenericPayload jsonData; 6 | 7 | /** Globally unique ID to avoid re-importing data */ 8 | private String idempotentId; 9 | 10 | /** Human-readable item name */ 11 | private String name; 12 | 13 | public ImportableData(GenericPayload jsonData, String idempotentId, String name) { 14 | this.jsonData = jsonData; 15 | this.idempotentId = idempotentId; 16 | this.name = name; 17 | } 18 | 19 | public GenericPayload getJsonData() { 20 | return jsonData; 21 | } 22 | 23 | public String getIdempotentId() { 24 | return idempotentId; 25 | } 26 | 27 | public String getName() { 28 | return name; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-generic/src/main/java/org/datatransferproject/datatransfer/generic/ImportableFileData.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.generic; 2 | 3 | import org.datatransferproject.types.common.DownloadableItem; 4 | 5 | public class ImportableFileData extends ImportableData { 6 | /** File file to POST * */ 7 | private DownloadableItem file; 8 | 9 | private String fileMimeType; 10 | 11 | public ImportableFileData( 12 | DownloadableItem file, 13 | String fileMimeType, 14 | GenericPayload jsonData, 15 | String idempotentId, 16 | String name) { 17 | super(jsonData, idempotentId, name); 18 | this.file = file; 19 | this.fileMimeType = fileMimeType; 20 | } 21 | 22 | public DownloadableItem getFile() { 23 | return file; 24 | } 25 | 26 | public String getFileMimeType() { 27 | return fileMimeType; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-generic/src/test/java/org/datatransferproject/datatransfer/generic/BlobIdToNameTest.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.generic; 2 | 3 | 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.junit.Test; 6 | 7 | public class BlobIdToNameTest { 8 | 9 | @Test 10 | public void simpleImportTest() throws Exception { 11 | BlobIdToName initial = new BlobIdToName(); 12 | initial.add("key1", "value1"); 13 | initial.add("key2", "value2"); 14 | 15 | ObjectMapper objectMapper = new ObjectMapper(); 16 | BlobIdToName afterSerDer = objectMapper.readValue(objectMapper.writeValueAsBytes(initial), BlobIdToName.class); 17 | assert(afterSerDer.get("key1").equals("value1")); 18 | assert(afterSerDer.get("key2").equals("value2")); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/FailedToListAlbumsException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.google.common; 2 | 3 | /** 4 | * FailedToListAlbumsException is thrown when we try to call PhotosInterface.listAlbums and are 5 | * unsuccessful. 6 | */ 7 | public class FailedToListAlbumsException extends Exception { 8 | public FailedToListAlbumsException(String message, Exception cause) { 9 | super(message, cause); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/FailedToListMediaItemsException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.google.common; 2 | 3 | /** 4 | * FailedToListMediaItemsException is thrown when we try to call PhotosInterface.listMediaItems and are 5 | * unsuccessful. 6 | */ 7 | public class FailedToListMediaItemsException extends Exception { 8 | public FailedToListMediaItemsException(String message, Exception cause) { 9 | super(message, cause); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/common/InvalidExportedItemException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.google.common; 2 | 3 | // An exception to throw when converting a photo/video/mediaitem into a shared data class fails, 4 | // or any of the data on the exported item is invalid. 5 | public class InvalidExportedItemException extends Exception { 6 | public InvalidExportedItemException(String message) { 7 | super(message); 8 | } 9 | 10 | public InvalidExportedItemException(String message, Throwable cause) { 11 | super(message, cause); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/drive/DriveFolderMapping.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.google.drive; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeName; 6 | import org.datatransferproject.types.common.models.DataModel; 7 | 8 | @JsonTypeName("org.dataportability:GoogleDriveFolderMapping") 9 | class DriveFolderMapping extends DataModel { 10 | 11 | private final String oldId; 12 | private final String newId; 13 | 14 | @JsonCreator 15 | DriveFolderMapping( 16 | @JsonProperty("oldId") String oldId, 17 | @JsonProperty("newId") String newId) { 18 | 19 | this.oldId = oldId; 20 | this.newId = newId; 21 | } 22 | 23 | public String getNewId() { 24 | return newId; 25 | } 26 | 27 | public String getOldId() { 28 | return oldId; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Google services. */ 17 | package org.datatransferproject.datatransfer.google; 18 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/photos/PhotoResult.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.google.photos; 2 | 3 | import java.io.Serializable; 4 | 5 | public class PhotoResult implements Serializable { 6 | private String id; 7 | private Long bytes; 8 | 9 | public PhotoResult(String id, Long bytes) { 10 | this.id = id; 11 | this.bytes = bytes == null ? 0 : bytes; 12 | } 13 | 14 | public String getId() { 15 | return id; 16 | } 17 | 18 | public Long getBytes() { 19 | return bytes; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/videos/VideoResult.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.datatransfer.google.videos; 2 | 3 | import java.io.Serializable; 4 | 5 | class VideoResult implements Serializable { 6 | private String id; 7 | private Long bytes; 8 | 9 | public VideoResult(String id, Long bytes) { 10 | this.id = id; 11 | this.bytes = bytes; 12 | } 13 | 14 | public String getId() { 15 | return id; 16 | } 17 | 18 | public Long getBytes() { 19 | return bytes; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.datatransfer.google.GoogleTransferExtension 2 | org.datatransferproject.datatransfer.google.BloggerTransferExtension 3 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/test/java/org/datatransferproject/datatransfer/google/tasks/GoogleTasksExporterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.datatransfer.google.tasks; 18 | 19 | public class GoogleTasksExporterTest { 20 | 21 | } -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-google/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-imgur/Readme.md: -------------------------------------------------------------------------------- 1 | # Imgur 2 | This folder contains the extension implementation for the 3 | [Imgur](https://imgur.com/) service. 4 | 5 | 6 | ## Data Supported 7 | 8 | - Albums Import & Export 9 | - Photos Import & Export 10 | 11 | ## Current State 12 | No known issues, however like much of DTP still needs some in depth testing to ensure corner 13 | cases are working. 14 | 15 | ## Keys & Auth 16 | 17 | Apply for a key at [https://api.imgur.com/oauth2/addclient](https://api.imgur.com/oauth2/addclient) 18 | 19 | Imgur uses OAuth 2.0 for authorization. 20 | 21 | ## Maintained By 22 | 23 | The Imgur extension was created by the 24 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 25 | and is not an official product of Imgur. 26 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-imgur/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.datatransfer.imgur.ImgurTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-instagram/Readme.md: -------------------------------------------------------------------------------- 1 | # Instagram 2 | This folder contains the extension implementation for the 3 | [Instagram](https://www.instagram.com) service. 4 | 5 | ## Data Supported 6 | 7 | - [Photos](src/main/java/org/datatransferproject/datatransfer/instagram/photos) Export only, no import. 8 | 9 | ## Current State 10 | No known issues, however like much of DTP still needs some in depth testing to ensure corner 11 | cases are working. 12 | 13 | ## Keys 14 | 15 | Apply for a key at [www.instagram.com/developer](https://www.instagram.com/developer/) 16 | 17 | Google uses OAuth 2 for authorization. 18 | 19 | ## Maintained By 20 | 21 | The Instagram extension was created by the 22 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 23 | and is not an official product of Instagram/Facebook. 24 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-instagram/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-cloud') 23 | compile project(':portability-spi-transfer') 24 | 25 | compile("com.google.api-client:google-api-client:${googleApiClient}") 26 | } 27 | 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-instagram/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transfer.instagram.InstagramTransferExtension 2 | 3 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-koofr/README.md: -------------------------------------------------------------------------------- 1 | # Koofr 2 | 3 | This folder contains the extension implementation for [Koofr](https://koofr.eu). 4 | 5 | ## Data Supported 6 | 7 | - Photos import 8 | - Videos import 9 | 10 | ## Current State 11 | 12 | - Does not support exports (#933). 13 | 14 | ## Keys & Auth 15 | 16 | Sign up through Koofr's [Developers portal](https://app.koofr.net/app/developers). 17 | 18 | Koofr uses OAuth 2 for authorization. 19 | 20 | ## Maintained By 21 | 22 | The Koofr extension was created and is maintained by 23 | [Koofr developers](https://koofr.eu/team/). 24 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-koofr/src/main/java/org/datatransferproject/transfer/koofr/exceptions/KoofrClientIOException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.koofr.exceptions; 2 | 3 | import okhttp3.Response; 4 | import okhttp3.ResponseBody; 5 | 6 | import java.io.IOException; 7 | 8 | public class KoofrClientIOException extends IOException { 9 | 10 | private final int httpResponseCode; 11 | 12 | public KoofrClientIOException(Response response) { 13 | super( 14 | String.format( 15 | "Got error code: %s message: %s body: %s", 16 | response.code(), response.message(), getResponseBody(response))); 17 | 18 | this.httpResponseCode = response.code(); 19 | } 20 | 21 | public int getCode() { 22 | return httpResponseCode; 23 | } 24 | 25 | private static String getResponseBody(Response response) { 26 | try { 27 | ResponseBody body = response.body(); 28 | return body != null ? body.string() : null; 29 | } catch (Exception e) { 30 | return "Failed to get response body"; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-koofr/src/main/java/org/datatransferproject/transfer/koofr/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers for Koofr. */ 17 | package org.datatransferproject.transfer.koofr; 18 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-koofr/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transfer.koofr.KoofrTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-koofr/src/test/resources/blank.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinit/data-transfer-project/3a37025eded8c8e50006de79cb58bfa269543471/extensions/data-transfer/portability-data-transfer-koofr/src/test/resources/blank.jpg -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-microsoft/Readme.md: -------------------------------------------------------------------------------- 1 | # Microsoft 2 | This folder contains the extension implementation for the 3 | [Microsoft](https://www.microsoft.com). 4 | 5 | ## Data Supported 6 | 7 | - Calendar Import & Export 8 | - Contacts Import & Export 9 | - Photos Import & Export 10 | 11 | ## Current State 12 | 13 | - Calendar doesn't support recurring events yet 14 | - All services still need some final testing 15 | 16 | ## Keys & Auth 17 | 18 | Sign up through Microsoft's [Application Registration Portal](https://apps.dev.microsoft.com/). 19 | 20 | Microsoft uses OAuth 2 for authorization. 21 | 22 | ## Maintained By 23 | 24 | The Google extension was created and maintained by the 25 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 26 | (this includes developers from Microsoft). 27 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-microsoft/src/main/java/org/datatransferproject/transfer/microsoft/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Implements importers and exporters for Microsoft services. */ 17 | package org.datatransferproject.transfer.microsoft; 18 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-microsoft/src/main/java/org/datatransferproject/transfer/microsoft/transformer/TransformConstants.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.microsoft.transformer; 2 | 3 | /** Defines constants used for Calendar export/import. */ 4 | public interface TransformConstants { 5 | 6 | String CALENDAR_ID = "calendar.id"; 7 | } 8 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-microsoft/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transfer.microsoft.MicrosoftTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-offline-demo/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-cloud') 23 | compile project(':portability-spi-transfer') 24 | compile project(':extensions:data-transfer:portability-data-transfer-microsoft') 25 | 26 | 27 | } 28 | 29 | configurePublication(project) -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-offline-demo/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transfer.offline.OfflineDemoTransferExtension 2 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-rememberthemilk/Readme.md: -------------------------------------------------------------------------------- 1 | # Remember the Milk 2 | This folder contains the extension implementation for the 3 | [Remember the Milk](https://www.rememberthemilk.com) service. 4 | 5 | ## Data Supported 6 | 7 | - Tasks import and export. 8 | 9 | ## Current State 10 | No known issues, however like much of DTP still needs some in depth testing to ensure corner 11 | cases are working. 12 | 13 | ## Keys 14 | 15 | Apply for a key at [www.rememberthemilk.com/services/api/keys.rtm](https://www.rememberthemilk.com/services/api/keys.rtm) 16 | 17 | ## Maintained By 18 | 19 | The Instagram extension was created by the 20 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 21 | and is not an official product of Remember the Milk. 22 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-rememberthemilk/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transfer.rememberthemilk.RememberTheMilkTransferExtension -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-smugmug/Readme.md: -------------------------------------------------------------------------------- 1 | # SumgMug 2 | This folder contains the extension implementation for the 3 | [SumgMug](https://www.smugmug.com/) service. 4 | 5 | ## Data Supported 6 | 7 | - Photos import and export. 8 | 9 | ## Current State 10 | No known issues, however like much of DTP still needs some in depth testing to ensure corner 11 | cases are working. 12 | 13 | ## Keys 14 | 15 | Apply for a key at [api.smugmug.com/api/developer/apply](https://api.smugmug.com/api/developer/apply). 16 | 17 | SumgMug uses OAuth 1 for authorization. 18 | 19 | ## Maintained By 20 | 21 | The Instagram extension was created by the 22 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 23 | and is not an official product of SumgMug. 24 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-smugmug/src/main/java/org/datatransferproject/transfer/smugmug/SmugMugTransmogrificationConfig.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer.smugmug; 2 | 3 | import org.datatransferproject.types.common.models.TransmogrificationConfig; 4 | 5 | // This class defines transmogrification paramaters for SmugMug imports 6 | public class SmugMugTransmogrificationConfig extends TransmogrificationConfig { 7 | // Smugmug doesn't allow photos to exist outside of a folder 8 | private static final boolean ALBUM_ALLOW_ROOT_PHOTOS = false; 9 | 10 | // Album size specified here: 11 | // https://news.smugmug.com/uploading-to-smugmug-what-how-big-and-how-many-d9df14b07bda 12 | private static final int ALBUM_MAX_SIZE = 5000; 13 | 14 | /** We need to override the methods to return the updated limits */ 15 | @Override 16 | public boolean getAlbumAllowRootPhotos() { 17 | return ALBUM_ALLOW_ROOT_PHOTOS; 18 | } 19 | 20 | @Override 21 | public int getAlbumMaxSize() { 22 | return ALBUM_MAX_SIZE; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-smugmug/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | org.datatransferproject.transfer.smugmug.SmugMugTransferExtension -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-spotify/Readme.md: -------------------------------------------------------------------------------- 1 | # Spotify 2 | This folder contains the extension implementation for the 3 | [Spotify](https://spotify.com) API. 4 | 5 | 6 | ## Data Supported 7 | 8 | - Playlists 9 | 10 | ## Current State 11 | 12 | - Mostly working 13 | 14 | The base cfuncionality for import and export work, there are some rough edges still. 15 | 16 | **Known Issues:** 17 | 18 | - Playlist art is not transfered 19 | - Playlist order is not preserved 20 | - Song matching is only on ISRC 21 | 22 | ## Keys & Auth 23 | 24 | https://developer.spotify.com/documentation/general/guides/app-settings/ 25 | 26 | ## Maintained By 27 | 28 | The Google extension was created and maintained by the 29 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 30 | (this includes developers from Google). 31 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-spotify/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | org.datatransferproject.transfer.spotify.SpotifyTransferExtension 18 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-twitter/Readme.md: -------------------------------------------------------------------------------- 1 | # Twitter 2 | This folder contains the extension implementation for the 3 | [Twitter](https://twitter.com/). 4 | 5 | ## Data Supported 6 | 7 | - Photos export. It extracts photos from your tweets. 8 | 9 | ## Current State 10 | No known issues, however like much of DTP still needs some in depth testing to ensure corner 11 | cases are working. 12 | 13 | ## Keys & Auth 14 | 15 | Apply for a key at [apps.twitter.com](https://apps.twitter.com/). 16 | 17 | Twitter uses OAuth 2 for authorization. 18 | 19 | ## Maintained By 20 | 21 | The Twitter extension was created by the 22 | [DTP maintainers](mailto:portability-maintainers@googlegroups.com) 23 | and is not an official product of Twitter. 24 | -------------------------------------------------------------------------------- /extensions/data-transfer/portability-data-transfer-twitter/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | org.datatransferproject.transfer.twitter.TwitterTransferExtension -------------------------------------------------------------------------------- /extensions/security/portability-security-cleartext/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-transfer') 23 | } 24 | 25 | configurePublication(project) 26 | -------------------------------------------------------------------------------- /extensions/security/portability-security-cleartext/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.security.SecurityExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.security.cleartext.ClearTextSecurityExtension 2 | -------------------------------------------------------------------------------- /extensions/security/portability-security-jwe/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-spi-transfer') 23 | compile group: 'com.nimbusds', name: 'nimbus-jose-jwt', version: '9.21.1' 24 | } 25 | 26 | configurePublication(project) 27 | -------------------------------------------------------------------------------- /extensions/security/portability-security-jwe/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.security.SecurityExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.security.jwe.JWESecurityExtension -------------------------------------------------------------------------------- /extensions/transport/portability-transport-jettyrest/src/main/java/org/datatransferproject/transport/jettyrest/rest/RESTConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.transport.jettyrest.rest; 17 | 18 | /** */ 19 | public interface RESTConstants { 20 | String EXPORT_AUTH_DATA_KEY = "ead_id"; 21 | String IMPORT_AUTH_DATA_KEY = "iad_id"; 22 | } 23 | -------------------------------------------------------------------------------- /extensions/transport/portability-transport-jettyrest/src/main/resources/META-INF/services/org.datatransferproject.spi.service.extension.ServiceExtension: -------------------------------------------------------------------------------- 1 | org.datatransferproject.transport.jettyrest.JettyRestExtension -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinit/data-transfer-project/3a37025eded8c8e50006de79cb58bfa269543471/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /libraries/README.md: -------------------------------------------------------------------------------- 1 | # Libraries 2 | 3 | This folder contains shared logic/libraries used by core DTP code. 4 | 5 | - **config** contains utilities for configuring DTP via flags and config files. 6 | 7 | - **security** contains cryptographic helper libraries. 8 | 9 | - **transfer** contains helper utilities for facilitating the transfer of data. 10 | 11 | -------------------------------------------------------------------------------- /libraries/config/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(":portability-api-launcher") 23 | compile("com.google.inject:guice:${guiceVersion}") 24 | } 25 | 26 | configurePublication(project) 27 | -------------------------------------------------------------------------------- /libraries/security/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | plugins { 17 | id 'maven' 18 | id 'signing' 19 | } 20 | 21 | dependencies { 22 | compile project(':portability-api-launcher') 23 | compile project(':portability-spi-transfer') 24 | } 25 | 26 | configurePublication(project) 27 | -------------------------------------------------------------------------------- /libraries/security/src/main/java/org/datatransferproject/security/CryptoTransformation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.security; 17 | 18 | /** Provides compile-time constants for secure crypto transformations. For use 19 | in creation of Encrypter/Decrypter */ 20 | public enum CryptoTransformation { 21 | AES_CBC_NOPADDING, 22 | RSA_ECB_PKCS1 23 | } 24 | 25 | -------------------------------------------------------------------------------- /libraries/security/src/main/java/org/datatransferproject/security/Decrypter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.security; 17 | 18 | /** Provides functionality to decrypt encrypted data. */ 19 | public interface Decrypter { 20 | 21 | /** Decrypts the given {@code encryptedData}. */ 22 | String decrypt(String encryptedData); 23 | } 24 | -------------------------------------------------------------------------------- /libraries/security/src/main/java/org/datatransferproject/security/Encrypter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.security; 17 | 18 | /** Provides functionality to encrypt raw data. */ 19 | public interface Encrypter { 20 | 21 | /** Encrypts the given {@code data}. */ 22 | String encrypt(String data); 23 | } 24 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "lockfileVersion": 1 3 | } 4 | -------------------------------------------------------------------------------- /portability-api-launcher/build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | dependencies { 3 | classpath 'org.eclipse.jgit:org.eclipse.jgit:5.2.1.201812262042-r' 4 | } 5 | } 6 | 7 | plugins { 8 | id 'maven' 9 | id 'signing' 10 | } 11 | 12 | dependencies { 13 | compile project(':portability-types-common') 14 | } 15 | 16 | configurePublication(project) 17 | 18 | 19 | jar { 20 | dependsOn "generateGitHash" 21 | from "$buildDir/resources/generated" 22 | } 23 | 24 | //noinspection GroovyAssignabilityCheck 25 | task generateGitHash() { 26 | def dest = new File("${project.buildDir}/resources/generated/META-INF") 27 | dest.mkdirs() 28 | def propFile = new File(dest, "launcher.properties") 29 | propFile.createNewFile(); 30 | 31 | propFile.write("version=${project.version}\nhash=${project.rootProject.ext.gitVersion}") 32 | } 33 | -------------------------------------------------------------------------------- /portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/BootExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.api.launcher; 17 | 18 | /** Implementations provide services required to boot a runtime system. */ 19 | public interface BootExtension extends SystemExtension { 20 | /** Initializes the extension. Implementations prepare core bootstrap services. */ 21 | void initialize(); 22 | } 23 | -------------------------------------------------------------------------------- /portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.api.launcher; 18 | 19 | public class Constants { 20 | public enum Environment { 21 | LOCAL, 22 | TEST, 23 | QA, 24 | PROD 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/JobAwareMonitor.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.api.launcher; 2 | 3 | public interface JobAwareMonitor extends Monitor { 4 | 5 | /** 6 | * Sets the jobId for the monitor, this will be included with all log messages. 7 | */ 8 | void setJobId(String jobId); 9 | } 10 | -------------------------------------------------------------------------------- /portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/MonitorExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.api.launcher; 17 | 18 | /** Implementations provide custom monitoring sinks. */ 19 | public interface MonitorExtension extends BootExtension { 20 | 21 | /** Returns a monitor. */ 22 | Monitor getMonitor(); 23 | } 24 | -------------------------------------------------------------------------------- /portability-api/README.md: -------------------------------------------------------------------------------- 1 | # portability-api 2 | 3 | This folder contains the binary for the API server used by DTP. 4 | 5 | DTP has two servers: the API, and the Worker. The API server exposes DTP's 6 | API (hence the name). This allows clients to submit jobs, and check job 7 | statuses. The [Worker](../portability-transfer) does the actual transfer 8 | of the data. 9 | 10 | -------------------------------------------------------------------------------- /portability-api/src/main/java/org/datatransferproject/api/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** Contains the core API server implementation. */ 17 | package org.datatransferproject.api; 18 | -------------------------------------------------------------------------------- /portability-bootstrap-vm/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | plugins { 19 | id 'maven' 20 | id 'signing' 21 | } 22 | 23 | dependencies { 24 | compile project(':portability-api') 25 | compile project(':portability-transfer') 26 | } 27 | 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /portability-spi-api/README.md: -------------------------------------------------------------------------------- 1 | # portability-spi-api 2 | 3 | This folder contains the Service Provider Interface (SPI) for DTP's 4 | API server. 5 | 6 | Service providers looking to be included in DTP will need to implement 7 | [AuthDataGenerator](src/main/java/org/datatransferproject/spi/api/auth/AuthDataGenerator.java) 8 | and [AuthServiceExtension](src/main/java/org/datatransferproject/spi/api/auth/extension/AuthServiceExtension.java). 9 | 10 | -------------------------------------------------------------------------------- /portability-spi-api/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | plugins { 18 | id 'maven' 19 | id 'signing' 20 | } 21 | 22 | dependencies { 23 | compile project(':portability-types-common') 24 | compile project(':portability-types-transfer') 25 | compile project(':portability-spi-cloud') 26 | compile project(':portability-api-launcher') 27 | } 28 | configurePublication(project) 29 | -------------------------------------------------------------------------------- /portability-spi-api/src/main/java/org/datatransferproject/spi/api/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Defines the api SPI. Extensions use this module to implement custom auth and api 18 | * functionality for the system. 19 | */ 20 | package org.datatransferproject.spi.api; 21 | -------------------------------------------------------------------------------- /portability-spi-api/src/main/java/org/datatransferproject/spi/api/transport/DiscardingStreamCounter.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.api.transport; 2 | 3 | import com.google.common.io.CountingInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | 7 | public class DiscardingStreamCounter { 8 | private DiscardingStreamCounter() {} 9 | 10 | /** Returns byte size of stream, discarding its contents and closing the stream. */ 11 | public static long discardForLength(InputStream stream) throws IOException { 12 | CountingInputStream counter = new CountingInputStream(stream); 13 | while (true) { 14 | if (counter.skip(Integer.MAX_VALUE) < Integer.MAX_VALUE) { 15 | counter.close(); 16 | return counter.getCount(); 17 | } 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /portability-spi-api/src/main/java/org/datatransferproject/spi/api/transport/FileStreamer.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.api.transport; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import org.datatransferproject.types.common.DownloadableItem; 6 | 7 | public interface FileStreamer { 8 | InputStream get(DownloadableItem downloadableItem) throws IOException; 9 | } 10 | -------------------------------------------------------------------------------- /portability-spi-api/src/main/java/org/datatransferproject/spi/api/transport/RemoteFileStreamer.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.api.transport; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import org.datatransferproject.types.common.DownloadableItem; 6 | 7 | public interface RemoteFileStreamer { 8 | public InputStream get(String remoteUrl) throws IOException; 9 | 10 | public InputStream get(DownloadableItem downloadableItem) throws IOException; 11 | } 12 | -------------------------------------------------------------------------------- /portability-spi-cloud/README.md: -------------------------------------------------------------------------------- 1 | # portability-spi-cloud 2 | 3 | This folder contains the Service Provider Interface (SPI) for DTP's 4 | Cloud abstractions. 5 | 6 | DTP is designed to be runnable from any cloud infrastructure, or locally. 7 | 8 | This folder contains the SPIs needed from any cloud implementation. 9 | 10 | [Current cloud implementations](../extensions/cloud). 11 | 12 | If you want to add support for a new cloud platform, simply add 13 | implementations of these interfaces to a new project in the above folder. 14 | -------------------------------------------------------------------------------- /portability-spi-cloud/src/main/java/org/datatransferproject/spi/cloud/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Defines the data portability client SPI. Extensions use this module to implement custom 18 | * functionality for the system. 19 | */ 20 | package org.datatransferproject.spi.cloud; 21 | -------------------------------------------------------------------------------- /portability-spi-service/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | plugins { 19 | id 'maven' 20 | id 'signing' 21 | } 22 | 23 | dependencies { 24 | 25 | compile project(':portability-api-launcher') 26 | 27 | } 28 | 29 | configurePublication(project) 30 | -------------------------------------------------------------------------------- /portability-spi-transfer/README.md: -------------------------------------------------------------------------------- 1 | # portability-spi-transfer 2 | 3 | This folder contains the Service Provider Interface (SPI) for DTP's 4 | transfer abstractions. 5 | 6 | If you are [adding a new service provider integration](../Documentation/Integration.md) 7 | these are the interfaces needed. 8 | 9 | See our [current integrations](../extensions/data-transfer) for example of implementation of 10 | these interfaces. 11 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/hooks/DefaultJobHooks.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.spi.transfer.hooks; 17 | 18 | /** A default implementation of the job hooks. */ 19 | public class DefaultJobHooks implements JobHooks {} 20 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/i18n/BaseMultilingualDictionary.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.spi.transfer.i18n; 18 | 19 | public class BaseMultilingualDictionary extends AbstractMultilingualDictionary { 20 | public BaseMultilingualDictionary(String locale) { 21 | super(locale, "MultilingualStringsBundle", BaseMultilingualString.values()); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/i18n/MultilingualString.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.spi.transfer.i18n; 18 | 19 | public interface MultilingualString { 20 | String getKey(); 21 | 22 | String getDefaultValue(); 23 | } 24 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/idempotentexecutor/IdempotentImportExecutorExtension.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.idempotentexecutor; 2 | 3 | import org.datatransferproject.api.launcher.BootExtension; 4 | import org.datatransferproject.api.launcher.ExtensionContext; 5 | import org.datatransferproject.api.launcher.Monitor; 6 | 7 | public interface IdempotentImportExecutorExtension extends BootExtension { 8 | IdempotentImportExecutor getIdempotentImportExecutor(ExtensionContext extensionContext); 9 | public default IdempotentImportExecutor getRetryingIdempotentImportExecutor( 10 | ExtensionContext extensionContext) { 11 | return null; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/idempotentexecutor/ImportFunction.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.idempotentexecutor; 2 | 3 | @FunctionalInterface 4 | public interface ImportFunction { 5 | ItemImportResult apply(T t) throws Exception; 6 | } 7 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Defines the transfer SPI. Extensions use this module to implement custom import/export 18 | * functionality for a service provider. 19 | */ 20 | package org.datatransferproject.spi.transfer; 21 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/security/SecurityException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.spi.transfer.security; 17 | 18 | /** */ 19 | public class SecurityException extends RuntimeException { 20 | public SecurityException(String message, Throwable cause) { 21 | super(message, cause); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/store/TransferStore.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.store; 2 | 3 | import org.datatransferproject.types.common.PortableType; 4 | 5 | /** 6 | * Used by {@link org.datatransferproject.spi.transfer.provider.Importer}s and {@link 7 | * org.datatransferproject.spi.transfer.provider.Exporter}s to save scratch data needed during 8 | * the transfer process. 9 | */ 10 | public interface TransferStore { 11 | 12 | /** 13 | * Creates a data entry. 14 | * 15 | * @param entity the entry 16 | */ 17 | void create(PortableType entity); 18 | 19 | /** 20 | * Updates an existing entry. 21 | * 22 | * @param entity the entry 23 | */ 24 | void update(PortableType entity); 25 | 26 | /** 27 | * Returns an existing entry or null if not found. 28 | * 29 | * @param id the entry id 30 | */ 31 | PortableType find(String id); 32 | 33 | /** 34 | * Removes and entry 35 | * 36 | * @param id the entry id 37 | */ 38 | void remove(String id); 39 | } 40 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/CopyException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.spi.transfer.types; 17 | 18 | public class CopyException extends Exception { 19 | public CopyException(String message, Throwable cause) { 20 | super(message, cause); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/CopyExceptionWithFailureReason.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | /** 6 | * Exception for a single known category of failure-reason, thrown during a copy that should be 7 | * caught by the JobProcessor (to then add the failure reason to the job). 8 | * 9 | *

{@link FailureReasons} is intended to assist in keeping implementations of this class 10 | * accessible. 11 | * 12 | *

Note if an exception case is needed to express multiple classes of distinct failuiure cases, 13 | * then deriving from {@link CopyException} instead is probably better. 14 | */ 15 | public abstract class CopyExceptionWithFailureReason extends CopyException { 16 | 17 | public CopyExceptionWithFailureReason(String message, Throwable cause) { 18 | super(message, cause); 19 | } 20 | 21 | @Nonnull 22 | public abstract String getFailureReason(); 23 | } 24 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/FailureReasons.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | public enum FailureReasons { 4 | CREDS_TIMEOUT("CREDS_TIMEOUT"), 5 | DESTINATION_FULL("DESTINATION_FULL"), 6 | INVALID_TOKEN("INVALID_TOKEN"), 7 | PERMISSION_DENIED("PERMISSION_DENIED"), 8 | SESSION_INVALIDATED("SESSION_INVALIDATED"), 9 | UNCONFIRMED_USER("UNCONFIRMED_USER"), 10 | UPLOAD_ERROR("UPLOAD_ERROR"), 11 | USER_CHECKPOINTED("USER_CHECKPOINTED"), 12 | DESTINATION_NOT_FOUND("DESTINATION_NOT_FOUND"), 13 | UPSTREAM_API_UNEXPECTED_RESPONSE_EXCEPTION("UPSTREAM_API_UNEXPECTED_RESPONSE_EXCEPTION"); 14 | private final String string; 15 | 16 | FailureReasons(String string) { 17 | this.string = string; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return string; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/InvalidTokenException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | public class InvalidTokenException extends CopyExceptionWithFailureReason { 6 | 7 | public InvalidTokenException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | @Nonnull 12 | @Override 13 | public String getFailureReason() { 14 | return FailureReasons.INVALID_TOKEN.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/PermissionDeniedException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | public class PermissionDeniedException extends CopyExceptionWithFailureReason { 6 | 7 | public PermissionDeniedException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | @Nonnull 12 | @Override 13 | public String getFailureReason() { 14 | return FailureReasons.PERMISSION_DENIED.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/SessionInvalidatedException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | public class SessionInvalidatedException extends CopyExceptionWithFailureReason { 6 | 7 | public SessionInvalidatedException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | @Nonnull 12 | @Override 13 | public String getFailureReason() { 14 | return FailureReasons.SESSION_INVALIDATED.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/UnconfirmedUserException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | public class UnconfirmedUserException extends CopyExceptionWithFailureReason { 6 | 7 | public UnconfirmedUserException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | @Nonnull 12 | @Override 13 | public String getFailureReason() { 14 | return FailureReasons.UNCONFIRMED_USER.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/UploadErrorException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | public class UploadErrorException extends CopyExceptionWithFailureReason { 6 | 7 | public UploadErrorException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | @Nonnull 12 | @Override 13 | public String getFailureReason() { 14 | return FailureReasons.UPLOAD_ERROR.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/UpstreamApiUnexpectedResponseException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | public class UpstreamApiUnexpectedResponseException extends CopyExceptionWithFailureReason { 6 | 7 | public UpstreamApiUnexpectedResponseException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | @Nonnull 12 | @Override 13 | public String getFailureReason() { 14 | return FailureReasons.UPSTREAM_API_UNEXPECTED_RESPONSE_EXCEPTION.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/java/org/datatransferproject/spi/transfer/types/UserCheckpointedException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.types; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | public class UserCheckpointedException extends CopyExceptionWithFailureReason { 6 | 7 | public UserCheckpointedException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | @Nonnull 12 | @Override 13 | public String getFailureReason() { 14 | return FailureReasons.USER_CHECKPOINTED.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Copy of {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_ar.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=\u0646\u0633\u062E\u0629 \u0645\u0646 {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_cs.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Kopie {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_de.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Kopie von {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_el.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=\u0391\u03BD\u03C4\u03AF\u03B3\u03C1\u03B1\u03C6\u03BF {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_es.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Copia de {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_fr.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Texte de {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_id.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Teks {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_it.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Copia di {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_ja.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof={0}\u306E\u30B3\u30D4\u30FC 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_ms.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Salinan {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_pt.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Copy de {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_ru.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=\u041A\u043E\u043F\u0438\u044F {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_sk.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=K\u00F3pia s\u00FAboru {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_tl.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=Kopya ng {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_uk.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof=\u041A\u043E\u043F\u0456\u044F {0} 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/main/resources/MultilingualStringsBundle_zh.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The Data Transfer Project Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | copyof={0}\u526F\u672C 17 | -------------------------------------------------------------------------------- /portability-spi-transfer/src/test/java/org/datatransferproject/spi/transfer/provider/ExportResultTest.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.spi.transfer.provider; 2 | 3 | import static org.datatransferproject.spi.transfer.provider.ExportResult.ResultType.CONTINUE; 4 | import static org.datatransferproject.spi.transfer.provider.ExportResult.ResultType.END; 5 | import static org.datatransferproject.spi.transfer.provider.ExportResult.ResultType.ERROR; 6 | import static org.junit.jupiter.api.Assertions.assertEquals; 7 | 8 | import org.datatransferproject.spi.transfer.provider.ExportResult.ResultType; 9 | import org.junit.jupiter.api.Test; 10 | 11 | public class ExportResultTest { 12 | 13 | @Test 14 | public void resultTypeMerge() { 15 | assertEquals(ERROR, ResultType.merge(ERROR, END)); 16 | assertEquals(ERROR, ResultType.merge(END, ERROR)); 17 | assertEquals(CONTINUE, ResultType.merge(END, CONTINUE)); 18 | assertEquals(CONTINUE, ResultType.merge(CONTINUE, END)); 19 | assertEquals(END, ResultType.merge(END, END)); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /portability-test-utilities/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | dependencies { 19 | compile project(':portability-spi-transfer') 20 | } 21 | -------------------------------------------------------------------------------- /portability-test-utilities/src/main/java/org/datatransferproject/test/types/FakeIdempotentImportExecutor.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.test.types; 2 | 3 | import org.datatransferproject.api.launcher.Monitor; 4 | import org.datatransferproject.spi.transfer.idempotentexecutor.InMemoryIdempotentImportExecutor; 5 | 6 | /** Unit-test friendly, fully-functional IdempotentImportExecutor, entirely in memory. */ 7 | // TODO migrate tests constructing InMemoryIdempotentImportExecutor to this class to avoid future 8 | // noise where we have to touch every test (should InMemoryIdempotentImportExecutor start doing 9 | // something unfriendly to tests). 10 | public class FakeIdempotentImportExecutor extends InMemoryIdempotentImportExecutor { 11 | // TODO figure out how to get Gradle to let us mock(Monitor.class) ourselves here? 12 | private static final Monitor fakeMonitor = new Monitor() {}; 13 | 14 | public FakeIdempotentImportExecutor() { 15 | super(FakeIdempotentImportExecutor.fakeMonitor); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /portability-test-utilities/src/main/java/org/datatransferproject/test/types/ObjectMapperFactory.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.test.types; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 5 | 6 | /** */ 7 | public final class ObjectMapperFactory { 8 | 9 | private ObjectMapperFactory() {} 10 | 11 | public static ObjectMapper createObjectMapper() { 12 | ObjectMapper objectMapper = new ObjectMapper(); 13 | objectMapper.registerModule(new JavaTimeModule()); 14 | return objectMapper; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-transfer/README.md: -------------------------------------------------------------------------------- 1 | # portability-transfer 2 | 3 | This folder contains the binary for the Worker server used by DTP. 4 | 5 | DTP has two servers: the API, and the Worker. The [API server](../portability-api) 6 | exposes DTP's API (hence the name). This allows clients to submit jobs, 7 | and check job statuses. The Worker does the actual transfer 8 | of the data. 9 | -------------------------------------------------------------------------------- /portability-transfer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'portability-transfer' 2 | -------------------------------------------------------------------------------- /portability-transfer/src/main/java/org/datatransferproject/transfer/CredsTimeoutException.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.transfer; 2 | 3 | import java.util.UUID; 4 | 5 | public class CredsTimeoutException extends RuntimeException { 6 | private UUID jobId; 7 | 8 | public CredsTimeoutException(String message, UUID jobId) { 9 | super(message); 10 | this.jobId = jobId; 11 | } 12 | 13 | public UUID getJobId() { 14 | return jobId; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /portability-transfer/src/main/java/org/datatransferproject/transfer/copier/InMemoryDataCopierExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.transfer.copier; 18 | 19 | import org.datatransferproject.api.launcher.BootExtension; 20 | 21 | public interface InMemoryDataCopierExtension extends BootExtension { 22 | Class getInMemoryDataCopierClass(); 23 | } 24 | -------------------------------------------------------------------------------- /portability-types-client/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | plugins { 19 | id 'maven' 20 | id 'signing' 21 | } 22 | 23 | dependencies { 24 | compile project(":portability-types-common") 25 | compile("io.swagger:swagger-annotations:${swaggerVersion}") 26 | testCompile("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}") 27 | } 28 | 29 | configurePublication(project) 30 | -------------------------------------------------------------------------------- /portability-types-client/src/main/java/org/datatransferproject/types/client/datatype/GetDataTypes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.types.client.datatype; 17 | 18 | /** A request to list data types available for export and import. */ 19 | public class GetDataTypes {} 20 | -------------------------------------------------------------------------------- /portability-types-client/src/main/java/org/datatransferproject/types/client/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Defines the data transfer client model objects. Model objects represent the system data 18 | * resources a client has access to. 19 | */ 20 | package org.datatransferproject.types.client; 21 | -------------------------------------------------------------------------------- /portability-types-client/src/main/java/org/datatransferproject/types/client/transfer/ReserveWorker.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.client.transfer; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import io.swagger.annotations.ApiModelProperty; 6 | 7 | /** Request to reserve a worker to process a transfer job. */ 8 | public class ReserveWorker { 9 | private final String id; 10 | 11 | @JsonCreator 12 | public ReserveWorker(@JsonProperty(value = "id", required = true) String id) { 13 | this.id = id; 14 | } 15 | 16 | @ApiModelProperty(value = "The id of the job for which we are requesting a worker", dataType = "string", 17 | required = true) 18 | public String getId() { 19 | return id; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/DownloadableFile.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | public interface DownloadableFile extends DownloadableItem, FolderItem { } 4 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/DownloadableItem.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | /** 4 | * Represent an item we can download through a URL and store in a temporary storage. 5 | * 6 | *

PhotoModel is a good example. Often, we check if the item is in the job store and download it 7 | * if it isn't. 8 | */ 9 | public interface DownloadableItem extends ImportableItem { 10 | 11 | /** Remote or local URL used to download and identify an item. */ 12 | String getFetchableUrl(); 13 | 14 | boolean isInTempStore(); 15 | } 16 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/Fileable.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | /** 4 | * Represent an item we expect to be recognizable as an ordinary file on someone's computer. 5 | */ 6 | public interface Fileable { 7 | /** 8 | * Returns a standard MIME-type string one would expect to find if they inspected the downloaded 9 | * version of this file with standard tools on their PC. 10 | * 11 | * See https://en.wikipedia.org/wiki/Media_type 12 | * 13 | * NOTE: this should match {@link MediaObject#getEncodingFormat} behavior. 14 | */ 15 | String getMimeType(); 16 | } 17 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/FolderItem.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | /** 4 | * Represent an item that belongs to some user-visible (and possibly user-defined) collection 5 | * like a folder on a filesystem, an album in a photo album, etc. 6 | */ 7 | public interface FolderItem extends Fileable { 8 | /** Returns the unique ID of the parent container this item lives in. */ 9 | String getFolderId(); 10 | } 11 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/ImportableItem.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | import javax.annotation.Nonnull; 5 | import javax.annotation.Nullable; 6 | 7 | public interface ImportableItem { 8 | 9 | @JsonIgnore 10 | @Nonnull 11 | String getIdempotentId(); 12 | 13 | @JsonIgnore 14 | @Nullable 15 | default String getName() { 16 | return null; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/IntPaginationToken.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeName; 6 | import org.datatransferproject.types.common.PaginationData; 7 | 8 | /** Numeric pagination data. */ 9 | @JsonTypeName("org.dataportability:NumberPagination") 10 | public class IntPaginationToken extends PaginationData { 11 | private final int start; 12 | 13 | /** 14 | * Ctor. 15 | * 16 | * @param start the number the next page starts on. 17 | */ 18 | @JsonCreator 19 | public IntPaginationToken(@JsonProperty("start") int start) { 20 | this.start = start; 21 | } 22 | 23 | public int getStart() { 24 | return start; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/PaginationData.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | /** 4 | * Contains pagination data for an export operation. 5 | * 6 | *

Concrete subtypes must use {@link com.fasterxml.jackson.annotation.JsonTypeName} to specify a 7 | * type descriminator used for deserialization. 8 | */ 9 | 10 | import com.fasterxml.jackson.annotation.JsonSubTypes; 11 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 12 | 13 | @JsonTypeInfo(use = JsonTypeInfo.Id.NAME) 14 | @JsonSubTypes({ 15 | @JsonSubTypes.Type(IntPaginationToken.class), 16 | @JsonSubTypes.Type(StringPaginationToken.class) 17 | }) 18 | public abstract class PaginationData extends PortableType {} 19 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/PortabilityCommon.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | public class PortabilityCommon { 4 | /** The protocol used for authorization. */ 5 | public enum AuthProtocol { 6 | UNKNOWN, 7 | OAUTH_1, 8 | OAUTH_2, 9 | CUSTOM 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/StringPaginationToken.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeName; 6 | import org.datatransferproject.types.common.PaginationData; 7 | 8 | /** 9 | * String pagination data. 10 | */ 11 | @JsonTypeName("org.dataportability:StringPagination") 12 | public class StringPaginationToken extends PaginationData { 13 | private final String token; 14 | 15 | /** 16 | * Ctor. 17 | * 18 | * @param token the token to get the next page 19 | */ 20 | @JsonCreator 21 | public StringPaginationToken(@JsonProperty("token") String token) { 22 | this.token = token; 23 | } 24 | 25 | public String getToken() { 26 | return token; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/DataModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.types.common.models; 17 | 18 | import java.util.Map; 19 | import org.datatransferproject.types.common.PortableType; 20 | 21 | /** The base data model type. */ 22 | public abstract class DataModel extends PortableType { 23 | public Map getCounts() { 24 | return null; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/ItemResource.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.transfer.models; 2 | 3 | import org.datatransferproject.types.common.models.DataModel; 4 | 5 | /** 6 | * A resource that contains a non-container resource, such as a photo or calendar event. 7 | * 8 | *

Concrete subtypes must use {@link com.fasterxml.jackson.annotation.JsonTypeName} to specify a 9 | * type descriminator used for deserialization. 10 | */ 11 | public abstract class ItemResource extends DataModel { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/contacts/ContactsModelWrapper.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common.models.contacts; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import org.datatransferproject.types.common.models.DataModel; 6 | 7 | /** A collection of contacts as serialized vCards. */ 8 | public class ContactsModelWrapper extends DataModel { 9 | private final String vCards; 10 | 11 | @JsonCreator 12 | public ContactsModelWrapper(@JsonProperty("vCards") String vCards) { 13 | this.vCards = vCards; 14 | } 15 | 16 | public String getVCards() { 17 | return vCards; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/playlists/CreativeWork.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common.models.playlists; 2 | 3 | /** 4 | * POJO of https://schema.org/CreativeWork 5 | */ 6 | public class CreativeWork { 7 | // NOTE: only a subset of fields are used so far, feel free to add more fields from the spec as 8 | // needed. 9 | protected String headline; 10 | 11 | public CreativeWork(String headline) { 12 | this.headline = headline; 13 | } 14 | 15 | public String headline() { 16 | return headline; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/playlists/MusicAlbum.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common.models.playlists; 2 | 3 | import org.datatransferproject.types.common.models.CreativeWork; 4 | /** 5 | * POJO https://schema.org/MusicAlbum 6 | */ 7 | public class MusicAlbum extends CreativeWork { 8 | // NOTE: only a subset of fields are used so far, feel free to add more fields from the spec as 9 | // needed. 10 | 11 | public MusicAlbum(String identifier, String headline) { 12 | super(identifier); 13 | setHeadline(headline); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/playlists/MusicGroup.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common.models.playlists; 2 | 3 | /** 4 | * POJO for https://schema.org/MusicGroup 5 | */ 6 | public class MusicGroup { 7 | // NOTE: only a subset of fields are used so far, feel free to add more fields from the spec as 8 | // needed. 9 | private String headline; 10 | 11 | public MusicGroup(String headline) { 12 | this.headline = headline; 13 | } 14 | 15 | public String getHeadline() { 16 | return headline; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/playlists/MusicPlaylist.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common.models.playlists; 2 | 3 | import com.google.common.collect.ImmutableList; 4 | import org.datatransferproject.types.common.models.CreativeWork; 5 | 6 | public class MusicPlaylist extends CreativeWork { 7 | // NOTE: only a subset of fields are used so far, feel free to add more fields from the spec as 8 | // needed. 9 | private ImmutableList track; 10 | 11 | public MusicPlaylist( 12 | String identifier, 13 | String headline, 14 | Iterable tracks) { 15 | super(identifier); 16 | setHeadline(headline); 17 | this.track = ImmutableList.copyOf(tracks); 18 | } 19 | 20 | public ImmutableList getTrack() { 21 | return track; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/playlists/PlaylistContainerResource.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common.models.playlists; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeName; 6 | import com.google.common.collect.ImmutableList; 7 | import java.util.Collection; 8 | import org.datatransferproject.types.common.models.ContainerResource; 9 | 10 | /** A Wrapper for a group of playlists. */ 11 | @JsonTypeName("PlaylistContainerResource") 12 | public class PlaylistContainerResource extends ContainerResource { 13 | private final Collection playlists; 14 | 15 | @JsonCreator 16 | public PlaylistContainerResource( 17 | @JsonProperty("playlists") Collection playlists) { 18 | this.playlists = playlists == null ? ImmutableList.of() : playlists; 19 | } 20 | 21 | public Collection getLists() { 22 | return playlists; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/social/SocialActivityAttachmentType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.types.common.models.social; 18 | 19 | public enum SocialActivityAttachmentType { 20 | LINK, 21 | IMAGE, 22 | VIDEO 23 | } 24 | -------------------------------------------------------------------------------- /portability-types-common/src/main/java/org/datatransferproject/types/common/models/social/SocialActivityType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.datatransferproject.types.common.models.social; 18 | 19 | public enum SocialActivityType { 20 | CHECKIN, 21 | POST, 22 | NOTE, 23 | } 24 | -------------------------------------------------------------------------------- /portability-types-common/src/test/java/org/datatransferproject/types/common/models/DataVerticalTest.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.common.models; 2 | 3 | import static org.datatransferproject.types.common.models.DataVertical.SOCIAL_POSTS; 4 | import static org.junit.jupiter.api.Assertions.assertEquals; 5 | 6 | import com.fasterxml.jackson.core.JsonProcessingException; 7 | import com.fasterxml.jackson.databind.ObjectMapper; 8 | import org.junit.jupiter.api.Test; 9 | 10 | class DataVerticalTest { 11 | 12 | @Test 13 | void shouldSerializeSocialPostsCorrectly() throws JsonProcessingException { 14 | ObjectMapper om = new ObjectMapper(); 15 | String json = "\"SOCIAL-POSTS\""; 16 | assertEquals(SOCIAL_POSTS, om.readValue(json, DataVertical.class)); 17 | assertEquals(SOCIAL_POSTS, DataVertical.fromDataType("SOCIAL-POSTS")); 18 | assertEquals(json, om.writeValueAsString(SOCIAL_POSTS)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /portability-types-transfer/README.md: -------------------------------------------------------------------------------- 1 | # portability-types-transfer 2 | 3 | This folder contains the core DTP-framework interfaces used throughout the DTP's codebase. 4 | For more general, common models see the `org.datatransferproject.types.common.models.*` 5 | packages over in [`portability-types-common/src/main/java/org/datatransferproject/types/common/models`](/portability-types-common/src/main/java/org/datatransferproject/types/common/models). 6 | 7 | - [auth](src/main/java/org/datatransferproject/types/transfer/auth) - 8 | common data structures for passing auth data around the DTP system 9 | - [errors](src/main/java/org/datatransferproject/types/transfer/errors) - 10 | common data structure for errors passed around or serialized 11 | - [retry](src/main/java/org/datatransferproject/types/transfer/retry) - 12 | common data structures for encoding the retry behavior for a server when 13 | there is an error. 14 | -------------------------------------------------------------------------------- /portability-types-transfer/src/main/java/org/datatransferproject/types/transfer/auth/AuthData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.datatransferproject.types.transfer.auth; 17 | 18 | import org.datatransferproject.types.common.PortableType; 19 | 20 | /** Base type for authentication data. */ 21 | public abstract class AuthData extends PortableType { 22 | public abstract String getToken(); 23 | } 24 | -------------------------------------------------------------------------------- /portability-types-transfer/src/main/java/org/datatransferproject/types/transfer/auth/TokenAuthData.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.transfer.auth; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeName; 6 | 7 | /** Token-based authentication data. */ 8 | @JsonTypeName("org.dataportability:TokenAuthData") 9 | public class TokenAuthData extends AuthData { 10 | private final String token; 11 | 12 | @JsonCreator 13 | public TokenAuthData(@JsonProperty("token") String token) { 14 | this.token = token; 15 | } 16 | 17 | @Override 18 | public String getToken() { 19 | return token; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /portability-types-transfer/src/main/java/org/datatransferproject/types/transfer/auth/TokenSecretAuthData.java: -------------------------------------------------------------------------------- 1 | package org.datatransferproject.types.transfer.auth; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeName; 6 | 7 | /** Token-secret-based authentication data. */ 8 | @JsonTypeName("org.dataportability:TokenSecretAuthData") 9 | public class TokenSecretAuthData extends AuthData { 10 | private final String token; 11 | private final String secret; 12 | 13 | @JsonCreator 14 | public TokenSecretAuthData( 15 | @JsonProperty("token") String token, @JsonProperty("secret") String secret) { 16 | this.token = token; 17 | this.secret = secret; 18 | } 19 | 20 | @Override 21 | public String getToken() { 22 | return token; 23 | } 24 | 25 | public String getSecret() { 26 | return secret; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /portability-types-transfer/src/main/java/org/datatransferproject/types/transfer/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 The Data Transfer Project Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Defines the data transfer model objects. Model objects represent the flow of data through the 18 | * system. 19 | */ 20 | package org.datatransferproject.types.transfer; 21 | -------------------------------------------------------------------------------- /scripts/docker_push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Script used to push new docker images to docker hub by travis. 4 | 5 | # These environment variables are set via the Travis UI/CLI 6 | echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 7 | docker push datatransferproject/demo 8 | --------------------------------------------------------------------------------