Develop Reference

software.amazon.awssdk.services.s3.model.S3Exception: Invalid response status from request while using aws java sdk v2 "S3TRANSFERMANAGER"

am trying to use S3TransferManager to upload file to s3. but my unit test fails due to the below error,
java.util.concurrent.CompletionException: software.amazon.awssdk.services.s3.model.S3Exception: Invalid response status from request
here's my code,
public class AwsTransferService {
private final S3TransferManager s3TransferManager;
private final AwsS3Config s3Config;
public AwsTransferService(AwsS3Config s3Config, AwsConfig awsConfig) {
this.s3Config = s3Config;
AwsBasicCredentials awsCredentials = create(awsConfig.getAccessKey(), awsConfig.getSecretKey());
this.s3TransferManager = S3TransferManager.builder()
.s3ClientConfiguration(builder -> builder.credentialsProvider(create(awsCredentials))
.region(s3Config.getRegion())
.minimumPartSizeInBytes(10 * MB)
.targetThroughputInGbps(20.0))
.build();
}
public AwsTransferService(S3TransferManager s3TransferManager, AwsS3Config s3Config) {
this.s3TransferManager = s3TransferManager;
this.s3Config = s3Config;
}
public void transferObject(#NonNull String bucketName, #NonNull String transferKey, #NonNull File file) {
validateS3Key(transferKey);
validatePath(file.toPath());
log.info("Transfering s3 object from :{} to :{}", file.getPath(), transferKey);
try {
Upload upload =
s3TransferManager.upload(b -> b.putObjectRequest(r -> r.bucket(bucketName).key(transferKey))
.source(file.toPath()));
CompletedUpload completedUpload = upload.completionFuture().join();
log.info("PutObjectResponse: " + completedUpload.response());
} catch (Exception e) {
e.printStackTrace();
}
}
and here is my unit test for the above code,
#RegisterExtension
public static final S3MockExtension S3_MOCK = builder()
.silent()
.withSecureConnection(false)
.build();
private S3ClientConfiguration s3ClientConfiguration;
private AwsTransferService service;
private AwsS3Service awsS3Service;
private S3TransferManager s3TransferManager;
private static S3Client s3Client;
#BeforeAll
public static void beforeAll() {
s3Client = S3_MOCK.createS3ClientV2();
}
#BeforeEach
public void beforeEach() throws IOException {
s3ClientConfiguration =mock(S3ClientConfiguration.class);
s3TransferManager = S3TransferManager.builder().s3ClientConfiguration(s3ClientConfiguration).build();
AwsS3Config s3Config = AwsS3Config.builder()
.region(Region.AP_SOUTHEAST_2)
.s3BucketName(S3Factory.VALID_S3_BUCKET)
.build();
awsS3Service = new AwsS3Service(s3Config, s3Client);
awsS3Service.createBucket(VALID_S3_BUCKET);
service = new AwsTransferService(s3TransferManager, s3Config);
}
#Test
public void transferObject_singleFile_ShouldUploadFiletoS3() throws IOException {
String transferKey = TRANSFER_KEY_UPLOAD;
String fileName = FILE_PATH + TRANSFER_FILE_NAME;
writeFile(fileName);
File transferFile = new File(fileName);
service.transferObject(VALID_S3_BUCKET, transferKey + TRANSFER_FILE_NAME, transferFile);
int expectedObjectsSize = 1;
Log.initLoggingToFile(Log.LogLevel.Error, "log.txt");
List<S3Object> matchedObjects = awsS3Service.listObjectsWithPrefix(transferKey + TRANSFER_FILE_NAME);
assertEquals(expectedObjectsSize, matchedObjects.size());
assertEquals(transferKey + TRANSFER_FILE_NAME, matchedObjects.get(0).key());
}
please let me know why the unit test fails with the above mentioned error?
also please let me know is there any other way to mock "s3ClientConfiguration"? in aws java sdk v2

Mockito.doThrow() not throwing any exception

I am trying to stub a void method of a mocked object to return an exception. This mocked object passed as dependency to the service which I am writing tests for.
Service:
#Component
public class FileHandler {
private static final Logger log = LoggerFactory.getLogger(FileHandler.class);
private final SSHAccess sshAccess;
#Value("${datastore.location}")
private String dataStoreDir;
public FileHandler(SSHAccess sshAccess){
this.sshAccess = sshAccess;
}
public Either<Pair<Exception, FileRecord>, FileRecord> transferFile(FileRecord fileRecord){
try {
var sourceURI = new URI(fileRecord.getSourceURI());
var digest = sshAccess.execute("md5sum " + sourceURI.getPath())
.replaceFirst("([^\\s]+)[\\d\\D]*", "$1");
if (digest.equals(fileRecord.getSourceDigest())) {
log.info(Thread.currentThread().getName() + ": Copying file: " + fileRecord.getId() + " of submission: " + fileRecord.getOwnedBy());
sshAccess.download(sourceURI.getPath(),new File(mkdir(dataStoreDir, digest), digest));
log.info(Thread.currentThread().getName() + ": Copying of file: " + fileRecord.getId() + " of submission: " + fileRecord.getOwnedBy() + " finished.");
return Either.Right(fileRecord);
}else{
log.error("MD5 mismatch for source file {}", sourceURI.getPath());
return Either.Left(Pair.of(new FileHandlerException("MD5 mismatch"), fileRecord));
}
} catch (URISyntaxException
| IOException
e) {
return Either.Left(Pair.of(new FileHandlerException(e), fileRecord));
}
}
private File mkdir(String dataStoreDir, String digest) throws IOException {
File dir = new File(dataStoreDir, digest.substring(0, 3));
if (!dir.exists() && !dir.mkdirs()) {
log.error("Unable to create directory {}", dir);
throw new IOException("Unable to create directory " + dir);
}
return dir;
}
}
Test Class:
#SpringBootTest
#TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class FileHandlerTest {
private FileHandler fileHandler;
#Mock
private SSHAccess sshAccess;
#BeforeAll
public void init(){
fileHandler = Mockito.spy(new FileHandler(sshAccess));
}
#Test
public void transferFileShouldReturnFileHandlerExceptionOnEitherLeftWhenSSHclientThrowsIOException() throws IOException {
FileRecord fileRecord = getFileRecord();
var digest = "6e484ac23110fae10021e";
when(sshAccess.execute(anyString())).thenReturn(digest);
doThrow(IOException.class).when(sshAccess).download(anyString(), any(File.class));
var result = fileHandler.transferFile(fileRecord);
Assertions.assertTrue(result.getLeft().isPresent()
&& result.getLeft().get().getFirst() instanceof FileHandlerException);
}
private FileRecord getFileRecord() {
var fileRecord = new FileRecord();
fileRecord.setId(1L);
fileRecord.setOwnedBy(1000);
fileRecord.setSourceURI("scp:host/test/uri/filename");
fileRecord.setSourceDigest("6e484ac23110fae10021e");
return fileRecord;
}
}
But when I run this test case, doThrow() doesn't throw any exception. Method executed without any exception and test failed. I am not sure what I am doing wrong here. Please help.

Not sure why are you using the #SpringBootTest annotation which will try to raise a similar context with the one when you are running the app. So in this case you could stop instantiating your FileHandler and just spy on it and on your SSHAccess beans or use #MockBean instead of #Mock.
Basically you should have something like this
#SpyBean
private FileHandler fileHandler;
#MockBean
private SSHAccess sshAccess;

You are using Junit 5. For Junit 5 you don't need the method #SpringBootTest, you need to use #ExtendWith(MockitoExtension.class)
#SpringBootTest
#TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class FileHandlerTest {
private FileHandler fileHandler;
#Mock
private SSHAccess sshAccess;
#BeforeAll
public void init(){
fileHandler = Mockito.spy(new FileHandler(sshAccess));
}
.....
.....
.....
}
Also instead of Mockito.spy(new FileHandler(sshAccess)) you can try Mockito.mock(new FileHandler(sshAccess))

How to test Rest API and mock url using spring boot and mockbeam

I have a Rest API
The class code is :
#SpringBootTest
#RunWith(SpringRunner.class)
public class FetchCoreVersionsListIT {
#MockBean
private RestTemplateBuilder restTemplateBuilder;
#MockBean
private RestTemplate restTemplate;
private VersionsService versionsService;
#Autowired
private FetchCoreVersionsList fetchCoreVersionsList;
private VersionList versionList;
private ArtifactoryFolderInfoChild version;
#Before
public void setUp() throws Exception {
this.version = new ArtifactoryFolderInfoChild();
this.version.setUri("2.1.0");
this.version.setFolder(true);
when(restTemplateBuilder.build()).thenReturn(restTemplate);
}
#Test
public void testCoreVersionsJsonHandle() throws Exception{
when(restTemplate.getForObject("https://openmrs.jfrog.io/openmrs/api/storage/public/org/openmrs/api/openmrs-api/",
String.class))
.thenReturn(getFileAsString("core-versions.json"));
("2.1.0"));*/
}
This is the core-versions.json . This is nothing else but the data received from this Rest API.
Basically I'm trying to run a test and I have a spring schedule that will parse the json received from that Rest url. Now, while testing the schedule, I want to return the same data but without connecting to the internet and hence want to return the contents of core-versions.json. I get the following error unfortunately :
java.lang.IllegalStateException: File downloaded from could not be parsed
My schedule class is this:
#Component
public class FetchCoreVersionsList {
private final Logger logger = LoggerFactory.getLogger(getClass());
private static final String[] STRINGS_TO_EXCLUDE = {"alpha", "beta", "RC", "SNAPSHOT"};
#Value("${core_version_list.url}")
private String url;
//#Value("${core_version_list.strategy}")
//private FetchCoreVersionsList.Strategy strategy = FetchCoreVersionsList.Strategy.FETCH;
private RestTemplateBuilder restTemplateBuilder;
private ObjectMapper mapper;
private VersionsService versionsService;
#Autowired
public FetchCoreVersionsList(RestTemplateBuilder restTemplateBuilder,
ObjectMapper mapper,
VersionsService versionsService) {
this.restTemplateBuilder = restTemplateBuilder;
this.mapper = mapper;
this.versionsService = versionsService;
}
#Scheduled(
initialDelayString = "${scheduler.fetch_core_versions_list.initial_delay}",
fixedDelayString = "${scheduler.fetch_core_versions_list.period}")
public void fetchCoreVersionsList() throws Exception {
logger.info("Fetching list of OpenMRS-Core versions");
// FetchCoreVersionsList.Strategy strategy = FetchCoreVersionsList.Strategy.FETCH;
String json;
/* if (strategy == Strategy.LOCAL) {
logger.debug("LOCAL strategy");
json = StreamUtils.copyToString(getClass().getClassLoader().getResourceAsStream("openmrs-core-versions.json"),
Charset.defaultCharset());
} else {*/
json = restTemplateBuilder.build().getForObject(url, String.class);
logger.info("FETCH strategy: " + json);
ArtifactoryFolderInfo versionlist;
try { logger.info("FETCH strategy: " + json);
logger.debug("papa strategy: " + url);
versionlist = mapper.readValue(json, ArtifactoryFolderInfo.class);
} catch (Exception ex) {
throw new IllegalStateException("File downloaded from " + url + " could not be parsed", ex);
}
if (logger.isInfoEnabled()) {
logger.info("There are " + versionlist.getChildren().size() + " openmrs-core versions");
}
if (versionlist.size() > 0) {
List<String> versions = new ArrayList<>();
List<ArtifactoryFolderInfoChild> allversions = versionlist.getChildren();
for (ArtifactoryFolderInfoChild candidateVersion : allversions) {
if (candidateVersion.getFolder() && !stringContainsItemFromList(candidateVersion.getUri(), STRINGS_TO_EXCLUDE)) {
versions.add(candidateVersion.getUri().replaceAll("/", ""));
}
}
versionsService.setVersions(versions);
} else {
logger.warn("File downloaded from " + url + " does not list any Core Versions to index. Keeping our current list");
}
}
private static boolean stringContainsItemFromList(String inputStr, String[] items) {
return Arrays.stream(items).parallel().anyMatch(inputStr::contains);
}
public enum Strategy {
FETCH, LOCAL
}
}
Kindly bear with me if this is a silly error as I am completely new to testing.

LOG4J2 Use multiple config files using java

can log4j2 use multiple config files. I wanna run my project and load one default config file - logger.xml and after that to check if there is a second configuration from another file logger_1.xml and to add it and not to override the first one.
Here is some dummy code. In short I wanna fill up the arrayList with file paths and then to load all of them.
public class LoggerConfiguratorManager
{
public static final String LOG4J_PATH = "etc/confs/logger.xml";
private static LoggerContext context = null;
private static final ConfigurationFactory factory = XmlConfigurationFactory.getInstance();
private static ConfigurationSource configurationSource = null;
private static Configuration configuration = null;
private static final ArrayList<String> registred_logger = new ArrayList<>();
private static void loadLoggerConfig(String logger_path)
{
InputStream is = null;
try
{
if(logger_path.endsWith(".xml"))
is = new FileInputStream(logger_path);
else
{
final ZipFile archive = new ZipFile(logger_path);
final ZipEntry logger_entry = archive.getEntry(LOG4J_PATH);
if(logger_entry == null) throw new IOException("Cannot find 'logger.xml' in " + logger_path);
is = archive.getInputStream(logger_entry);
}
configurationSource = new ConfigurationSource(is);
configuration = factory.getConfiguration(configurationSource);
}
catch(IOException ex)
{
System.err.println("=============================================================================");
System.err.println("=============================== LOGGER CONFIG ===============================");
System.err.println("=============================================================================");
System.err.println("=== [ERROR] " + ex);
}
finally
{
if (configurationSource != null)
{
context = Configurator.initialize(null, configurationSource);
context.start(configuration);
try { is.close(); } catch(IOException ex) { }
}
}
}
public static void load()
{
registred_logger.add(Globals.getClassLocation(LoggerConfiguratorManager.class));
for(final String conf : registred_logger)
loadLoggerConfig(conf);
}
public static void regLoggerConf(String conf_path) { registred_logger.add(conf_path); }

I would suggest doing instead:
public class LoggerConfiguratorManager {
private static final String LOG4J_PATH = "etc/confs/log4j2.xml";
private static final StringBuffer paths = new StringBuffer(LOG4J_PATH);
public static void registerConfiguration(String confPath) {
paths.append(",").append(confPath);
}
public static void initLog4j() {
Configurator.initializer("My Config", null, paths.toString(), null);
}
}
For a full working example please see https://github.com/rgoers/CompositeConfigurationExample.

Is it possible to have persistence.xml in a location other than META-INF?

I want to have my persistence.xml in conf folder of my app. How can I tell Persistence.createEntityManagerFactory that it should read it from there?

If you are using EclipseLink you can set the persistence.xml location with the persistence unit property, "eclipselink.persistencexml".
properties.put("eclipselink.persistencexml", "/org/acme/acme-persistence.xml");
EntityManagerFactory factory = Persistence.createEntityManagerFactory("acme", properties);

This solution worked for me
Thread.currentThread().setContextClassLoader(new ClassLoader() {
#Override
public Enumeration<URL> getResources(String name) throws IOException {
if (name.equals("META-INF/persistence.xml")) {
return Collections.enumeration(Arrays.asList(new File("conf/persistence.xml")
.toURI().toURL()));
}
return super.getResources(name);
}
});
Persistence.createEntityManagerFactory("test");

The createEntityManagerFactory methods search for persistence.xml files within the META-INF directory of any CLASSPATH element.
if your CLASSPATH contains the conf directory, you could place an EntityManagerFactory definition in conf/META-INF/persistence.xml

The ClassLoader may be a URLClassLoader, so try it this way:
final URL alternativePersistenceXmlUrl = new File("conf/persistence.xml").toURI().toURL();
ClassLoader output;
ClassLoader current = Thread.currentThread().getContextClassLoader();
try{
URLClassLoader parent = (URLClassLoader)current;
output = new URLClassLoader(parent.getURLs(), parent){
#Override
public Enumeration<URL> getResources(String name) throws IOException {
if (name.equals("META-INF/persistence.xml")) {
return Collections.enumeration(Arrays.asList(alternativePersistenceXmlUrl));
}
return super.getResources(name);
}
};
}catch(ClassCastException ignored) {
output = new ClassLoader() {
#Override
public Enumeration<URL> getResources(String name) throws IOException {
if (name.equals("META-INF/persistence.xml")) {
return Collections.enumeration(Arrays.asList(alternativePersistenceXmlUrl));
}
return super.getResources(name);
}
};
}
It should work. Works for me under certain test etc conditions.
Please this is a hack and should not be used in production.

My solution is for EclipseLink 2.7.0 and Java 9 and it is modified and detailed version of #Evgeniy Dorofeev answer.
In org.eclipse.persistence.internal.jpa.deployment.PersistenceUnitProcessor on line 236 we see the following code:
URL puRootUrl = computePURootURL(descUrl, descriptorPath);
This code is used by EclipseLink to compute root url of the persistence.xml path. That's very important because final path will be made by adding descriptorPath to puRootUrl.
So, let's suppose we have file on /home/Smith/program/some-folder/persistence.xml, then we have:
Thread currentThread = Thread.currentThread();
ClassLoader previousClassLoader = currentThread.getContextClassLoader();
Thread.currentThread().setContextClassLoader(new ClassLoader(previousClassLoader) {
#Override
public Enumeration<URL> getResources(String name) throws IOException {
if (name.equals("some-folder/persistence.xml")) {
URL url = new File("/home/Smith/program/some-folder/persistence.xml").toURI().toURL();
return Collections.enumeration(Arrays.asList(url));
}
return super.getResources(name);
}
});
Map<String, String> properties = new HashMap<>();
properties.put("eclipselink.persistencexml", "some-folder/persistence.xml");
try {
entityManagerFactory = Persistence.createEntityManagerFactory("unit-name", properties);
} catch (Exception ex) {
logger.error("Error occured creating EMF", ex);
} finally {
currentThread.setContextClassLoader(previousClassLoader);
}
Details:
Pay attention that when creating new class loader I pass there previous classloader otherwise it doesn't work.
We set property eclipselink.persistencexml. If we don't do that then default descriptorPath will be equal to META-INF/persistence.xml and we would need to keep our persistence.xml on /home/Smith/program/META-INF/persistence.xml to be found.

I tried these ways when the program is starting (at first line of main function):
Write your persistence.xml to the resources/META-INF/persistence.xml of the jar
I had problem with this way: Java write .txt file in resource folder
Create META-INF folder in the jar directory and put your persistence.xml into it, then execute this command:
jar uf $jarName META-INF/persistence.xml
This command will replace META-INF/persistence.xml (your file) in the jar
private fun persistence() {
val fileName = "META-INF/persistence.xml"
val jarName: String?
val done = try {
jarName = javaClass.protectionDomain.codeSource.location.path
if (File(fileName).exists() && !jarName.isNullOrBlank()
&& jarName.endsWith(".jar") && File(jarName).exists()) {
Command().exec("jar uf $jarName META-INF/persistence.xml", timeoutSec = 30)
true
} else false
} catch (e: Exception) {
false
}
if (done) {
logger.info { "$fileName exist and will be loaded :)" }
} else {
logger.info {
"$fileName not exist in current folder so it will be read from .jar :(" +
" you can run: jar uf jarName.jar META-INF/persistence.xml"
}
}
}
Running Command Line in Java

A solution by creating tweaked PersistenceUnitDescriptor.
import org.hibernate.jpa.boot.internal.ParsedPersistenceXmlDescriptor;
import org.hibernate.jpa.boot.internal.PersistenceXmlParser;
import org.hibernate.jpa.boot.spi.Bootstrap;
import org.hibernate.jpa.boot.spi.EntityManagerFactoryBuilder;
public class HibernateEntityManagerFactoryBuilder {
public static final EntityManagerFactory build(URL xmlUrl) {
final ParsedPersistenceXmlDescriptor xmlDescriptor = PersistenceXmlParser.locateIndividualPersistenceUnit(xmlUrl);
final HibernatePersistenceUnitDescriptor hibernateDescriptor = new HibernatePersistenceUnitDescriptor(xmlDescriptor);
final EntityManagerFactoryBuilder builder = Bootstrap.getEntityManagerFactoryBuilder(hibernateDescriptor, Collections.emptyMap(), (ClassLoader) null);
final EntityManagerFactory factory = builder.build();
return factory;
}
public static final EntityManagerFactory build(URL xmlUrl, final String name) {
final ParsedPersistenceXmlDescriptor xmlDescriptor = PersistenceXmlParser.locateNamedPersistenceUnit(xmlUrl, name);
if(xmlDescriptor == null) throw new RuntimeException("Persistence unit with name '"+name+ "' not found.");
final HibernatePersistenceUnitDescriptor hibernateDescriptor = new HibernatePersistenceUnitDescriptor(xmlDescriptor);
final EntityManagerFactoryBuilder builder = Bootstrap.getEntityManagerFactoryBuilder(hibernateDescriptor, Collections.emptyMap(), (ClassLoader) null);
final EntityManagerFactory factory = builder.build();
return factory;
}
public static void main(String[] args) {
try {
final EntityManagerFactory factory = build(new File("D:/ini/persistence.xml").toURI().toURL());
} catch (Exception e) {e.printStackTrace();}
}
}
public class HibernatePersistenceUnitDescriptor implements PersistenceUnitDescriptor {
private final PersistenceUnitDescriptor descriptor;
public HibernatePersistenceUnitDescriptor(PersistenceUnitDescriptor descriptor) {
this.descriptor = descriptor;
}
#Override
public URL getPersistenceUnitRootUrl() {
return null;
}
#Override
public String getName() {
return descriptor.getName();
}
#Override
public String getProviderClassName() {
return descriptor.getProviderClassName();
}
#Override
public boolean isUseQuotedIdentifiers() {
return descriptor.isUseQuotedIdentifiers();
}
#Override
public boolean isExcludeUnlistedClasses() {
return descriptor.isExcludeUnlistedClasses();
}
#Override
public PersistenceUnitTransactionType getTransactionType() {
return descriptor.getTransactionType();
}
#Override
public ValidationMode getValidationMode() {
return descriptor.getValidationMode();
}
#Override
public SharedCacheMode getSharedCacheMode() {
return descriptor.getSharedCacheMode();
}
#Override
public List<String> getManagedClassNames() {
return descriptor.getManagedClassNames();
}
#Override
public List<String> getMappingFileNames() {
return descriptor.getMappingFileNames();
}
#Override
public List<URL> getJarFileUrls() {
return descriptor.getJarFileUrls();
}
#Override
public Object getNonJtaDataSource() {
return descriptor.getNonJtaDataSource();
}
#Override
public Object getJtaDataSource() {
return descriptor.getJtaDataSource();
}
#Override
public Properties getProperties() {
return descriptor.getProperties();
}
#Override
public ClassLoader getClassLoader() {
return descriptor.getClassLoader();
}
#Override
public ClassLoader getTempClassLoader() {
return descriptor.getTempClassLoader();
}
#Override
public void pushClassTransformer(EnhancementContext enhancementContext) {
descriptor.pushClassTransformer(enhancementContext);
}
}