Elasticsearch version:v5.0.1
Plugins installed: [repository-hdfs]
JVM version:
java version "1.8.0_92"
Java(TM) SE Runtime Environment (build 1.8.0_92-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode)
OS version:
CentOS release 6.7 (Final)
Linux version 2.6.32-573.26.1.el6.x86_64 (mockbuild#c6b8.bsys.dev.centos.org) (gcc version 4.4.7 20120313 (Red Hat 4.4.7-16) (GCC) ) #1 SMP Wed May 4 00:57:44 UTC 2016
Description of the problem including expected versus actual behavior:
When I create repositories, ES response
"{
"acknowledged": true
}",
but when I create snapshot of index, it throws exception:
[2016-12-12T11:38:04,417][WARN ][r.suppressed ] path: /_snapshot/my_hdfs_repo/20161209-snapshot, params: {repository=my_hdfs_repo, snapshot=20161209-snapshot}
org.elasticsearch.transport.RemoteTransportException: [node-2][10.90.6.234:9340][cluster:admin/snapshot/create]
Caused by: org.elasticsearch.repositories.RepositoryException: [my_hdfs_repo] could not read repository data from index blob
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryData(BlobStoreRepository.java:751) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.snapshots.SnapshotsService.createSnapshot(SnapshotsService.java:226) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:82) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:41) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:86) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.doRun(TransportMasterNodeAction.java:170) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:520) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.0.1.jar:5.0.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_92]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_92]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_92]
Caused by: java.io.IOException: com.google.protobuf.ServiceException: java.security.AccessControlException: access denied ("javax.security.auth.PrivateCredentialPermission" "org.apache.hadoop.security.Credentials" "read")
at org.apache.hadoop.ipc.ProtobufHelper.getRemoteException(ProtobufHelper.java:47) ~[?:?]
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getListing(ClientNamenodeProtocolTranslatorPB.java:580) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_92]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) ~[?:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) ~[?:?]
at com.sun.proxy.$Proxy34.getListing(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2094) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2077) ~[?:?]
at org.apache.hadoop.fs.Hdfs.listStatus(Hdfs.java:254) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util$1.next(FileContext.java:1798) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util$1.next(FileContext.java:1794) ~[?:?]
at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1800) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1759) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1718) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer$6.run(HdfsBlobContainer.java:145) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer$6.run(HdfsBlobContainer.java:142) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobStore$4.run(HdfsBlobStore.java:136) ~[?:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_92]
at java.security.AccessController.doPrivileged(AccessController.java:713) ~[?:1.8.0_92]
at org.elasticsearch.repositories.hdfs.HdfsBlobStore.execute(HdfsBlobStore.java:133) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer.listBlobsByPrefix(HdfsBlobContainer.java:142) ~[?:?]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.listBlobsToGetLatestIndexId(BlobStoreRepository.java:849) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.latestIndexBlobId(BlobStoreRepository.java:818) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryData(BlobStoreRepository.java:721) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.snapshots.SnapshotsService.createSnapshot(SnapshotsService.java:226) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:82) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:41) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:86) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.doRun(TransportMasterNodeAction.java:170) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:520) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.0.1.jar:5.0.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_92]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_92]
at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_92]
Caused by: org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper: service_exception: java.security.AccessControlException: access denied ("javax.security.auth.PrivateCredentialPermission" "org.apache.hadoop.security.Credentials" "read")
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:243) ~[?:?]
at com.sun.proxy.$Proxy33.getListing(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getListing(ClientNamenodeProtocolTranslatorPB.java:573) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_92]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) ~[?:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) ~[?:?]
at com.sun.proxy.$Proxy34.getListing(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2094) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2077) ~[?:?]
at org.apache.hadoop.fs.Hdfs.listStatus(Hdfs.java:254) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util$1.next(FileContext.java:1798) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util$1.next(FileContext.java:1794) ~[?:?]
at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1800) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1759) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1718) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer$6.run(HdfsBlobContainer.java:145) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer$6.run(HdfsBlobContainer.java:142) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobStore$4.run(HdfsBlobStore.java:136) ~[?:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_92]
at java.security.AccessController.doPrivileged(AccessController.java:713) ~[?:1.8.0_92]
at org.elasticsearch.repositories.hdfs.HdfsBlobStore.execute(HdfsBlobStore.java:133) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer.listBlobsByPrefix(HdfsBlobContainer.java:142) ~[?:?]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.listBlobsToGetLatestIndexId(BlobStoreRepository.java:849) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.latestIndexBlobId(BlobStoreRepository.java:818) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryData(BlobStoreRepository.java:721) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.snapshots.SnapshotsService.createSnapshot(SnapshotsService.java:226) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:82) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:41) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:86) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.doRun(TransportMasterNodeAction.java:170) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:520) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.0.1.jar:5.0.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_92]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_92]
at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_92]
Caused by: java.lang.SecurityException: access denied ("javax.security.auth.PrivateCredentialPermission" "org.apache.hadoop.security.Credentials" "read")
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472) ~[?:1.8.0_92]
at java.security.AccessControlContext.checkPermission2(AccessControlContext.java:538) ~[?:1.8.0_92]
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:481) ~[?:1.8.0_92]
at java.security.AccessController.checkPermission(AccessController.java:884) ~[?:1.8.0_92]
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549) ~[?:1.8.0_92]
at javax.security.auth.Subject$ClassSet.populateSet(Subject.java:1414) ~[?:1.8.0_92]
at javax.security.auth.Subject$ClassSet.<init>(Subject.java:1372) ~[?:1.8.0_92]
at javax.security.auth.Subject.getPrivateCredentials(Subject.java:767) ~[?:1.8.0_92]
at org.apache.hadoop.security.UserGroupInformation.getCredentialsInternal(UserGroupInformation.java:1499) ~[?:?]
at org.apache.hadoop.security.UserGroupInformation.getTokens(UserGroupInformation.java:1464) ~[?:?]
at org.apache.hadoop.ipc.Client$Connection.<init>(Client.java:436) ~[?:?]
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1519) ~[?:?]
at org.apache.hadoop.ipc.Client.call(Client.java:1446) ~[?:?]
at org.apache.hadoop.ipc.Client.call(Client.java:1407) ~[?:?]
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) ~[?:?]
at com.sun.proxy.$Proxy33.getListing(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getListing(ClientNamenodeProtocolTranslatorPB.java:573) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_92]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) ~[?:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) ~[?:?]
at com.sun.proxy.$Proxy34.getListing(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2094) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2077) ~[?:?]
at org.apache.hadoop.fs.Hdfs.listStatus(Hdfs.java:254) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util$1.next(FileContext.java:1798) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util$1.next(FileContext.java:1794) ~[?:?]
at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1800) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1759) ~[?:?]
at org.apache.hadoop.fs.FileContext$Util.listStatus(FileContext.java:1718) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer$6.run(HdfsBlobContainer.java:145) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer$6.run(HdfsBlobContainer.java:142) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobStore$4.run(HdfsBlobStore.java:136) ~[?:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_92]
at java.security.AccessController.doPrivileged(AccessController.java:713) ~[?:1.8.0_92]
at org.elasticsearch.repositories.hdfs.HdfsBlobStore.execute(HdfsBlobStore.java:133) ~[?:?]
at org.elasticsearch.repositories.hdfs.HdfsBlobContainer.listBlobsByPrefix(HdfsBlobContainer.java:142) ~[?:?]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.listBlobsToGetLatestIndexId(BlobStoreRepository.java:849) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.latestIndexBlobId(BlobStoreRepository.java:818) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryData(BlobStoreRepository.java:721) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.snapshots.SnapshotsService.createSnapshot(SnapshotsService.java:226) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:82) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction.masterOperation(TransportCreateSnapshotAction.java:41) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:86) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.doRun(TransportMasterNodeAction.java:170) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:520) ~[elasticsearch-5.0.1.jar:5.0.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.0.1.jar:5.0.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_92]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_92]
at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_92]
Steps to reproduce:
1.create repositories
PUT /_snapshot/my_backup
{
"type": "hdfs",
"settings": {
"path": "/path/on/hadoop",
"uri": "hdfs://hadoop_cluster_domain:[port]",
"conf_location":"/hadoop/hdfs-site.xml,/hadoop/core-site.xml",
"user":"hadoop"
}
}
2.snapshot my index
PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=true
3.exception is throwed
Solution: Change the default java security policy of the HDFS plugin.
The policy file is located at ES_HOME/plugins/repository-hdfs/plugin-security.policy, where ES_HOME is your installation path of Elasticsearch. Add the following policy:
grant {
// ... existing policies
permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
}
Then, in the start script of Elasticsearch (ES_HOME/bin/elasticsearch), add the following Java option to the command line arguments:
-Djava.security.policy=ES_HOME/plugins/repository-hdfs/plugin-security.policy
Synchronize the modified files to all cluster nodes.
Finally, restart all cluster nodes.
Reference
[ElasticSearch填坑] 创建Snapshot: SecurityException
Related
Migrated application from JRE-8 to JRE-17. Jar is build and ran on JRE-17. No code change was required. No clues why this exception occurs.
Jersey version is 2.35. The same code works fine when build and ran on JRE-8
javax.ws.rs.ProcessingException: java.net.ConnectException: Connection refused
at org.glassfish.jersey.client.internal.HttpUrlConnector.apply(HttpUrlConnector.java:269) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.ClientRuntime.invoke(ClientRuntime.java:297) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.JerseyInvocation.lambda$invoke$0(JerseyInvocation.java:662) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.JerseyInvocation.call(JerseyInvocation.java:697) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.JerseyInvocation.lambda$runInScope$3(JerseyInvocation.java:691) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:292) ~[jersey-common-2.35.jar!/:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:274) ~[jersey-common-2.35.jar!/:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:205) ~[jersey-common-2.35.jar!/:?]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:390) ~[jersey-common-2.35.jar!/:?]
at org.glassfish.jersey.client.JerseyInvocation.runInScope(JerseyInvocation.java:691) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.JerseyInvocation.invoke(JerseyInvocation.java:661) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.JerseyInvocation$Builder.method(JerseyInvocation.java:439) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.JerseyInvocation$Builder.post(JerseyInvocation.java:345) ~[jersey-client-2.35.jar!/:?]
//removed the calling code here
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.Net.connect0(Native Method) ~[?:?]
at sun.nio.ch.Net.connect(Unknown Source) ~[?:?]
at sun.nio.ch.Net.connect(Unknown Source) ~[?:?]
at sun.nio.ch.NioSocketImpl.connect(Unknown Source) ~[?:?]
at java.net.Socket.connect(Unknown Source) ~[?:?]
at sun.net.NetworkClient.doConnect(Unknown Source) ~[?:?]
at sun.net.www.http.HttpClient.openServer(Unknown Source) ~[?:?]
at sun.net.www.http.HttpClient$1.run(Unknown Source) ~[?:?]
at sun.net.www.http.HttpClient$1.run(Unknown Source) ~[?:?]
at java.security.AccessController.doPrivileged(Unknown Source) ~[?:?]
at sun.net.www.http.HttpClient.privilegedOpenServer(Unknown Source) ~[?:?]
at sun.net.www.http.HttpClient.openServer(Unknown Source) ~[?:?]
at sun.net.www.http.HttpClient.<init>(Unknown Source) ~[?:?]
at sun.net.www.http.HttpClient.New(Unknown Source) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(Unknown Source) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.plainConnect0(Unknown Source) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.plainConnect(Unknown Source) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.connect(Unknown Source) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(Unknown Source) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(Unknown Source) ~[?:?]
at java.net.HttpURLConnection.getResponseCode(Unknown Source) ~[?:?]
at org.glassfish.jersey.client.internal.HttpUrlConnector.handleException(HttpUrlConnector.java:549) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.internal.HttpUrlConnector._apply(HttpUrlConnector.java:375) ~[jersey-client-2.35.jar!/:?]
at org.glassfish.jersey.client.internal.HttpUrlConnector.apply(HttpUrlConnector.java:267) ~[jersey-client-2.35.jar!/:?]
... 26 more
I am in the middle of converting a Java 8 project to Java 11, and as a part of that conversion I have made a slew of different code changes and upgrades (most notably upgrading to Hibernate 5.3.18.FINAL from 4.X). I've gotten the entire, large project to compile successfully in Java 11. However, when I kick off the project using Karaf, I am getting a persistent OSGi error, and it's driving me crazy.
[FelixDispatchQueue] ERROR org.osgi.framework.FrameworkEvent - FrameworkEvent ERROR
javax.persistence.PersistenceException: [PersistenceUnit: abc] Unable to build Hibernate SessionFactory
at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.persistenceException(EntityManagerFactoryBuilderImpl.java:1327) ~[?:?]
at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.build(EntityManagerFactoryBuilderImpl.java:1253) ~[?:?]
at org.hibernate.osgi.OsgiPersistenceProvider.createContainerEntityManagerFactory(OsgiPersistenceProvider.java:96) ~[?:?]
at org.apache.aries.jpa.container.impl.AriesEntityManagerFactoryBuilder.createAndPublishEMF(AriesEntityManagerFactoryBuilder.java:371) ~[?:?]
at org.apache.aries.jpa.container.impl.AriesEntityManagerFactoryBuilder.createEntityManagerFactory(AriesEntityManagerFactoryBuilder.java:183) ~[?:?]
at org.apache.aries.jpa.container.impl.AriesEntityManagerFactoryBuilder.dataSourceReady(AriesEntityManagerFactoryBuilder.java:538) ~[?:?]
at org.apache.aries.jpa.container.impl.AriesEntityManagerFactoryBuilder.foundDS(AriesEntityManagerFactoryBuilder.java:510) ~[?:?]
at org.apache.aries.jpa.container.impl.DataSourceTracker.addingService(DataSourceTracker.java:86) ~[?:?]
at org.apache.aries.jpa.container.impl.DataSourceTracker.addingService(DataSourceTracker.java:34) ~[?:?]
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:941) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:870) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.AbstractTracked.trackAdding(AbstractTracked.java:256) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.AbstractTracked.trackInitial(AbstractTracked.java:183) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:318) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:261) ~[osgi.core-6.0.0.jar:?]
at org.apache.aries.jpa.container.impl.AriesEntityManagerFactoryBuilder.<init>(AriesEntityManagerFactoryBuilder.java:99) ~[?:?]
at org.apache.aries.jpa.container.impl.PersistenceProviderTracker.addingService(PersistenceProviderTracker.java:88) ~[?:?]
at org.apache.aries.jpa.container.impl.PersistenceProviderTracker.addingService(PersistenceProviderTracker.java:44) ~[?:?]
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:941) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:870) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.AbstractTracked.trackAdding(AbstractTracked.java:256) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.AbstractTracked.trackInitial(AbstractTracked.java:183) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:318) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:261) ~[osgi.core-6.0.0.jar:?]
at org.apache.aries.jpa.container.impl.PersistenceBundleTracker.trackProvider(PersistenceBundleTracker.java:229) ~[?:?]
at org.apache.aries.jpa.container.impl.PersistenceBundleTracker.findPersistenceUnits(PersistenceBundleTracker.java:213) ~[?:?]
at org.apache.aries.jpa.container.impl.PersistenceBundleTracker.addingBundle(PersistenceBundleTracker.java:98) ~[?:?]
at org.apache.aries.jpa.container.impl.PersistenceBundleTracker.addingBundle(PersistenceBundleTracker.java:44) ~[?:?]
at org.osgi.util.tracker.BundleTracker$Tracked.customizerAdding(BundleTracker.java:469) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.BundleTracker$Tracked.customizerAdding(BundleTracker.java:415) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.AbstractTracked.trackAdding(AbstractTracked.java:256) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.AbstractTracked.track(AbstractTracked.java:229) ~[osgi.core-6.0.0.jar:?]
at org.osgi.util.tracker.BundleTracker$Tracked.bundleChanged(BundleTracker.java:444) ~[osgi.core-6.0.0.jar:?]
at org.apache.felix.framework.EventDispatcher.invokeBundleListenerCallback(EventDispatcher.java:915) ~[?:?]
at org.apache.felix.framework.EventDispatcher.fireEventImmediately(EventDispatcher.java:834) ~[?:?]
at org.apache.felix.framework.EventDispatcher.fireBundleEvent(EventDispatcher.java:516) ~[?:?]
at org.apache.felix.framework.Felix.fireBundleEvent(Felix.java:4579) ~[?:?]
at org.apache.felix.framework.Felix.activateBundle(Felix.java:2231) ~[?:?]
at org.apache.felix.framework.Felix.startBundle(Felix.java:2146) ~[?:?]
at org.apache.felix.framework.BundleImpl.start(BundleImpl.java:998) ~[?:?]
at org.apache.felix.framework.BundleImpl.start(BundleImpl.java:984) ~[?:?]
at org.apache.karaf.features.internal.service.BundleInstallSupportImpl.startBundle(BundleInstallSupportImpl.java:165) ~[?:?]
at org.apache.karaf.features.internal.service.FeaturesServiceImpl.startBundle(FeaturesServiceImpl.java:1154) ~[?:?]
at org.apache.karaf.features.internal.service.Deployer.deploy(Deployer.java:1044) ~[?:?]
at org.apache.karaf.features.internal.service.FeaturesServiceImpl.doProvision(FeaturesServiceImpl.java:1063) ~[?:?]
at org.apache.karaf.features.internal.service.FeaturesServiceImpl.lambda$doProvisionInThread$13(FeaturesServiceImpl.java:998) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:829) [?:?]
...
...
Caused by: java.lang.NoClassDefFoundError: org/hibernate/proxy/HibernateProxy
at java.lang.ClassLoader.defineClass1(Native Method) ~[?:?]
at java.lang.System$2.defineClass(System.java:2131) ~[?:?]
at java.lang.invoke.MethodHandles$Lookup.defineClass(MethodHandles.java:962) ~[?:?]
at jdk.internal.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at net.bytebuddy.dynamic.loading.ClassInjector$UsingLookup$Dispatcher$ForJava9CapableVm.defineClass(ClassInjector.java:1685) ~[?:?]
at net.bytebuddy.dynamic.loading.ClassInjector$UsingLookup.injectRaw(ClassInjector.java:1418) ~[?:?]
at net.bytebuddy.dynamic.loading.ClassInjector$AbstractBase.inject(ClassInjector.java:112) ~[?:?]
at net.bytebuddy.dynamic.loading.ClassLoadingStrategy$UsingLookup.load(ClassLoadingStrategy.java:466) ~[?:?]
at net.bytebuddy.dynamic.TypeResolutionStrategy$Passive.initialize(TypeResolutionStrategy.java:100) ~[?:?]
at net.bytebuddy.dynamic.DynamicType$Default$Unloaded.load(DynamicType.java:5662) ~[?:?]
at org.hibernate.bytecode.internal.bytebuddy.ByteBuddyState.lambda$load$0(ByteBuddyState.java:187) ~[?:?]
at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:152) ~[?:?]
at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:365) ~[?:?]
at net.bytebuddy.TypeCache.findOrInsert(TypeCache.java:174) ~[?:?]
at net.bytebuddy.TypeCache$WithInlineExpunction.findOrInsert(TypeCache.java:376) ~[?:?]
at org.hibernate.bytecode.internal.bytebuddy.ByteBuddyState.load(ByteBuddyState.java:183) ~[?:?]
at org.hibernate.bytecode.internal.bytebuddy.ByteBuddyState.loadProxy(ByteBuddyState.java:107) ~[?:?]
at org.hibernate.proxy.pojo.bytebuddy.ByteBuddyProxyHelper.buildProxy(ByteBuddyProxyHelper.java:54) ~[?:?]
at org.hibernate.proxy.pojo.bytebuddy.ByteBuddyProxyFactory.postInstantiate(ByteBuddyProxyFactory.java:62) ~[?:?]
at org.hibernate.tuple.entity.PojoEntityTuplizer.buildProxyFactory(PojoEntityTuplizer.java:103) ~[?:?]
at org.hibernate.tuple.entity.AbstractEntityTuplizer.<init>(AbstractEntityTuplizer.java:155) ~[?:?]
at org.hibernate.tuple.entity.PojoEntityTuplizer.<init>(PojoEntityTuplizer.java:51) ~[?:?]
at jdk.internal.reflect.GeneratedConstructorAccessor59.newInstance(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:490) ~[?:?]
at org.hibernate.tuple.entity.EntityTuplizerFactory.constructTuplizer(EntityTuplizerFactory.java:88) ~[?:?]
at org.hibernate.tuple.entity.EntityTuplizerFactory.constructDefaultTuplizer(EntityTuplizerFactory.java:116) ~[?:?]
at org.hibernate.tuple.entity.EntityMetamodel.<init>(EntityMetamodel.java:413) ~[?:?]
at org.hibernate.persister.entity.AbstractEntityPersister.<init>(AbstractEntityPersister.java:557) ~[?:?]
at org.hibernate.persister.entity.SingleTableEntityPersister.<init>(SingleTableEntityPersister.java:124) ~[?:?]
at jdk.internal.reflect.GeneratedConstructorAccessor58.newInstance(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:490) ~[?:?]
at org.hibernate.persister.internal.PersisterFactoryImpl.createEntityPersister(PersisterFactoryImpl.java:96) ~[?:?]
at org.hibernate.persister.internal.PersisterFactoryImpl.createEntityPersister(PersisterFactoryImpl.java:77) ~[?:?]
at org.hibernate.metamodel.internal.MetamodelImpl.initialize(MetamodelImpl.java:154) ~[?:?]
at org.hibernate.internal.SessionFactoryImpl.<init>(SessionFactoryImpl.java:295) ~[?:?]
at org.hibernate.boot.internal.SessionFactoryBuilderImpl.build(SessionFactoryBuilderImpl.java:467) ~[?:?]
at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.build(EntityManagerFactoryBuilderImpl.java:1250) ~[?:?]
... 48 more
Caused by: java.lang.ClassNotFoundException: org.hibernate.proxy.HibernateProxy
Before you ask, I've put every single combination of matching bytebuddy and javassist in the pom files (matching the version required by 5.3.18.FINAL) and it doesn't fix the issue. Note that my project is building successfully in Maven but simply won't find HibernateProxy when I kick it off using karaf. Downgrading to Java 8 is not an option - this needs to run in Java 11.
More tech details:
Using Spring Boot
Using Blueprint
Hibernate.xml is updated to persistence_2_1.xsd
<jta-data-source>osgi:service/javax.sql.DataSource/(osgi.jndi.service.name=abc)</jta-data-source>
Database is h2
Although it's not a perfect solution, the way I was able to temporarily get around this issue was to force the hibernate-core jar in the main classpath. I'm sure the main issue is that hibernate-core is not in the right place, but I'm unclear as to how. I did a lot of manipulations of poms and adding exceptions to maven-bundle-plugin, but that didn't work - only putting hibernate-core in the main classpath.
After updating Infinispan from version 9.4.11.Final to 10.0.1.Final the starting of multiple caches produces this error when trying to start the fourth cache. With the previous version it runs smoothly.
Caused by: java.lang.RuntimeException: org.infinispan.commons.CacheConfigurationException: Error starting component org.infinispan.persistence.manager.PersistenceManager
at com.metaproc.core.datagrid.impl.InfinispanDatagridServiceImpl.<init>(InfinispanDatagridServiceImpl.java:139) ~[?:?]
... 71 more
Caused by: org.infinispan.commons.CacheConfigurationException: Error starting component org.infinispan.persistence.manager.PersistenceManager
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:555) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:561) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:344) ~[?:?]
at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:240) ~[?:?]
at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:207) ~[?:?]
at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1091) ~[?:?]
at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:513) ~[?:?]
at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:693) ~[?:?]
at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:632) ~[?:?]
at org.infinispan.manager.DefaultCacheManager.startCache(DefaultCacheManager.java:1024) ~[?:?]
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
at java.lang.Thread.start0(Native Method) ~[?:?]
at java.lang.Thread.start(Thread.java:717) ~[?:?]
at org.infinispan.persistence.sifs.Index.start(Index.java:75) ~[?:?]
at org.infinispan.persistence.sifs.SoftIndexFileStore.startIndex(SoftIndexFileStore.java:242) ~[?:?]
at org.infinispan.persistence.sifs.SoftIndexFileStore.start(SoftIndexFileStore.java:161) ~[?:?]
at org.infinispan.persistence.support.ComposedSegmentedLoadWriteStore.startNewStoreForSegment(ComposedSegmentedLoadWriteStore.java:258) ~[?:?]
at org.infinispan.persistence.support.ComposedSegmentedLoadWriteStore.start(ComposedSegmentedLoadWriteStore.java:244) ~[?:?]
at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$startWriter$42(PersistenceManagerImpl.java:1304) ~[?:?]
at org.infinispan.persistence.manager.PersistenceManagerImpl.startStore(PersistenceManagerImpl.java:1333) ~[?:?]
at org.infinispan.persistence.manager.PersistenceManagerImpl.startWriter(PersistenceManagerImpl.java:1296) ~[?:?]
at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$start$0(PersistenceManagerImpl.java:193) ~[?:?]
at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:?]
at org.infinispan.persistence.manager.PersistenceManagerImpl.start(PersistenceManagerImpl.java:193) ~[?:?]
at org.infinispan.persistence.support.DelegatingPersistenceManager.start(DelegatingPersistenceManager.java:39) ~[?:?]
at org.infinispan.persistence.support.CorePackageImpl$1.start(CorePackageImpl.java:30) ~[?:?]
at org.infinispan.persistence.support.CorePackageImpl$1.start(CorePackageImpl.java:24) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:587) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:584) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:578) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:561) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:605) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:569) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:547) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[?:?]
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:770) ~[?:?]
at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:344) ~[?:?]
at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:240) ~[?:?]
at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:207) ~[?:?]
at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1091) ~[?:?]
at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:513) ~[?:?]
at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:693) ~[?:?]
at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:632) ~[?:?]
at org.infinispan.manager.DefaultCacheManager.startCache(DefaultCacheManager.java:1024) ~[?:?]
We are using this configuration:
final SoftIndexFileStoreConfigurationBuilder persistentStoreConfiguration =
new SoftIndexFileStoreConfigurationBuilder(configurationBuilder)
.indexLocation(indexPath)
.dataLocation(dataPath)
.maxNodeSize(8192)
.fetchPersistentState(true);
new ConfigurationBuilder()
.locking()
.isolationLevel(IsolationLevel.REPEATABLE_READ)
.lockAcquisitionTimeout(30000)
.useLockStriping(false)
.transaction()
.autoCommit(true)
.completedTxTimeout(60000)
.lockingMode(LockingMode.OPTIMISTIC)
.transactionMode(TransactionMode.TRANSACTIONAL)
.notifications(true)
.reaperWakeUpInterval(30000)
.cacheStopTimeout(30000)
.transactionManagerLookup(new GenericTransactionManagerLookup())
.clustering()
.cacheMode(CacheMode.DIST_SYNC)
.stateTransfer()
.fetchInMemoryState(true)
.hash().numOwners(2)
.l1()
.lifespan(-1)
.memory()
.size(1)
.persistence()
.passivation(true)
.addStore(persistentStoreConfiguration).build();
It looks like something of the configuration of the soft index file store is not correct anymore, I just cannot figure out what it might be. I tried to decrease the maxNodeSize, without any success. Can someone point me to the part where I need to adjust the configurations after the upgrade or is there anything else I'm missing?
I am pretty certain this is caused by the SoftIndexFileStore being segmented and creating a bunch of extra threads. There should have been a WARN message logged to your console output stating something to that extent.
You can also read about this at Upgrading Guide.
You can explicitly define the store to not be segmented by setting the segmented="false" attribute on your xml configuration. Since you are using programmatic configuration it would just be adding .segmented(false) to your code. Source Reference
Every time I try to run migrations, I get these massive exceptions and have been unsuccessful in finding the cause. I am using it as a part of an application that is an add-on to another application. When I tried to run it my itself it worked fine. However I need it to work in the add-on environment and don't understand why it wouldn't as I have never run into a similar problem with any other external libraries I have used with the add-on. I have checked to confirm that Flyway and all of those classes are available on the class path, they are.
java.lang.ExceptionInInitializerError: null
at org.flywaydb.core.internal.util.FeatureDetector.isSlf4jAvailable(FeatureDetector.java:96) ~[?:?]
at org.flywaydb.core.internal.logging.LogCreatorFactory.getLogCreator(LogCreatorFactory.java:39) ~[?:?]
at org.flywaydb.core.api.logging.LogFactory.getLog(LogFactory.java:78) ~[?:?]
at org.flywaydb.core.internal.util.FeatureDetector.<clinit>(FeatureDetector.java:25) ~[?:?]
at org.flywaydb.core.internal.logging.LogCreatorFactory.getLogCreator(LogCreatorFactory.java:35) ~[?:?]
at org.flywaydb.core.api.logging.LogFactory.getLog(LogFactory.java:78) ~[?:?]
at org.flywaydb.core.Flyway.<clinit>(Flyway.java:86) ~[?:?]
at me.calebbassham.stats.Stats.setupDatabase(Stats.java:22) ~[?:?]
at com.oblivionuhc.uhc.UHC$onEnable$3.run(UHC.kt:104) ~[?:?]
at org.bukkit.craftbukkit.v1_13_R2.scheduler.CraftTask.run(CraftTask.java:82) ~[patched_1.13.2.jar:git-Paper-486]
at org.bukkit.craftbukkit.v1_13_R2.scheduler.CraftAsyncTask.run(CraftAsyncTask.java:52) ~[patched_1.13.2.jar:git-Paper-486]
at com.destroystokyo.paper.ServerSchedulerReportingWrapper.run(ServerSchedulerReportingWrapper.java:22) ~[patched_1.13.2.jar:git-Paper-486]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.flywaydb.core.api.FlywayException: Unable to instantiate class org.flywaydb.core.internal.logging.slf4j.Slf4jLogCreator : org.flywaydb.core.internal.logging.slf4j.Slf4jLogCreator
at org.flywaydb.core.internal.util.ClassUtils.instantiate(ClassUtils.java:63) ~[?:?]
at org.flywaydb.core.internal.logging.LogCreatorFactory.getLogCreator(LogCreatorFactory.java:40) ~[?:?]
at org.flywaydb.core.api.logging.LogFactory.getLog(LogFactory.java:78) ~[?:?]
at org.flywaydb.core.internal.util.ClassUtils.<clinit>(ClassUtils.java:39) ~[?:?]
... 15 more
Caused by: java.lang.ClassNotFoundException: org.flywaydb.core.internal.logging.slf4j.Slf4jLogCreator
at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:583) ~[?:?]
at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) ~[?:?]
at java.lang.ClassLoader.loadClass(ClassLoader.java:521) ~[?:?]
at java.lang.Class.forName0(Native Method) ~[?:?]
at java.lang.Class.forName(Class.java:398) ~[?:?]
at org.flywaydb.core.internal.util.ClassUtils.instantiate(ClassUtils.java:61) ~[?:?]
at org.flywaydb.core.internal.logging.LogCreatorFactory.getLogCreator(LogCreatorFactory.java:40) ~[?:?]
at org.flywaydb.core.api.logging.LogFactory.getLog(LogFactory.java:78) ~[?:?]
at org.flywaydb.core.internal.util.ClassUtils.<clinit>(ClassUtils.java:39) ~[?:?]
... 15 more
I have the next error in production cluster Mule 3.6. it occur sometimes during the day, but does not reach detect the original cause.
2015-11-17 08:01:54,819 ERROR [org.mule.exception.DefaultSystemExceptionStrategy] - Caught exception in Exception Strategy: Only owner thread can write to message: Thread[[ComponentAA-PRD-v3.2.1.RELEASE].HTTP_Listener_Config.worker.130,5,main]/Thread[[ComponentAA-PRD-v3.2.1.RELEASE].http.listener(9) SelectorRunner,5,main]
java.lang.IllegalStateException: Only owner thread can write to message: Thread[[axcess-document-platform-PRD-v3.2.1.RELEASE].HTTP_Listener_Config.worker.130,5,main]/Thread[[axcess-document-platform-PRD-v3.2.1.RELEASE].http.listener(9) SelectorRunner,5,main]
at org.mule.DefaultMuleMessage.newException(DefaultMuleMessage.java:1665) ~[?:?]
at org.mule.DefaultMuleMessage.checkMutable(DefaultMuleMessage.java:1651) ~[?:?]
at org.mule.DefaultMuleMessage.assertAccess(DefaultMuleMessage.java:1580) ~[?:?]
at org.mule.DefaultMuleMessage.setExceptionPayload(DefaultMuleMessage.java:1003) ~[?:?]
at org.mule.exception.TemplateMessagingExceptionStrategy.handleException(TemplateMessagingExceptionStrategy.java:79) ~[?:?]
at org.mule.execution.AsyncResponseFlowProcessingPhase$3$1.execute(AsyncResponseFlowProcessingPhase.java:105) ~[?:?]
at org.mule.execution.AsyncResponseFlowProcessingPhase.executeCallback(AsyncResponseFlowProcessingPhase.java:117) ~[?:?]
at org.mule.execution.AsyncResponseFlowProcessingPhase.access$000(AsyncResponseFlowProcessingPhase.java:23) ~[?:?]
at org.mule.execution.AsyncResponseFlowProcessingPhase$3.responseSentWithFailure(AsyncResponseFlowProcessingPhase.java:100) ~[?:?]
at org.mule.module.http.internal.listener.HttpMessageProcessorTemplate$1.responseSendFailure(HttpMessageProcessorTemplate.java:114) ~[?:?]
at org.mule.module.http.internal.listener.grizzly.ResponseCompletionHandler.failed(ResponseCompletionHandler.java:153) ~[?:?]
at org.glassfish.grizzly.asyncqueue.AsyncQueueRecord.notifyFailure(AsyncQueueRecord.java:110) ~[?:?]
at org.glassfish.grizzly.nio.AbstractNIOAsyncQueueWriter.onWriteFailure(AbstractNIOAsyncQueueWriter.java:489) ~[?:?]
at org.glassfish.grizzly.nio.AbstractNIOAsyncQueueWriter.processAsync(AbstractNIOAsyncQueueWriter.java:400) ~[?:?]
at org.glassfish.grizzly.filterchain.DefaultFilterChain.process(DefaultFilterChain.java:108) ~[?:?]
at org.glassfish.grizzly.ProcessorExecutor.execute(ProcessorExecutor.java:77) ~[?:?]
at org.glassfish.grizzly.nio.transport.TCPNIOTransport.fireIOEvent(TCPNIOTransport.java:561) ~[?:?]
at org.glassfish.grizzly.strategies.AbstractIOStrategy.fireIOEvent(AbstractIOStrategy.java:112) ~[?:?]
at org.mule.module.http.internal.listener.grizzly.ExecutorPerServerAddressIOStrategy.run0(ExecutorPerServerAddressIOStrategy.java:102) ~[?:?]
at org.mule.module.http.internal.listener.grizzly.ExecutorPerServerAddressIOStrategy.executeIoEvent(ExecutorPerServerAddressIOStrategy.java:75) ~[?:?]
at org.glassfish.grizzly.strategies.AbstractIOStrategy.executeIoEvent(AbstractIOStrategy.java:89) ~[?:?]
at org.glassfish.grizzly.nio.SelectorRunner.iterateKeyEvents(SelectorRunner.java:414) ~[?:?]
at org.glassfish.grizzly.nio.SelectorRunner.iterateKeys(SelectorRunner.java:383) ~[?:?]
at org.glassfish.grizzly.nio.SelectorRunner.doSelect(SelectorRunner.java:347) ~[?:?]
at org.glassfish.grizzly.nio.SelectorRunner.run(SelectorRunner.java:278) ~[?:?]
at org.glassfish.grizzly.threadpool.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:565) ~[?:?]
at org.glassfish.grizzly.threadpool.AbstractThreadPool$Worker.run(AbstractThreadPool.java:545) ~[?:?]
at java.lang.Thread.run(Thread.java:745) [?:1.7.0_80]
That issue was reported as https://www.mulesoft.org/jira/browse/MULE-8961 and it is fixed since Mule 3.7.3 and 3.8.0.