Enonic version: 7.2.0
OS: Linux
Hi!
I get a weird error when modifying or publishing content. It happens completely randomly, however, it makes application completely unreliable. I tried doing optimizations, but it did not help. I assume it could appear because of the new audit log. The function itself makes a lot of external requests(~5k-7k) and does modify and publish to a low amount of data(10-50 entries). Here is the log:
15:53:36.717 WARN org.eclipse.jetty.server.HttpChannel - /_/service/com.norsedigital.nibio/debug
java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached
at java.base/java.lang.Thread.start0(Native Method)
at java.base/java.lang.Thread.start(Thread.java:803)
at java.base/java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:937)
at java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1352)
at com.enonic.xp.core.impl.content.ContentAuditLogSupport.update(ContentAuditLogSupport.java:158)
at com.enonic.xp.core.impl.content.ContentServiceImpl.update(ContentServiceImpl.java:361)
at com.enonic.xp.lib.content.ModifyContentHandler.doExecute(ModifyContentHandler.java:43)
at com.enonic.xp.lib.content.BaseContextHandler.execute(BaseContextHandler.java:14)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12201$11402A$content.L:1#modify(com.norsedigital.nibio:/lib/xp/content.js:297)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12200$37338$publications.L:1#markPublicationRemovedCristin#publication(com.norsedigital.nibio:/site/lib/import/publications.js:1319)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptFunctionData.invoke(ScriptFunctionData.java:655)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptFunction.invoke(ScriptFunction.java:513)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:527)
at jdk.nashorn.javaadapters.java_util_concurrent_Callable.call(Unknown Source)
at com.enonic.xp.context.ContextImpl.callWith(ContextImpl.java:102)
at com.enonic.xp.lib.context.ContextHandlerBean.run(ContextHandlerBean.java:36)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12177$1216AA$context.L:1#run(com.norsedigital.nibio:/lib/xp/context.js:56)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12199$392A$contextLib/0x0000000801195040.L:1#executeInDraft(com.norsedigital.nibio:/site/lib/contextLib.js:20)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12198$37204AZ$publications.L:1#markPublicationRemovedCristin(com.norsedigital.nibio:/site/lib/import/publications.js:1318)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12179$12833$publications.L:1#checkPublications(com.norsedigital.nibio:/site/lib/import/publications.js:533)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12178$2153$publications.L:1#runPublicationsCheck-1#result(com.norsedigital.nibio:/site/lib/import/publications.js:93)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptFunctionData.invoke(ScriptFunctionData.java:655)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptFunction.invoke(ScriptFunction.java:513)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:527)
at jdk.nashorn.javaadapters.java_util_concurrent_Callable.call(Unknown Source)
at com.enonic.xp.context.ContextImpl.callWith(ContextImpl.java:102)
at com.enonic.xp.lib.context.ContextHandlerBean.run(ContextHandlerBean.java:36)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12177$1216AA$context.L:1#run(com.norsedigital.nibio:/lib/xp/context.js:56)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12176$1965$publications.L:1#runPublicationsCheck-1(com.norsedigital.nibio:/site/lib/import/publications.js:84)
at jdk.scripting.nashorn.scripts/jdk.nashorn.internal.scripts.Script$Recompilation$12175$438A$debug.L:1#get(com.norsedigital.nibio:/services/debug/debug.js:33)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptFunctionData.invoke(ScriptFunctionData.java:657)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptFunction.invoke(ScriptFunction.java:513)
at jdk.scripting.nashorn/jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:527)
at jdk.scripting.nashorn/jdk.nashorn.api.scripting.ScriptObjectMirror.call(ScriptObjectMirror.java:120)
at com.enonic.xp.script.impl.value.FunctionScriptValue.call(FunctionScriptValue.java:36)
at com.enonic.xp.script.impl.executor.ScriptExportsImpl.executeMethod(ScriptExportsImpl.java:59)
at com.enonic.xp.portal.impl.controller.ControllerScriptImpl.doExecute(ControllerScriptImpl.java:71)
at com.enonic.xp.portal.impl.controller.ControllerScriptImpl.lambda$execute$0(ControllerScriptImpl.java:36)
at com.enonic.xp.trace.Tracer.traceEx(Tracer.java:72)
at com.enonic.xp.trace.Tracer.trace(Tracer.java:51)
at com.enonic.xp.trace.Tracer.trace(Tracer.java:98)
at com.enonic.xp.portal.impl.controller.ControllerScriptImpl.execute(ControllerScriptImpl.java:36)
at com.enonic.xp.portal.impl.handler.service.ServiceHandlerWorker.execute(ServiceHandlerWorker.java:105)
at com.enonic.xp.portal.impl.handler.service.ServiceHandler.doHandle(ServiceHandler.java:63)
at com.enonic.xp.portal.impl.handler.service.ServiceHandler.doHandle(ServiceHandler.java:22)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:66)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.portal.handler.BasePortalHandler.doHandle(BasePortalHandler.java:46)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:66)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.impl.trace.TraceWebFilter.doHandle(TraceWebFilter.java:38)
at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:66)
at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
at com.enonic.xp.web.impl.handler.WebDispatcherImpl.dispatch(WebDispatcherImpl.java:44)
at com.enonic.xp.web.impl.handler.WebDispatcherServlet.doHandle(WebDispatcherServlet.java:135)
at com.enonic.xp.web.impl.handler.WebDispatcherServlet.service(WebDispatcherServlet.java:63)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at com.enonic.xp.web.impl.dispatch.mapping.ServletDefinitionImpl.service(ServletDefinitionImpl.java:40)
at com.enonic.xp.web.impl.dispatch.pipeline.ServletPipelineImpl.service(ServletPipelineImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:45)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.portal.impl.idprovider.IdProviderFilter.doHandle(IdProviderFilter.java:45)
at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.impl.auth.BasicAuthFilter.doHandle(BasicAuthFilter.java:34)
at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.impl.context.ContextFilter.lambda$doHandle$0(ContextFilter.java:33)
at com.enonic.xp.context.ContextImpl.callWith(ContextImpl.java:102)
at com.enonic.xp.web.impl.context.ContextFilter.doHandle(ContextFilter.java:32)
at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:26)
at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.impl.dos.DosFilterWrapper.doFilter(DosFilterWrapper.java:65)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:26)
at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterPipelineImpl.filter(FilterPipelineImpl.java:40)
at com.enonic.xp.web.impl.dispatch.DispatchServletImpl.service(DispatchServletImpl.java:49)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:760)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:547)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:625)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1592)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1296)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1562)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1211)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:220)
at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:79)
at com.enonic.xp.web.vhost.impl.VirtualHostFilter.doHandle(VirtualHostFilter.java:53)
at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.impl.dos.DosFilterWrapper.doFilter(DosFilterWrapper.java:65)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at org.eclipse.jetty.servlets.HeaderFilter.doFilter(HeaderFilter.java:117)
at com.enonic.xp.web.impl.header.HeaderFilterWrapper.doHandle(HeaderFilterWrapper.java:51)
at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:41)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
at com.enonic.xp.web.impl.dispatch.pipeline.FilterPipelineImpl.filter(FilterPipelineImpl.java:40)
at com.enonic.xp.web.impl.dispatch.DispatchServletImpl.service(DispatchServletImpl.java:49)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:760)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:547)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:767)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1592)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1296)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1562)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1211)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:386)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:562)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:378)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:270)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:388)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base/java.lang.Thread.run(Thread.java:834)
15:53:36.736 WARN o.e.j.util.thread.QueuedThreadPool -
java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached
at java.base/java.lang.Thread.start0(Native Method)
at java.base/java.lang.Thread.start(Thread.java:803)
at org.eclipse.jetty.util.thread.QueuedThreadPool.startThread(QueuedThreadPool.java:660)
at org.eclipse.jetty.util.thread.QueuedThreadPool.execute(QueuedThreadPool.java:547)
at org.eclipse.jetty.io.SelectorManager.execute(SelectorManager.java:163)
at org.eclipse.jetty.io.ManagedSelector.execute(ManagedSelector.java:217)
at org.eclipse.jetty.io.ManagedSelector.destroyEndPoint(ManagedSelector.java:287)
at org.eclipse.jetty.io.ChannelEndPoint.onClose(ChannelEndPoint.java:219)
at org.eclipse.jetty.io.AbstractEndPoint.doOnClose(AbstractEndPoint.java:225)
at org.eclipse.jetty.io.AbstractEndPoint.close(AbstractEndPoint.java:192)
at org.eclipse.jetty.io.AbstractEndPoint.close(AbstractEndPoint.java:175)
at org.eclipse.jetty.server.HttpConnection.abort(HttpConnection.java:557)
at org.eclipse.jetty.server.HttpChannel.abort(HttpChannel.java:963)
at org.eclipse.jetty.server.HttpChannelOverHttp.abort(HttpChannelOverHttp.java:494)
at org.eclipse.jetty.server.HttpChannel$SendCallback.failed(HttpChannel.java:1247)
at org.eclipse.jetty.io.AbstractConnection.lambda$failedCallback$0(AbstractConnection.java:92)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.base/java.lang.Thread.run(Thread.java:834)