All shards failed error

Enonic version: 7.0.2
OS: Linux

Hi!

We are getting tons of errors in our logs(~500 per day). The server itself is up and running and seem to work properly. What could cause this errors and what is the best way to fix this?

Thanks!

01:10:08.843 ERROR c.e.x.p.i.e.ExceptionRendererImpl - all shards failed
com.enonic.xp.web.WebException: all shards failed
	at com.enonic.xp.web.impl.exception.ExceptionMapperImpl.map(ExceptionMapperImpl.java:32)
	at com.enonic.xp.portal.handler.BasePortalHandler.handleError(BasePortalHandler.java:60)
	at com.enonic.xp.portal.handler.BasePortalHandler.doHandle(BasePortalHandler.java:52)
	at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:66)
	at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
	at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
	at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
	at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
	at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
	at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:75)
	at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
	at com.enonic.xp.web.impl.trace.TraceWebFilter.doHandle(TraceWebFilter.java:38)
	at com.enonic.xp.web.handler.BaseWebHandler.handle(BaseWebHandler.java:66)
	at com.enonic.xp.web.impl.handler.WebHandlerChainImpl.handle(WebHandlerChainImpl.java:30)
	at com.enonic.xp.web.impl.handler.WebDispatcherImpl.dispatch(WebDispatcherImpl.java:50)
	at com.enonic.xp.web.impl.handler.WebDispatcherServlet.doHandle(WebDispatcherServlet.java:135)
	at com.enonic.xp.web.impl.handler.WebDispatcherServlet.service(WebDispatcherServlet.java:63)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
	at com.enonic.xp.web.impl.dispatch.mapping.ServletDefinitionImpl.service(ServletDefinitionImpl.java:39)
	at com.enonic.xp.web.impl.dispatch.pipeline.ServletPipelineImpl.service(ServletPipelineImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:45)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.portal.impl.idprovider.IdProviderFilter.doHandle(IdProviderFilter.java:45)
	at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
	at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
	at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.web.impl.auth.BasicAuthFilter.doHandle(BasicAuthFilter.java:33)
	at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
	at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
	at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.web.impl.context.ContextFilter.lambda$doHandle$0(ContextFilter.java:33)
	at com.enonic.xp.context.ContextImpl.callWith(ContextImpl.java:101)
	at com.enonic.xp.web.impl.context.ContextFilter.doHandle(ContextFilter.java:32)
	at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
	at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
	at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:26)
	at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
	at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.web.impl.dos.DosFilterWrapper.doFilter(DosFilterWrapper.java:65)
	at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterPipelineImpl.filter(FilterPipelineImpl.java:39)
	at com.enonic.xp.web.impl.dispatch.DispatchServletImpl.service(DispatchServletImpl.java:49)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:860)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:535)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:420)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:190)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:203)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:73)
	at com.enonic.xp.web.vhost.impl.VirtualHostFilter.doHandle(VirtualHostFilter.java:53)
	at com.enonic.xp.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:31)
	at com.enonic.xp.web.filter.BaseWebFilter.doFilter(BaseWebFilter.java:33)
	at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.web.impl.dos.DosFilterWrapper.doFilter(DosFilterWrapper.java:65)
	at com.enonic.xp.web.impl.dispatch.mapping.FilterDefinitionImpl.doFilter(FilterDefinitionImpl.java:40)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:50)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterChainImpl.doFilter(FilterChainImpl.java:36)
	at com.enonic.xp.web.impl.dispatch.pipeline.FilterPipelineImpl.filter(FilterPipelineImpl.java:39)
	at com.enonic.xp.web.impl.dispatch.DispatchServletImpl.service(DispatchServletImpl.java:49)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:860)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:535)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:190)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:219)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
	at org.eclipse.jetty.server.Server.handle(Server.java:530)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
	at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
	at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
	at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
	at org.elasticsearch.action.search.AbstractSearchAsyncAction.onFirstPhaseResult(AbstractSearchAsyncAction.java:206)
	at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:152)
	at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:46)
	at org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:874)
	at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:852)
	at org.elasticsearch.transport.TransportService$4.onFailure(TransportService.java:389)
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	... 1 common frames omitted
Caused by: org.elasticsearch.search.SearchParseException: from is set to [-17] and is expected to be higher or equal to 0
	at org.elasticsearch.search.query.FromParseElement.parse(FromParseElement.java:38)
	at org.elasticsearch.search.SearchService.parseSource(SearchService.java:856)
	at org.elasticsearch.search.SearchService.createContext(SearchService.java:667)
	at org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:633)
	at org.elasticsearch.search.SearchService.executeFetchPhase(SearchService.java:472)
	at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryFetchTransportHandler.messageReceived(SearchServiceTransportAction.java:392)
	at org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryFetchTransportHandler.messageReceived(SearchServiceTransportAction.java:389)
	at org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
	at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378)
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
	... 3 common frames omitted

Short version - it does not look good.
Start by checking your cluster state, and also get an overview of the replication status.

For XP6 https://xp.readthedocs.io/en/stable/operations/monitoring.html
For XP7 https://developer.enonic.com/docs/xp/stable/api/statistics

From the exception. It seems to be linked to the value of a from in a search/query.