View Javadoc
1   package ca.uhn.fhir.jpa.search;
2   
3   /*-
4    * #%L
5    * HAPI FHIR JPA Server
6    * %%
7    * Copyright (C) 2014 - 2019 University Health Network
8    * %%
9    * Licensed under the Apache License, Version 2.0 (the "License");
10   * you may not use this file except in compliance with the License.
11   * You may obtain a copy of the License at
12   *
13   *      http://www.apache.org/licenses/LICENSE-2.0
14   *
15   * Unless required by applicable law or agreed to in writing, software
16   * distributed under the License is distributed on an "AS IS" BASIS,
17   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18   * See the License for the specific language governing permissions and
19   * limitations under the License.
20   * #L%family
21   */
22  
23  import ca.uhn.fhir.context.FhirContext;
24  import ca.uhn.fhir.interceptor.api.HookParams;
25  import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
26  import ca.uhn.fhir.interceptor.api.Pointcut;
27  import ca.uhn.fhir.jpa.dao.*;
28  import ca.uhn.fhir.jpa.dao.data.ISearchDao;
29  import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao;
30  import ca.uhn.fhir.jpa.dao.data.ISearchResultDao;
31  import ca.uhn.fhir.jpa.entity.Search;
32  import ca.uhn.fhir.jpa.entity.SearchInclude;
33  import ca.uhn.fhir.jpa.entity.SearchResult;
34  import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
35  import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
36  import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
37  import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
38  import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
39  import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
40  import ca.uhn.fhir.model.api.Include;
41  import ca.uhn.fhir.rest.api.CacheControlDirective;
42  import ca.uhn.fhir.rest.api.Constants;
43  import ca.uhn.fhir.rest.api.SearchTotalModeEnum;
44  import ca.uhn.fhir.rest.api.SummaryEnum;
45  import ca.uhn.fhir.rest.api.server.IBundleProvider;
46  import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
47  import ca.uhn.fhir.rest.api.server.RequestDetails;
48  import ca.uhn.fhir.rest.server.IPagingProvider;
49  import ca.uhn.fhir.rest.server.SimpleBundleProvider;
50  import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException;
51  import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
52  import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
53  import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException;
54  import ca.uhn.fhir.rest.server.method.PageMethodBinding;
55  import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
56  import ca.uhn.fhir.rest.server.util.ICachedSearchDetails;
57  import ca.uhn.fhir.util.StopWatch;
58  import com.google.common.annotations.VisibleForTesting;
59  import com.google.common.collect.Lists;
60  import org.apache.commons.lang3.Validate;
61  import org.apache.commons.lang3.exception.ExceptionUtils;
62  import org.apache.commons.lang3.time.DateUtils;
63  import org.hl7.fhir.instance.model.api.IBaseResource;
64  import org.jetbrains.annotations.NotNull;
65  import org.springframework.beans.factory.annotation.Autowired;
66  import org.springframework.data.domain.AbstractPageRequest;
67  import org.springframework.data.domain.Page;
68  import org.springframework.data.domain.Pageable;
69  import org.springframework.data.domain.Sort;
70  import org.springframework.orm.jpa.JpaDialect;
71  import org.springframework.orm.jpa.JpaTransactionManager;
72  import org.springframework.orm.jpa.vendor.HibernateJpaDialect;
73  import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
74  import org.springframework.stereotype.Component;
75  import org.springframework.transaction.PlatformTransactionManager;
76  import org.springframework.transaction.TransactionDefinition;
77  import org.springframework.transaction.TransactionStatus;
78  import org.springframework.transaction.annotation.Propagation;
79  import org.springframework.transaction.annotation.Transactional;
80  import org.springframework.transaction.support.TransactionCallbackWithoutResult;
81  import org.springframework.transaction.support.TransactionTemplate;
82  
83  import javax.annotation.Nullable;
84  import javax.annotation.PostConstruct;
85  import javax.persistence.EntityManager;
86  import java.io.IOException;
87  import java.util.*;
88  import java.util.concurrent.*;
89  
90  import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
91  
92  @Component("mySearchCoordinatorSvc")
93  public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc {
94  	public static final int DEFAULT_SYNC_SIZE = 250;
95  
96  	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchCoordinatorSvcImpl.class);
97  	private final ConcurrentHashMap<String, BaseTask> myIdToSearchTask = new ConcurrentHashMap<>();
98  	@Autowired
99  	private FhirContext myContext;
100 	@Autowired
101 	private DaoConfig myDaoConfig;
102 	@Autowired
103 	private EntityManager myEntityManager;
104 	private ExecutorService myExecutor;
105 	private Integer myLoadingThrottleForUnitTests = null;
106 	private long myMaxMillisToWaitForRemoteResults = DateUtils.MILLIS_PER_MINUTE;
107 	private boolean myNeverUseLocalSearchForUnitTests;
108 	@Autowired
109 	private ISearchDao mySearchDao;
110 	@Autowired
111 	private ISearchIncludeDao mySearchIncludeDao;
112 	@Autowired
113 	private ISearchResultDao mySearchResultDao;
114 	@Autowired
115 	private IInterceptorBroadcaster myInterceptorBroadcaster;
116 	@Autowired
117 	private PlatformTransactionManager myManagedTxManager;
118 	@Autowired
119 	private DaoRegistry myDaoRegistry;
120 	@Autowired
121 	private IPagingProvider myPagingProvider;
122 
123 	private int mySyncSize = DEFAULT_SYNC_SIZE;
124 	/**
125 	 * Set in {@link #start()}
126 	 */
127 	private boolean myCustomIsolationSupported;
128 
129 	/**
130 	 * Constructor
131 	 */
132 	public SearchCoordinatorSvcImpl() {
133 		CustomizableThreadFactory threadFactory = new CustomizableThreadFactory("search_coord_");
134 		myExecutor = Executors.newCachedThreadPool(threadFactory);
135 	}
136 
137 	@PostConstruct
138 	public void start() {
139 		if (myManagedTxManager instanceof JpaTransactionManager) {
140 			JpaDialect jpaDialect = ((JpaTransactionManager) myManagedTxManager).getJpaDialect();
141 			if (jpaDialect instanceof HibernateJpaDialect) {
142 				myCustomIsolationSupported = true;
143 			}
144 		}
145 		if (myCustomIsolationSupported == false) {
146 			ourLog.warn("JPA dialect does not support transaction isolation! This can have an impact on search performance.");
147 		}
148 	}
149 
150 	@Override
151 	public void cancelAllActiveSearches() {
152 		for (BaseTask next : myIdToSearchTask.values()) {
153 			next.requestImmediateAbort();
154 			try {
155 				next.getCompletionLatch().await(30, TimeUnit.SECONDS);
156 			} catch (InterruptedException e) {
157 				ourLog.warn("Failed to wait for completion", e);
158 			}
159 		}
160 	}
161 
162 	/**
163 	 * This method is called by the HTTP client processing thread in order to
164 	 * fetch resources.
165 	 */
166 	@Override
167 	@Transactional(propagation = Propagation.NEVER)
168 	public List<Long> getResources(final String theUuid, int theFrom, int theTo, @Nullable RequestDetails theRequestDetails) {
169 		TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
170 		txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
171 
172 		Search search;
173 		StopWatch sw = new StopWatch();
174 		while (true) {
175 
176 			if (myNeverUseLocalSearchForUnitTests == false) {
177 				BaseTask task = myIdToSearchTask.get(theUuid);
178 				if (task != null) {
179 					ourLog.trace("Local search found");
180 					List<Long> resourcePids = task.getResourcePids(theFrom, theTo);
181 					if (resourcePids != null) {
182 						return resourcePids;
183 					}
184 				}
185 			}
186 
187 			search = txTemplate.execute(t -> mySearchDao.findByUuid(theUuid));
188 
189 			if (search == null) {
190 				ourLog.debug("Client requested unknown paging ID[{}]", theUuid);
191 				String msg = myContext.getLocalizer().getMessage(PageMethodBinding.class, "unknownSearchId", theUuid);
192 				throw new ResourceGoneException(msg);
193 			}
194 
195 			verifySearchHasntFailedOrThrowInternalErrorException(search);
196 			if (search.getStatus() == SearchStatusEnum.FINISHED) {
197 				ourLog.debug("Search entity marked as finished with {} results", search.getNumFound());
198 				break;
199 			}
200 			if (search.getNumFound() >= theTo) {
201 				ourLog.debug("Search entity has {} results so far", search.getNumFound());
202 				break;
203 			}
204 
205 			if (sw.getMillis() > myMaxMillisToWaitForRemoteResults) {
206 				ourLog.error("Search {} of type {} for {}{} timed out after {}ms", search.getId(), search.getSearchType(), search.getResourceType(), search.getSearchQueryString(), sw.getMillis());
207 				throw new InternalErrorException("Request timed out after " + sw.getMillis() + "ms");
208 			}
209 
210 			// If the search was saved in "pass complete mode" it's probably time to
211 			// start a new pass
212 			if (search.getStatus() == SearchStatusEnum.PASSCMPLET) {
213 				Optional<Search> newSearch = tryToMarkSearchAsInProgress(search);
214 				if (newSearch.isPresent()) {
215 					search = newSearch.get();
216 					String resourceType = search.getResourceType();
217 					SearchParameterMap params = search.getSearchParameterMap();
218 					IFhirResourceDao<?> resourceDao = myDaoRegistry.getResourceDao(resourceType);
219 					SearchContinuationTask task = new SearchContinuationTask(search, resourceDao, params, resourceType, theRequestDetails);
220 					myIdToSearchTask.put(search.getUuid(), task);
221 					myExecutor.submit(task);
222 				}
223 			}
224 
225 			try {
226 				Thread.sleep(500);
227 			} catch (InterruptedException e) {
228 				// ignore
229 			}
230 		}
231 
232 		final Pageable page = toPage(theFrom, theTo);
233 		if (page == null) {
234 			return Collections.emptyList();
235 		}
236 
237 		final Search foundSearch = search;
238 
239 		ourLog.trace("Loading stored search");
240 		List<Long> retVal = txTemplate.execute(theStatus -> {
241 			final List<Long> resultPids = new ArrayList<>();
242 			Page<Long> searchResultPids = mySearchResultDao.findWithSearchUuid(foundSearch, page);
243 			for (Long next : searchResultPids) {
244 				resultPids.add(next);
245 			}
246 			return resultPids;
247 		});
248 		return retVal;
249 	}
250 
251 	private Optional<Search> tryToMarkSearchAsInProgress(Search theSearch) {
252 		ourLog.trace("Going to try to change search status from {} to {}", theSearch.getStatus(), SearchStatusEnum.LOADING);
253 		try {
254 			TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
255 			txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
256 			txTemplate.afterPropertiesSet();
257 			return txTemplate.execute(t -> {
258 				Search search = mySearchDao.findById(theSearch.getId()).orElse(theSearch);
259 
260 				if (search.getStatus() != SearchStatusEnum.PASSCMPLET) {
261 					throw new IllegalStateException("Can't change to LOADING because state is " + theSearch.getStatus());
262 				}
263 				search.setStatus(SearchStatusEnum.LOADING);
264 				Search newSearch = mySearchDao.save(search);
265 				return Optional.of(newSearch);
266 			});
267 		} catch (Exception e) {
268 			ourLog.warn("Failed to activate search: {}", e.toString());
269 			ourLog.trace("Failed to activate search", e);
270 			return Optional.empty();
271 		}
272 	}
273 
274 	private void populateBundleProvider(PersistedJpaBundleProvider theRetVal) {
275 		theRetVal.setContext(myContext);
276 		theRetVal.setEntityManager(myEntityManager);
277 		theRetVal.setPlatformTransactionManager(myManagedTxManager);
278 		theRetVal.setSearchDao(mySearchDao);
279 		theRetVal.setSearchCoordinatorSvc(this);
280 		theRetVal.setInterceptorBroadcaster(myInterceptorBroadcaster);
281 	}
282 
283 	@Override
284 	public IBundleProvider registerSearch(final IDao theCallingDao, final SearchParameterMap theParams, String theResourceType, CacheControlDirective theCacheControlDirective, RequestDetails theRequestDetails) {
285 		StopWatch w = new StopWatch();
286 		final String searchUuid = UUID.randomUUID().toString();
287 
288 		ourLog.debug("Registering new search {}", searchUuid);
289 
290 		Class<? extends IBaseResource> resourceTypeClass = myContext.getResourceDefinition(theResourceType).getImplementingClass();
291 		final ISearchBuilder sb = theCallingDao.newSearchBuilder();
292 		sb.setType(resourceTypeClass, theResourceType);
293 		sb.setFetchSize(mySyncSize);
294 
295 		final Integer loadSynchronousUpTo;
296 		if (theCacheControlDirective != null && theCacheControlDirective.isNoStore()) {
297 			if (theCacheControlDirective.getMaxResults() != null) {
298 				loadSynchronousUpTo = theCacheControlDirective.getMaxResults();
299 				if (loadSynchronousUpTo > myDaoConfig.getCacheControlNoStoreMaxResultsUpperLimit()) {
300 					throw new InvalidRequestException(Constants.HEADER_CACHE_CONTROL + " header " + Constants.CACHE_CONTROL_MAX_RESULTS + " value must not exceed " + myDaoConfig.getCacheControlNoStoreMaxResultsUpperLimit());
301 				}
302 			} else {
303 				loadSynchronousUpTo = 100;
304 			}
305 		} else {
306 			loadSynchronousUpTo = null;
307 		}
308 
309 		if (theParams.isLoadSynchronous() || loadSynchronousUpTo != null) {
310 
311 			ourLog.debug("Search {} is loading in synchronous mode", searchUuid);
312 			SearchRuntimeDetails searchRuntimeDetails = new SearchRuntimeDetails(theRequestDetails, searchUuid);
313 			searchRuntimeDetails.setLoadSynchronous(true);
314 
315 			// Execute the query and make sure we return distinct results
316 			TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
317 			txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
318 			return txTemplate.execute(t -> {
319 
320 				// Load the results synchronously
321 				final List<Long> pids = new ArrayList<>();
322 
323 				try (IResultIterator resultIter = sb.createQuery(theParams, searchRuntimeDetails, theRequestDetails)) {
324 					while (resultIter.hasNext()) {
325 						pids.add(resultIter.next());
326 						if (loadSynchronousUpTo != null && pids.size() >= loadSynchronousUpTo) {
327 							break;
328 						}
329 						if (theParams.getLoadSynchronousUpTo() != null && pids.size() >= theParams.getLoadSynchronousUpTo()) {
330 							break;
331 						}
332 					}
333 				} catch (IOException e) {
334 					ourLog.error("IO failure during database access", e);
335 					throw new InternalErrorException(e);
336 				}
337 
338 				JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(pids, () -> sb);
339 				HookParams params = new HookParams()
340 					.add(IPreResourceAccessDetails.class, accessDetails)
341 					.add(RequestDetails.class, theRequestDetails)
342 					.addIfMatchesType(ServletRequestDetails.class, theRequestDetails);
343 				JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
344 
345 				for (int i = pids.size() - 1; i >= 0; i--) {
346 					if (accessDetails.isDontReturnResourceAtIndex(i)) {
347 						pids.remove(i);
348 					}
349 				}
350 
351 				/*
352 				 * For synchronous queries, we load all the includes right away
353 				 * since we're returning a static bundle with all the results
354 				 * pre-loaded. This is ok because syncronous requests are not
355 				 * expected to be paged
356 				 *
357 				 * On the other hand for async queries we load includes/revincludes
358 				 * individually for pages as we return them to clients
359 				 */
360 				final Set<Long> includedPids = new HashSet<>();
361 				includedPids.addAll(sb.loadIncludes(myContext, myEntityManager, pids, theParams.getRevIncludes(), true, theParams.getLastUpdated(), "(synchronous)", theRequestDetails));
362 				includedPids.addAll(sb.loadIncludes(myContext, myEntityManager, pids, theParams.getIncludes(), false, theParams.getLastUpdated(), "(synchronous)", theRequestDetails));
363 				List<Long> includedPidsList = new ArrayList<>(includedPids);
364 
365 				List<IBaseResource> resources = new ArrayList<>();
366 				sb.loadResourcesByPid(pids, includedPidsList, resources, false, theRequestDetails);
367 				return new SimpleBundleProvider(resources);
368 			});
369 		}
370 
371 		/*
372 		 * See if there are any cached searches whose results we can return
373 		 * instead
374 		 */
375 		boolean useCache = true;
376 		if (theCacheControlDirective != null && theCacheControlDirective.isNoCache() == true) {
377 			useCache = false;
378 		}
379 		final String queryString = theParams.toNormalizedQueryString(myContext);
380 		if (theParams.getEverythingMode() == null) {
381 			if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null && useCache) {
382 
383 				final Date createdCutoff = new Date(System.currentTimeMillis() - myDaoConfig.getReuseCachedSearchResultsForMillis());
384 				final String resourceType = theResourceType;
385 
386 				TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
387 				PersistedJpaBundleProvider foundSearchProvider = txTemplate.execute(t -> {
388 					Search searchToUse = null;
389 
390 					// Interceptor call: STORAGE_PRECHECK_FOR_CACHED_SEARCH
391 					HookParams params = new HookParams()
392 						.add(SearchParameterMap.class, theParams)
393 						.add(RequestDetails.class, theRequestDetails)
394 						.addIfMatchesType(ServletRequestDetails.class, theRequestDetails);
395 					Object outcome = JpaInterceptorBroadcaster.doCallHooksAndReturnObject(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRECHECK_FOR_CACHED_SEARCH, params);
396 					if (Boolean.FALSE.equals(outcome)) {
397 						return null;
398 					}
399 
400 					// Check for a search matching the given hash
401 					int hashCode = queryString.hashCode();
402 					Collection<Search> candidates = mySearchDao.find(resourceType, hashCode, createdCutoff);
403 					for (Search nextCandidateSearch : candidates) {
404 						if (queryString.equals(nextCandidateSearch.getSearchQueryString())) {
405 							searchToUse = nextCandidateSearch;
406 							break;
407 						}
408 					}
409 
410 					PersistedJpaBundleProvider retVal = null;
411 					if (searchToUse != null) {
412 						ourLog.debug("Reusing search {} from cache", searchToUse.getUuid());
413 
414 						// Interceptor call: JPA_PERFTRACE_SEARCH_REUSING_CACHED
415 						params = new HookParams()
416 							.add(SearchParameterMap.class, theParams)
417 							.add(RequestDetails.class, theRequestDetails)
418 							.addIfMatchesType(ServletRequestDetails.class, theRequestDetails);
419 						JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_SEARCH_REUSING_CACHED, params);
420 
421 						searchToUse.setSearchLastReturned(new Date());
422 						mySearchDao.updateSearchLastReturned(searchToUse.getId(), new Date());
423 
424 						retVal = new PersistedJpaBundleProvider(theRequestDetails, searchToUse.getUuid(), theCallingDao);
425 						retVal.setCacheHit(true);
426 
427 						populateBundleProvider(retVal);
428 					}
429 
430 					return retVal;
431 				});
432 
433 				if (foundSearchProvider != null) {
434 					return foundSearchProvider;
435 				}
436 
437 			}
438 		}
439 
440 		Search search = new Search();
441 		populateSearchEntity(theParams, theResourceType, searchUuid, queryString, search);
442 
443 		// Interceptor call: STORAGE_PRESEARCH_REGISTERED
444 		HookParams params = new HookParams()
445 			.add(ICachedSearchDetails.class, search)
446 			.add(RequestDetails.class, theRequestDetails)
447 			.addIfMatchesType(ServletRequestDetails.class, theRequestDetails);
448 		JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRESEARCH_REGISTERED, params);
449 
450 		SearchTask task = new SearchTask(search, theCallingDao, theParams, theResourceType, theRequestDetails);
451 		myIdToSearchTask.put(search.getUuid(), task);
452 		myExecutor.submit(task);
453 
454 		PersistedJpaSearchFirstPageBundleProvider retVal = new PersistedJpaSearchFirstPageBundleProvider(search, theCallingDao, task, sb, myManagedTxManager, theRequestDetails);
455 		populateBundleProvider(retVal);
456 
457 		ourLog.debug("Search initial phase completed in {}ms", w.getMillis());
458 		return retVal;
459 
460 	}
461 
462 	private void callInterceptorStoragePreAccessResources(IInterceptorBroadcaster theInterceptorBroadcaster, RequestDetails theRequestDetails, ISearchBuilder theSb, List<Long> thePids) {
463 		if (thePids.isEmpty() == false) {
464 			JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(thePids, () -> theSb);
465 			HookParams params = new HookParams()
466 				.add(IPreResourceAccessDetails.class, accessDetails)
467 				.add(RequestDetails.class, theRequestDetails)
468 				.addIfMatchesType(ServletRequestDetails.class, theRequestDetails);
469 			JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
470 			for (int i = thePids.size() - 1; i >= 0; i--) {
471 				if (accessDetails.isDontReturnResourceAtIndex(i)) {
472 					thePids.remove(i);
473 				}
474 			}
475 		}
476 	}
477 
478 	@VisibleForTesting
479 	void setContextForUnitTest(FhirContext theCtx) {
480 		myContext = theCtx;
481 	}
482 
483 	@VisibleForTesting
484 	void setDaoConfigForUnitTest(DaoConfig theDaoConfig) {
485 		myDaoConfig = theDaoConfig;
486 	}
487 
488 	@VisibleForTesting
489 	void setEntityManagerForUnitTest(EntityManager theEntityManager) {
490 		myEntityManager = theEntityManager;
491 	}
492 
493 	@VisibleForTesting
494 	public void setLoadingThrottleForUnitTests(Integer theLoadingThrottleForUnitTests) {
495 		myLoadingThrottleForUnitTests = theLoadingThrottleForUnitTests;
496 	}
497 
498 	@VisibleForTesting
499 	public void setNeverUseLocalSearchForUnitTests(boolean theNeverUseLocalSearchForUnitTests) {
500 		myNeverUseLocalSearchForUnitTests = theNeverUseLocalSearchForUnitTests;
501 	}
502 
503 	@VisibleForTesting
504 	void setSearchDaoForUnitTest(ISearchDao theSearchDao) {
505 		mySearchDao = theSearchDao;
506 	}
507 
508 	@VisibleForTesting
509 	void setSearchDaoIncludeForUnitTest(ISearchIncludeDao theSearchIncludeDao) {
510 		mySearchIncludeDao = theSearchIncludeDao;
511 	}
512 
513 	@VisibleForTesting
514 	void setSearchDaoResultForUnitTest(ISearchResultDao theSearchResultDao) {
515 		mySearchResultDao = theSearchResultDao;
516 	}
517 
518 	@VisibleForTesting
519 	public void setSyncSizeForUnitTests(int theSyncSize) {
520 		mySyncSize = theSyncSize;
521 	}
522 
523 	@VisibleForTesting
524 	void setTransactionManagerForUnitTest(PlatformTransactionManager theTxManager) {
525 		myManagedTxManager = theTxManager;
526 	}
527 
528 	@VisibleForTesting
529 	void setDaoRegistryForUnitTest(DaoRegistry theDaoRegistry) {
530 		myDaoRegistry = theDaoRegistry;
531 	}
532 
533 	@VisibleForTesting
534 	void setInterceptorBroadcasterForUnitTest(IInterceptorBroadcaster theInterceptorBroadcaster) {
535 		myInterceptorBroadcaster = theInterceptorBroadcaster;
536 	}
537 
538 	public abstract class BaseTask implements Callable<Void> {
539 		private final SearchParameterMap myParams;
540 		private final IDao myCallingDao;
541 		private final String myResourceType;
542 		private final ArrayList<Long> mySyncedPids = new ArrayList<>();
543 		private final CountDownLatch myInitialCollectionLatch = new CountDownLatch(1);
544 		private final CountDownLatch myCompletionLatch;
545 		private final ArrayList<Long> myUnsyncedPids = new ArrayList<>();
546 		private final RequestDetails myRequest;
547 		private Search mySearch;
548 		private boolean myAbortRequested;
549 		private int myCountSavedTotal = 0;
550 		private int myCountSavedThisPass = 0;
551 		private int myCountBlockedThisPass = 0;
552 		private boolean myAdditionalPrefetchThresholdsRemaining;
553 		private List<Long> myPreviouslyAddedResourcePids;
554 		private Integer myMaxResultsToFetch;
555 		private SearchRuntimeDetails mySearchRuntimeDetails;
556 
557 		/**
558 		 * Constructor
559 		 */
560 		protected BaseTask(Search theSearch, IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequest) {
561 			mySearch = theSearch;
562 			myCallingDao = theCallingDao;
563 			myParams = theParams;
564 			myResourceType = theResourceType;
565 			myCompletionLatch = new CountDownLatch(1);
566 			mySearchRuntimeDetails = new SearchRuntimeDetails(theRequest, mySearch.getUuid());
567 			mySearchRuntimeDetails.setQueryString(theParams.toNormalizedQueryString(theCallingDao.getContext()));
568 			myRequest = theRequest;
569 		}
570 
571 		protected Search getSearch() {
572 			return mySearch;
573 		}
574 
575 		CountDownLatch getInitialCollectionLatch() {
576 			return myInitialCollectionLatch;
577 		}
578 
579 		void setPreviouslyAddedResourcePids(List<Long> thePreviouslyAddedResourcePids) {
580 			myPreviouslyAddedResourcePids = thePreviouslyAddedResourcePids;
581 			myCountSavedTotal = myPreviouslyAddedResourcePids.size();
582 		}
583 
584 		private ISearchBuilder newSearchBuilder() {
585 			Class<? extends IBaseResource> resourceTypeClass = myContext.getResourceDefinition(myResourceType).getImplementingClass();
586 			ISearchBuilder sb = myCallingDao.newSearchBuilder();
587 			sb.setType(resourceTypeClass, myResourceType);
588 
589 			return sb;
590 		}
591 
592 		public List<Long> getResourcePids(int theFromIndex, int theToIndex) {
593 			ourLog.debug("Requesting search PIDs from {}-{}", theFromIndex, theToIndex);
594 
595 			boolean keepWaiting;
596 			do {
597 				synchronized (mySyncedPids) {
598 					ourLog.trace("Search status is {}", mySearch.getStatus());
599 					boolean haveEnoughResults = mySyncedPids.size() >= theToIndex;
600 					if (!haveEnoughResults) {
601 						switch (mySearch.getStatus()) {
602 							case LOADING:
603 								keepWaiting = true;
604 								break;
605 							case PASSCMPLET:
606 								/*
607 								 * If we get here, it means that the user requested resources that crossed the
608 								 * current pre-fetch boundary. For example, if the prefetch threshold is 50 and the
609 								 * user has requested resources 0-60, then they would get 0-50 back but the search
610 								 * coordinator would then stop searching.SearchCoordinatorSvcImplTest
611 								 */
612 								keepWaiting = false;
613 								break;
614 							case FAILED:
615 							case FINISHED:
616 							default:
617 								keepWaiting = false;
618 								break;
619 						}
620 					} else {
621 						keepWaiting = false;
622 					}
623 				}
624 
625 				if (keepWaiting) {
626 					ourLog.info("Waiting as we only have {} results - Search status: {}", mySyncedPids.size(), mySearch.getStatus());
627 					try {
628 						Thread.sleep(500);
629 					} catch (InterruptedException theE) {
630 						// ignore
631 					}
632 				}
633 			} while (keepWaiting);
634 
635 			ourLog.debug("Proceeding, as we have {} results", mySyncedPids.size());
636 
637 			ArrayList<Long> retVal = new ArrayList<>();
638 			synchronized (mySyncedPids) {
639 				verifySearchHasntFailedOrThrowInternalErrorException(mySearch);
640 
641 				int toIndex = theToIndex;
642 				if (mySyncedPids.size() < toIndex) {
643 					toIndex = mySyncedPids.size();
644 				}
645 				for (int i = theFromIndex; i < toIndex; i++) {
646 					retVal.add(mySyncedPids.get(i));
647 				}
648 			}
649 
650 			ourLog.trace("Done syncing results - Wanted {}-{} and returning {} of {}", theFromIndex, theToIndex, retVal.size(), mySyncedPids.size());
651 
652 			return retVal;
653 		}
654 
655 		void saveSearch() {
656 			TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
657 			txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
658 			txTemplate.execute(new TransactionCallbackWithoutResult() {
659 				@Override
660 				protected void doInTransactionWithoutResult(@NotNull TransactionStatus theArg0) {
661 					doSaveSearch();
662 				}
663 
664 			});
665 		}
666 
667 		private void saveUnsynced(final IResultIterator theResultIter) {
668 			TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
669 			txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
670 			txTemplate.execute(new TransactionCallbackWithoutResult() {
671 				@Override
672 				protected void doInTransactionWithoutResult(@NotNull TransactionStatus theArg0) {
673 					if (mySearch.getId() == null) {
674 						doSaveSearch();
675 					}
676 
677 					ArrayList<Long> unsyncedPids = myUnsyncedPids;
678 
679 					// Interceptor call: STORAGE_PREACCESS_RESOURCES
680 					// This can be used to remove results from the search result details before
681 					// the user has a chance to know that they were in the results
682 					if (mySearchRuntimeDetails.getRequestDetails() != null && unsyncedPids.isEmpty() == false) {
683 						JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(unsyncedPids, () -> newSearchBuilder());
684 						HookParams params = new HookParams()
685 							.add(IPreResourceAccessDetails.class, accessDetails)
686 							.add(RequestDetails.class, mySearchRuntimeDetails.getRequestDetails())
687 							.addIfMatchesType(ServletRequestDetails.class, mySearchRuntimeDetails.getRequestDetails());
688 						JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
689 
690 						for (int i = unsyncedPids.size() - 1; i >= 0; i--) {
691 							if (accessDetails.isDontReturnResourceAtIndex(i)) {
692 								unsyncedPids.remove(i);
693 								myCountBlockedThisPass++;
694 								myCountSavedTotal++;
695 							}
696 						}
697 					}
698 
699 					List<SearchResult> resultsToSave = Lists.newArrayList();
700 					for (Long nextPid : unsyncedPids) {
701 						SearchResult nextResult = new SearchResult(mySearch);
702 						nextResult.setResourcePid(nextPid);
703 						nextResult.setOrder(myCountSavedTotal);
704 						resultsToSave.add(nextResult);
705 						int order = nextResult.getOrder();
706 						ourLog.trace("Saving ORDER[{}] Resource {}", order, nextResult.getResourcePid());
707 
708 						myCountSavedTotal++;
709 						myCountSavedThisPass++;
710 					}
711 
712 					mySearchResultDao.saveAll(resultsToSave);
713 
714 					synchronized (mySyncedPids) {
715 						int numSyncedThisPass = unsyncedPids.size();
716 						ourLog.trace("Syncing {} search results - Have more: {}", numSyncedThisPass, theResultIter.hasNext());
717 						mySyncedPids.addAll(unsyncedPids);
718 						unsyncedPids.clear();
719 
720 						if (theResultIter.hasNext() == false) {
721 							mySearch.setNumFound(myCountSavedTotal);
722 							int skippedCount = theResultIter.getSkippedCount();
723 							int totalFetched = skippedCount + myCountSavedThisPass + myCountBlockedThisPass;
724 							ourLog.trace("MaxToFetch[{}] SkippedCount[{}] CountSavedThisPass[{}] CountSavedThisTotal[{}] AdditionalPrefetchRemaining[{}]", myMaxResultsToFetch, skippedCount, myCountSavedThisPass, myCountSavedTotal, myAdditionalPrefetchThresholdsRemaining);
725 
726 							if (myMaxResultsToFetch != null && totalFetched < myMaxResultsToFetch) {
727 								ourLog.trace("Setting search status to FINISHED");
728 								mySearch.setStatus(SearchStatusEnum.FINISHED);
729 								mySearch.setTotalCount(myCountSavedTotal);
730 							} else if (myAdditionalPrefetchThresholdsRemaining) {
731 								ourLog.trace("Setting search status to PASSCMPLET");
732 								mySearch.setStatus(SearchStatusEnum.PASSCMPLET);
733 								mySearch.setSearchParameterMap(myParams);
734 							} else {
735 								ourLog.trace("Setting search status to FINISHED");
736 								mySearch.setStatus(SearchStatusEnum.FINISHED);
737 								mySearch.setTotalCount(myCountSavedTotal);
738 							}
739 						}
740 					}
741 
742 					mySearch.setNumFound(myCountSavedTotal);
743 
744 					int numSynced;
745 					synchronized (mySyncedPids) {
746 						numSynced = mySyncedPids.size();
747 					}
748 
749 					if (myDaoConfig.getCountSearchResultsUpTo() == null ||
750 						myDaoConfig.getCountSearchResultsUpTo() <= 0 ||
751 						myDaoConfig.getCountSearchResultsUpTo() <= numSynced) {
752 						myInitialCollectionLatch.countDown();
753 					}
754 
755 					doSaveSearch();
756 
757 				}
758 			});
759 
760 		}
761 
762 		boolean isNotAborted() {
763 			return myAbortRequested == false;
764 		}
765 
766 		void markComplete() {
767 			myCompletionLatch.countDown();
768 		}
769 
770 		CountDownLatch getCompletionLatch() {
771 			return myCompletionLatch;
772 		}
773 
774 		/**
775 		 * Request that the task abort as soon as possible
776 		 */
777 		void requestImmediateAbort() {
778 			myAbortRequested = true;
779 		}
780 
781 		/**
782 		 * This is the method which actually performs the search.
783 		 * It is called automatically by the thread pool.
784 		 */
785 		@Override
786 		public Void call() {
787 			StopWatch sw = new StopWatch();
788 
789 			try {
790 				// Create an initial search in the DB and give it an ID
791 				saveSearch();
792 
793 				TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
794 				txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
795 
796 				if (myCustomIsolationSupported) {
797 					txTemplate.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED);
798 				}
799 
800 				txTemplate.execute(new TransactionCallbackWithoutResult() {
801 					@Override
802 					protected void doInTransactionWithoutResult(TransactionStatus theStatus) {
803 						doSearch();
804 					}
805 				});
806 
807 				mySearchRuntimeDetails.setSearchStatus(mySearch.getStatus());
808 				if (mySearch.getStatus() == SearchStatusEnum.FINISHED) {
809 					HookParams params = new HookParams()
810 						.add(RequestDetails.class, myRequest)
811 						.addIfMatchesType(ServletRequestDetails.class, myRequest)
812 						.add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
813 					JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_COMPLETE, params);
814 				} else {
815 					HookParams params = new HookParams()
816 						.add(RequestDetails.class, myRequest)
817 						.addIfMatchesType(ServletRequestDetails.class, myRequest)
818 						.add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
819 					JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_PASS_COMPLETE, params);
820 				}
821 
822 				ourLog.trace("Have completed search for [{}{}] and found {} resources in {}ms - Status is {}", mySearch.getResourceType(), mySearch.getSearchQueryString(), mySyncedPids.size(), sw.getMillis(), mySearch.getStatus());
823 
824 			} catch (Throwable t) {
825 
826 				/*
827 				 * Don't print a stack trace for client errors (i.e. requests that
828 				 * aren't valid because the client screwed up).. that's just noise
829 				 * in the logs and who needs that.
830 				 */
831 				boolean logged = false;
832 				if (t instanceof BaseServerResponseException) {
833 					BaseServerResponseException exception = (BaseServerResponseException) t;
834 					if (exception.getStatusCode() >= 400 && exception.getStatusCode() < 500) {
835 						logged = true;
836 						ourLog.warn("Failed during search due to invalid request: {}", t.toString());
837 					}
838 				}
839 
840 				if (!logged) {
841 					ourLog.error("Failed during search loading after {}ms", sw.getMillis(), t);
842 				}
843 				myUnsyncedPids.clear();
844 
845 				Throwable rootCause = ExceptionUtils.getRootCause(t);
846 				rootCause = defaultIfNull(rootCause, t);
847 
848 				String failureMessage = rootCause.getMessage();
849 
850 				int failureCode = InternalErrorException.STATUS_CODE;
851 				if (t instanceof BaseServerResponseException) {
852 					failureCode = ((BaseServerResponseException) t).getStatusCode();
853 				}
854 
855 				mySearch.setFailureMessage(failureMessage);
856 				mySearch.setFailureCode(failureCode);
857 				mySearch.setStatus(SearchStatusEnum.FAILED);
858 
859 				mySearchRuntimeDetails.setSearchStatus(mySearch.getStatus());
860 				HookParams params = new HookParams()
861 					.add(RequestDetails.class, myRequest)
862 					.addIfMatchesType(ServletRequestDetails.class, myRequest)
863 					.add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
864 				JpaInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FAILED, params);
865 
866 				saveSearch();
867 
868 			} finally {
869 
870 				myIdToSearchTask.remove(mySearch.getUuid());
871 				myInitialCollectionLatch.countDown();
872 				markComplete();
873 
874 			}
875 			return null;
876 		}
877 
878 		private void doSaveSearch() {
879 
880 			Search newSearch;
881 			if (mySearch.getId() == null) {
882 				newSearch = mySearchDao.save(mySearch);
883 				for (SearchInclude next : mySearch.getIncludes()) {
884 					mySearchIncludeDao.save(next);
885 				}
886 			} else {
887 				newSearch = mySearchDao.save(mySearch);
888 			}
889 
890 			// mySearchDao.save is not supposed to return null, but in unit tests
891 			// it can if the mock search dao isn't set up to handle that
892 			if (newSearch != null) {
893 				mySearch = newSearch;
894 			}
895 		}
896 
897 		/**
898 		 * This method actually creates the database query to perform the
899 		 * search, and starts it.
900 		 */
901 		private void doSearch() {
902 
903 			/*
904 			 * If the user has explicitly requested a _count, perform a
905 			 *
906 			 * SELECT COUNT(*) ....
907 			 *
908 			 * before doing anything else.
909 			 */
910 			boolean wantOnlyCount = SummaryEnum.COUNT.equals(myParams.getSummaryMode());
911 			boolean wantCount =
912 				wantOnlyCount ||
913 					SearchTotalModeEnum.ACCURATE.equals(myParams.getSearchTotalMode()) ||
914 					(myParams.getSearchTotalMode() == null && SearchTotalModeEnum.ACCURATE.equals(myDaoConfig.getDefaultTotalMode()));
915 			if (wantCount) {
916 				ourLog.trace("Performing count");
917 				ISearchBuilder sb = newSearchBuilder();
918 				Iterator<Long> countIterator = sb.createCountQuery(myParams, mySearch.getUuid(), myRequest);
919 				Long count = countIterator.next();
920 				ourLog.trace("Got count {}", count);
921 
922 				TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
923 				txTemplate.execute(new TransactionCallbackWithoutResult() {
924 					@Override
925 					protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
926 						mySearch.setTotalCount(count.intValue());
927 						if (wantOnlyCount) {
928 							mySearch.setStatus(SearchStatusEnum.FINISHED);
929 						}
930 						doSaveSearch();
931 						mySearchDao.flush();
932 					}
933 				});
934 				if (wantOnlyCount) {
935 					return;
936 				}
937 			}
938 
939 			ourLog.trace("Done count");
940 			ISearchBuilder sb = newSearchBuilder();
941 
942 			/*
943 			 * Figure out how many results we're actually going to fetch from the
944 			 * database in this pass. This calculation takes into consideration the
945 			 * "pre-fetch thresholds" specified in DaoConfig#getSearchPreFetchThresholds()
946 			 * as well as the value of the _count parameter.
947 			 */
948 			int currentlyLoaded = defaultIfNull(mySearch.getNumFound(), 0);
949 			int minWanted = 0;
950 			if (myParams.getCount() != null) {
951 				minWanted = myParams.getCount();
952 				minWanted = Math.max(minWanted, myPagingProvider.getMaximumPageSize());
953 				minWanted += currentlyLoaded;
954 			}
955 
956 			for (Iterator<Integer> iter = myDaoConfig.getSearchPreFetchThresholds().iterator(); iter.hasNext(); ) {
957 				int next = iter.next();
958 				if (next != -1 && next <= currentlyLoaded) {
959 					continue;
960 				}
961 
962 				if (next == -1) {
963 					sb.setMaxResultsToFetch(null);
964 				} else {
965 					myMaxResultsToFetch = Math.max(next, minWanted);
966 					sb.setMaxResultsToFetch(myMaxResultsToFetch);
967 				}
968 
969 				if (iter.hasNext()) {
970 					myAdditionalPrefetchThresholdsRemaining = true;
971 				}
972 
973 				// If we get here's we've found an appropriate threshold
974 				break;
975 			}
976 
977 			/*
978 			 * Provide any PID we loaded in previous search passes to the
979 			 * SearchBuilder so that we don't get duplicates coming from running
980 			 * the same query again.
981 			 *
982 			 * We could possibly accomplish this in a different way by using sorted
983 			 * results in our SQL query and specifying an offset. I don't actually
984 			 * know if that would be faster or not. At some point should test this
985 			 * idea.
986 			 */
987 			if (myPreviouslyAddedResourcePids != null) {
988 				sb.setPreviouslyAddedResourcePids(myPreviouslyAddedResourcePids);
989 				mySyncedPids.addAll(myPreviouslyAddedResourcePids);
990 			}
991 
992 			/*
993 			 * Construct the SQL query we'll be sending to the database
994 			 */
995 			try (IResultIterator resultIterator = sb.createQuery(myParams, mySearchRuntimeDetails, myRequest)) {
996 				assert (resultIterator != null);
997 
998 				/*
999 				 * The following loop actually loads the PIDs of the resources
1000 				 * matching the search off of the disk and into memory. After
1001 				 * every X results, we commit to the HFJ_SEARCH table.
1002 				 */
1003 				int syncSize = mySyncSize;
1004 				while (resultIterator.hasNext()) {
1005 					myUnsyncedPids.add(resultIterator.next());
1006 
1007 					boolean shouldSync = myUnsyncedPids.size() >= syncSize;
1008 
1009 					if (myDaoConfig.getCountSearchResultsUpTo() != null &&
1010 						myDaoConfig.getCountSearchResultsUpTo() > 0 &&
1011 						myDaoConfig.getCountSearchResultsUpTo() < myUnsyncedPids.size()) {
1012 						shouldSync = false;
1013 					}
1014 
1015 					if (myUnsyncedPids.size() > 50000) {
1016 						shouldSync = true;
1017 					}
1018 
1019 					// If no abort was requested, bail out
1020 					Validate.isTrue(isNotAborted(), "Abort has been requested");
1021 
1022 					if (shouldSync) {
1023 						saveUnsynced(resultIterator);
1024 					}
1025 
1026 					if (myLoadingThrottleForUnitTests != null) {
1027 						try {
1028 							Thread.sleep(myLoadingThrottleForUnitTests);
1029 						} catch (InterruptedException e) {
1030 							// ignore
1031 						}
1032 					}
1033 
1034 				}
1035 
1036 				// If no abort was requested, bail out
1037 				Validate.isTrue(isNotAborted(), "Abort has been requested");
1038 
1039 				saveUnsynced(resultIterator);
1040 
1041 			} catch (IOException e) {
1042 				ourLog.error("IO failure during database access", e);
1043 				throw new InternalErrorException(e);
1044 			}
1045 		}
1046 	}
1047 
1048 
1049 	public class SearchContinuationTask extends BaseTask {
1050 
1051 		public SearchContinuationTask(Search theSearch, IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequest) {
1052 			super(theSearch, theCallingDao, theParams, theResourceType, theRequest);
1053 		}
1054 
1055 		@Override
1056 		public Void call() {
1057 			try {
1058 				TransactionTemplate txTemplate = new TransactionTemplate(myManagedTxManager);
1059 				txTemplate.afterPropertiesSet();
1060 				txTemplate.execute(t -> {
1061 					List<Long> previouslyAddedResourcePids = mySearchResultDao.findWithSearchUuidOrderIndependent(getSearch());
1062 					ourLog.debug("Have {} previously added IDs in search: {}", previouslyAddedResourcePids.size(), getSearch().getUuid());
1063 					setPreviouslyAddedResourcePids(previouslyAddedResourcePids);
1064 					return null;
1065 				});
1066 			} catch (Throwable e) {
1067 				ourLog.error("Failure processing search", e);
1068 				getSearch().setFailureMessage(e.toString());
1069 				getSearch().setStatus(SearchStatusEnum.FAILED);
1070 
1071 				saveSearch();
1072 				return null;
1073 			}
1074 
1075 			return super.call();
1076 		}
1077 
1078 		@Override
1079 		public List<Long> getResourcePids(int theFromIndex, int theToIndex) {
1080 			return super.getResourcePids(theFromIndex, theToIndex);
1081 		}
1082 	}
1083 
1084 	/**
1085 	 * A search task is a Callable task that runs in
1086 	 * a thread pool to handle an individual search. One instance
1087 	 * is created for any requested search and runs from the
1088 	 * beginning to the end of the search.
1089 	 * <p>
1090 	 * Understand:
1091 	 * This class executes in its own thread separate from the
1092 	 * web server client thread that made the request. We do that
1093 	 * so that we can return to the client as soon as possible,
1094 	 * but keep the search going in the background (and have
1095 	 * the next page of results ready to go when the client asks).
1096 	 */
1097 	class SearchTask extends BaseTask {
1098 
1099 		/**
1100 		 * Constructor
1101 		 */
1102 		SearchTask(Search theSearch, IDao theCallingDao, SearchParameterMap theParams, String theResourceType, RequestDetails theRequestDetails) {
1103 			super(theSearch, theCallingDao, theParams, theResourceType, theRequestDetails);
1104 		}
1105 
1106 		/**
1107 		 * This method is called by the server HTTP thread, and
1108 		 * will block until at least one page of results have been
1109 		 * fetched from the DB, and will never block after that.
1110 		 */
1111 		Integer awaitInitialSync() {
1112 			ourLog.trace("Awaiting initial sync");
1113 			do {
1114 				try {
1115 					if (getInitialCollectionLatch().await(250, TimeUnit.MILLISECONDS)) {
1116 						break;
1117 					}
1118 				} catch (InterruptedException e) {
1119 					// Shouldn't happen
1120 					throw new InternalErrorException(e);
1121 				}
1122 			} while (getSearch().getStatus() == SearchStatusEnum.LOADING);
1123 			ourLog.trace("Initial sync completed");
1124 
1125 			return getSearch().getTotalCount();
1126 		}
1127 
1128 	}
1129 
1130 	public static void populateSearchEntity(SearchParameterMap theParams, String theResourceType, String theSearchUuid, String theQueryString, Search theSearch) {
1131 		theSearch.setDeleted(false);
1132 		theSearch.setUuid(theSearchUuid);
1133 		theSearch.setCreated(new Date());
1134 		theSearch.setSearchLastReturned(new Date());
1135 		theSearch.setTotalCount(null);
1136 		theSearch.setNumFound(0);
1137 		theSearch.setPreferredPageSize(theParams.getCount());
1138 		theSearch.setSearchType(theParams.getEverythingMode() != null ? SearchTypeEnum.EVERYTHING : SearchTypeEnum.SEARCH);
1139 		theSearch.setLastUpdated(theParams.getLastUpdated());
1140 		theSearch.setResourceType(theResourceType);
1141 		theSearch.setStatus(SearchStatusEnum.LOADING);
1142 
1143 		theSearch.setSearchQueryString(theQueryString);
1144 		theSearch.setSearchQueryStringHash(theQueryString.hashCode());
1145 
1146 		for (Include next : theParams.getIncludes()) {
1147 			theSearch.addInclude(new SearchInclude(theSearch, next.getValue(), false, next.isRecurse()));
1148 		}
1149 		for (Include next : theParams.getRevIncludes()) {
1150 			theSearch.addInclude(new SearchInclude(theSearch, next.getValue(), true, next.isRecurse()));
1151 		}
1152 	}
1153 
1154 	/**
1155 	 * Creates a {@link Pageable} using a start and end index
1156 	 */
1157 	@SuppressWarnings("WeakerAccess")
1158 	public static @Nullable
1159 	Pageable toPage(final int theFromIndex, int theToIndex) {
1160 		int pageSize = theToIndex - theFromIndex;
1161 		if (pageSize < 1) {
1162 			return null;
1163 		}
1164 
1165 		int pageIndex = theFromIndex / pageSize;
1166 
1167 		Pageable page = new AbstractPageRequest(pageIndex, pageSize) {
1168 			private static final long serialVersionUID = 1L;
1169 
1170 			@Override
1171 			public long getOffset() {
1172 				return theFromIndex;
1173 			}
1174 
1175 			@Override
1176 			public Sort getSort() {
1177 				return Sort.unsorted();
1178 			}
1179 
1180 			@Override
1181 			public Pageable next() {
1182 				return null;
1183 			}
1184 
1185 			@Override
1186 			public Pageable previous() {
1187 				return null;
1188 			}
1189 
1190 			@Override
1191 			public Pageable first() {
1192 				return null;
1193 			}
1194 		};
1195 
1196 		return page;
1197 	}
1198 
1199 	static void verifySearchHasntFailedOrThrowInternalErrorException(Search theSearch) {
1200 		if (theSearch.getStatus() == SearchStatusEnum.FAILED) {
1201 			Integer status = theSearch.getFailureCode();
1202 			status = defaultIfNull(status, 500);
1203 
1204 			String message = theSearch.getFailureMessage();
1205 			throw BaseServerResponseException.newInstance(status, message);
1206 		}
1207 	}
1208 
1209 }