001package ca.uhn.fhir.jpa.search;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2021 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 * http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.FhirContext;
024import ca.uhn.fhir.interceptor.api.HookParams;
025import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
026import ca.uhn.fhir.interceptor.api.Pointcut;
027import ca.uhn.fhir.interceptor.model.RequestPartitionId;
028import ca.uhn.fhir.jpa.api.config.DaoConfig;
029import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
030import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
031import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
032import ca.uhn.fhir.jpa.dao.HistoryBuilder;
033import ca.uhn.fhir.jpa.dao.HistoryBuilderFactory;
034import ca.uhn.fhir.jpa.dao.ISearchBuilder;
035import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
036import ca.uhn.fhir.jpa.entity.Search;
037import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
038import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
039import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
040import ca.uhn.fhir.jpa.partition.RequestPartitionHelperSvc;
041import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
042import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum;
043import ca.uhn.fhir.jpa.util.InterceptorUtil;
044import ca.uhn.fhir.model.api.Include;
045import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
046import ca.uhn.fhir.jpa.util.MemoryCacheService;
047import ca.uhn.fhir.model.primitive.InstantDt;
048import ca.uhn.fhir.rest.api.server.IBundleProvider;
049import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
050import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
051import ca.uhn.fhir.rest.api.server.RequestDetails;
052import ca.uhn.fhir.rest.api.server.SimplePreResourceAccessDetails;
053import ca.uhn.fhir.rest.api.server.SimplePreResourceShowDetails;
054import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
055import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
056import com.google.common.annotations.VisibleForTesting;
057import org.hl7.fhir.instance.model.api.IBaseResource;
058import org.hl7.fhir.instance.model.api.IIdType;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061import org.springframework.beans.factory.annotation.Autowired;
062import org.springframework.transaction.PlatformTransactionManager;
063import org.springframework.transaction.TransactionDefinition;
064import org.springframework.transaction.TransactionStatus;
065import org.springframework.transaction.support.TransactionCallbackWithoutResult;
066import org.springframework.transaction.support.TransactionTemplate;
067
068import javax.annotation.Nonnull;
069import javax.persistence.EntityManager;
070import javax.persistence.PersistenceContext;
071import java.util.ArrayList;
072import java.util.Collections;
073import java.util.HashSet;
074import java.util.List;
075import java.util.Optional;
076import java.util.Set;
077import java.util.function.Function;
078import java.util.stream.Collectors;
079
080public class PersistedJpaBundleProvider implements IBundleProvider {
081
082        private static final Logger ourLog = LoggerFactory.getLogger(PersistedJpaBundleProvider.class);
083
084        /*
085         * Autowired fields
086         */
087        private final RequestDetails myRequest;
088        @Autowired
089        protected PlatformTransactionManager myTxManager;
090        @PersistenceContext
091        private EntityManager myEntityManager;
092        @Autowired
093        private IInterceptorBroadcaster myInterceptorBroadcaster;
094        @Autowired
095        private SearchBuilderFactory mySearchBuilderFactory;
096        @Autowired
097        private HistoryBuilderFactory myHistoryBuilderFactory;
098        @Autowired
099        private DaoRegistry myDaoRegistry;
100        @Autowired
101        private FhirContext myContext;
102        @Autowired
103        private ISearchCoordinatorSvc mySearchCoordinatorSvc;
104        @Autowired
105        private ISearchCacheSvc mySearchCacheSvc;
106        @Autowired
107        private RequestPartitionHelperSvc myRequestPartitionHelperSvc;
108        @Autowired
109        private DaoConfig myDaoConfig;
110
111        /*
112         * Non autowired fields (will be different for every instance
113         * of this class, since it's a prototype
114         */
115        @Autowired
116        private MemoryCacheService myMemoryCacheService;
117        private Search mySearchEntity;
118        private String myUuid;
119        private SearchCacheStatusEnum myCacheStatus;
120        private RequestPartitionId myRequestPartitionId;
121
122        /**
123         * Constructor
124         */
125        public PersistedJpaBundleProvider(RequestDetails theRequest, String theSearchUuid) {
126                myRequest = theRequest;
127                myUuid = theSearchUuid;
128        }
129
130        /**
131         * Constructor
132         */
133        public PersistedJpaBundleProvider(RequestDetails theRequest, Search theSearch) {
134                myRequest = theRequest;
135                mySearchEntity = theSearch;
136        }
137
138        /**
139         * When HAPI FHIR server is running "for real", a new
140         * instance of the bundle provider is created to serve
141         * every HTTP request, so it's ok for us to keep
142         * state in here and expect that it will go away. But
143         * in unit tests we keep this object around for longer
144         * sometimes.
145         */
146        public void clearCachedDataForUnitTest() {
147                mySearchEntity = null;
148        }
149
150        /**
151         * Perform a history search
152         */
153        private List<IBaseResource> doHistoryInTransaction(Integer theOffset, int theFromIndex, int theToIndex) {
154
155                HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(mySearchEntity.getResourceType(), mySearchEntity.getResourceId(), mySearchEntity.getLastUpdatedLow(), mySearchEntity.getLastUpdatedHigh());
156
157                RequestPartitionId partitionId = getRequestPartitionIdForHistory();
158                List<ResourceHistoryTable> results = historyBuilder.fetchEntities(partitionId, theOffset, theFromIndex, theToIndex);
159
160                List<IBaseResource> retVal = new ArrayList<>();
161                for (ResourceHistoryTable next : results) {
162                        BaseHasResource resource;
163                        resource = next;
164
165                        IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(next.getResourceType());
166                        retVal.add(dao.toResource(resource, true));
167                }
168
169
170                // Interceptor call: STORAGE_PREACCESS_RESOURCES
171                {
172                        SimplePreResourceAccessDetails accessDetails = new SimplePreResourceAccessDetails(retVal);
173                        HookParams params = new HookParams()
174                                .add(IPreResourceAccessDetails.class, accessDetails)
175                                .add(RequestDetails.class, myRequest)
176                                .addIfMatchesType(ServletRequestDetails.class, myRequest);
177                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
178
179                        for (int i = retVal.size() - 1; i >= 0; i--) {
180                                if (accessDetails.isDontReturnResourceAtIndex(i)) {
181                                        retVal.remove(i);
182                                }
183                        }
184                }
185
186                // Interceptor broadcast: STORAGE_PRESHOW_RESOURCES
187                {
188                        SimplePreResourceShowDetails showDetails = new SimplePreResourceShowDetails(retVal);
189                        HookParams params = new HookParams()
190                                .add(IPreResourceShowDetails.class, showDetails)
191                                .add(RequestDetails.class, myRequest)
192                                .addIfMatchesType(ServletRequestDetails.class, myRequest);
193                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PRESHOW_RESOURCES, params);
194                        retVal = showDetails.toList();
195                }
196
197
198                return retVal;
199        }
200
201        @Nonnull
202        private RequestPartitionId getRequestPartitionIdForHistory() {
203                if (myRequestPartitionId == null) {
204                        if (mySearchEntity.getResourceId() != null) {
205                                // If we have an ID, we've already checked the partition and made sure it's appropriate
206                                myRequestPartitionId = RequestPartitionId.allPartitions();
207                        } else {
208                                myRequestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequest(myRequest, mySearchEntity.getResourceType(), null);
209                        }
210                }
211                return myRequestPartitionId;
212        }
213
214        protected List<IBaseResource> doSearchOrEverything(final int theFromIndex, final int theToIndex) {
215                if (mySearchEntity.getTotalCount() != null && mySearchEntity.getNumFound() <= 0) {
216                        // No resources to fetch (e.g. we did a _summary=count search)
217                        return Collections.emptyList();
218                }
219                String resourceName = mySearchEntity.getResourceType();
220                Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(resourceName).getImplementingClass();
221                IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceName);
222
223                final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(dao, resourceName, resourceType);
224
225                final List<ResourcePersistentId> pidsSubList = mySearchCoordinatorSvc.getResources(myUuid, theFromIndex, theToIndex, myRequest);
226
227                TransactionTemplate template = new TransactionTemplate(myTxManager);
228                template.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
229                return template.execute(theStatus -> toResourceList(sb, pidsSubList));
230        }
231
232
233        /**
234         * Returns false if the entity can't be found
235         */
236        public boolean ensureSearchEntityLoaded() {
237                if (mySearchEntity == null) {
238                        Optional<Search> searchOpt = mySearchCacheSvc.fetchByUuid(myUuid);
239                        if (!searchOpt.isPresent()) {
240                                return false;
241                        }
242
243                        setSearchEntity(searchOpt.get());
244
245                        ourLog.trace("Retrieved search with version {} and total {}", mySearchEntity.getVersion(), mySearchEntity.getTotalCount());
246
247                        return true;
248                }
249
250                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
251                        if (mySearchEntity.getTotalCount() == null) {
252                                calculateHistoryCount();
253                        }
254                }
255
256                return true;
257        }
258
259        /**
260         * Note that this method is called outside a DB transaction, and uses a loading cache
261         * (assuming the default {@literal COUNT_CACHED} mode) so this effectively throttles
262         * access to the database by preventing multiple concurrent DB calls for an expensive
263         * count operation.
264         */
265        private void calculateHistoryCount() {
266                MemoryCacheService.HistoryCountKey key;
267                if (mySearchEntity.getResourceId() != null) {
268                        key = MemoryCacheService.HistoryCountKey.forInstance(mySearchEntity.getResourceId());
269                } else if (mySearchEntity.getResourceType() != null) {
270                        key = MemoryCacheService.HistoryCountKey.forType(mySearchEntity.getResourceType());
271                } else {
272                        key = MemoryCacheService.HistoryCountKey.forSystem();
273                }
274
275                Function<MemoryCacheService.HistoryCountKey, Integer> supplier = k -> new TransactionTemplate(myTxManager).execute(t -> {
276                        HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(mySearchEntity.getResourceType(), mySearchEntity.getResourceId(), mySearchEntity.getLastUpdatedLow(), mySearchEntity.getLastUpdatedHigh());
277                        Long count = historyBuilder.fetchCount(getRequestPartitionIdForHistory());
278                        return count.intValue();
279                });
280
281                boolean haveOffset = mySearchEntity.getLastUpdatedLow() != null || mySearchEntity.getLastUpdatedHigh() != null;
282
283                switch (myDaoConfig.getHistoryCountMode()) {
284                        case COUNT_ACCURATE: {
285                                int count = supplier.apply(key);
286                                mySearchEntity.setTotalCount(count);
287                                break;
288                        }
289                        case CACHED_ONLY_WITHOUT_OFFSET: {
290                                if (!haveOffset) {
291                                        int count = myMemoryCacheService.get(MemoryCacheService.CacheEnum.HISTORY_COUNT, key, supplier);
292                                        mySearchEntity.setTotalCount(count);
293                                }
294                                break;
295                        }
296                        case COUNT_DISABLED: {
297                                break;
298                        }
299                }
300
301        }
302
303        @Override
304        public InstantDt getPublished() {
305                ensureSearchEntityLoaded();
306                return new InstantDt(mySearchEntity.getCreated());
307        }
308
309        @Nonnull
310        @Override
311        public List<IBaseResource> getResources(final int theFromIndex, final int theToIndex) {
312                TransactionTemplate template = new TransactionTemplate(myTxManager);
313
314                template.execute(new TransactionCallbackWithoutResult() {
315                        @Override
316                        protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) {
317                                boolean entityLoaded = ensureSearchEntityLoaded();
318                                assert entityLoaded;
319                        }
320                });
321
322                assert mySearchEntity != null;
323                assert mySearchEntity.getSearchType() != null;
324
325                switch (mySearchEntity.getSearchType()) {
326                        case HISTORY:
327                                return template.execute(theStatus -> doHistoryInTransaction(mySearchEntity.getOffset(), theFromIndex, theToIndex));
328                        case SEARCH:
329                        case EVERYTHING:
330                        default:
331                                List<IBaseResource> retVal = doSearchOrEverything(theFromIndex, theToIndex);
332                                /*
333                                 * If we got fewer resources back than we asked for, it's possible that the search
334                                 * completed. If that's the case, the cached version of the search entity is probably
335                                 * no longer valid so let's force a reload if it gets asked for again (most likely
336                                 * because someone is calling size() on us)
337                                 */
338                                if (retVal.size() < theToIndex - theFromIndex) {
339                                        mySearchEntity = null;
340                                }
341                                return retVal;
342                }
343        }
344
345        @Override
346        public String getUuid() {
347                return myUuid;
348        }
349
350        public SearchCacheStatusEnum getCacheStatus() {
351                return myCacheStatus;
352        }
353
354        void setCacheStatus(SearchCacheStatusEnum theSearchCacheStatusEnum) {
355                myCacheStatus = theSearchCacheStatusEnum;
356        }
357
358        @Override
359        public Integer preferredPageSize() {
360                ensureSearchEntityLoaded();
361                return mySearchEntity.getPreferredPageSize();
362        }
363
364        public void setContext(FhirContext theContext) {
365                myContext = theContext;
366        }
367
368        public void setEntityManager(EntityManager theEntityManager) {
369                myEntityManager = theEntityManager;
370        }
371
372        @VisibleForTesting
373        public void setSearchCoordinatorSvcForUnitTest(ISearchCoordinatorSvc theSearchCoordinatorSvc) {
374                mySearchCoordinatorSvc = theSearchCoordinatorSvc;
375        }
376
377        @VisibleForTesting
378        public void setTxManagerForUnitTest(PlatformTransactionManager theTxManager) {
379                myTxManager = theTxManager;
380        }
381
382        // Note: Leave as protected, HSPC depends on this
383        @SuppressWarnings("WeakerAccess")
384        protected void setSearchEntity(Search theSearchEntity) {
385                mySearchEntity = theSearchEntity;
386        }
387
388        @Override
389        public Integer size() {
390                ensureSearchEntityLoaded();
391                SearchCoordinatorSvcImpl.verifySearchHasntFailedOrThrowInternalErrorException(mySearchEntity);
392
393                Integer size = mySearchEntity.getTotalCount();
394                if (size != null) {
395                        return Math.max(0, size);
396                }
397
398                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
399                        return null;
400                } else {
401                        return mySearchCoordinatorSvc.getSearchTotal(myUuid).orElse(null);
402                }
403
404        }
405
406        protected boolean hasIncludes() {
407                ensureSearchEntityLoaded();
408                return !mySearchEntity.getIncludes().isEmpty();
409        }
410
411        // Note: Leave as protected, HSPC depends on this
412        @SuppressWarnings("WeakerAccess")
413        protected List<IBaseResource> toResourceList(ISearchBuilder theSearchBuilder, List<ResourcePersistentId> thePids) {
414
415                List<ResourcePersistentId> includedPidList = new ArrayList<>();
416                if (mySearchEntity.getSearchType() == SearchTypeEnum.SEARCH) {
417                        Integer maxIncludes = myDaoConfig.getMaximumIncludesToLoadPerPage();
418
419                        // Load _revincludes
420                        Set<ResourcePersistentId> includedPids = theSearchBuilder.loadIncludes(myContext, myEntityManager, thePids, mySearchEntity.toRevIncludesList(), true, mySearchEntity.getLastUpdated(), myUuid, myRequest, maxIncludes);
421                        if (maxIncludes != null) {
422                                maxIncludes -= includedPids.size();
423                        }
424                        thePids.addAll(includedPids);
425                        includedPidList.addAll(includedPids);
426
427                        // Load _includes
428                        Set<ResourcePersistentId> revIncludedPids = theSearchBuilder.loadIncludes(myContext, myEntityManager, thePids, mySearchEntity.toIncludesList(), false, mySearchEntity.getLastUpdated(), myUuid, myRequest, maxIncludes);
429                        thePids.addAll(revIncludedPids);
430                        includedPidList.addAll(revIncludedPids);
431
432                }
433
434                // Execute the query and make sure we return distinct results
435                List<IBaseResource> resources = new ArrayList<>();
436                theSearchBuilder.loadResourcesByPid(thePids, includedPidList, resources, false, myRequest);
437
438                resources = InterceptorUtil.fireStoragePreshowResource(resources, myRequest, myInterceptorBroadcaster);
439
440                return resources;
441        }
442
443        public void setInterceptorBroadcaster(IInterceptorBroadcaster theInterceptorBroadcaster) {
444                myInterceptorBroadcaster = theInterceptorBroadcaster;
445        }
446
447        @VisibleForTesting
448        public void setSearchCacheSvcForUnitTest(ISearchCacheSvc theSearchCacheSvc) {
449                mySearchCacheSvc = theSearchCacheSvc;
450        }
451
452        @VisibleForTesting
453        public void setDaoRegistryForUnitTest(DaoRegistry theDaoRegistry) {
454                myDaoRegistry = theDaoRegistry;
455        }
456
457        @VisibleForTesting
458        public void setDaoConfigForUnitTest(DaoConfig theDaoConfig) {
459                myDaoConfig = theDaoConfig;
460        }
461
462        @VisibleForTesting
463        public void setSearchBuilderFactoryForUnitTest(SearchBuilderFactory theSearchBuilderFactory) {
464                mySearchBuilderFactory = theSearchBuilderFactory;
465        }
466}