Source code

001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.search;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.interceptor.api.HookParams;
024import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
025import ca.uhn.fhir.interceptor.api.Pointcut;
026import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
027import ca.uhn.fhir.interceptor.model.RequestPartitionId;
028import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
029import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
030import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
031import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
032import ca.uhn.fhir.jpa.dao.HistoryBuilder;
033import ca.uhn.fhir.jpa.dao.HistoryBuilderFactory;
034import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser;
035import ca.uhn.fhir.jpa.dao.ISearchBuilder;
036import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
037import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
038import ca.uhn.fhir.jpa.entity.Search;
039import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
040import ca.uhn.fhir.jpa.model.dao.JpaPid;
041import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
042import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
043import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
044import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
045import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum;
046import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
047import ca.uhn.fhir.jpa.util.MemoryCacheService;
048import ca.uhn.fhir.jpa.util.QueryParameterUtils;
049import ca.uhn.fhir.model.primitive.InstantDt;
050import ca.uhn.fhir.rest.api.server.IBundleProvider;
051import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
052import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
053import ca.uhn.fhir.rest.api.server.RequestDetails;
054import ca.uhn.fhir.rest.api.server.SimplePreResourceAccessDetails;
055import ca.uhn.fhir.rest.api.server.SimplePreResourceShowDetails;
056import ca.uhn.fhir.rest.server.interceptor.ServerInterceptorUtil;
057import ca.uhn.fhir.rest.server.method.ResponsePage;
058import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
059import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
060import com.google.common.annotations.VisibleForTesting;
061import jakarta.annotation.Nonnull;
062import jakarta.persistence.EntityManager;
063import jakarta.persistence.PersistenceContext;
064import org.hl7.fhir.instance.model.api.IBaseResource;
065import org.slf4j.Logger;
066import org.slf4j.LoggerFactory;
067import org.springframework.beans.factory.annotation.Autowired;
068
069import java.util.ArrayList;
070import java.util.Collections;
071import java.util.List;
072import java.util.Optional;
073import java.util.Set;
074import java.util.function.Function;
075
076public class PersistedJpaBundleProvider implements IBundleProvider {
077
078        private static final Logger ourLog = LoggerFactory.getLogger(PersistedJpaBundleProvider.class);
079
080        /*
081         * Autowired fields
082         */
083        protected final RequestDetails myRequest;
084
085        @Autowired
086        protected HapiTransactionService myTxService;
087
088        @PersistenceContext
089        private EntityManager myEntityManager;
090
091        @Autowired
092        private IInterceptorBroadcaster myInterceptorBroadcaster;
093
094        @Autowired
095        private SearchBuilderFactory<JpaPid> mySearchBuilderFactory;
096
097        @Autowired
098        private HistoryBuilderFactory myHistoryBuilderFactory;
099
100        @Autowired
101        private DaoRegistry myDaoRegistry;
102
103        @Autowired
104        private FhirContext myContext;
105
106        @Autowired
107        private ISearchCoordinatorSvc<JpaPid> mySearchCoordinatorSvc;
108
109        @Autowired
110        private ISearchCacheSvc mySearchCacheSvc;
111
112        @Autowired
113        private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
114
115        @Autowired
116        private JpaStorageSettings myStorageSettings;
117
118        @Autowired
119        private MemoryCacheService myMemoryCacheService;
120
121        @Autowired
122        private IJpaStorageResourceParser myJpaStorageResourceParser;
123        /*
124         * Non autowired fields (will be different for every instance
125         * of this class, since it's a prototype
126         */
127        private Search mySearchEntity;
128        private final String myUuid;
129        private SearchCacheStatusEnum myCacheStatus;
130        private RequestPartitionId myRequestPartitionId;
131
132        /**
133         * Constructor
134         */
135        public PersistedJpaBundleProvider(RequestDetails theRequest, String theSearchUuid) {
136                myRequest = theRequest;
137                myUuid = theSearchUuid;
138        }
139
140        /**
141         * Constructor
142         */
143        public PersistedJpaBundleProvider(RequestDetails theRequest, Search theSearch) {
144                myRequest = theRequest;
145                mySearchEntity = theSearch;
146                myUuid = theSearch.getUuid();
147        }
148
149        @VisibleForTesting
150        public void setRequestPartitionHelperSvcForUnitTest(IRequestPartitionHelperSvc theRequestPartitionHelperSvc) {
151                myRequestPartitionHelperSvc = theRequestPartitionHelperSvc;
152        }
153
154        @VisibleForTesting
155        public Search getSearchEntityForTesting() {
156                return getSearchEntity();
157        }
158
159        protected Search getSearchEntity() {
160                return mySearchEntity;
161        }
162
163        // Note: Leave as protected, HSPC depends on this
164        @SuppressWarnings("WeakerAccess")
165        protected void setSearchEntity(Search theSearchEntity) {
166                mySearchEntity = theSearchEntity;
167        }
168
169        /**
170         * Perform a history search
171         */
172        private List<IBaseResource> doHistoryInTransaction(Integer theOffset, int theFromIndex, int theToIndex) {
173
174                HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
175                                mySearchEntity.getResourceType(),
176                                mySearchEntity.getResourceId(),
177                                mySearchEntity.getLastUpdatedLow(),
178                                mySearchEntity.getLastUpdatedHigh());
179
180                RequestPartitionId partitionId = getRequestPartitionId();
181                List<ResourceHistoryTable> results = historyBuilder.fetchEntities(
182                                partitionId, theOffset, theFromIndex, theToIndex, mySearchEntity.getHistorySearchStyle());
183
184                List<IBaseResource> retVal = new ArrayList<>();
185                for (ResourceHistoryTable next : results) {
186                        BaseHasResource resource;
187                        resource = next;
188
189                        retVal.add(myJpaStorageResourceParser.toResource(resource, true));
190                }
191
192                // Interceptor call: STORAGE_PREACCESS_RESOURCES
193                {
194                        SimplePreResourceAccessDetails accessDetails = new SimplePreResourceAccessDetails(retVal);
195                        HookParams params = new HookParams()
196                                        .add(IPreResourceAccessDetails.class, accessDetails)
197                                        .add(RequestDetails.class, myRequest)
198                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
199                        CompositeInterceptorBroadcaster.doCallHooks(
200                                        myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
201
202                        for (int i = retVal.size() - 1; i >= 0; i--) {
203                                if (accessDetails.isDontReturnResourceAtIndex(i)) {
204                                        retVal.remove(i);
205                                }
206                        }
207                }
208
209                // Interceptor broadcast: STORAGE_PRESHOW_RESOURCES
210                {
211                        SimplePreResourceShowDetails showDetails = new SimplePreResourceShowDetails(retVal);
212                        HookParams params = new HookParams()
213                                        .add(IPreResourceShowDetails.class, showDetails)
214                                        .add(RequestDetails.class, myRequest)
215                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
216                        CompositeInterceptorBroadcaster.doCallHooks(
217                                        myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PRESHOW_RESOURCES, params);
218                        retVal = showDetails.toList();
219                }
220
221                return retVal;
222        }
223
224        @Nonnull
225        protected final RequestPartitionId getRequestPartitionId() {
226                if (myRequestPartitionId == null) {
227                        ReadPartitionIdRequestDetails details;
228                        if (mySearchEntity == null) {
229                                details = ReadPartitionIdRequestDetails.forSearchUuid(myUuid);
230                        } else if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
231                                details = ReadPartitionIdRequestDetails.forHistory(mySearchEntity.getResourceType(), null);
232                        } else {
233                                SearchParameterMap params =
234                                                mySearchEntity.getSearchParameterMap().orElse(null);
235                                details = ReadPartitionIdRequestDetails.forSearchType(mySearchEntity.getResourceType(), params, null);
236                        }
237                        myRequestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequest(myRequest, details);
238                }
239                return myRequestPartitionId;
240        }
241
242        public void setRequestPartitionId(RequestPartitionId theRequestPartitionId) {
243                myRequestPartitionId = theRequestPartitionId;
244        }
245
246        protected List<IBaseResource> doSearchOrEverything(
247                        final int theFromIndex,
248                        final int theToIndex,
249                        @Nonnull ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
250                if (mySearchEntity.getTotalCount() != null && mySearchEntity.getNumFound() <= 0) {
251                        // No resources to fetch (e.g. we did a _summary=count search)
252                        return Collections.emptyList();
253                }
254                String resourceName = mySearchEntity.getResourceType();
255                Class<? extends IBaseResource> resourceType =
256                                myContext.getResourceDefinition(resourceName).getImplementingClass();
257                IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceName);
258
259                final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(dao, resourceName, resourceType);
260
261                RequestPartitionId requestPartitionId = getRequestPartitionId();
262                // we request 1 more resource than we need
263                // this is so we can be sure of when we hit the last page
264                // (when doing offset searches)
265                final List<JpaPid> pidsSubList = mySearchCoordinatorSvc.getResources(
266                                myUuid, theFromIndex, theToIndex + 1, myRequest, requestPartitionId);
267                // max list size should be either the entire list, or from - to length
268                int maxSize = Math.min(theToIndex - theFromIndex, pidsSubList.size());
269                theResponsePageBuilder.setTotalRequestedResourcesFetched(pidsSubList.size());
270
271                List<JpaPid> firstBatchOfPids = pidsSubList.subList(0, maxSize);
272                List<IBaseResource> resources = myTxService
273                                .withRequest(myRequest)
274                                .withRequestPartitionId(requestPartitionId)
275                                .execute(() -> {
276                                        return toResourceList(sb, firstBatchOfPids, theResponsePageBuilder);
277                                });
278
279                return resources;
280        }
281
282        /**
283         * Returns false if the entity can't be found
284         */
285        public boolean ensureSearchEntityLoaded() {
286                if (mySearchEntity == null) {
287                        Optional<Search> searchOpt = myTxService
288                                        .withRequest(myRequest)
289                                        .withRequestPartitionId(myRequestPartitionId)
290                                        .execute(() -> mySearchCacheSvc.fetchByUuid(myUuid, myRequestPartitionId));
291                        if (!searchOpt.isPresent()) {
292                                return false;
293                        }
294
295                        setSearchEntity(searchOpt.get());
296
297                        ourLog.trace(
298                                        "Retrieved search with version {} and total {}",
299                                        mySearchEntity.getVersion(),
300                                        mySearchEntity.getTotalCount());
301
302                        return true;
303                }
304
305                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
306                        if (mySearchEntity.getTotalCount() == null) {
307                                calculateHistoryCount();
308                        }
309                }
310
311                return true;
312        }
313
314        /**
315         * Note that this method is called outside a DB transaction, and uses a loading cache
316         * (assuming the default {@literal COUNT_CACHED} mode) so this effectively throttles
317         * access to the database by preventing multiple concurrent DB calls for an expensive
318         * count operation.
319         */
320        private void calculateHistoryCount() {
321                MemoryCacheService.HistoryCountKey key;
322                if (mySearchEntity.getResourceId() != null) {
323                        key = MemoryCacheService.HistoryCountKey.forInstance(mySearchEntity.getResourceId());
324                } else if (mySearchEntity.getResourceType() != null) {
325                        key = MemoryCacheService.HistoryCountKey.forType(mySearchEntity.getResourceType());
326                } else {
327                        key = MemoryCacheService.HistoryCountKey.forSystem();
328                }
329
330                Function<MemoryCacheService.HistoryCountKey, Integer> supplier = k -> myTxService
331                                .withRequest(myRequest)
332                                .withRequestPartitionId(getRequestPartitionId())
333                                .execute(() -> {
334                                        HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
335                                                        mySearchEntity.getResourceType(),
336                                                        mySearchEntity.getResourceId(),
337                                                        mySearchEntity.getLastUpdatedLow(),
338                                                        mySearchEntity.getLastUpdatedHigh());
339                                        Long count = historyBuilder.fetchCount(getRequestPartitionId());
340                                        return count.intValue();
341                                });
342
343                boolean haveOffset = mySearchEntity.getLastUpdatedLow() != null || mySearchEntity.getLastUpdatedHigh() != null;
344
345                switch (myStorageSettings.getHistoryCountMode()) {
346                        case COUNT_ACCURATE: {
347                                int count = supplier.apply(key);
348                                mySearchEntity.setTotalCount(count);
349                                break;
350                        }
351                        case CACHED_ONLY_WITHOUT_OFFSET: {
352                                if (!haveOffset) {
353                                        int count = myMemoryCacheService.get(MemoryCacheService.CacheEnum.HISTORY_COUNT, key, supplier);
354                                        mySearchEntity.setTotalCount(count);
355                                }
356                                break;
357                        }
358                        case COUNT_DISABLED: {
359                                break;
360                        }
361                }
362        }
363
364        @Override
365        public InstantDt getPublished() {
366                ensureSearchEntityLoaded();
367                return new InstantDt(mySearchEntity.getCreated());
368        }
369
370        @Nonnull
371        @Override
372        public List<IBaseResource> getResources(int theFromIndex, int theToIndex) {
373                return getResources(theFromIndex, theToIndex, new ResponsePage.ResponsePageBuilder());
374        }
375
376        @Override
377        public List<IBaseResource> getResources(
378                        int theFromIndex, int theToIndex, @Nonnull ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
379                boolean entityLoaded = ensureSearchEntityLoaded();
380                assert entityLoaded;
381                assert mySearchEntity != null;
382                assert mySearchEntity.getSearchType() != null;
383
384                switch (mySearchEntity.getSearchType()) {
385                        case HISTORY:
386                                return myTxService
387                                                .withRequest(myRequest)
388                                                .withRequestPartitionId(getRequestPartitionId())
389                                                .execute(() -> doHistoryInTransaction(mySearchEntity.getOffset(), theFromIndex, theToIndex));
390                        case SEARCH:
391                        case EVERYTHING:
392                        default:
393                                List<IBaseResource> retVal = doSearchOrEverything(theFromIndex, theToIndex, theResponsePageBuilder);
394                                /*
395                                 * If we got fewer resources back than we asked for, it's possible that the search
396                                 * completed. If that's the case, the cached version of the search entity is probably
397                                 * no longer valid so let's force a reload if it gets asked for again (most likely
398                                 * because someone is calling size() on us)
399                                 */
400                                if (retVal.size() < theToIndex - theFromIndex) {
401                                        mySearchEntity = null;
402                                }
403                                return retVal;
404                }
405        }
406
407        @Override
408        public String getUuid() {
409                return myUuid;
410        }
411
412        public SearchCacheStatusEnum getCacheStatus() {
413                return myCacheStatus;
414        }
415
416        void setCacheStatus(SearchCacheStatusEnum theSearchCacheStatusEnum) {
417                myCacheStatus = theSearchCacheStatusEnum;
418        }
419
420        @Override
421        public Integer preferredPageSize() {
422                ensureSearchEntityLoaded();
423                return mySearchEntity.getPreferredPageSize();
424        }
425
426        public void setContext(FhirContext theContext) {
427                myContext = theContext;
428        }
429
430        public void setEntityManager(EntityManager theEntityManager) {
431                myEntityManager = theEntityManager;
432        }
433
434        @VisibleForTesting
435        public void setSearchCoordinatorSvcForUnitTest(ISearchCoordinatorSvc theSearchCoordinatorSvc) {
436                mySearchCoordinatorSvc = theSearchCoordinatorSvc;
437        }
438
439        @VisibleForTesting
440        public void setTxServiceForUnitTest(HapiTransactionService theTxManager) {
441                myTxService = theTxManager;
442        }
443
444        @Override
445        public Integer size() {
446                ensureSearchEntityLoaded();
447                QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearchEntity);
448
449                Integer size = mySearchEntity.getTotalCount();
450                if (size != null) {
451                        return Math.max(0, size);
452                }
453
454                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
455                        return null;
456                } else {
457                        return mySearchCoordinatorSvc
458                                        .getSearchTotal(myUuid, myRequest, myRequestPartitionId)
459                                        .orElse(null);
460                }
461        }
462
463        protected boolean hasIncludes() {
464                ensureSearchEntityLoaded();
465                return !mySearchEntity.getIncludes().isEmpty();
466        }
467
468        // Note: Leave as protected, HSPC depends on this
469        @SuppressWarnings("WeakerAccess")
470        protected List<IBaseResource> toResourceList(
471                        ISearchBuilder theSearchBuilder,
472                        List<JpaPid> thePids,
473                        ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
474                List<JpaPid> includedPidList = new ArrayList<>();
475                if (mySearchEntity.getSearchType() == SearchTypeEnum.SEARCH) {
476                        Integer maxIncludes = myStorageSettings.getMaximumIncludesToLoadPerPage();
477
478                        // Load non-iterate _revincludes
479                        Set<JpaPid> nonIterateRevIncludedPids = theSearchBuilder.loadIncludes(
480                                        myContext,
481                                        myEntityManager,
482                                        thePids,
483                                        mySearchEntity.toRevIncludesList(false),
484                                        true,
485                                        mySearchEntity.getLastUpdated(),
486                                        myUuid,
487                                        myRequest,
488                                        maxIncludes);
489                        if (maxIncludes != null) {
490                                maxIncludes -= nonIterateRevIncludedPids.size();
491                        }
492                        thePids.addAll(nonIterateRevIncludedPids);
493                        includedPidList.addAll(nonIterateRevIncludedPids);
494
495                        // Load non-iterate _includes
496                        Set<JpaPid> nonIterateIncludedPids = theSearchBuilder.loadIncludes(
497                                        myContext,
498                                        myEntityManager,
499                                        thePids,
500                                        mySearchEntity.toIncludesList(false),
501                                        false,
502                                        mySearchEntity.getLastUpdated(),
503                                        myUuid,
504                                        myRequest,
505                                        maxIncludes);
506                        if (maxIncludes != null) {
507                                maxIncludes -= nonIterateIncludedPids.size();
508                        }
509                        thePids.addAll(nonIterateIncludedPids);
510                        includedPidList.addAll(nonIterateIncludedPids);
511
512                        // Load iterate _revinclude
513                        Set<JpaPid> iterateRevIncludedPids = theSearchBuilder.loadIncludes(
514                                        myContext,
515                                        myEntityManager,
516                                        thePids,
517                                        mySearchEntity.toRevIncludesList(true),
518                                        true,
519                                        mySearchEntity.getLastUpdated(),
520                                        myUuid,
521                                        myRequest,
522                                        maxIncludes);
523                        if (maxIncludes != null) {
524                                maxIncludes -= iterateRevIncludedPids.size();
525                        }
526                        thePids.addAll(iterateRevIncludedPids);
527                        includedPidList.addAll(iterateRevIncludedPids);
528
529                        // Load iterate _includes
530                        Set<JpaPid> iterateIncludedPids = theSearchBuilder.loadIncludes(
531                                        myContext,
532                                        myEntityManager,
533                                        thePids,
534                                        mySearchEntity.toIncludesList(true),
535                                        false,
536                                        mySearchEntity.getLastUpdated(),
537                                        myUuid,
538                                        myRequest,
539                                        maxIncludes);
540                        thePids.addAll(iterateIncludedPids);
541                        includedPidList.addAll(iterateIncludedPids);
542                }
543
544                // Execute the query and make sure we return distinct results
545                List<IBaseResource> resources = new ArrayList<>();
546                theSearchBuilder.loadResourcesByPid(thePids, includedPidList, resources, false, myRequest);
547
548                // we will send the resource list to our interceptors
549                // this can (potentially) change the results being returned.
550                int precount = resources.size();
551                resources = ServerInterceptorUtil.fireStoragePreshowResource(resources, myRequest, myInterceptorBroadcaster);
552                // we only care about omitted results from this page
553                theResponsePageBuilder.setOmittedResourceCount(precount - resources.size());
554                theResponsePageBuilder.setResources(resources);
555                theResponsePageBuilder.setIncludedResourceCount(includedPidList.size());
556
557                return resources;
558        }
559
560        public void setInterceptorBroadcaster(IInterceptorBroadcaster theInterceptorBroadcaster) {
561                myInterceptorBroadcaster = theInterceptorBroadcaster;
562        }
563
564        @VisibleForTesting
565        public void setSearchCacheSvcForUnitTest(ISearchCacheSvc theSearchCacheSvc) {
566                mySearchCacheSvc = theSearchCacheSvc;
567        }
568
569        @VisibleForTesting
570        public void setDaoRegistryForUnitTest(DaoRegistry theDaoRegistry) {
571                myDaoRegistry = theDaoRegistry;
572        }
573
574        @VisibleForTesting
575        public void setStorageSettingsForUnitTest(JpaStorageSettings theStorageSettings) {
576                myStorageSettings = theStorageSettings;
577        }
578
579        @VisibleForTesting
580        public void setSearchBuilderFactoryForUnitTest(SearchBuilderFactory theSearchBuilderFactory) {
581                mySearchBuilderFactory = theSearchBuilderFactory;
582        }
583}