Source code

001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2025 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.search;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.interceptor.api.HookParams;
024import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
025import ca.uhn.fhir.interceptor.api.Pointcut;
026import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
027import ca.uhn.fhir.interceptor.model.RequestPartitionId;
028import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
029import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
030import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
031import ca.uhn.fhir.jpa.dao.HistoryBuilder;
032import ca.uhn.fhir.jpa.dao.HistoryBuilderFactory;
033import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser;
034import ca.uhn.fhir.jpa.dao.ISearchBuilder;
035import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
036import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
037import ca.uhn.fhir.jpa.entity.Search;
038import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
039import ca.uhn.fhir.jpa.model.dao.JpaPid;
040import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
041import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
042import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
043import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
044import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum;
045import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
046import ca.uhn.fhir.jpa.util.MemoryCacheService;
047import ca.uhn.fhir.jpa.util.QueryParameterUtils;
048import ca.uhn.fhir.model.primitive.InstantDt;
049import ca.uhn.fhir.rest.api.server.IBundleProvider;
050import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
051import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
052import ca.uhn.fhir.rest.api.server.RequestDetails;
053import ca.uhn.fhir.rest.api.server.SimplePreResourceAccessDetails;
054import ca.uhn.fhir.rest.api.server.SimplePreResourceShowDetails;
055import ca.uhn.fhir.rest.server.interceptor.ServerInterceptorUtil;
056import ca.uhn.fhir.rest.server.method.ResponsePage;
057import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
058import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
059import com.google.common.annotations.VisibleForTesting;
060import jakarta.annotation.Nonnull;
061import jakarta.persistence.EntityManager;
062import jakarta.persistence.PersistenceContext;
063import org.hl7.fhir.instance.model.api.IBaseResource;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066import org.springframework.beans.factory.annotation.Autowired;
067
068import java.util.ArrayList;
069import java.util.Collections;
070import java.util.List;
071import java.util.Optional;
072import java.util.Set;
073import java.util.function.Function;
074
075public class PersistedJpaBundleProvider implements IBundleProvider {
076
077        private static final Logger ourLog = LoggerFactory.getLogger(PersistedJpaBundleProvider.class);
078
079        /*
080         * Autowired fields
081         */
082        protected final RequestDetails myRequest;
083
084        @Autowired
085        protected HapiTransactionService myTxService;
086
087        @PersistenceContext
088        private EntityManager myEntityManager;
089
090        @Autowired
091        private IInterceptorBroadcaster myInterceptorBroadcaster;
092
093        @Autowired
094        private SearchBuilderFactory<JpaPid> mySearchBuilderFactory;
095
096        @Autowired
097        private HistoryBuilderFactory myHistoryBuilderFactory;
098
099        @Autowired
100        private DaoRegistry myDaoRegistry;
101
102        @Autowired
103        private FhirContext myContext;
104
105        @Autowired
106        private ISearchCoordinatorSvc<JpaPid> mySearchCoordinatorSvc;
107
108        @Autowired
109        private ISearchCacheSvc mySearchCacheSvc;
110
111        @Autowired
112        private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
113
114        @Autowired
115        private JpaStorageSettings myStorageSettings;
116
117        @Autowired
118        private MemoryCacheService myMemoryCacheService;
119
120        @Autowired
121        private IJpaStorageResourceParser myJpaStorageResourceParser;
122        /*
123         * Non autowired fields (will be different for every instance
124         * of this class, since it's a prototype
125         */
126        private Search mySearchEntity;
127        private final String myUuid;
128        private SearchCacheStatusEnum myCacheStatus;
129        private RequestPartitionId myRequestPartitionId;
130
131        /**
132         * Constructor
133         */
134        public PersistedJpaBundleProvider(RequestDetails theRequest, String theSearchUuid) {
135                myRequest = theRequest;
136                myUuid = theSearchUuid;
137        }
138
139        /**
140         * Constructor
141         */
142        public PersistedJpaBundleProvider(RequestDetails theRequest, Search theSearch) {
143                myRequest = theRequest;
144                mySearchEntity = theSearch;
145                myUuid = theSearch.getUuid();
146        }
147
148        @VisibleForTesting
149        public void setRequestPartitionHelperSvcForUnitTest(IRequestPartitionHelperSvc theRequestPartitionHelperSvc) {
150                myRequestPartitionHelperSvc = theRequestPartitionHelperSvc;
151        }
152
153        @VisibleForTesting
154        public Search getSearchEntityForTesting() {
155                return getSearchEntity();
156        }
157
158        protected Search getSearchEntity() {
159                return mySearchEntity;
160        }
161
162        // Note: Leave as protected, HSPC depends on this
163        @SuppressWarnings("WeakerAccess")
164        protected void setSearchEntity(Search theSearchEntity) {
165                mySearchEntity = theSearchEntity;
166        }
167
168        /**
169         * Perform a history search
170         */
171        private List<IBaseResource> doHistoryInTransaction(Integer theOffset, int theFromIndex, int theToIndex) {
172
173                HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
174                                mySearchEntity.getResourceType(),
175                                mySearchEntity.getResourceId(),
176                                mySearchEntity.getLastUpdatedLow(),
177                                mySearchEntity.getLastUpdatedHigh());
178
179                RequestPartitionId partitionId = getRequestPartitionId();
180                List<ResourceHistoryTable> results = historyBuilder.fetchEntities(
181                                partitionId, theOffset, theFromIndex, theToIndex, mySearchEntity.getHistorySearchStyle());
182
183                List<IBaseResource> retVal = new ArrayList<>();
184                for (ResourceHistoryTable next : results) {
185                        BaseHasResource resource;
186                        resource = next;
187
188                        retVal.add(myJpaStorageResourceParser.toResource(resource, true));
189                }
190
191                IInterceptorBroadcaster compositeBroadcaster =
192                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, myRequest);
193
194                // Interceptor call: STORAGE_PREACCESS_RESOURCES
195                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_PREACCESS_RESOURCES)) {
196                        SimplePreResourceAccessDetails accessDetails = new SimplePreResourceAccessDetails(retVal);
197                        HookParams params = new HookParams()
198                                        .add(IPreResourceAccessDetails.class, accessDetails)
199                                        .add(RequestDetails.class, myRequest)
200                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
201                        compositeBroadcaster.callHooks(Pointcut.STORAGE_PREACCESS_RESOURCES, params);
202
203                        for (int i = retVal.size() - 1; i >= 0; i--) {
204                                if (accessDetails.isDontReturnResourceAtIndex(i)) {
205                                        retVal.remove(i);
206                                }
207                        }
208                }
209
210                // Interceptor broadcast: STORAGE_PRESHOW_RESOURCES
211                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_PRESHOW_RESOURCES)) {
212                        SimplePreResourceShowDetails showDetails = new SimplePreResourceShowDetails(retVal);
213                        HookParams params = new HookParams()
214                                        .add(IPreResourceShowDetails.class, showDetails)
215                                        .add(RequestDetails.class, myRequest)
216                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
217                        compositeBroadcaster.callHooks(Pointcut.STORAGE_PRESHOW_RESOURCES, params);
218                        retVal = showDetails.toList();
219                }
220
221                return retVal;
222        }
223
224        @Nonnull
225        protected final RequestPartitionId getRequestPartitionId() {
226                if (myRequestPartitionId == null) {
227                        ReadPartitionIdRequestDetails details;
228                        if (mySearchEntity == null) {
229                                details = ReadPartitionIdRequestDetails.forSearchUuid(myUuid);
230                        } else if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
231                                details = ReadPartitionIdRequestDetails.forHistory(mySearchEntity.getResourceType(), null);
232                        } else {
233                                SearchParameterMap params =
234                                                mySearchEntity.getSearchParameterMap().orElse(null);
235                                details = ReadPartitionIdRequestDetails.forSearchType(mySearchEntity.getResourceType(), params, null);
236                        }
237                        myRequestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequest(myRequest, details);
238                }
239                assert myRequestPartitionId != null;
240                return myRequestPartitionId;
241        }
242
243        public void setRequestPartitionId(RequestPartitionId theRequestPartitionId) {
244                myRequestPartitionId = theRequestPartitionId;
245        }
246
247        protected List<IBaseResource> doSearchOrEverything(
248                        final int theFromIndex,
249                        final int theToIndex,
250                        @Nonnull ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
251                if (mySearchEntity.getTotalCount() != null && mySearchEntity.getNumFound() <= 0) {
252                        // No resources to fetch (e.g. we did a _summary=count search)
253                        return Collections.emptyList();
254                }
255                String resourceName = mySearchEntity.getResourceType();
256                Class<? extends IBaseResource> resourceType =
257                                myContext.getResourceDefinition(resourceName).getImplementingClass();
258
259                final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(resourceName, resourceType);
260
261                RequestPartitionId requestPartitionId = getRequestPartitionId();
262                // we request 1 more resource than we need
263                // this is so we can be sure of when we hit the last page
264                // (when doing offset searches)
265                final List<JpaPid> pidsSubList = mySearchCoordinatorSvc.getResources(
266                                myUuid, theFromIndex, theToIndex + 1, myRequest, requestPartitionId);
267                // max list size should be either the entire list, or from - to length
268                int maxSize = Math.min(theToIndex - theFromIndex, pidsSubList.size());
269                theResponsePageBuilder.setTotalRequestedResourcesFetched(pidsSubList.size());
270
271                List<JpaPid> firstBatchOfPids = pidsSubList.subList(0, maxSize);
272                List<IBaseResource> resources = myTxService
273                                .withRequest(myRequest)
274                                .withRequestPartitionId(requestPartitionId)
275                                .execute(() -> {
276                                        return toResourceList(sb, firstBatchOfPids, theResponsePageBuilder);
277                                });
278
279                return resources;
280        }
281
282        /**
283         * Returns false if the entity can't be found
284         */
285        public boolean ensureSearchEntityLoaded() {
286                if (mySearchEntity == null) {
287                        Optional<Search> searchOpt = myTxService
288                                        .withRequest(myRequest)
289                                        .withRequestPartitionId(myRequestPartitionId)
290                                        .execute(() -> mySearchCacheSvc.fetchByUuid(myUuid, myRequestPartitionId));
291                        if (!searchOpt.isPresent()) {
292                                return false;
293                        }
294
295                        setSearchEntity(searchOpt.get());
296
297                        ourLog.trace(
298                                        "Retrieved search with version {} and total {}",
299                                        mySearchEntity.getVersion(),
300                                        mySearchEntity.getTotalCount());
301
302                        return true;
303                }
304
305                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
306                        if (mySearchEntity.getTotalCount() == null) {
307                                calculateHistoryCount();
308                        }
309                }
310
311                return true;
312        }
313
314        /**
315         * Note that this method is called outside a DB transaction, and uses a loading cache
316         * (assuming the default {@literal COUNT_CACHED} mode) so this effectively throttles
317         * access to the database by preventing multiple concurrent DB calls for an expensive
318         * count operation.
319         */
320        private void calculateHistoryCount() {
321                MemoryCacheService.HistoryCountKey key;
322                if (mySearchEntity.getResourceId() != null) {
323                        key = MemoryCacheService.HistoryCountKey.forInstance(mySearchEntity.getResourceId());
324                } else if (mySearchEntity.getResourceType() != null) {
325                        key = MemoryCacheService.HistoryCountKey.forType(mySearchEntity.getResourceType());
326                } else {
327                        key = MemoryCacheService.HistoryCountKey.forSystem();
328                }
329
330                Function<MemoryCacheService.HistoryCountKey, Integer> supplier = k -> myTxService
331                                .withRequest(myRequest)
332                                .withRequestPartitionId(getRequestPartitionId())
333                                .execute(() -> {
334                                        HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
335                                                        mySearchEntity.getResourceType(),
336                                                        mySearchEntity.getResourceId(),
337                                                        mySearchEntity.getLastUpdatedLow(),
338                                                        mySearchEntity.getLastUpdatedHigh());
339                                        Long count = historyBuilder.fetchCount(getRequestPartitionId());
340                                        return count.intValue();
341                                });
342
343                boolean haveOffset = mySearchEntity.getLastUpdatedLow() != null || mySearchEntity.getLastUpdatedHigh() != null;
344
345                switch (myStorageSettings.getHistoryCountMode()) {
346                        case COUNT_ACCURATE: {
347                                int count = supplier.apply(key);
348                                mySearchEntity.setTotalCount(count);
349                                break;
350                        }
351                        case CACHED_ONLY_WITHOUT_OFFSET: {
352                                if (!haveOffset) {
353                                        int count = myMemoryCacheService.get(MemoryCacheService.CacheEnum.HISTORY_COUNT, key, supplier);
354                                        mySearchEntity.setTotalCount(count);
355                                }
356                                break;
357                        }
358                        case COUNT_DISABLED: {
359                                break;
360                        }
361                }
362        }
363
364        @Override
365        public InstantDt getPublished() {
366                ensureSearchEntityLoaded();
367                return new InstantDt(mySearchEntity.getCreated());
368        }
369
370        @Nonnull
371        @Override
372        public List<IBaseResource> getResources(int theFromIndex, int theToIndex) {
373                return getResources(theFromIndex, theToIndex, new ResponsePage.ResponsePageBuilder());
374        }
375
376        @Override
377        public List<IBaseResource> getResources(
378                        int theFromIndex, int theToIndex, @Nonnull ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
379                boolean entityLoaded = ensureSearchEntityLoaded();
380                assert entityLoaded;
381                assert mySearchEntity != null;
382                assert mySearchEntity.getSearchType() != null;
383
384                switch (mySearchEntity.getSearchType()) {
385                        case HISTORY:
386                                return myTxService
387                                                .withRequest(myRequest)
388                                                .withRequestPartitionId(getRequestPartitionId())
389                                                .execute(() -> doHistoryInTransaction(mySearchEntity.getOffset(), theFromIndex, theToIndex));
390                        case SEARCH:
391                        case EVERYTHING:
392                        default:
393                                List<IBaseResource> retVal = doSearchOrEverything(theFromIndex, theToIndex, theResponsePageBuilder);
394                                /*
395                                 * If we got fewer resources back than we asked for, it's possible that the search
396                                 * completed. If that's the case, the cached version of the search entity is probably
397                                 * no longer valid so let's force a reload if it gets asked for again (most likely
398                                 * because someone is calling size() on us)
399                                 */
400                                if (retVal.size() < theToIndex - theFromIndex) {
401                                        mySearchEntity = null;
402                                }
403                                return retVal;
404                }
405        }
406
407        @Override
408        public String getUuid() {
409                return myUuid;
410        }
411
412        public SearchCacheStatusEnum getCacheStatus() {
413                return myCacheStatus;
414        }
415
416        void setCacheStatus(SearchCacheStatusEnum theSearchCacheStatusEnum) {
417                myCacheStatus = theSearchCacheStatusEnum;
418        }
419
420        @Override
421        public Integer preferredPageSize() {
422                ensureSearchEntityLoaded();
423                return mySearchEntity.getPreferredPageSize();
424        }
425
426        public void setContext(FhirContext theContext) {
427                myContext = theContext;
428        }
429
430        public void setEntityManager(EntityManager theEntityManager) {
431                myEntityManager = theEntityManager;
432        }
433
434        @VisibleForTesting
435        public void setSearchCoordinatorSvcForUnitTest(ISearchCoordinatorSvc theSearchCoordinatorSvc) {
436                mySearchCoordinatorSvc = theSearchCoordinatorSvc;
437        }
438
439        @VisibleForTesting
440        public void setTxServiceForUnitTest(HapiTransactionService theTxManager) {
441                myTxService = theTxManager;
442        }
443
444        @Override
445        public Integer size() {
446                ensureSearchEntityLoaded();
447                QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearchEntity);
448
449                Integer size = mySearchEntity.getTotalCount();
450                if (size != null) {
451                        return Math.max(0, size);
452                }
453
454                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
455                        return null;
456                } else {
457                        return mySearchCoordinatorSvc
458                                        .getSearchTotal(myUuid, myRequest, myRequestPartitionId)
459                                        .orElse(null);
460                }
461        }
462
463        protected boolean hasIncludes() {
464                ensureSearchEntityLoaded();
465                return !mySearchEntity.getIncludes().isEmpty();
466        }
467
468        // Note: Leave as protected, HSPC depends on this
469        @SuppressWarnings("WeakerAccess")
470        protected List<IBaseResource> toResourceList(
471                        ISearchBuilder theSearchBuilder,
472                        List<JpaPid> thePids,
473                        ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
474                List<JpaPid> includedPidList = new ArrayList<>();
475                if (mySearchEntity.getSearchType() == SearchTypeEnum.SEARCH) {
476                        Integer maxIncludes = myStorageSettings.getMaximumIncludesToLoadPerPage();
477
478                        // Load non-iterate _revincludes
479                        Set<JpaPid> nonIterateRevIncludedPids = theSearchBuilder.loadIncludes(
480                                        myContext,
481                                        myEntityManager,
482                                        thePids,
483                                        mySearchEntity.toRevIncludesList(false),
484                                        true,
485                                        mySearchEntity.getLastUpdated(),
486                                        myUuid,
487                                        myRequest,
488                                        maxIncludes);
489                        if (maxIncludes != null) {
490                                maxIncludes -= nonIterateRevIncludedPids.size();
491                        }
492                        thePids.addAll(nonIterateRevIncludedPids);
493                        includedPidList.addAll(nonIterateRevIncludedPids);
494
495                        // Load non-iterate _includes
496                        Set<JpaPid> nonIterateIncludedPids = theSearchBuilder.loadIncludes(
497                                        myContext,
498                                        myEntityManager,
499                                        thePids,
500                                        mySearchEntity.toIncludesList(false),
501                                        false,
502                                        mySearchEntity.getLastUpdated(),
503                                        myUuid,
504                                        myRequest,
505                                        maxIncludes);
506                        if (maxIncludes != null) {
507                                maxIncludes -= nonIterateIncludedPids.size();
508                        }
509                        thePids.addAll(nonIterateIncludedPids);
510                        includedPidList.addAll(nonIterateIncludedPids);
511
512                        // Load iterate _revinclude
513                        Set<JpaPid> iterateRevIncludedPids = theSearchBuilder.loadIncludes(
514                                        myContext,
515                                        myEntityManager,
516                                        thePids,
517                                        mySearchEntity.toRevIncludesList(true),
518                                        true,
519                                        mySearchEntity.getLastUpdated(),
520                                        myUuid,
521                                        myRequest,
522                                        maxIncludes);
523                        if (maxIncludes != null) {
524                                maxIncludes -= iterateRevIncludedPids.size();
525                        }
526                        thePids.addAll(iterateRevIncludedPids);
527                        includedPidList.addAll(iterateRevIncludedPids);
528
529                        // Load iterate _includes
530                        Set<JpaPid> iterateIncludedPids = theSearchBuilder.loadIncludes(
531                                        myContext,
532                                        myEntityManager,
533                                        thePids,
534                                        mySearchEntity.toIncludesList(true),
535                                        false,
536                                        mySearchEntity.getLastUpdated(),
537                                        myUuid,
538                                        myRequest,
539                                        maxIncludes);
540                        thePids.addAll(iterateIncludedPids);
541                        includedPidList.addAll(iterateIncludedPids);
542                }
543
544                // Execute the query and make sure we return distinct results
545                List<IBaseResource> resources = new ArrayList<>();
546                theSearchBuilder.loadResourcesByPid(thePids, includedPidList, resources, false, myRequest);
547
548                // we will send the resource list to our interceptors
549                // this can (potentially) change the results being returned.
550                int precount = resources.size();
551                resources = ServerInterceptorUtil.fireStoragePreshowResource(resources, myRequest, myInterceptorBroadcaster);
552                // we only care about omitted results from this page
553                theResponsePageBuilder.setOmittedResourceCount(precount - resources.size());
554                theResponsePageBuilder.setResources(resources);
555                theResponsePageBuilder.setIncludedResourceCount(includedPidList.size());
556
557                return resources;
558        }
559
560        public void setInterceptorBroadcaster(IInterceptorBroadcaster theInterceptorBroadcaster) {
561                myInterceptorBroadcaster = theInterceptorBroadcaster;
562        }
563
564        @VisibleForTesting
565        public void setSearchCacheSvcForUnitTest(ISearchCacheSvc theSearchCacheSvc) {
566                mySearchCacheSvc = theSearchCacheSvc;
567        }
568
569        @VisibleForTesting
570        public void setDaoRegistryForUnitTest(DaoRegistry theDaoRegistry) {
571                myDaoRegistry = theDaoRegistry;
572        }
573
574        @VisibleForTesting
575        public void setStorageSettingsForUnitTest(JpaStorageSettings theStorageSettings) {
576                myStorageSettings = theStorageSettings;
577        }
578
579        @VisibleForTesting
580        public void setSearchBuilderFactoryForUnitTest(SearchBuilderFactory theSearchBuilderFactory) {
581                mySearchBuilderFactory = theSearchBuilderFactory;
582        }
583}