001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2025 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.dao;
021
022import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
023import ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition;
024import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
025import ca.uhn.fhir.context.FhirContext;
026import ca.uhn.fhir.context.FhirVersionEnum;
027import ca.uhn.fhir.context.RuntimeChildResourceDefinition;
028import ca.uhn.fhir.context.RuntimeResourceDefinition;
029import ca.uhn.fhir.i18n.Msg;
030import ca.uhn.fhir.interceptor.api.HookParams;
031import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
032import ca.uhn.fhir.interceptor.api.Pointcut;
033import ca.uhn.fhir.interceptor.model.RequestPartitionId;
034import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
035import ca.uhn.fhir.jpa.api.dao.IDao;
036import ca.uhn.fhir.jpa.api.dao.IJpaDao;
037import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
038import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
039import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
040import ca.uhn.fhir.jpa.cache.IResourceTypeCacheSvc;
041import ca.uhn.fhir.jpa.dao.data.IResourceHistoryTableDao;
042import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao;
043import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
044import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
045import ca.uhn.fhir.jpa.dao.expunge.ExpungeService;
046import ca.uhn.fhir.jpa.dao.index.DaoSearchParamSynchronizer;
047import ca.uhn.fhir.jpa.dao.index.SearchParamWithInlineReferencesExtractor;
048import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
049import ca.uhn.fhir.jpa.delete.DeleteConflictService;
050import ca.uhn.fhir.jpa.esr.ExternallyStoredResourceAddress;
051import ca.uhn.fhir.jpa.esr.ExternallyStoredResourceAddressMetadataKey;
052import ca.uhn.fhir.jpa.esr.ExternallyStoredResourceServiceRegistry;
053import ca.uhn.fhir.jpa.model.config.PartitionSettings;
054import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
055import ca.uhn.fhir.jpa.model.cross.IResourceLookup;
056import ca.uhn.fhir.jpa.model.dao.JpaPid;
057import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
058import ca.uhn.fhir.jpa.model.entity.BaseTag;
059import ca.uhn.fhir.jpa.model.entity.EntityIndexStatusEnum;
060import ca.uhn.fhir.jpa.model.entity.ResourceEncodingEnum;
061import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
062import ca.uhn.fhir.jpa.model.entity.ResourceLink;
063import ca.uhn.fhir.jpa.model.entity.ResourceTable;
064import ca.uhn.fhir.jpa.model.entity.ResourceTag;
065import ca.uhn.fhir.jpa.model.entity.TagDefinition;
066import ca.uhn.fhir.jpa.model.entity.TagTypeEnum;
067import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData;
068import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
069import ca.uhn.fhir.jpa.model.util.JpaConstants;
070import ca.uhn.fhir.jpa.partition.IPartitionLookupSvc;
071import ca.uhn.fhir.jpa.searchparam.extractor.LogicalReferenceHelper;
072import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
073import ca.uhn.fhir.jpa.searchparam.fulltext.FullTextExtractionRequest;
074import ca.uhn.fhir.jpa.searchparam.fulltext.FullTextExtractionResponse;
075import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult;
076import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher;
077import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
078import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
079import ca.uhn.fhir.jpa.util.AddRemoveCount;
080import ca.uhn.fhir.jpa.util.QueryChunker;
081import ca.uhn.fhir.model.api.IResource;
082import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
083import ca.uhn.fhir.model.api.Tag;
084import ca.uhn.fhir.model.api.TagList;
085import ca.uhn.fhir.model.base.composite.BaseCodingDt;
086import ca.uhn.fhir.model.primitive.IdDt;
087import ca.uhn.fhir.parser.DataFormatException;
088import ca.uhn.fhir.rest.api.Constants;
089import ca.uhn.fhir.rest.api.InterceptorInvocationTimingEnum;
090import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
091import ca.uhn.fhir.rest.api.server.RequestDetails;
092import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
093import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
094import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
095import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
096import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
097import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
098import ca.uhn.fhir.util.CoverageIgnore;
099import ca.uhn.fhir.util.HapiExtensions;
100import ca.uhn.fhir.util.MetaUtil;
101import ca.uhn.fhir.util.StopWatch;
102import ca.uhn.fhir.util.XmlUtil;
103import com.google.common.annotations.VisibleForTesting;
104import com.google.common.base.Charsets;
105import com.google.common.collect.Sets;
106import com.google.common.hash.HashCode;
107import jakarta.annotation.Nonnull;
108import jakarta.annotation.Nullable;
109import jakarta.annotation.PostConstruct;
110import jakarta.persistence.EntityManager;
111import jakarta.persistence.PersistenceContext;
112import jakarta.persistence.PersistenceContextType;
113import org.apache.commons.collections4.CollectionUtils;
114import org.apache.commons.lang3.NotImplementedException;
115import org.apache.commons.lang3.StringUtils;
116import org.apache.commons.lang3.Validate;
117import org.hl7.fhir.instance.model.api.IAnyResource;
118import org.hl7.fhir.instance.model.api.IBase;
119import org.hl7.fhir.instance.model.api.IBaseCoding;
120import org.hl7.fhir.instance.model.api.IBaseExtension;
121import org.hl7.fhir.instance.model.api.IBaseHasExtensions;
122import org.hl7.fhir.instance.model.api.IBaseMetaType;
123import org.hl7.fhir.instance.model.api.IBaseReference;
124import org.hl7.fhir.instance.model.api.IBaseResource;
125import org.hl7.fhir.instance.model.api.IDomainResource;
126import org.hl7.fhir.instance.model.api.IIdType;
127import org.hl7.fhir.instance.model.api.IPrimitiveType;
128import org.slf4j.Logger;
129import org.slf4j.LoggerFactory;
130import org.springframework.beans.BeansException;
131import org.springframework.beans.factory.annotation.Autowired;
132import org.springframework.context.ApplicationContext;
133import org.springframework.context.ApplicationContextAware;
134import org.springframework.stereotype.Repository;
135
136import java.util.ArrayList;
137import java.util.Collection;
138import java.util.Collections;
139import java.util.Date;
140import java.util.HashSet;
141import java.util.IdentityHashMap;
142import java.util.List;
143import java.util.Set;
144import java.util.StringTokenizer;
145import java.util.function.Consumer;
146import java.util.function.Supplier;
147import java.util.stream.Collectors;
148import javax.xml.stream.events.Characters;
149import javax.xml.stream.events.XMLEvent;
150
151import static java.util.Objects.nonNull;
152import static org.apache.commons.collections4.CollectionUtils.isEqualCollection;
153import static org.apache.commons.lang3.StringUtils.isBlank;
154import static org.apache.commons.lang3.StringUtils.isNotBlank;
155import static org.apache.commons.lang3.StringUtils.left;
156import static org.apache.commons.lang3.StringUtils.trim;
157
158/**
159 * TODO: JA - This class has only one subclass now. Historically it was a common
160 * ancestor for BaseHapiFhirSystemDao and BaseHapiFhirResourceDao but I've untangled
161 * the former from this hierarchy in order to simplify moving common functionality
162 * for resource DAOs into the hapi-fhir-storage project. This class should be merged
163 * into BaseHapiFhirResourceDao, but that should be done in its own dedicated PR
164 * since it'll be a noisy change.
165 */
166@SuppressWarnings("WeakerAccess")
167@Repository
168public abstract class BaseHapiFhirDao<T extends IBaseResource> extends BaseStorageResourceDao<T>
169                implements IDao, IJpaDao<T>, ApplicationContextAware {
170
171        public static final String NS_JPA_PROFILE = "https://github.com/hapifhir/hapi-fhir/ns/jpa/profile";
172        private static final Logger ourLog = LoggerFactory.getLogger(BaseHapiFhirDao.class);
173        private static boolean ourValidationDisabledForUnitTest;
174        private static boolean ourDisableIncrementOnUpdateForUnitTest = false;
175
176        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
177        protected EntityManager myEntityManager;
178
179        @Autowired
180        protected IIdHelperService<JpaPid> myIdHelperService;
181
182        @Autowired
183        protected ISearchCoordinatorSvc<JpaPid> mySearchCoordinatorSvc;
184
185        @Autowired
186        protected ITermReadSvc myTerminologySvc;
187
188        @Autowired
189        protected IResourceHistoryTableDao myResourceHistoryTableDao;
190
191        @Autowired
192        protected IResourceTableDao myResourceTableDao;
193
194        @Autowired
195        protected IResourceLinkDao myResourceLinkDao;
196
197        @Autowired
198        protected IResourceTagDao myResourceTagDao;
199
200        @Autowired
201        protected DeleteConflictService myDeleteConflictService;
202
203        @Autowired
204        protected IInterceptorBroadcaster myInterceptorBroadcaster;
205
206        @Autowired
207        protected InMemoryResourceMatcher myInMemoryResourceMatcher;
208
209        @Autowired
210        protected IJpaStorageResourceParser myJpaStorageResourceParser;
211
212        @Autowired
213        protected PartitionSettings myPartitionSettings;
214
215        @Autowired
216        ExpungeService myExpungeService;
217
218        @Autowired
219        private ExternallyStoredResourceServiceRegistry myExternallyStoredResourceServiceRegistry;
220
221        @Autowired
222        private ISearchParamPresenceSvc mySearchParamPresenceSvc;
223
224        @Autowired
225        private SearchParamWithInlineReferencesExtractor mySearchParamWithInlineReferencesExtractor;
226
227        @Autowired
228        private DaoSearchParamSynchronizer myDaoSearchParamSynchronizer;
229
230        private FhirContext myContext;
231        private ApplicationContext myApplicationContext;
232
233        @Autowired
234        private IPartitionLookupSvc myPartitionLookupSvc;
235
236        @Autowired(required = false)
237        private IFulltextSearchSvc myFulltextSearchSvc;
238
239        @Autowired
240        protected ResourceHistoryCalculator myResourceHistoryCalculator;
241
242        @Autowired
243        protected CacheTagDefinitionDao cacheTagDefinitionDao;
244
245        @Autowired
246        protected IResourceTypeCacheSvc myResourceTypeCacheSvc;
247
248        protected final CodingSpy myCodingSpy = new CodingSpy();
249
250        @VisibleForTesting
251        public void setExternallyStoredResourceServiceRegistryForUnitTest(
252                        ExternallyStoredResourceServiceRegistry theExternallyStoredResourceServiceRegistry) {
253                myExternallyStoredResourceServiceRegistry = theExternallyStoredResourceServiceRegistry;
254        }
255
256        @VisibleForTesting
257        public void setSearchParamPresenceSvc(ISearchParamPresenceSvc theSearchParamPresenceSvc) {
258                mySearchParamPresenceSvc = theSearchParamPresenceSvc;
259        }
260
261        @VisibleForTesting
262        public void setResourceHistoryCalculator(ResourceHistoryCalculator theResourceHistoryCalculator) {
263                myResourceHistoryCalculator = theResourceHistoryCalculator;
264        }
265
266        @VisibleForTesting
267        public void setInterceptorBroadcasterForUnitTest(IInterceptorBroadcaster theInterceptorBroadcaster) {
268                myInterceptorBroadcaster = theInterceptorBroadcaster;
269        }
270
271        @Override
272        protected IInterceptorBroadcaster getInterceptorBroadcaster() {
273                return myInterceptorBroadcaster;
274        }
275
276        protected ApplicationContext getApplicationContext() {
277                return myApplicationContext;
278        }
279
280        @Override
281        public void setApplicationContext(@Nonnull ApplicationContext theApplicationContext) throws BeansException {
282                /*
283                 * We do a null check here because Smile's module system tries to
284                 * initialize the application context twice if two modules depend on
285                 * the persistence module. The second time sets the dependency's appctx.
286                 */
287                if (myApplicationContext == null) {
288                        myApplicationContext = theApplicationContext;
289                }
290        }
291
292        private void extractHapiTags(
293                        TransactionDetails theTransactionDetails,
294                        IResource theResource,
295                        ResourceTable theEntity,
296                        Set<ResourceTag> allDefs) {
297                TagList tagList = ResourceMetadataKeyEnum.TAG_LIST.get(theResource);
298                if (tagList != null) {
299                        for (Tag next : tagList) {
300                                TagDefinition def = cacheTagDefinitionDao.getTagOrNull(
301                                                theTransactionDetails,
302                                                TagTypeEnum.TAG,
303                                                next.getScheme(),
304                                                next.getTerm(),
305                                                next.getLabel(),
306                                                next.getVersion(),
307                                                myCodingSpy.getBooleanObject(next));
308                                if (def != null) {
309                                        ResourceTag tag = theEntity.addTag(def);
310                                        allDefs.add(tag);
311                                        theEntity.setHasTags(true);
312                                }
313                        }
314                }
315
316                List<BaseCodingDt> securityLabels = ResourceMetadataKeyEnum.SECURITY_LABELS.get(theResource);
317                if (securityLabels != null) {
318                        for (BaseCodingDt next : securityLabels) {
319                                TagDefinition def = cacheTagDefinitionDao.getTagOrNull(
320                                                theTransactionDetails,
321                                                TagTypeEnum.SECURITY_LABEL,
322                                                next.getSystemElement().getValue(),
323                                                next.getCodeElement().getValue(),
324                                                next.getDisplayElement().getValue(),
325                                                next.getVersionElement().getValue(),
326                                                next.getUserSelectedElement().getValue());
327                                if (def != null) {
328                                        ResourceTag tag = theEntity.addTag(def);
329                                        allDefs.add(tag);
330                                        theEntity.setHasTags(true);
331                                }
332                        }
333                }
334
335                List<IdDt> profiles = ResourceMetadataKeyEnum.PROFILES.get(theResource);
336                if (profiles != null) {
337                        for (IIdType next : profiles) {
338                                TagDefinition def = cacheTagDefinitionDao.getTagOrNull(
339                                                theTransactionDetails, TagTypeEnum.PROFILE, NS_JPA_PROFILE, next.getValue(), null, null, null);
340                                if (def != null) {
341                                        ResourceTag tag = theEntity.addTag(def);
342                                        allDefs.add(tag);
343                                        theEntity.setHasTags(true);
344                                }
345                        }
346                }
347        }
348
349        private void extractRiTags(
350                        TransactionDetails theTransactionDetails,
351                        IAnyResource theResource,
352                        ResourceTable theEntity,
353                        Set<ResourceTag> theAllTags) {
354                List<? extends IBaseCoding> tagList = theResource.getMeta().getTag();
355                if (tagList != null) {
356                        for (IBaseCoding next : tagList) {
357                                TagDefinition def = cacheTagDefinitionDao.getTagOrNull(
358                                                theTransactionDetails,
359                                                TagTypeEnum.TAG,
360                                                next.getSystem(),
361                                                next.getCode(),
362                                                next.getDisplay(),
363                                                next.getVersion(),
364                                                myCodingSpy.getBooleanObject(next));
365                                if (def != null) {
366                                        ResourceTag tag = theEntity.addTag(def);
367                                        theAllTags.add(tag);
368                                        theEntity.setHasTags(true);
369                                }
370                        }
371                }
372
373                List<? extends IBaseCoding> securityLabels = theResource.getMeta().getSecurity();
374                if (securityLabels != null) {
375                        for (IBaseCoding next : securityLabels) {
376                                TagDefinition def = cacheTagDefinitionDao.getTagOrNull(
377                                                theTransactionDetails,
378                                                TagTypeEnum.SECURITY_LABEL,
379                                                next.getSystem(),
380                                                next.getCode(),
381                                                next.getDisplay(),
382                                                next.getVersion(),
383                                                myCodingSpy.getBooleanObject(next));
384                                if (def != null) {
385                                        ResourceTag tag = theEntity.addTag(def);
386                                        theAllTags.add(tag);
387                                        theEntity.setHasTags(true);
388                                }
389                        }
390                }
391
392                List<? extends IPrimitiveType<String>> profiles = theResource.getMeta().getProfile();
393                if (profiles != null) {
394                        for (IPrimitiveType<String> next : profiles) {
395                                TagDefinition def = cacheTagDefinitionDao.getTagOrNull(
396                                                theTransactionDetails, TagTypeEnum.PROFILE, NS_JPA_PROFILE, next.getValue(), null, null, null);
397                                if (def != null) {
398                                        ResourceTag tag = theEntity.addTag(def);
399                                        theAllTags.add(tag);
400                                        theEntity.setHasTags(true);
401                                }
402                        }
403                }
404        }
405
406        private void extractProfileTags(
407                        TransactionDetails theTransactionDetails,
408                        IBaseResource theResource,
409                        ResourceTable theEntity,
410                        Set<ResourceTag> theAllTags) {
411                RuntimeResourceDefinition def = myContext.getResourceDefinition(theResource);
412                if (!def.isStandardType()) {
413                        String profile = def.getResourceProfile("");
414                        if (isNotBlank(profile)) {
415                                TagDefinition profileDef = cacheTagDefinitionDao.getTagOrNull(
416                                                theTransactionDetails, TagTypeEnum.PROFILE, NS_JPA_PROFILE, profile, null, null, null);
417
418                                ResourceTag tag = theEntity.addTag(profileDef);
419                                theAllTags.add(tag);
420                                theEntity.setHasTags(true);
421                        }
422                }
423        }
424
425        private Set<ResourceTag> getAllTagDefinitions(ResourceTable theEntity) {
426                HashSet<ResourceTag> retVal = Sets.newHashSet();
427                if (theEntity.isHasTags()) {
428                        retVal.addAll(theEntity.getTags());
429                }
430                return retVal;
431        }
432
433        @Override
434        public JpaStorageSettings getStorageSettings() {
435                return myStorageSettings;
436        }
437
438        @Override
439        public FhirContext getContext() {
440                return myContext;
441        }
442
443        @Autowired
444        public void setContext(FhirContext theContext) {
445                super.myFhirContext = theContext;
446                myContext = theContext;
447        }
448
449        void incrementId(T theResource, ResourceTable theSavedEntity, IIdType theResourceId) {
450                if (theResourceId == null || theResourceId.getVersionIdPart() == null) {
451                        theSavedEntity.initializeVersion();
452                } else {
453                        theSavedEntity.markVersionUpdatedInCurrentTransaction();
454                }
455
456                assert theResourceId != null;
457                String newVersion = Long.toString(theSavedEntity.getVersion());
458                IIdType newId = theResourceId.withVersion(newVersion);
459                theResource.getIdElement().setValue(newId.getValue());
460        }
461
462        public boolean isLogicalReference(IIdType theId) {
463                return LogicalReferenceHelper.isLogicalReference(myStorageSettings, theId);
464        }
465
466        /**
467         * Returns {@literal true} if the resource has changed (either the contents or the tags)
468         */
469        protected EncodedResource populateResourceIntoEntity(
470                        TransactionDetails theTransactionDetails,
471                        RequestDetails theRequest,
472                        IBaseResource theResource,
473                        ResourceTable theEntity,
474                        boolean thePerformIndexing) {
475                if (theEntity.getResourceType() == null) {
476                        theEntity.setResourceType(toResourceName(theResource));
477                }
478                if (theEntity.getResourceTypeId() == null && theResource != null) {
479                        theEntity.setResourceTypeId(myResourceTypeCacheSvc.getResourceTypeId(toResourceName(theResource)));
480                }
481
482                byte[] resourceBinary;
483                String resourceText;
484                ResourceEncodingEnum encoding;
485                boolean changed = false;
486
487                if (theEntity.getDeleted() == null) {
488
489                        if (thePerformIndexing) {
490
491                                ExternallyStoredResourceAddress address = null;
492                                if (myExternallyStoredResourceServiceRegistry.hasProviders()) {
493                                        address = ExternallyStoredResourceAddressMetadataKey.INSTANCE.get(theResource);
494                                }
495
496                                if (address != null) {
497
498                                        encoding = ResourceEncodingEnum.ESR;
499                                        resourceBinary = null;
500                                        resourceText = address.getProviderId() + ":" + address.getLocation();
501                                        changed = true;
502
503                                } else {
504
505                                        encoding = myStorageSettings.getResourceEncoding();
506
507                                        String resourceType = theEntity.getResourceType();
508
509                                        List<String> excludeElements = new ArrayList<>(8);
510                                        IBaseMetaType meta = theResource.getMeta();
511
512                                        IBaseExtension<?, ?> sourceExtension = getExcludedElements(resourceType, excludeElements, meta);
513
514                                        theEntity.setFhirVersion(myContext.getVersion().getVersion());
515
516                                        // TODO:  LD: Once 2024-02 it out the door we should consider further refactoring here to move
517                                        // more of this logic within the calculator and eliminate more local variables
518                                        final ResourceHistoryState calculate = myResourceHistoryCalculator.calculateResourceHistoryState(
519                                                        theResource, encoding, excludeElements);
520
521                                        resourceText = calculate.getResourceText();
522                                        resourceBinary = calculate.getResourceBinary();
523                                        encoding = calculate.getEncoding(); // This may be a no-op
524                                        final HashCode hashCode = calculate.getHashCode();
525
526                                        String hashSha256 = hashCode.toString();
527                                        if (!hashSha256.equals(theEntity.getHashSha256())) {
528                                                changed = true;
529                                        }
530                                        theEntity.setHashSha256(hashSha256);
531
532                                        if (sourceExtension != null) {
533                                                IBaseExtension<?, ?> newSourceExtension = ((IBaseHasExtensions) meta).addExtension();
534                                                newSourceExtension.setUrl(sourceExtension.getUrl());
535                                                newSourceExtension.setValue(sourceExtension.getValue());
536                                        }
537                                }
538
539                        } else {
540
541                                encoding = null;
542                                resourceBinary = null;
543                                resourceText = null;
544                        }
545
546                        boolean skipUpdatingTags = myStorageSettings.isMassIngestionMode() && theEntity.isHasTags();
547                        skipUpdatingTags |= myStorageSettings.getTagStorageMode() == JpaStorageSettings.TagStorageModeEnum.INLINE;
548
549                        if (!skipUpdatingTags) {
550                                changed |= updateTags(theTransactionDetails, theRequest, theResource, theEntity);
551                        }
552
553                } else {
554
555                        if (nonNull(theEntity.getHashSha256())) {
556                                theEntity.setHashSha256(null);
557                                changed = true;
558                        }
559
560                        resourceBinary = null;
561                        resourceText = null;
562                        encoding = ResourceEncodingEnum.DEL;
563                }
564
565                if (thePerformIndexing && !changed) {
566                        if (theEntity.getId() == null) {
567                                changed = true;
568                        } else if (myStorageSettings.isMassIngestionMode()) {
569
570                                // Don't check existing - We'll rely on the SHA256 hash only
571
572                        } else if (theEntity.getVersion() == 1L && theEntity.getCurrentVersionEntity() == null) {
573
574                                // No previous version if this is the first version
575
576                        } else {
577                                ResourceHistoryTable currentHistoryVersion = theEntity.getCurrentVersionEntity();
578                                if (currentHistoryVersion == null) {
579                                        currentHistoryVersion = myResourceHistoryTableDao.findForIdAndVersion(
580                                                        theEntity.getId().toFk(), theEntity.getVersion());
581                                }
582                                if (currentHistoryVersion == null || !currentHistoryVersion.hasResource()) {
583                                        changed = true;
584                                } else {
585                                        // TODO:  LD: Once 2024-02 it out the door we should consider further refactoring here to move
586                                        // more of this logic within the calculator and eliminate more local variables
587                                        changed = myResourceHistoryCalculator.isResourceHistoryChanged(
588                                                        currentHistoryVersion, resourceBinary, resourceText);
589                                }
590                        }
591                }
592
593                EncodedResource retVal = new EncodedResource();
594                retVal.setEncoding(encoding);
595                retVal.setResourceBinary(resourceBinary);
596                retVal.setResourceText(resourceText);
597                retVal.setChanged(changed);
598
599                return retVal;
600        }
601
602        /**
603         * helper to format the meta element for serialization of the resource.
604         *
605         * @param theResourceType    the resource type of the resource
606         * @param theExcludeElements list of extensions in the meta element to exclude from serialization
607         * @param theMeta            the meta element of the resource
608         * @return source extension if present in the meta element
609         */
610        private IBaseExtension<?, ?> getExcludedElements(
611                        String theResourceType, List<String> theExcludeElements, IBaseMetaType theMeta) {
612                boolean hasExtensions = false;
613                IBaseExtension<?, ?> sourceExtension = null;
614                if (theMeta instanceof IBaseHasExtensions) {
615                        List<? extends IBaseExtension<?, ?>> extensions = ((IBaseHasExtensions) theMeta).getExtension();
616                        if (!extensions.isEmpty()) {
617                                hasExtensions = true;
618
619                                /*
620                                 * FHIR DSTU3 did not have the Resource.meta.source field, so we use a
621                                 * custom HAPI FHIR extension in Resource.meta to store that field. However,
622                                 * we put the value for that field in a separate table, so we don't want to serialize
623                                 * it into the stored BLOB. Therefore: remove it from the resource temporarily
624                                 * and restore it afterward.
625                                 */
626                                if (myFhirContext.getVersion().getVersion().equals(FhirVersionEnum.DSTU3)) {
627                                        for (int i = 0; i < extensions.size(); i++) {
628                                                if (extensions.get(i).getUrl().equals(HapiExtensions.EXT_META_SOURCE)) {
629                                                        sourceExtension = extensions.remove(i);
630                                                        i--;
631                                                }
632                                        }
633                                }
634                                boolean allExtensionsRemoved = extensions.isEmpty();
635                                if (allExtensionsRemoved) {
636                                        hasExtensions = false;
637                                }
638                        }
639                }
640
641                theExcludeElements.add("id");
642                boolean inlineTagMode =
643                                getStorageSettings().getTagStorageMode() == JpaStorageSettings.TagStorageModeEnum.INLINE;
644                if (hasExtensions || inlineTagMode) {
645                        if (!inlineTagMode) {
646                                theExcludeElements.add(theResourceType + ".meta.profile");
647                                theExcludeElements.add(theResourceType + ".meta.tag");
648                                theExcludeElements.add(theResourceType + ".meta.security");
649                        }
650                        theExcludeElements.add(theResourceType + ".meta.versionId");
651                        theExcludeElements.add(theResourceType + ".meta.lastUpdated");
652                        theExcludeElements.add(theResourceType + ".meta.source");
653                } else {
654                        /*
655                         * If there are no extensions in the meta element, we can just exclude the
656                         * whole meta element, which avoids adding an empty "meta":{}
657                         * from showing up in the serialized JSON.
658                         */
659                        theExcludeElements.add(theResourceType + ".meta");
660                }
661                return sourceExtension;
662        }
663
664        protected boolean updateTags(
665                        TransactionDetails theTransactionDetails,
666                        RequestDetails theRequest,
667                        IBaseResource theResource,
668                        ResourceTable theEntity) {
669                Set<ResourceTag> allResourceTagsFromTheResource = new HashSet<>();
670                Set<ResourceTag> allOriginalResourceTagsFromTheEntity = getAllTagDefinitions(theEntity);
671
672                if (theResource instanceof IResource) {
673                        extractHapiTags(theTransactionDetails, (IResource) theResource, theEntity, allResourceTagsFromTheResource);
674                } else {
675                        extractRiTags(theTransactionDetails, (IAnyResource) theResource, theEntity, allResourceTagsFromTheResource);
676                }
677
678                extractProfileTags(theTransactionDetails, theResource, theEntity, allResourceTagsFromTheResource);
679
680                // the extract[Hapi|Ri|Profile]Tags methods above will have populated the allResourceTagsFromTheResource Set
681                // AND
682                // added all tags from theResource.meta.tags to theEntity.meta.tags.  the next steps are to:
683                // 1- remove duplicates;
684                // 2- remove tags from theEntity that are not present in theResource if header HEADER_META_SNAPSHOT_MODE
685                // is present in the request;
686                //
687                Set<ResourceTag> allResourceTagsNewAndOldFromTheEntity = getAllTagDefinitions(theEntity);
688                Set<TagDefinition> allTagDefinitionsPresent = new HashSet<>();
689
690                allResourceTagsNewAndOldFromTheEntity.forEach(tag -> {
691
692                        // Don't keep duplicate tags or tags with a missing definition
693                        TagDefinition tagDefinition = tag.getTag();
694                        if (tagDefinition == null || !allTagDefinitionsPresent.add(tagDefinition)) {
695                                theEntity.getTags().remove(tag);
696                        }
697
698                        // Drop any tags that have been removed
699                        if (tagDefinition != null && !allResourceTagsFromTheResource.contains(tag)) {
700                                if (shouldDroppedTagBeRemovedOnUpdate(theRequest, tag)) {
701                                        theEntity.getTags().remove(tag);
702                                } else if (HapiExtensions.EXT_SUBSCRIPTION_MATCHING_STRATEGY.equals(tagDefinition.getSystem())) {
703                                        theEntity.getTags().remove(tag);
704                                }
705                        }
706                });
707
708                // at this point, theEntity.meta.tags will be up to date:
709                // 1- it was stripped from tags that needed removing;
710                // 2- it has new tags from a resource update through theResource;
711                // 3- it has tags from the previous version;
712                //
713                // Since tags are merged on updates, we add tags from theEntity that theResource does not have
714                Set<ResourceTag> allUpdatedResourceTagsNewAndOldMinusRemovalsFromTheEntity = getAllTagDefinitions(theEntity);
715
716                allUpdatedResourceTagsNewAndOldMinusRemovalsFromTheEntity.forEach(aResourcetag -> {
717                        if (!allResourceTagsFromTheResource.contains(aResourcetag)) {
718                                IBaseCoding iBaseCoding = theResource
719                                                .getMeta()
720                                                .addTag()
721                                                .setCode(aResourcetag.getTag().getCode())
722                                                .setSystem(aResourcetag.getTag().getSystem())
723                                                .setVersion(aResourcetag.getTag().getVersion());
724
725                                allResourceTagsFromTheResource.add(aResourcetag);
726
727                                if (aResourcetag.getTag().getUserSelected() != null) {
728                                        iBaseCoding.setUserSelected(aResourcetag.getTag().getUserSelected());
729                                }
730                        }
731                });
732
733                theEntity.setHasTags(!allUpdatedResourceTagsNewAndOldMinusRemovalsFromTheEntity.isEmpty());
734                return !isEqualCollection(allOriginalResourceTagsFromTheEntity, allResourceTagsFromTheResource);
735        }
736
737        /**
738         * Subclasses may override to provide behaviour. Called when a pre-existing resource has been updated in the database
739         *
740         * @param theEntity The resource
741         */
742        protected void postDelete(ResourceTable theEntity) {
743                // nothing
744        }
745
746        /**
747         * Subclasses may override to provide behaviour. Called when a resource has been inserted into the database for the first time.
748         *
749         * @param theEntity         The entity being updated (Do not modify the entity! Undefined behaviour will occur!)
750         * @param theResource       The resource being persisted
751         * @param theRequestDetails The request details, needed for partition support
752         */
753        protected void postPersist(ResourceTable theEntity, T theResource, RequestDetails theRequestDetails) {
754                // nothing
755        }
756
757        /**
758         * Subclasses may override to provide behaviour. Called when a pre-existing resource has been updated in the database
759         *
760         * @param theEntity         The resource
761         * @param theResource       The resource being persisted
762         * @param theRequestDetails The request details, needed for partition support
763         */
764        protected void postUpdate(ResourceTable theEntity, T theResource, RequestDetails theRequestDetails) {
765                // nothing
766        }
767
768        @Override
769        @CoverageIgnore
770        public BaseHasResource readEntity(IIdType theValueId, RequestDetails theRequest) {
771                throw new NotImplementedException(Msg.code(927));
772        }
773
774        /**
775         * This method is called when an update to an existing resource detects that the resource supplied for update is missing a tag/profile/security label that the currently persisted resource holds.
776         * <p>
777         * The default implementation removes any profile declarations, but leaves tags and security labels in place. Subclasses may choose to override and change this behaviour.
778         * </p>
779         * <p>
780         * See <a href="http://hl7.org/fhir/resource.html#tag-updates">Updates to Tags, Profiles, and Security Labels</a> for a description of the logic that the default behaviour follows.
781         * </p>
782         *
783         * @param theTag The tag
784         * @return Returns <code>true</code> if the tag should be removed
785         */
786        protected boolean shouldDroppedTagBeRemovedOnUpdate(RequestDetails theRequest, ResourceTag theTag) {
787                if (theTag.getTag() == null) {
788                        return true;
789                }
790
791                Set<TagTypeEnum> metaSnapshotModeTokens = null;
792
793                if (theRequest != null) {
794                        List<String> metaSnapshotMode = theRequest.getHeaders(JpaConstants.HEADER_META_SNAPSHOT_MODE);
795                        if (metaSnapshotMode != null && !metaSnapshotMode.isEmpty()) {
796                                metaSnapshotModeTokens = new HashSet<>();
797                                for (String nextHeaderValue : metaSnapshotMode) {
798                                        StringTokenizer tok = new StringTokenizer(nextHeaderValue, ",");
799                                        while (tok.hasMoreTokens()) {
800                                                switch (trim(tok.nextToken())) {
801                                                        case "TAG":
802                                                                metaSnapshotModeTokens.add(TagTypeEnum.TAG);
803                                                                break;
804                                                        case "PROFILE":
805                                                                metaSnapshotModeTokens.add(TagTypeEnum.PROFILE);
806                                                                break;
807                                                        case "SECURITY_LABEL":
808                                                                metaSnapshotModeTokens.add(TagTypeEnum.SECURITY_LABEL);
809                                                                break;
810                                                }
811                                        }
812                                }
813                        }
814                }
815
816                if (metaSnapshotModeTokens == null) {
817                        metaSnapshotModeTokens = Collections.singleton(TagTypeEnum.PROFILE);
818                }
819
820                return metaSnapshotModeTokens.contains(theTag.getTag().getTagType());
821        }
822
823        String toResourceName(IBaseResource theResource) {
824                return myContext.getResourceType(theResource);
825        }
826
827        @VisibleForTesting
828        public void setEntityManager(EntityManager theEntityManager) {
829                myEntityManager = theEntityManager;
830        }
831
832        @VisibleForTesting
833        public void setSearchParamWithInlineReferencesExtractor(
834                        SearchParamWithInlineReferencesExtractor theSearchParamWithInlineReferencesExtractor) {
835                mySearchParamWithInlineReferencesExtractor = theSearchParamWithInlineReferencesExtractor;
836        }
837
838        @VisibleForTesting
839        public void setResourceHistoryTableDao(IResourceHistoryTableDao theResourceHistoryTableDao) {
840                myResourceHistoryTableDao = theResourceHistoryTableDao;
841        }
842
843        @VisibleForTesting
844        public void setDaoSearchParamSynchronizer(DaoSearchParamSynchronizer theDaoSearchParamSynchronizer) {
845                myDaoSearchParamSynchronizer = theDaoSearchParamSynchronizer;
846        }
847
848        private void verifyMatchUrlForConditionalCreateOrUpdate(
849                        CreateOrUpdateByMatch theCreateOrUpdate,
850                        IBaseResource theResource,
851                        String theIfNoneExist,
852                        ResourceIndexedSearchParams theParams,
853                        RequestDetails theRequestDetails) {
854                // Make sure that the match URL was actually appropriate for the supplied resource
855                InMemoryMatchResult outcome =
856                                myInMemoryResourceMatcher.match(theIfNoneExist, theResource, theParams, theRequestDetails);
857
858                if (outcome.supported() && !outcome.matched()) {
859                        String errorMsg =
860                                        getConditionalCreateOrUpdateErrorMsg(theCreateOrUpdate, theIfNoneExist, theResource.fhirType());
861                        throw new InvalidRequestException(Msg.code(929) + errorMsg);
862                }
863        }
864
865        private String getConditionalCreateOrUpdateErrorMsg(
866                        CreateOrUpdateByMatch theCreateOrUpdate, String url, String resourceType) {
867                return String.format(
868                                "Failed to process conditional %s. "
869                                                + "The supplied resource %s of type %s did not satisfy the conditional URL.",
870                                theCreateOrUpdate.name().toLowerCase(), url, resourceType);
871        }
872
873        @SuppressWarnings("unchecked")
874        @Override
875        public ResourceTable updateEntity(
876                        RequestDetails theRequest,
877                        final IBaseResource theResource,
878                        IBasePersistedResource theEntity,
879                        Date theDeletedTimestampOrNull,
880                        boolean thePerformIndexing,
881                        boolean theUpdateVersion,
882                        TransactionDetails theTransactionDetails,
883                        boolean theForceUpdate,
884                        boolean theCreateNewHistoryEntry) {
885                Validate.notNull(theEntity, "entity must not be null");
886                Validate.isTrue(
887                                theDeletedTimestampOrNull != null || theResource != null,
888                                "Must have either a resource[%s] or a deleted timestamp[%s] for resource PID[%s]",
889                                theDeletedTimestampOrNull != null,
890                                theResource != null,
891                                theEntity.getPersistentId());
892
893                ourLog.debug("Starting entity update");
894
895                ResourceTable entity = (ResourceTable) theEntity;
896
897                /*
898                 * This should be the very first thing..
899                 */
900                if (theResource != null) {
901                        if (thePerformIndexing && theDeletedTimestampOrNull == null) {
902                                if (!ourValidationDisabledForUnitTest) {
903                                        validateResourceForStorage((T) theResource, entity);
904                                }
905                        }
906                        if (!StringUtils.isBlank(entity.getResourceType())) {
907                                String resourceType = myContext.getResourceType(theResource);
908                                // This is just a sanity check and should never actually fail.
909                                // We resolve the ID using IdLookupService, and there should be
910                                // no way to get it to give you a mismatched type for an ID.
911                                Validate.isTrue(resourceType.equals(entity.getResourceType()));
912                        }
913                }
914
915                if (entity.getPublished() == null) {
916                        ourLog.debug("Entity has published time: {}", theTransactionDetails.getTransactionDate());
917                        entity.setPublished(theTransactionDetails.getTransactionDate());
918                }
919
920                ResourceIndexedSearchParams existingParams = null;
921
922                ResourceIndexedSearchParams newParams = null;
923
924                EncodedResource changed;
925                if (theDeletedTimestampOrNull != null) {
926                        // DELETE
927
928                        entity.setDeleted(theDeletedTimestampOrNull);
929                        entity.setUpdated(theDeletedTimestampOrNull);
930                        changed = populateResourceIntoEntity(theTransactionDetails, theRequest, theResource, entity, true);
931
932                } else {
933
934                        // CREATE or UPDATE
935
936                        IdentityHashMap<ResourceTable, ResourceIndexedSearchParams> existingSearchParams =
937                                        getSearchParamsMapFromTransaction(theTransactionDetails);
938                        existingParams = existingSearchParams.get(entity);
939                        if (existingParams == null) {
940                                existingParams = ResourceIndexedSearchParams.withLists(entity);
941                                /*
942                                 * If we have lots of resource links, this proactively fetches the targets so
943                                 * that we don't look them up one-by-one when comparing the new set to the
944                                 * old set later on
945                                 */
946                                if (existingParams.getResourceLinks().size() >= 10) {
947                                        List<Long> allPids = existingParams.getResourceLinks().stream()
948                                                        .map(ResourceLink::getId)
949                                                        .collect(Collectors.toList());
950                                        new QueryChunker<Long>().chunk(allPids, chunkPids -> {
951                                                List<ResourceLink> targets = myResourceLinkDao.findByPidAndFetchTargetDetails(chunkPids);
952                                                ourLog.trace("Prefetched targets: {}", targets);
953                                        });
954                                }
955                                existingSearchParams.put(entity, existingParams);
956                        }
957                        entity.setDeleted(null);
958
959                        // TODO: is this IF statement always true? Try removing it
960                        if (thePerformIndexing || theEntity.getVersion() == 1) {
961
962                                newParams = ResourceIndexedSearchParams.withSets();
963
964                                RequestPartitionId requestPartitionId;
965                                if (!myPartitionSettings.isPartitioningEnabled()) {
966                                        requestPartitionId = RequestPartitionId.allPartitions();
967                                } else if (entity.getPartitionId() != null) {
968                                        requestPartitionId = entity.getPartitionId().toPartitionId();
969                                } else {
970                                        requestPartitionId = myPartitionSettings.getDefaultRequestPartitionId();
971                                }
972
973                                // Extract search params for resource
974                                mySearchParamWithInlineReferencesExtractor.populateFromResource(
975                                                requestPartitionId,
976                                                newParams,
977                                                theTransactionDetails,
978                                                entity,
979                                                theResource,
980                                                existingParams,
981                                                theRequest,
982                                                thePerformIndexing);
983
984                                if (CollectionUtils.isNotEmpty(newParams.myLinks)) {
985                                        setTargetResourceTypeIdForResourceLinks(newParams.myLinks);
986                                }
987
988                                // Actually persist the ResourceTable and ResourceHistoryTable entities
989                                changed = populateResourceIntoEntity(theTransactionDetails, theRequest, theResource, entity, true);
990
991                                if (theForceUpdate) {
992                                        changed.setChanged(true);
993                                }
994
995                                if (changed.isChanged()) {
996                                        checkConditionalMatch(
997                                                        entity, theUpdateVersion, theResource, thePerformIndexing, newParams, theRequest);
998
999                                        if (CURRENTLY_REINDEXING.get(theResource) != Boolean.TRUE) {
1000                                                entity.setUpdated(theTransactionDetails.getTransactionDate());
1001                                        }
1002                                        newParams.populateResourceTableSearchParamsPresentFlags(entity);
1003                                }
1004
1005                        } else {
1006
1007                                entity.setUpdated(theTransactionDetails.getTransactionDate());
1008                                changed = populateResourceIntoEntity(theTransactionDetails, theRequest, theResource, entity, false);
1009                        }
1010                }
1011
1012                if (changed != null && changed.isChanged()) {
1013                        populateFullTextFieldsAndSetEntityStatus(theRequest, myContext, theResource, entity, newParams);
1014                }
1015
1016                if (thePerformIndexing
1017                                && changed != null
1018                                && !changed.isChanged()
1019                                && !theForceUpdate
1020                                && myStorageSettings.isSuppressUpdatesWithNoChange()
1021                                && (entity.getVersion() > 1 || theUpdateVersion)) {
1022                        ourLog.debug(
1023                                        "Resource {} has not changed",
1024                                        entity.getIdDt().toUnqualified().getValue());
1025                        if (theResource != null) {
1026                                myJpaStorageResourceParser.updateResourceMetadata(entity, theResource);
1027                        }
1028                        entity.setUnchangedInCurrentOperation(true);
1029                        return entity;
1030                }
1031
1032                if (entity.getId().getId() != null && theUpdateVersion) {
1033                        entity.markVersionUpdatedInCurrentTransaction();
1034                }
1035
1036                /*
1037                 * Save the resource itself
1038                 */
1039                if (entity.getId().getId() == null) {
1040                        myEntityManager.persist(entity);
1041
1042                        if (entity.getFhirId() == null) {
1043                                entity.setFhirId(Long.toString(entity.getId().getId()));
1044                        }
1045
1046                        postPersist(entity, (T) theResource, theRequest);
1047
1048                } else if (entity.getDeleted() != null) {
1049                        entity = myEntityManager.merge(entity);
1050
1051                        postDelete(entity);
1052
1053                } else {
1054                        entity = myEntityManager.merge(entity);
1055
1056                        postUpdate(entity, (T) theResource, theRequest);
1057                }
1058
1059                if (theCreateNewHistoryEntry) {
1060                        createHistoryEntry(theRequest, theResource, entity, changed);
1061                }
1062
1063                /*
1064                 * Update the "search param present" table which is used for the
1065                 * ?foo:missing=true queries
1066                 *
1067                 * Note that we're only populating this for reference params
1068                 * because the index tables for all other types have a MISSING column
1069                 * right on them for handling the :missing queries. We can't use the
1070                 * index table for resource links (reference indexes) because we index
1071                 * those by path and not by parameter name.
1072                 */
1073                if (thePerformIndexing && newParams != null) {
1074                        AddRemoveCount presenceCount =
1075                                        mySearchParamPresenceSvc.updatePresence(entity, newParams.mySearchParamPresentEntities);
1076
1077                        // Interceptor broadcast: JPA_PERFTRACE_INFO
1078                        if (!presenceCount.isEmpty()) {
1079                                IInterceptorBroadcaster compositeBroadcaster =
1080                                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequest);
1081                                if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO)) {
1082                                        StorageProcessingMessage message = new StorageProcessingMessage();
1083                                        message.setMessage(
1084                                                        "For " + entity.getIdDt().toUnqualifiedVersionless().getValue() + " added "
1085                                                                        + presenceCount.getAddCount() + " and removed " + presenceCount.getRemoveCount()
1086                                                                        + " resource search parameter presence entries");
1087                                        HookParams params = new HookParams()
1088                                                        .add(RequestDetails.class, theRequest)
1089                                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
1090                                                        .add(StorageProcessingMessage.class, message);
1091                                        compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
1092                                }
1093                        }
1094                }
1095
1096                /*
1097                 * Indexing
1098                 */
1099                if (thePerformIndexing) {
1100                        if (newParams == null) {
1101                                myExpungeService.deleteAllSearchParams(entity.getPersistentId());
1102                                entity.clearAllParamsPopulated();
1103                        } else {
1104
1105                                // Synchronize search param indexes
1106                                AddRemoveCount searchParamAddRemoveCount =
1107                                                myDaoSearchParamSynchronizer.synchronizeSearchParamsToDatabase(
1108                                                                newParams, entity, existingParams);
1109
1110                                newParams.populateResourceTableParamCollections(entity);
1111
1112                                // Interceptor broadcast: JPA_PERFTRACE_INFO
1113                                if (!searchParamAddRemoveCount.isEmpty()) {
1114                                        IInterceptorBroadcaster compositeBroadcaster =
1115                                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(
1116                                                                        myInterceptorBroadcaster, theRequest);
1117                                        if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO)) {
1118                                                StorageProcessingMessage message = new StorageProcessingMessage();
1119                                                message.setMessage("For "
1120                                                                + entity.getIdDt().toUnqualifiedVersionless().getValue() + " added "
1121                                                                + searchParamAddRemoveCount.getAddCount() + " and removed "
1122                                                                + searchParamAddRemoveCount.getRemoveCount()
1123                                                                + " resource search parameter index entries");
1124                                                HookParams params = new HookParams()
1125                                                                .add(RequestDetails.class, theRequest)
1126                                                                .addIfMatchesType(ServletRequestDetails.class, theRequest)
1127                                                                .add(StorageProcessingMessage.class, message);
1128                                                compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
1129                                        }
1130                                }
1131
1132                                // Put the final set of search params into the transaction
1133                                getSearchParamsMapFromTransaction(theTransactionDetails).put(entity, newParams);
1134                        }
1135                }
1136
1137                if (theResource != null) {
1138                        myJpaStorageResourceParser.updateResourceMetadata(entity, theResource);
1139                }
1140
1141                return entity;
1142        }
1143
1144        private static IdentityHashMap<ResourceTable, ResourceIndexedSearchParams> getSearchParamsMapFromTransaction(
1145                        TransactionDetails theTransactionDetails) {
1146                return theTransactionDetails.getOrCreateUserData(
1147                                HapiTransactionService.XACT_USERDATA_KEY_EXISTING_SEARCH_PARAMS, IdentityHashMap::new);
1148        }
1149
1150        /**
1151         * Make sure that the match URL was actually appropriate for the supplied
1152         * resource, if so configured, or do it only for first version, since technically it
1153         * is possible (and legal) for someone to be using a conditional update
1154         * to match a resource and then update it in a way that it no longer
1155         * matches.
1156         */
1157        private void checkConditionalMatch(
1158                        ResourceTable theEntity,
1159                        boolean theUpdateVersion,
1160                        IBaseResource theResource,
1161                        boolean thePerformIndexing,
1162                        ResourceIndexedSearchParams theNewParams,
1163                        RequestDetails theRequest) {
1164
1165                if (!thePerformIndexing) {
1166                        return;
1167                }
1168
1169                if (theEntity.getCreatedByMatchUrl() == null && theEntity.getUpdatedByMatchUrl() == null) {
1170                        return;
1171                }
1172
1173                // version is not updated at this point, but could be pending for update, which we consider here
1174                long pendingVersion = theEntity.getVersion();
1175                if (theUpdateVersion && !theEntity.isVersionUpdatedInCurrentTransaction()) {
1176                        pendingVersion++;
1177                }
1178
1179                if (myStorageSettings.isPreventInvalidatingConditionalMatchCriteria() || pendingVersion <= 1L) {
1180                        String createOrUpdateUrl;
1181                        CreateOrUpdateByMatch createOrUpdate;
1182
1183                        if (theEntity.getCreatedByMatchUrl() != null) {
1184                                createOrUpdateUrl = theEntity.getCreatedByMatchUrl();
1185                                createOrUpdate = CreateOrUpdateByMatch.CREATE;
1186                        } else {
1187                                createOrUpdateUrl = theEntity.getUpdatedByMatchUrl();
1188                                createOrUpdate = CreateOrUpdateByMatch.UPDATE;
1189                        }
1190
1191                        verifyMatchUrlForConditionalCreateOrUpdate(
1192                                        createOrUpdate, theResource, createOrUpdateUrl, theNewParams, theRequest);
1193                }
1194        }
1195
1196        public IBasePersistedResource<?> updateHistoryEntity(
1197                        RequestDetails theRequest,
1198                        T theResource,
1199                        IBasePersistedResource<?> theEntity,
1200                        IBasePersistedResource<?> theHistoryEntity,
1201                        IIdType theResourceId,
1202                        TransactionDetails theTransactionDetails,
1203                        boolean isUpdatingCurrent) {
1204                Validate.notNull(theEntity);
1205                Validate.isTrue(
1206                                theResource != null,
1207                                "Must have either a resource[%s] for resource PID[%s]",
1208                                theResource != null,
1209                                theEntity.getPersistentId());
1210
1211                ourLog.debug("Starting history entity update");
1212                EncodedResource encodedResource = new EncodedResource();
1213                ResourceHistoryTable historyEntity;
1214
1215                if (isUpdatingCurrent) {
1216                        ResourceTable entity = (ResourceTable) theEntity;
1217
1218                        IBaseResource oldResource;
1219                        if (getStorageSettings().isMassIngestionMode()) {
1220                                oldResource = null;
1221                        } else {
1222                                oldResource = myJpaStorageResourceParser.toResource(entity, false);
1223                        }
1224
1225                        notifyInterceptors(theRequest, theResource, oldResource, theTransactionDetails, true);
1226
1227                        ResourceTable savedEntity = updateEntity(
1228                                        theRequest, theResource, entity, null, true, false, theTransactionDetails, false, false);
1229                        // Have to call populate again for the encodedResource, since using createHistoryEntry() will cause version
1230                        // constraint failure, ie updating the same resource at the same time
1231                        encodedResource = populateResourceIntoEntity(theTransactionDetails, theRequest, theResource, entity, true);
1232                        // For some reason the current version entity is not attached until after using updateEntity
1233                        historyEntity = ((ResourceTable) readEntity(theResourceId, theRequest)).getCurrentVersionEntity();
1234
1235                        // Update version/lastUpdated so that interceptors see the correct version
1236                        myJpaStorageResourceParser.updateResourceMetadata(savedEntity, theResource);
1237                        // Populate the PID in the resource, so it is available to hooks
1238                        addPidToResource(savedEntity, theResource);
1239
1240                        if (!savedEntity.isUnchangedInCurrentOperation()) {
1241                                notifyInterceptors(theRequest, theResource, oldResource, theTransactionDetails, false);
1242                        }
1243                } else {
1244                        historyEntity = (ResourceHistoryTable) theHistoryEntity;
1245                        if (!StringUtils.isBlank(historyEntity.getResourceType())) {
1246                                String resourceType = myContext.getResourceType(theResource);
1247                                if (!resourceType.equals(historyEntity.getResourceType())) {
1248                                        throw new UnprocessableEntityException(Msg.code(930) + "Existing resource ID["
1249                                                        + historyEntity.getIdDt().toUnqualifiedVersionless() + "] is of type["
1250                                                        + historyEntity.getResourceType()
1251                                                        + "] - Cannot update with [" + resourceType + "]");
1252                                }
1253                        }
1254
1255                        historyEntity.setDeleted(null);
1256
1257                        // Check if resource is the same
1258                        ResourceEncodingEnum encoding = myStorageSettings.getResourceEncoding();
1259                        List<String> excludeElements = new ArrayList<>(8);
1260                        getExcludedElements(historyEntity.getResourceType(), excludeElements, theResource.getMeta());
1261                        String encodedResourceString =
1262                                        myResourceHistoryCalculator.encodeResource(theResource, encoding, excludeElements);
1263                        byte[] resourceBinary = ResourceHistoryCalculator.getResourceBinary(encoding, encodedResourceString);
1264                        final boolean changed = myResourceHistoryCalculator.isResourceHistoryChanged(
1265                                        historyEntity, resourceBinary, encodedResourceString);
1266
1267                        historyEntity.setUpdated(theTransactionDetails.getTransactionDate());
1268
1269                        if (!changed && myStorageSettings.isSuppressUpdatesWithNoChange() && (historyEntity.getVersion() > 1)) {
1270                                ourLog.debug(
1271                                                "Resource {} has not changed",
1272                                                historyEntity.getIdDt().toUnqualified().getValue());
1273                                myJpaStorageResourceParser.updateResourceMetadata(historyEntity, theResource);
1274                                return historyEntity;
1275                        }
1276
1277                        myResourceHistoryCalculator.populateEncodedResource(
1278                                        encodedResource, encodedResourceString, resourceBinary, encoding);
1279                }
1280                /*
1281                 * Save the resource itself to the resourceHistoryTable
1282                 */
1283                historyEntity = myEntityManager.merge(historyEntity);
1284                historyEntity.setEncoding(encodedResource.getEncoding());
1285                historyEntity.setResource(encodedResource.getResourceBinary());
1286                historyEntity.setResourceTextVc(encodedResource.getResourceText());
1287                myResourceHistoryTableDao.save(historyEntity);
1288
1289                myJpaStorageResourceParser.updateResourceMetadata(historyEntity, theResource);
1290
1291                return historyEntity;
1292        }
1293
1294        private void populateEncodedResource(
1295                        EncodedResource encodedResource,
1296                        String encodedResourceString,
1297                        byte[] theResourceBinary,
1298                        ResourceEncodingEnum theEncoding) {
1299                encodedResource.setResourceText(encodedResourceString);
1300                encodedResource.setResourceBinary(theResourceBinary);
1301                encodedResource.setEncoding(theEncoding);
1302        }
1303
1304        private void createHistoryEntry(
1305                        RequestDetails theRequest, IBaseResource theResource, ResourceTable theEntity, EncodedResource theChanged) {
1306                boolean versionedTags =
1307                                getStorageSettings().getTagStorageMode() == JpaStorageSettings.TagStorageModeEnum.VERSIONED;
1308
1309                ResourceHistoryTable historyEntry = null;
1310                long resourceVersion = theEntity.getVersion();
1311                if (!myStorageSettings.isResourceDbHistoryEnabled() && resourceVersion > 1L) {
1312                        /*
1313                         * If we're not storing history, then just pull the current history
1314                         * table row and update it. Note that there is always a chance that
1315                         * this could return null if the current resourceVersion has been expunged
1316                         * in which case we'll still create a new one
1317                         */
1318                        historyEntry = myResourceHistoryTableDao.findForIdAndVersion(
1319                                        theEntity.getResourceId().toFk(), resourceVersion - 1);
1320                        if (historyEntry != null) {
1321                                theEntity.populateHistoryEntityVersionAndDates(historyEntry);
1322                                if (versionedTags && theEntity.isHasTags()) {
1323                                        for (ResourceTag next : theEntity.getTags()) {
1324                                                historyEntry.addTag(next.getTag());
1325                                        }
1326                                }
1327                        }
1328                }
1329
1330                /*
1331                 * This should basically always be null unless resource history
1332                 * is disabled on this server. In that case, we'll just be reusing
1333                 * the previous version entity.
1334                 */
1335                if (historyEntry == null) {
1336                        historyEntry = theEntity.toHistory(versionedTags && theEntity.getDeleted() == null);
1337                }
1338
1339                historyEntry.setEncoding(theChanged.getEncoding());
1340                historyEntry.setResource(theChanged.getResourceBinary());
1341                historyEntry.setResourceTextVc(theChanged.getResourceText());
1342
1343                ourLog.debug("Saving history entry ID[{}] for RES_ID[{}]", historyEntry.getId(), historyEntry.getResourceId());
1344                myEntityManager.persist(historyEntry);
1345                theEntity.setCurrentVersionEntity(historyEntry);
1346
1347                // Save resource source
1348                String source = null;
1349
1350                if (theResource != null) {
1351                        if (myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.R4)) {
1352                                IBaseMetaType meta = theResource.getMeta();
1353                                source = MetaUtil.getSource(myContext, meta);
1354                        }
1355                        if (myContext.getVersion().getVersion().equals(FhirVersionEnum.DSTU3)) {
1356                                source = ((IBaseHasExtensions) theResource.getMeta())
1357                                                .getExtension().stream()
1358                                                                .filter(t -> HapiExtensions.EXT_META_SOURCE.equals(t.getUrl()))
1359                                                                .filter(t -> t.getValue() instanceof IPrimitiveType)
1360                                                                .map(t -> ((IPrimitiveType<?>) t.getValue()).getValueAsString())
1361                                                                .findFirst()
1362                                                                .orElse(null);
1363                        }
1364                }
1365
1366                String requestId = getRequestId(theRequest, source);
1367                source = MetaUtil.extractSourceUriOrEmpty(source);
1368
1369                boolean shouldStoreSource =
1370                                myStorageSettings.getStoreMetaSourceInformation().isStoreSourceUri();
1371                boolean shouldStoreRequestId =
1372                                myStorageSettings.getStoreMetaSourceInformation().isStoreRequestId();
1373                boolean haveSource = isNotBlank(source) && shouldStoreSource;
1374                boolean haveRequestId = isNotBlank(requestId) && shouldStoreRequestId;
1375                if (haveSource || haveRequestId) {
1376                        if (haveRequestId) {
1377                                String persistedRequestId = left(requestId, Constants.REQUEST_ID_LENGTH);
1378                                historyEntry.setRequestId(persistedRequestId);
1379                        }
1380                        if (haveSource) {
1381                                String persistedSource = left(source, ResourceHistoryTable.SOURCE_URI_LENGTH);
1382                                historyEntry.setSourceUri(persistedSource);
1383                        }
1384                        if (theResource != null) {
1385                                MetaUtil.populateResourceSource(
1386                                                myFhirContext,
1387                                                shouldStoreSource ? source : null,
1388                                                shouldStoreRequestId ? requestId : null,
1389                                                theResource);
1390                        }
1391                }
1392        }
1393
1394        private String getRequestId(RequestDetails theRequest, String theSource) {
1395                if (myStorageSettings.isPreserveRequestIdInResourceBody()) {
1396                        return StringUtils.substringAfter(theSource, "#");
1397                }
1398                return theRequest != null ? theRequest.getRequestId() : null;
1399        }
1400
1401        @Override
1402        public DaoMethodOutcome updateInternal(
1403                        RequestDetails theRequestDetails,
1404                        T theResource,
1405                        String theMatchUrl,
1406                        boolean thePerformIndexing,
1407                        boolean theForceUpdateVersion,
1408                        IBasePersistedResource theEntity,
1409                        IIdType theResourceId,
1410                        @Nullable IBaseResource theOldResource,
1411                        RestOperationTypeEnum theOperationType,
1412                        TransactionDetails theTransactionDetails) {
1413
1414                ResourceTable entity = (ResourceTable) theEntity;
1415
1416                // We'll update the resource ID with the correct version later but for
1417                // now at least set it to something useful for the interceptors
1418                theResource.setId(entity.getIdDt());
1419
1420                // Notify IServerOperationInterceptors about pre-action call
1421                notifyInterceptors(theRequestDetails, theResource, theOldResource, theTransactionDetails, true);
1422
1423                entity.setUpdatedByMatchUrl(theMatchUrl);
1424
1425                // Perform update
1426                ResourceTable savedEntity = updateEntity(
1427                                theRequestDetails,
1428                                theResource,
1429                                entity,
1430                                null,
1431                                thePerformIndexing,
1432                                thePerformIndexing,
1433                                theTransactionDetails,
1434                                theForceUpdateVersion,
1435                                thePerformIndexing);
1436
1437                /*
1438                 * If we aren't indexing (meaning we're probably executing a sub-operation within a transaction),
1439                 * we'll manually increase the version. This is important because we want the updated version number
1440                 * to be reflected in the resource shared with interceptors
1441                 */
1442                if (!thePerformIndexing
1443                                && !savedEntity.isUnchangedInCurrentOperation()
1444                                && !ourDisableIncrementOnUpdateForUnitTest) {
1445                        if (!theResourceId.hasVersionIdPart()) {
1446                                theResourceId = theResourceId.withVersion(Long.toString(savedEntity.getVersion()));
1447                        }
1448                        incrementId(theResource, savedEntity, theResourceId);
1449                }
1450
1451                // Update version/lastUpdated so that interceptors see the correct version
1452                myJpaStorageResourceParser.updateResourceMetadata(savedEntity, theResource);
1453
1454                // Populate the PID in the resource so it is available to hooks
1455                addPidToResource(savedEntity, theResource);
1456
1457                // Notify interceptors
1458                if (!savedEntity.isUnchangedInCurrentOperation()) {
1459                        notifyInterceptors(theRequestDetails, theResource, theOldResource, theTransactionDetails, false);
1460                }
1461
1462                Collection<? extends BaseTag> tagList = Collections.emptyList();
1463                if (entity.isHasTags()) {
1464                        tagList = entity.getTags();
1465                }
1466                long version = entity.getVersion();
1467                myJpaStorageResourceParser.populateResourceMetadata(entity, false, tagList, version, theResource);
1468
1469                boolean wasDeleted = false;
1470                if (theOldResource != null) {
1471                        wasDeleted = theOldResource.isDeleted();
1472                }
1473
1474                if (wasDeleted && !myStorageSettings.isDeleteEnabled()) {
1475                        String msg = myContext.getLocalizer().getMessage(BaseHapiFhirDao.class, "cantUndeleteWithDeletesDisabled");
1476                        throw new InvalidRequestException(Msg.code(2573) + msg);
1477                }
1478
1479                DaoMethodOutcome outcome = toMethodOutcome(
1480                                                theRequestDetails, savedEntity, theResource, theMatchUrl, theOperationType)
1481                                .setCreated(wasDeleted);
1482
1483                if (!thePerformIndexing) {
1484                        IIdType id = getContext().getVersion().newIdType();
1485                        id.setValue(entity.getIdDt().getValue());
1486                        outcome.setId(id);
1487                }
1488
1489                // Only include a task timer if we're not in a sub-request (i.e. a transaction)
1490                // since individual item times don't actually make much sense in the context
1491                // of a transaction
1492                StopWatch w = null;
1493                if (theRequestDetails != null && !theRequestDetails.isSubRequest()) {
1494                        if (theTransactionDetails != null && !theTransactionDetails.isFhirTransaction()) {
1495                                w = new StopWatch(theTransactionDetails.getTransactionDate());
1496                        }
1497                }
1498
1499                populateOperationOutcomeForUpdate(w, outcome, theMatchUrl, outcome.getOperationType(), theTransactionDetails);
1500
1501                return outcome;
1502        }
1503
1504        private void notifyInterceptors(
1505                        RequestDetails theRequestDetails,
1506                        T theResource,
1507                        IBaseResource theOldResource,
1508                        TransactionDetails theTransactionDetails,
1509                        boolean isUnchanged) {
1510                Pointcut interceptorPointcut = Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED;
1511
1512                HookParams hookParams = new HookParams()
1513                                .add(IBaseResource.class, theOldResource)
1514                                .add(IBaseResource.class, theResource)
1515                                .add(RequestDetails.class, theRequestDetails)
1516                                .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
1517                                .add(TransactionDetails.class, theTransactionDetails);
1518
1519                if (!isUnchanged) {
1520                        hookParams.add(
1521                                        InterceptorInvocationTimingEnum.class,
1522                                        theTransactionDetails.getInvocationTiming(Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED));
1523                        interceptorPointcut = Pointcut.STORAGE_PRECOMMIT_RESOURCE_UPDATED;
1524                }
1525
1526                doCallHooks(theTransactionDetails, theRequestDetails, interceptorPointcut, hookParams);
1527        }
1528
1529        protected void addPidToResource(IResourceLookup<JpaPid> theEntity, IBaseResource theResource) {
1530                if (theResource instanceof IAnyResource) {
1531                        IDao.RESOURCE_PID.put(theResource, theEntity.getPersistentId());
1532                } else if (theResource instanceof IResource) {
1533                        IDao.RESOURCE_PID.put(theResource, theEntity.getPersistentId());
1534                }
1535        }
1536
1537        private void validateChildReferenceTargetTypes(IBase theElement, String thePath) {
1538                if (theElement == null) {
1539                        return;
1540                }
1541                BaseRuntimeElementDefinition<?> def = myContext.getElementDefinition(theElement.getClass());
1542                if (!(def instanceof BaseRuntimeElementCompositeDefinition)) {
1543                        return;
1544                }
1545
1546                BaseRuntimeElementCompositeDefinition<?> cdef = (BaseRuntimeElementCompositeDefinition<?>) def;
1547                for (BaseRuntimeChildDefinition nextChildDef : cdef.getChildren()) {
1548
1549                        List<IBase> values = nextChildDef.getAccessor().getValues(theElement);
1550                        if (values == null || values.isEmpty()) {
1551                                continue;
1552                        }
1553
1554                        String newPath = thePath + "." + nextChildDef.getElementName();
1555
1556                        for (IBase nextChild : values) {
1557                                validateChildReferenceTargetTypes(nextChild, newPath);
1558                        }
1559
1560                        if (nextChildDef instanceof RuntimeChildResourceDefinition) {
1561                                RuntimeChildResourceDefinition nextChildDefRes = (RuntimeChildResourceDefinition) nextChildDef;
1562                                Set<String> validTypes = new HashSet<>();
1563                                boolean allowAny = false;
1564                                for (Class<? extends IBaseResource> nextValidType : nextChildDefRes.getResourceTypes()) {
1565                                        if (nextValidType.isInterface()) {
1566                                                allowAny = true;
1567                                                break;
1568                                        }
1569                                        validTypes.add(getContext().getResourceType(nextValidType));
1570                                }
1571
1572                                if (allowAny) {
1573                                        continue;
1574                                }
1575
1576                                if (getStorageSettings().isEnforceReferenceTargetTypes()) {
1577                                        for (IBase nextChild : values) {
1578                                                IBaseReference nextRef = (IBaseReference) nextChild;
1579                                                IIdType referencedId = nextRef.getReferenceElement();
1580                                                if (!isBlank(referencedId.getResourceType())) {
1581                                                        if (!isLogicalReference(referencedId)) {
1582                                                                if (!referencedId.getValue().contains("?")) {
1583                                                                        if (!validTypes.contains(referencedId.getResourceType())) {
1584                                                                                throw new UnprocessableEntityException(Msg.code(931)
1585                                                                                                + "Invalid reference found at path '" + newPath + "'. Resource type '"
1586                                                                                                + referencedId.getResourceType() + "' is not valid for this path");
1587                                                                        }
1588                                                                }
1589                                                        }
1590                                                }
1591                                        }
1592                                }
1593                        }
1594                }
1595        }
1596
1597        protected void validateMetaCount(int theMetaCount) {
1598                if (myStorageSettings.getResourceMetaCountHardLimit() != null) {
1599                        if (theMetaCount > myStorageSettings.getResourceMetaCountHardLimit()) {
1600                                throw new UnprocessableEntityException(Msg.code(932) + "Resource contains " + theMetaCount
1601                                                + " meta entries (tag/profile/security label), maximum is "
1602                                                + myStorageSettings.getResourceMetaCountHardLimit());
1603                        }
1604                }
1605        }
1606
1607        /**
1608         * This method is invoked immediately before storing a new resource, or an update to an existing resource to allow the DAO to ensure that it is valid for persistence. By default, checks for the
1609         * "subsetted" tag and rejects resources which have it. Subclasses should call the superclass implementation to preserve this check.
1610         *
1611         * @param theResource     The resource that is about to be persisted
1612         * @param theEntityToSave TODO
1613         */
1614        protected void validateResourceForStorage(T theResource, ResourceTable theEntityToSave) {
1615                Object tag = null;
1616
1617                int totalMetaCount = 0;
1618
1619                if (theResource instanceof IResource) {
1620                        IResource res = (IResource) theResource;
1621                        TagList tagList = ResourceMetadataKeyEnum.TAG_LIST.get(res);
1622                        if (tagList != null) {
1623                                tag = tagList.getTag(Constants.TAG_SUBSETTED_SYSTEM_DSTU3, Constants.TAG_SUBSETTED_CODE);
1624                                totalMetaCount += tagList.size();
1625                        }
1626                        List<IdDt> profileList = ResourceMetadataKeyEnum.PROFILES.get(res);
1627                        if (profileList != null) {
1628                                totalMetaCount += profileList.size();
1629                        }
1630                } else {
1631                        IAnyResource res = (IAnyResource) theResource;
1632                        tag = res.getMeta().getTag(Constants.TAG_SUBSETTED_SYSTEM_DSTU3, Constants.TAG_SUBSETTED_CODE);
1633                        totalMetaCount += res.getMeta().getTag().size();
1634                        totalMetaCount += res.getMeta().getProfile().size();
1635                        totalMetaCount += res.getMeta().getSecurity().size();
1636                }
1637
1638                if (tag != null) {
1639                        throw new UnprocessableEntityException(
1640                                        Msg.code(933)
1641                                                        + "Resource contains the 'subsetted' tag, and must not be stored as it may contain a subset of available data");
1642                }
1643
1644                if (getStorageSettings().isEnforceReferenceTargetTypes()) {
1645                        String resName = getContext().getResourceType(theResource);
1646                        validateChildReferenceTargetTypes(theResource, resName);
1647                }
1648
1649                validateMetaCount(totalMetaCount);
1650        }
1651
1652        @PostConstruct
1653        public void start() {}
1654
1655        @VisibleForTesting
1656        public void setStorageSettingsForUnitTest(JpaStorageSettings theStorageSettings) {
1657                myStorageSettings = theStorageSettings;
1658        }
1659
1660        /**
1661         * If configured to do so, extracts the FullText indexes for the given
1662         * entity. The {@link ResourceTable#setIndexStatus(EntityIndexStatusEnum) Index Status}
1663         * is updated to reflect whether fulltext indexing is being used on this entity.
1664         */
1665        private void populateFullTextFieldsAndSetEntityStatus(
1666                        RequestDetails theRequestDetails,
1667                        final FhirContext theContext,
1668                        final IBaseResource theResource,
1669                        ResourceTable theEntity,
1670                        ResourceIndexedSearchParams theNewParams) {
1671                if (myFulltextSearchSvc == null || myFulltextSearchSvc.isDisabled()) {
1672                        theEntity.setIndexStatus(EntityIndexStatusEnum.INDEXED_RDBMS_ONLY);
1673                        return;
1674                }
1675
1676                // This will get changed if we end up setting either
1677                theEntity.setIndexStatus(EntityIndexStatusEnum.INDEXED_RDBMS_ONLY);
1678
1679                // Standard FullText indexing
1680                if (myStorageSettings.isHibernateSearchIndexFullText()) {
1681
1682                        // _content
1683                        if (mySearchParamRegistry.hasActiveSearchParam(
1684                                        theEntity.getResourceType(),
1685                                        Constants.PARAM_CONTENT,
1686                                        ISearchParamRegistry.SearchParamLookupContextEnum.INDEX)) {
1687                                Supplier<String> contentSupplier = () -> parseContentTextIntoWords(theContext, theResource);
1688                                Pointcut pointcut = Pointcut.JPA_INDEX_EXTRACT_FULLTEXT_CONTENT;
1689                                Consumer<String> contentEntitySetter = theEntity::setContentText;
1690                                extractFullTextIndexData(
1691                                                theRequestDetails, theResource, theEntity, pointcut, contentSupplier, contentEntitySetter);
1692                        }
1693
1694                        // _text
1695                        if (mySearchParamRegistry.hasActiveSearchParam(
1696                                        theEntity.getResourceType(),
1697                                        Constants.PARAM_TEXT,
1698                                        ISearchParamRegistry.SearchParamLookupContextEnum.INDEX)) {
1699                                Supplier<String> textSupplier = () -> parseNarrativeTextIntoWords(theResource);
1700                                Pointcut pointcut = Pointcut.JPA_INDEX_EXTRACT_FULLTEXT_TEXT;
1701                                Consumer<String> textEntitySetter = theEntity::setNarrativeText;
1702                                extractFullTextIndexData(
1703                                                theRequestDetails, theResource, theEntity, pointcut, textSupplier, textEntitySetter);
1704                        }
1705                }
1706
1707                // Advanced indexing - Index standard search params in the FullText index
1708                if (myStorageSettings.isHibernateSearchIndexSearchParams()) {
1709                        if (theResource != null) {
1710                                ExtendedHSearchIndexData hSearchIndexData =
1711                                                myFulltextSearchSvc.extractLuceneIndexData(theResource, theEntity, theNewParams);
1712                                theEntity.setLuceneIndexData(hSearchIndexData);
1713                        } else {
1714                                theEntity.setLuceneIndexData(null);
1715                        }
1716                        theEntity.setIndexStatus(EntityIndexStatusEnum.INDEXED_ALL);
1717                }
1718        }
1719
1720        private void extractFullTextIndexData(
1721                        RequestDetails theRequestDetails,
1722                        IBaseResource theResource,
1723                        ResourceTable theEntity,
1724                        Pointcut thePointcut,
1725                        Supplier<String> theContentSupplier,
1726                        Consumer<String> theEntityIndexSetter) {
1727                IInterceptorBroadcaster compositeBroadcaster =
1728                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
1729                FullTextExtractionResponse contentOutcome = null;
1730                if (compositeBroadcaster.hasHooks(thePointcut)) {
1731                        FullTextExtractionRequest contentRequest = new FullTextExtractionRequest(
1732                                        theEntity.getIdType(myContext), theResource, getResourceName(), theContentSupplier);
1733                        HookParams contentParams = new HookParams().add(FullTextExtractionRequest.class, contentRequest);
1734                        contentOutcome = (FullTextExtractionResponse)
1735                                        compositeBroadcaster.callHooksAndReturnObject(thePointcut, contentParams);
1736                }
1737
1738                if (contentOutcome == null || contentOutcome.isIndexNormally()) {
1739                        theEntityIndexSetter.accept(theContentSupplier.get());
1740                        theEntity.setIndexStatus(EntityIndexStatusEnum.INDEXED_ALL);
1741                } else if (!contentOutcome.isDoNotIndex()) {
1742                        theEntityIndexSetter.accept(contentOutcome.getPayload());
1743                        theEntity.setIndexStatus(EntityIndexStatusEnum.INDEXED_ALL);
1744                }
1745        }
1746
1747        @VisibleForTesting
1748        public void setPartitionSettingsForUnitTest(PartitionSettings thePartitionSettings) {
1749                myPartitionSettings = thePartitionSettings;
1750        }
1751
1752        /**
1753         * Do not call this method outside of unit tests
1754         */
1755        @VisibleForTesting
1756        public void setJpaStorageResourceParserForUnitTest(IJpaStorageResourceParser theJpaStorageResourceParser) {
1757                myJpaStorageResourceParser = theJpaStorageResourceParser;
1758        }
1759
1760        @VisibleForTesting
1761        public void setResourceTypeCacheSvc(IResourceTypeCacheSvc theResourceTypeCacheSvc) {
1762                myResourceTypeCacheSvc = theResourceTypeCacheSvc;
1763        }
1764
1765        @Nullable
1766        @SuppressWarnings("unchecked")
1767        public static String parseContentTextIntoWords(
1768                        @Nonnull FhirContext theContext, @Nullable IBaseResource theResource) {
1769                if (theResource == null) {
1770                        return null;
1771                }
1772
1773                Class<IPrimitiveType<String>> stringType = (Class<IPrimitiveType<String>>)
1774                                theContext.getElementDefinition("string").getImplementingClass();
1775
1776                StringBuilder retVal = new StringBuilder();
1777                List<IPrimitiveType<String>> childElements =
1778                                theContext.newTerser().getAllPopulatedChildElementsOfType(theResource, stringType);
1779                for (IPrimitiveType<String> nextType : childElements) {
1780                        if (stringType.equals(nextType.getClass())) {
1781                                String nextValue = nextType.getValueAsString();
1782                                if (isNotBlank(nextValue)) {
1783                                        retVal.append(nextValue.replace("\n", " ").replace("\r", " "));
1784                                        retVal.append("\n");
1785                                }
1786                        }
1787                }
1788                return retVal.toString();
1789        }
1790
1791        public static String decodeResource(byte[] theResourceBytes, ResourceEncodingEnum theResourceEncoding) {
1792                String resourceText = null;
1793                switch (theResourceEncoding) {
1794                        case JSON:
1795                                resourceText = new String(theResourceBytes, Charsets.UTF_8);
1796                                break;
1797                        case JSONC:
1798                                resourceText = GZipUtil.decompress(theResourceBytes);
1799                                break;
1800                        case DEL:
1801                        case ESR:
1802                                break;
1803                }
1804                return resourceText;
1805        }
1806
1807        @Nullable
1808        private static String parseNarrativeTextIntoWords(@Nullable IBaseResource theResource) {
1809                if (theResource == null) {
1810                        return null;
1811                }
1812                StringBuilder b = new StringBuilder();
1813                if (theResource instanceof IResource) {
1814                        IResource resource = (IResource) theResource;
1815                        List<XMLEvent> xmlEvents = XmlUtil.parse(resource.getText().getDiv().getValue());
1816                        if (xmlEvents != null) {
1817                                for (XMLEvent next : xmlEvents) {
1818                                        if (next.isCharacters()) {
1819                                                Characters characters = next.asCharacters();
1820                                                b.append(characters.getData()).append(" ");
1821                                        }
1822                                }
1823                        }
1824                } else if (theResource instanceof IDomainResource) {
1825                        IDomainResource resource = (IDomainResource) theResource;
1826                        try {
1827                                String divAsString = resource.getText().getDivAsString();
1828                                List<XMLEvent> xmlEvents = XmlUtil.parse(divAsString);
1829                                if (xmlEvents != null) {
1830                                        for (XMLEvent next : xmlEvents) {
1831                                                if (next.isCharacters()) {
1832                                                        Characters characters = next.asCharacters();
1833                                                        b.append(characters.getData()).append(" ");
1834                                                }
1835                                        }
1836                                }
1837                        } catch (Exception e) {
1838                                throw new DataFormatException(Msg.code(934) + "Unable to convert DIV to string", e);
1839                        }
1840                }
1841                return b.toString();
1842        }
1843
1844        @VisibleForTesting
1845        public static void setDisableIncrementOnUpdateForUnitTest(boolean theDisableIncrementOnUpdateForUnitTest) {
1846                ourDisableIncrementOnUpdateForUnitTest = theDisableIncrementOnUpdateForUnitTest;
1847        }
1848
1849        /**
1850         * Do not call this method outside of unit tests
1851         */
1852        @VisibleForTesting
1853        public static void setValidationDisabledForUnitTest(boolean theValidationDisabledForUnitTest) {
1854                ourValidationDisabledForUnitTest = theValidationDisabledForUnitTest;
1855        }
1856
1857        private enum CreateOrUpdateByMatch {
1858                CREATE,
1859                UPDATE
1860        }
1861
1862        private void setTargetResourceTypeIdForResourceLinks(Collection<ResourceLink> resourceLinks) {
1863                resourceLinks.stream()
1864                                .filter(link -> link.getTargetResourceType() != null)
1865                                .forEach(link -> link.setTargetResourceTypeId(
1866                                                myResourceTypeCacheSvc.getResourceTypeId(link.getTargetResourceType())));
1867        }
1868}