001/*-
002 * #%L
003 * HAPI FHIR Storage api
004 * %%
005 * Copyright (C) 2014 - 2025 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.dao;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.FhirVersionEnum;
024import ca.uhn.fhir.context.RuntimeResourceDefinition;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.HookParams;
027import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
028import ca.uhn.fhir.interceptor.api.Pointcut;
029import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
030import ca.uhn.fhir.interceptor.model.RequestPartitionId;
031import ca.uhn.fhir.interceptor.model.TransactionWriteOperationsDetails;
032import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
033import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
034import ca.uhn.fhir.jpa.api.dao.IJpaDao;
035import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
036import ca.uhn.fhir.jpa.api.model.DeleteConflict;
037import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
038import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome;
039import ca.uhn.fhir.jpa.api.model.LazyDaoMethodOutcome;
040import ca.uhn.fhir.jpa.cache.IResourceVersionSvc;
041import ca.uhn.fhir.jpa.cache.ResourcePersistentIdMap;
042import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
043import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService;
044import ca.uhn.fhir.jpa.delete.DeleteConflictUtil;
045import ca.uhn.fhir.jpa.model.config.PartitionSettings;
046import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
047import ca.uhn.fhir.jpa.model.entity.ResourceTable;
048import ca.uhn.fhir.jpa.model.entity.StorageSettings;
049import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
050import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
051import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
052import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult;
053import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher;
054import ca.uhn.fhir.jpa.searchparam.matcher.SearchParamMatcher;
055import ca.uhn.fhir.jpa.util.TransactionSemanticsHeader;
056import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
057import ca.uhn.fhir.model.valueset.BundleEntryTransactionMethodEnum;
058import ca.uhn.fhir.parser.DataFormatException;
059import ca.uhn.fhir.parser.IParser;
060import ca.uhn.fhir.rest.api.Constants;
061import ca.uhn.fhir.rest.api.PatchTypeEnum;
062import ca.uhn.fhir.rest.api.PreferReturnEnum;
063import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
064import ca.uhn.fhir.rest.api.server.RequestDetails;
065import ca.uhn.fhir.rest.api.server.storage.DeferredInterceptorBroadcasts;
066import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
067import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
068import ca.uhn.fhir.rest.param.ParameterUtil;
069import ca.uhn.fhir.rest.server.RestfulServerUtils;
070import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException;
071import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
072import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
073import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
074import ca.uhn.fhir.rest.server.exceptions.NotModifiedException;
075import ca.uhn.fhir.rest.server.exceptions.PayloadTooLargeException;
076import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
077import ca.uhn.fhir.rest.server.method.BaseMethodBinding;
078import ca.uhn.fhir.rest.server.method.BaseResourceReturningMethodBinding;
079import ca.uhn.fhir.rest.server.method.UpdateMethodBinding;
080import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
081import ca.uhn.fhir.rest.server.servlet.ServletSubRequestDetails;
082import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
083import ca.uhn.fhir.rest.server.util.ServletRequestUtil;
084import ca.uhn.fhir.util.AsyncUtil;
085import ca.uhn.fhir.util.BundleUtil;
086import ca.uhn.fhir.util.ElementUtil;
087import ca.uhn.fhir.util.FhirTerser;
088import ca.uhn.fhir.util.ResourceReferenceInfo;
089import ca.uhn.fhir.util.StopWatch;
090import ca.uhn.fhir.util.UrlUtil;
091import com.google.common.annotations.VisibleForTesting;
092import com.google.common.collect.ArrayListMultimap;
093import com.google.common.collect.ListMultimap;
094import jakarta.annotation.Nonnull;
095import jakarta.annotation.Nullable;
096import org.apache.commons.lang3.RandomUtils;
097import org.apache.commons.lang3.StringUtils;
098import org.apache.commons.lang3.ThreadUtils;
099import org.apache.commons.lang3.Validate;
100import org.hl7.fhir.dstu3.model.Bundle;
101import org.hl7.fhir.exceptions.FHIRException;
102import org.hl7.fhir.instance.model.api.IBase;
103import org.hl7.fhir.instance.model.api.IBaseBinary;
104import org.hl7.fhir.instance.model.api.IBaseBundle;
105import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
106import org.hl7.fhir.instance.model.api.IBaseParameters;
107import org.hl7.fhir.instance.model.api.IBaseReference;
108import org.hl7.fhir.instance.model.api.IBaseResource;
109import org.hl7.fhir.instance.model.api.IIdType;
110import org.hl7.fhir.instance.model.api.IPrimitiveType;
111import org.hl7.fhir.r4.model.IdType;
112import org.slf4j.Logger;
113import org.slf4j.LoggerFactory;
114import org.springframework.beans.factory.annotation.Autowired;
115import org.springframework.core.task.TaskExecutor;
116import org.springframework.transaction.PlatformTransactionManager;
117import org.springframework.transaction.TransactionDefinition;
118import org.springframework.transaction.support.TransactionCallback;
119import org.springframework.transaction.support.TransactionTemplate;
120
121import java.time.Duration;
122import java.util.ArrayList;
123import java.util.Collection;
124import java.util.Collections;
125import java.util.Comparator;
126import java.util.Date;
127import java.util.HashMap;
128import java.util.HashSet;
129import java.util.IdentityHashMap;
130import java.util.Iterator;
131import java.util.LinkedHashMap;
132import java.util.LinkedHashSet;
133import java.util.List;
134import java.util.Map;
135import java.util.Optional;
136import java.util.Set;
137import java.util.TreeSet;
138import java.util.concurrent.ConcurrentHashMap;
139import java.util.concurrent.CountDownLatch;
140import java.util.concurrent.TimeUnit;
141import java.util.regex.Pattern;
142import java.util.stream.Collectors;
143
144import static ca.uhn.fhir.util.StringUtil.toUtf8String;
145import static java.util.Objects.isNull;
146import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
147import static org.apache.commons.lang3.StringUtils.defaultString;
148import static org.apache.commons.lang3.StringUtils.isBlank;
149import static org.apache.commons.lang3.StringUtils.isNotBlank;
150
151public abstract class BaseTransactionProcessor {
152
153        public static final String URN_PREFIX = "urn:";
154        public static final String URN_PREFIX_ESCAPED = UrlUtil.escapeUrlParam(URN_PREFIX);
155        public static final Pattern UNQUALIFIED_MATCH_URL_START = Pattern.compile("^[a-zA-Z0-9_-]+=");
156        public static final Pattern INVALID_PLACEHOLDER_PATTERN = Pattern.compile("[a-zA-Z]+:.*");
157        private static final Logger ourLog = LoggerFactory.getLogger(BaseTransactionProcessor.class);
158        private static final String POST = "POST";
159        private static final String PUT = "PUT";
160        private static final String DELETE = "DELETE";
161        private static final String PATCH = "PATCH";
162
163        @Autowired
164        private IRequestPartitionHelperSvc myRequestPartitionHelperService;
165
166        @Autowired
167        private PlatformTransactionManager myTxManager;
168
169        @Autowired
170        private FhirContext myContext;
171
172        @Autowired
173        @SuppressWarnings("rawtypes")
174        private ITransactionProcessorVersionAdapter myVersionAdapter;
175
176        @Autowired
177        private DaoRegistry myDaoRegistry;
178
179        @Autowired
180        private IInterceptorBroadcaster myInterceptorBroadcaster;
181
182        @Autowired
183        private IHapiTransactionService myHapiTransactionService;
184
185        @Autowired
186        private StorageSettings myStorageSettings;
187
188        @Autowired
189        PartitionSettings myPartitionSettings;
190
191        @Autowired
192        private InMemoryResourceMatcher myInMemoryResourceMatcher;
193
194        @Autowired
195        private SearchParamMatcher mySearchParamMatcher;
196
197        @Autowired
198        private ThreadPoolFactory myThreadPoolFactory;
199
200        private TaskExecutor myExecutor;
201
202        @Autowired
203        private IResourceVersionSvc myResourceVersionSvc;
204
205        @VisibleForTesting
206        public void setStorageSettings(StorageSettings theStorageSettings) {
207                myStorageSettings = theStorageSettings;
208        }
209
210        public ITransactionProcessorVersionAdapter getVersionAdapter() {
211                return myVersionAdapter;
212        }
213
214        @VisibleForTesting
215        public void setVersionAdapter(ITransactionProcessorVersionAdapter theVersionAdapter) {
216                myVersionAdapter = theVersionAdapter;
217        }
218
219        private TaskExecutor getTaskExecutor() {
220                if (myExecutor == null) {
221                        myExecutor = myThreadPoolFactory.newThreadPool(
222                                        myStorageSettings.getBundleBatchPoolSize(),
223                                        myStorageSettings.getBundleBatchMaxPoolSize(),
224                                        "bundle-batch-");
225                }
226                return myExecutor;
227        }
228
229        public <BUNDLE extends IBaseBundle> BUNDLE transaction(
230                        RequestDetails theRequestDetails, BUNDLE theRequest, boolean theNestedMode) {
231                String actionName = "Transaction";
232                IBaseBundle response = processTransactionAsSubRequest(theRequestDetails, theRequest, actionName, theNestedMode);
233
234                List<IBase> entries = myVersionAdapter.getEntries(response);
235                for (int i = 0; i < entries.size(); i++) {
236                        if (ElementUtil.isEmpty(entries.get(i))) {
237                                entries.remove(i);
238                                i--;
239                        }
240                }
241
242                return (BUNDLE) response;
243        }
244
245        public IBaseBundle collection(final RequestDetails theRequestDetails, IBaseBundle theRequest) {
246                String transactionType = myVersionAdapter.getBundleType(theRequest);
247
248                if (!org.hl7.fhir.r4.model.Bundle.BundleType.COLLECTION.toCode().equals(transactionType)) {
249                        throw new InvalidRequestException(
250                                        Msg.code(526) + "Can not process collection Bundle of type: " + transactionType);
251                }
252
253                ourLog.info(
254                                "Beginning storing collection with {} resources",
255                                myVersionAdapter.getEntries(theRequest).size());
256
257                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
258                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
259
260                IBaseBundle resp =
261                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
262
263                List<IBaseResource> resources = new ArrayList<>();
264                for (final Object nextRequestEntry : myVersionAdapter.getEntries(theRequest)) {
265                        IBaseResource resource = myVersionAdapter.getResource((IBase) nextRequestEntry);
266                        resources.add(resource);
267                }
268
269                IBaseBundle transactionBundle = myVersionAdapter.createBundle("transaction");
270                for (IBaseResource next : resources) {
271                        IBase entry = myVersionAdapter.addEntry(transactionBundle);
272                        myVersionAdapter.setResource(entry, next);
273                        myVersionAdapter.setRequestVerb(entry, "PUT");
274                        myVersionAdapter.setRequestUrl(
275                                        entry, next.getIdElement().toUnqualifiedVersionless().getValue());
276                }
277
278                transaction(theRequestDetails, transactionBundle, false);
279
280                return resp;
281        }
282
283        @SuppressWarnings("unchecked")
284        private void populateEntryWithOperationOutcome(BaseServerResponseException caughtEx, IBase nextEntry) {
285                myVersionAdapter.populateEntryWithOperationOutcome(caughtEx, nextEntry);
286        }
287
288        @SuppressWarnings("unchecked")
289        private void handleTransactionCreateOrUpdateOutcome(
290                        IdSubstitutionMap theIdSubstitutions,
291                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
292                        IIdType theNextResourceId,
293                        DaoMethodOutcome theOutcome,
294                        IBase theNewEntry,
295                        String theResourceType,
296                        IBaseResource theRes,
297                        RequestDetails theRequestDetails) {
298                IIdType newId = theOutcome.getId().toUnqualified();
299                IIdType resourceId =
300                                isPlaceholder(theNextResourceId) ? theNextResourceId : theNextResourceId.toUnqualifiedVersionless();
301                if (!newId.equals(resourceId)) {
302                        if (!theNextResourceId.isEmpty()) {
303                                theIdSubstitutions.put(resourceId, newId);
304                        }
305                        if (isPlaceholder(resourceId)) {
306                                /*
307                                 * The correct way for substitution IDs to be is to be with no resource type, but we'll accept the qualified kind too just to be lenient.
308                                 */
309                                IIdType id = myContext.getVersion().newIdType();
310                                id.setValue(theResourceType + '/' + resourceId.getValue());
311                                theIdSubstitutions.put(id, newId);
312                        }
313                }
314
315                populateIdToPersistedOutcomeMap(theIdToPersistedOutcome, newId, theOutcome);
316
317                if (shouldSwapBinaryToActualResource(theRes, theResourceType, theNextResourceId)) {
318                        theRes = theIdToPersistedOutcome.get(newId).getResource();
319                }
320
321                if (theOutcome.getCreated()) {
322                        myVersionAdapter.setResponseStatus(theNewEntry, toStatusString(Constants.STATUS_HTTP_201_CREATED));
323                } else {
324                        myVersionAdapter.setResponseStatus(theNewEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
325                }
326
327                Date lastModified = getLastModified(theRes);
328                myVersionAdapter.setResponseLastModified(theNewEntry, lastModified);
329
330                if (theOutcome.getOperationOutcome() != null) {
331                        myVersionAdapter.setResponseOutcome(theNewEntry, theOutcome.getOperationOutcome());
332                }
333
334                if (theRequestDetails != null) {
335                        String prefer = theRequestDetails.getHeader(Constants.HEADER_PREFER);
336                        PreferReturnEnum preferReturn =
337                                        RestfulServerUtils.parsePreferHeader(null, prefer).getReturn();
338                        if (preferReturn != null) {
339                                if (preferReturn == PreferReturnEnum.REPRESENTATION) {
340                                        if (theOutcome.getResource() != null) {
341                                                theOutcome.fireResourceViewCallbacks();
342                                                myVersionAdapter.setResource(theNewEntry, theOutcome.getResource());
343                                        }
344                                }
345                        }
346                }
347        }
348
349        /**
350         * Method which populates entry in idToPersistedOutcome.
351         * Will store whatever outcome is sent, unless the key already exists, then we only replace an instance if we find that the instance
352         * we are replacing with is non-lazy. This allows us to evaluate later more easily, as we _know_ we need access to these.
353         */
354        private void populateIdToPersistedOutcomeMap(
355                        Map<IIdType, DaoMethodOutcome> idToPersistedOutcome, IIdType newId, DaoMethodOutcome outcome) {
356                // Prefer real method outcomes over lazy ones.
357                if (idToPersistedOutcome.containsKey(newId)) {
358                        if (!(outcome instanceof LazyDaoMethodOutcome)) {
359                                idToPersistedOutcome.put(newId, outcome);
360                        }
361                } else {
362                        idToPersistedOutcome.put(newId, outcome);
363                }
364        }
365
366        private Date getLastModified(IBaseResource theRes) {
367                return theRes.getMeta().getLastUpdated();
368        }
369
370        private IBaseBundle processTransactionAsSubRequest(
371                        RequestDetails theRequestDetails, IBaseBundle theRequest, String theActionName, boolean theNestedMode) {
372                BaseStorageDao.markRequestAsProcessingSubRequest(theRequestDetails);
373                try {
374
375                        // Interceptor call: STORAGE_TRANSACTION_PROCESSING
376                        IInterceptorBroadcaster compositeBroadcaster = CompositeInterceptorBroadcaster.newCompositeBroadcaster(
377                                        myInterceptorBroadcaster, theRequestDetails);
378                        if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING)) {
379                                HookParams params = new HookParams()
380                                                .add(RequestDetails.class, theRequestDetails)
381                                                .addIfMatchesType(ServletRequestDetails.class, theRequest)
382                                                .add(IBaseBundle.class, theRequest);
383                                compositeBroadcaster.callHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING, params);
384                        }
385
386                        return processTransaction(theRequestDetails, theRequest, theActionName, theNestedMode);
387                } finally {
388                        BaseStorageDao.clearRequestAsProcessingSubRequest(theRequestDetails);
389                }
390        }
391
392        @VisibleForTesting
393        public void setTxManager(PlatformTransactionManager theTxManager) {
394                myTxManager = theTxManager;
395        }
396
397        private IBaseBundle batch(final RequestDetails theRequestDetails, IBaseBundle theRequest, boolean theNestedMode) {
398                TransactionSemanticsHeader transactionSemantics = TransactionSemanticsHeader.DEFAULT;
399                if (theRequestDetails != null) {
400                        String transactionSemanticsString = theRequestDetails.getHeader("X-Transaction-Semantics");
401                        if (transactionSemanticsString != null) {
402                                transactionSemantics = TransactionSemanticsHeader.parse(transactionSemanticsString);
403                        }
404                }
405
406                int totalAttempts = defaultIfNull(transactionSemantics.getRetryCount(), 0) + 1;
407                boolean switchedToBatch = false;
408
409                int minRetryDelay = defaultIfNull(transactionSemantics.getMinRetryDelay(), 0);
410                int maxRetryDelay = defaultIfNull(transactionSemantics.getMaxRetryDelay(), 0);
411
412                // Don't let the user request a crazy delay, this could be used
413                // as a DOS attack
414                maxRetryDelay = Math.max(maxRetryDelay, minRetryDelay);
415                maxRetryDelay = Math.min(maxRetryDelay, 10000);
416                totalAttempts = Math.min(totalAttempts, 5);
417
418                IBaseBundle response;
419                for (int i = 1; ; i++) {
420                        try {
421                                if (i < totalAttempts && transactionSemantics.isTryBatchAsTransactionFirst()) {
422                                        BundleUtil.setBundleType(myContext, theRequest, "transaction");
423                                        response = processTransaction(theRequestDetails, theRequest, "Transaction", theNestedMode);
424                                } else {
425                                        BundleUtil.setBundleType(myContext, theRequest, "batch");
426                                        response = processBatch(theRequestDetails, theRequest, theNestedMode);
427                                }
428                                break;
429                        } catch (BaseServerResponseException e) {
430                                if (i >= totalAttempts || theNestedMode) {
431                                        throw e;
432                                }
433
434                                long delay = RandomUtils.insecure().randomLong(minRetryDelay, maxRetryDelay);
435                                String sleepMessage = "";
436                                if (delay > 0) {
437                                        sleepMessage = "Sleeping for " + delay + " ms. ";
438                                }
439
440                                String switchedToBatchMessage = "";
441                                if (switchedToBatch) {
442                                        switchedToBatchMessage = " (as batch)";
443                                }
444
445                                String switchingToBatchMessage = "";
446                                if (i + 1 == totalAttempts && transactionSemantics.isTryBatchAsTransactionFirst()) {
447                                        switchingToBatchMessage = "Performing final retry using batch semantics. ";
448                                        switchedToBatch = true;
449                                        BundleUtil.setBundleType(myContext, theRequest, "batch");
450                                }
451
452                                ourLog.warn(
453                                                "Transaction processing attempt{} {}/{} failed. {}{}",
454                                                switchedToBatchMessage,
455                                                i,
456                                                totalAttempts,
457                                                sleepMessage,
458                                                switchingToBatchMessage);
459
460                                if (delay > 0) {
461                                        ThreadUtils.sleepQuietly(Duration.ofMillis(delay));
462                                }
463                        }
464                }
465
466                return response;
467        }
468
469        private IBaseBundle processBatch(RequestDetails theRequestDetails, IBaseBundle theRequest, boolean theNestedMode) {
470                ourLog.info(
471                                "Beginning batch with {} resources",
472                                myVersionAdapter.getEntries(theRequest).size());
473
474                long start = System.currentTimeMillis();
475
476                IBaseBundle response =
477                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
478                Map<Integer, Object> responseMap = new ConcurrentHashMap<>();
479
480                List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest);
481                int requestEntriesSize = requestEntries.size();
482
483                // Now, run all non-gets sequentially, and all gets are submitted to the executor to run (potentially) in
484                // parallel
485                // The result is kept in the map to save the original position
486                List<RetriableBundleTask> getCalls = new ArrayList<>();
487                List<RetriableBundleTask> nonGetCalls = new ArrayList<>();
488
489                CountDownLatch completionLatch = new CountDownLatch(requestEntriesSize);
490                for (int i = 0; i < requestEntriesSize; i++) {
491                        IBase nextRequestEntry = requestEntries.get(i);
492                        RetriableBundleTask retriableBundleTask = new RetriableBundleTask(
493                                        completionLatch, theRequestDetails, responseMap, i, nextRequestEntry, theNestedMode);
494                        if (myVersionAdapter
495                                        .getEntryRequestVerb(myContext, nextRequestEntry)
496                                        .equalsIgnoreCase("GET")) {
497                                getCalls.add(retriableBundleTask);
498                        } else {
499                                nonGetCalls.add(retriableBundleTask);
500                        }
501                }
502                // Execute all non-gets on calling thread.
503                nonGetCalls.forEach(RetriableBundleTask::run);
504                // Execute all gets (potentially in a pool)
505                if (myStorageSettings.getBundleBatchPoolSize() == 1) {
506                        getCalls.forEach(RetriableBundleTask::run);
507                } else {
508                        getCalls.forEach(getCall -> getTaskExecutor().execute(getCall));
509                }
510
511                // waiting for all async tasks to be completed
512                AsyncUtil.awaitLatchAndIgnoreInterrupt(completionLatch, 300L, TimeUnit.SECONDS);
513
514                // Now, create the bundle response in original order
515                Object nextResponseEntry;
516                for (int i = 0; i < requestEntriesSize; i++) {
517
518                        nextResponseEntry = responseMap.get(i);
519                        if (nextResponseEntry instanceof ServerResponseExceptionHolder) {
520                                ServerResponseExceptionHolder caughtEx = (ServerResponseExceptionHolder) nextResponseEntry;
521                                if (caughtEx.getException() != null) {
522                                        IBase nextEntry = myVersionAdapter.addEntry(response);
523                                        populateEntryWithOperationOutcome(caughtEx.getException(), nextEntry);
524                                        myVersionAdapter.setResponseStatus(
525                                                        nextEntry, toStatusString(caughtEx.getException().getStatusCode()));
526                                }
527                        } else {
528                                myVersionAdapter.addEntry(response, (IBase) nextResponseEntry);
529                        }
530                }
531
532                long delay = System.currentTimeMillis() - start;
533                ourLog.info("Batch completed in {}ms", delay);
534
535                return response;
536        }
537
538        @VisibleForTesting
539        public void setHapiTransactionService(HapiTransactionService theHapiTransactionService) {
540                myHapiTransactionService = theHapiTransactionService;
541        }
542
543        private IBaseBundle processTransaction(
544                        final RequestDetails theRequestDetails,
545                        final IBaseBundle theRequest,
546                        final String theActionName,
547                        boolean theNestedMode) {
548                validateDependencies();
549
550                String transactionType = myVersionAdapter.getBundleType(theRequest);
551
552                if (org.hl7.fhir.r4.model.Bundle.BundleType.BATCH.toCode().equals(transactionType)) {
553                        return batch(theRequestDetails, theRequest, theNestedMode);
554                }
555
556                if (transactionType == null) {
557                        String message = "Transaction Bundle did not specify valid Bundle.type, assuming "
558                                        + Bundle.BundleType.TRANSACTION.toCode();
559                        ourLog.warn(message);
560                        transactionType = org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode();
561                }
562                if (!org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode().equals(transactionType)) {
563                        throw new InvalidRequestException(
564                                        Msg.code(527) + "Unable to process transaction where incoming Bundle.type = " + transactionType);
565                }
566
567                List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest);
568                int numberOfEntries = requestEntries.size();
569
570                if (myStorageSettings.getMaximumTransactionBundleSize() != null
571                                && numberOfEntries > myStorageSettings.getMaximumTransactionBundleSize()) {
572                        throw new PayloadTooLargeException(
573                                        Msg.code(528) + "Transaction Bundle Too large.  Transaction bundle contains " + numberOfEntries
574                                                        + " which exceedes the maximum permitted transaction bundle size of "
575                                                        + myStorageSettings.getMaximumTransactionBundleSize());
576                }
577
578                ourLog.debug("Beginning {} with {} resources", theActionName, numberOfEntries);
579
580                final TransactionDetails transactionDetails = new TransactionDetails();
581                transactionDetails.setFhirTransaction(true);
582                final StopWatch transactionStopWatch = new StopWatch();
583
584                // Do all entries have a verb?
585                for (int i = 0; i < numberOfEntries; i++) {
586                        IBase nextReqEntry = requestEntries.get(i);
587                        String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
588                        if (verb == null || !isValidVerb(verb)) {
589                                throw new InvalidRequestException(Msg.code(529)
590                                                + myContext
591                                                                .getLocalizer()
592                                                                .getMessage(BaseStorageDao.class, "transactionEntryHasInvalidVerb", verb, i));
593                        }
594                }
595
596                /*
597                 * We want to execute the transaction request bundle elements in the order
598                 * specified by the FHIR specification (see TransactionSorter) so we save the
599                 * original order in the request, then sort it.
600                 *
601                 * Entries with a type of GET are removed from the bundle so that they
602                 * can be processed at the very end. We do this because the incoming resources
603                 * are saved in a two-phase way in order to deal with interdependencies, and
604                 * we want the GET processing to use the final indexing state
605                 */
606                final IBaseBundle response =
607                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTIONRESPONSE.toCode());
608                List<IBase> getEntries = new ArrayList<>();
609                final IdentityHashMap<IBase, Integer> originalRequestOrder = new IdentityHashMap<>();
610                for (int i = 0; i < requestEntries.size(); i++) {
611                        IBase requestEntry = requestEntries.get(i);
612                        originalRequestOrder.put(requestEntry, i);
613                        myVersionAdapter.addEntry(response);
614                        if (myVersionAdapter.getEntryRequestVerb(myContext, requestEntry).equals("GET")) {
615                                getEntries.add(requestEntry);
616                        }
617                }
618
619                /*
620                 * See FhirSystemDaoDstu3Test#testTransactionWithPlaceholderIdInMatchUrl
621                 * Basically if the resource has a match URL that references a placeholder,
622                 * we try to handle the resource with the placeholder first.
623                 */
624                Set<String> placeholderIds = new HashSet<>();
625                for (IBase nextEntry : requestEntries) {
626                        String fullUrl = myVersionAdapter.getFullUrl(nextEntry);
627                        if (isNotBlank(fullUrl) && fullUrl.startsWith(URN_PREFIX)) {
628                                placeholderIds.add(fullUrl);
629                        }
630                }
631                requestEntries.sort(new TransactionSorter(placeholderIds));
632
633                // perform all writes
634                prepareThenExecuteTransactionWriteOperations(
635                                theRequestDetails,
636                                theActionName,
637                                transactionDetails,
638                                transactionStopWatch,
639                                response,
640                                originalRequestOrder,
641                                requestEntries);
642
643                // perform all gets
644                // (we do these last so that the gets happen on the final state of the DB;
645                // see above note)
646                doTransactionReadOperations(
647                                theRequestDetails, response, getEntries, originalRequestOrder, transactionStopWatch, theNestedMode);
648
649                // Interceptor broadcast: JPA_PERFTRACE_INFO
650                IInterceptorBroadcaster compositeBroadcaster =
651                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
652                if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO)) {
653                        String taskDurations = transactionStopWatch.formatTaskDurations();
654                        StorageProcessingMessage message = new StorageProcessingMessage();
655                        message.setMessage("Transaction timing:\n" + taskDurations);
656                        HookParams params = new HookParams()
657                                        .add(RequestDetails.class, theRequestDetails)
658                                        .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
659                                        .add(StorageProcessingMessage.class, message);
660                        compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
661                }
662
663                return response;
664        }
665
666        @SuppressWarnings("unchecked")
667        private void doTransactionReadOperations(
668                        final RequestDetails theRequestDetails,
669                        IBaseBundle theResponse,
670                        List<IBase> theGetEntries,
671                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
672                        StopWatch theTransactionStopWatch,
673                        boolean theNestedMode) {
674                if (theGetEntries.size() > 0) {
675                        theTransactionStopWatch.startTask("Process " + theGetEntries.size() + " GET entries");
676
677                        /*
678                         * Loop through the request and process any entries of type GET
679                         */
680                        for (IBase nextReqEntry : theGetEntries) {
681                                if (theNestedMode) {
682                                        throw new InvalidRequestException(
683                                                        Msg.code(530) + "Can not invoke read operation on nested transaction");
684                                }
685
686                                if (!(theRequestDetails instanceof ServletRequestDetails)) {
687                                        throw new MethodNotAllowedException(
688                                                        Msg.code(531) + "Can not call transaction GET methods from this context");
689                                }
690
691                                ServletRequestDetails srd = (ServletRequestDetails) theRequestDetails;
692                                Integer originalOrder = theOriginalRequestOrder.get(nextReqEntry);
693                                IBase nextRespEntry =
694                                                (IBase) myVersionAdapter.getEntries(theResponse).get(originalOrder);
695
696                                ArrayListMultimap<String, String> paramValues = ArrayListMultimap.create();
697
698                                String transactionUrl = extractTransactionUrlOrThrowException(nextReqEntry, "GET");
699
700                                ServletSubRequestDetails requestDetails =
701                                                ServletRequestUtil.getServletSubRequestDetails(srd, transactionUrl, paramValues);
702
703                                String url = requestDetails.getRequestPath();
704
705                                BaseMethodBinding method = srd.getServer().determineResourceMethod(requestDetails, url);
706                                if (method == null) {
707                                        throw new IllegalArgumentException(Msg.code(532) + "Unable to handle GET " + url);
708                                }
709
710                                if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) {
711                                        requestDetails.addHeader(
712                                                        Constants.HEADER_IF_MATCH, myVersionAdapter.getEntryRequestIfMatch(nextReqEntry));
713                                }
714                                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry))) {
715                                        requestDetails.addHeader(
716                                                        Constants.HEADER_IF_NONE_EXIST, myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry));
717                                }
718                                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry))) {
719                                        requestDetails.addHeader(
720                                                        Constants.HEADER_IF_NONE_MATCH, myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry));
721                                }
722
723                                Validate.isTrue(method instanceof BaseResourceReturningMethodBinding, "Unable to handle GET {}", url);
724                                try {
725                                        BaseResourceReturningMethodBinding methodBinding = (BaseResourceReturningMethodBinding) method;
726                                        requestDetails.setRestOperationType(methodBinding.getRestOperationType());
727
728                                        IBaseResource resource = methodBinding.doInvokeServer(srd.getServer(), requestDetails);
729                                        if (paramValues.containsKey(Constants.PARAM_SUMMARY)
730                                                        || paramValues.containsKey(Constants.PARAM_CONTENT)) {
731                                                resource = filterNestedBundle(requestDetails, resource);
732                                        }
733                                        myVersionAdapter.setResource(nextRespEntry, resource);
734                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
735                                } catch (NotModifiedException e) {
736                                        myVersionAdapter.setResponseStatus(
737                                                        nextRespEntry, toStatusString(Constants.STATUS_HTTP_304_NOT_MODIFIED));
738                                } catch (BaseServerResponseException e) {
739                                        ourLog.info("Failure processing transaction GET {}: {}", url, e.toString());
740                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(e.getStatusCode()));
741                                        populateEntryWithOperationOutcome(e, nextRespEntry);
742                                }
743                        }
744                        theTransactionStopWatch.endCurrentTask();
745                }
746        }
747
748        /**
749         * All of the write operations in the transaction (PUT, POST, etc.. basically anything
750         * except GET) are performed in their own database transaction before we do the reads.
751         * We do this because the reads (specifically the searches) often spawn their own
752         * secondary database transaction and if we allow that within the primary
753         * database transaction we can end up with deadlocks if the server is under
754         * heavy load with lots of concurrent transactions using all available
755         * database connections.
756         */
757        @SuppressWarnings("unchecked")
758        private void prepareThenExecuteTransactionWriteOperations(
759                        RequestDetails theRequestDetails,
760                        String theActionName,
761                        TransactionDetails theTransactionDetails,
762                        StopWatch theTransactionStopWatch,
763                        IBaseBundle theResponse,
764                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
765                        List<IBase> theEntries) {
766
767                TransactionWriteOperationsDetails writeOperationsDetails = null;
768                if (haveWriteOperationsHooks(theRequestDetails)) {
769                        writeOperationsDetails = buildWriteOperationsDetails(theEntries);
770                        callWriteOperationsHook(
771                                        Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE,
772                                        theRequestDetails,
773                                        theTransactionDetails,
774                                        writeOperationsDetails);
775                }
776
777                TransactionCallback<EntriesToProcessMap> txCallback = status -> {
778                        final Set<IIdType> allIds = new LinkedHashSet<>();
779                        final IdSubstitutionMap idSubstitutions = new IdSubstitutionMap();
780                        final Map<IIdType, DaoMethodOutcome> idToPersistedOutcome = new LinkedHashMap<>();
781
782                        EntriesToProcessMap retVal = doTransactionWriteOperations(
783                                        theRequestDetails,
784                                        theActionName,
785                                        theTransactionDetails,
786                                        allIds,
787                                        idSubstitutions,
788                                        idToPersistedOutcome,
789                                        theResponse,
790                                        theOriginalRequestOrder,
791                                        theEntries,
792                                        theTransactionStopWatch);
793
794                        theTransactionStopWatch.startTask("Commit writes to database");
795                        return retVal;
796                };
797                EntriesToProcessMap entriesToProcess;
798
799                RequestPartitionId requestPartitionId =
800                                determineRequestPartitionIdForWriteEntries(theRequestDetails, theEntries);
801
802                try {
803                        entriesToProcess = myHapiTransactionService
804                                        .withRequest(theRequestDetails)
805                                        .withRequestPartitionId(requestPartitionId)
806                                        .withTransactionDetails(theTransactionDetails)
807                                        .execute(txCallback);
808                } finally {
809                        if (haveWriteOperationsHooks(theRequestDetails)) {
810                                callWriteOperationsHook(
811                                                Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST,
812                                                theRequestDetails,
813                                                theTransactionDetails,
814                                                writeOperationsDetails);
815                        }
816                }
817
818                theTransactionStopWatch.endCurrentTask();
819
820                for (Map.Entry<IBase, IIdType> nextEntry : entriesToProcess.entrySet()) {
821                        String responseLocation = nextEntry.getValue().toUnqualified().getValue();
822                        String responseEtag = nextEntry.getValue().getVersionIdPart();
823                        myVersionAdapter.setResponseLocation(nextEntry.getKey(), responseLocation);
824                        myVersionAdapter.setResponseETag(nextEntry.getKey(), responseEtag);
825                }
826        }
827
828        /**
829         * This method looks at the FHIR actions being performed in a List of bundle entries,
830         * and determines the associated request partitions.
831         */
832        @Nullable
833        protected RequestPartitionId determineRequestPartitionIdForWriteEntries(
834                        RequestDetails theRequestDetails, List<IBase> theEntries) {
835                if (!myPartitionSettings.isPartitioningEnabled()) {
836                        return RequestPartitionId.allPartitions();
837                }
838
839                return theEntries.stream()
840                                .map(e -> getEntryRequestPartitionId(theRequestDetails, e))
841                                .reduce(null, (accumulator, nextPartition) -> {
842                                        if (accumulator == null) {
843                                                return nextPartition;
844                                        } else if (nextPartition == null) {
845                                                return accumulator;
846                                        } else if (myHapiTransactionService.isCompatiblePartition(accumulator, nextPartition)) {
847                                                return accumulator.mergeIds(nextPartition);
848                                        } else {
849                                                String msg = myContext
850                                                                .getLocalizer()
851                                                                .getMessage(
852                                                                                BaseTransactionProcessor.class, "multiplePartitionAccesses", theEntries.size());
853                                                throw new InvalidRequestException(Msg.code(2541) + msg);
854                                        }
855                                });
856        }
857
858        @Nullable
859        private RequestPartitionId getEntryRequestPartitionId(RequestDetails theRequestDetails, IBase nextEntry) {
860                RequestPartitionId nextWriteEntryRequestPartitionId = null;
861                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextEntry);
862                if (isNotBlank(verb)) {
863                        BundleEntryTransactionMethodEnum verbEnum = BundleEntryTransactionMethodEnum.valueOf(verb);
864                        switch (verbEnum) {
865                                case GET:
866                                        nextWriteEntryRequestPartitionId = null;
867                                        break;
868                                case DELETE: {
869                                        String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
870                                        if (isNotBlank(requestUrl)) {
871                                                IdType id = new IdType(requestUrl);
872                                                String resourceType = id.getResourceType();
873                                                ReadPartitionIdRequestDetails details =
874                                                                ReadPartitionIdRequestDetails.forDelete(resourceType, id);
875                                                nextWriteEntryRequestPartitionId =
876                                                                myRequestPartitionHelperService.determineReadPartitionForRequest(
877                                                                                theRequestDetails, details);
878                                        }
879                                        break;
880                                }
881                                case PATCH: {
882                                        String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
883                                        if (isNotBlank(requestUrl)) {
884                                                IdType id = new IdType(requestUrl);
885                                                String resourceType = id.getResourceType();
886                                                ReadPartitionIdRequestDetails details =
887                                                                ReadPartitionIdRequestDetails.forPatch(resourceType, id);
888                                                nextWriteEntryRequestPartitionId =
889                                                                myRequestPartitionHelperService.determineReadPartitionForRequest(
890                                                                                theRequestDetails, details);
891                                        }
892                                        break;
893                                }
894                                case POST:
895                                case PUT: {
896                                        IBaseResource resource = myVersionAdapter.getResource(nextEntry);
897                                        if (resource != null) {
898                                                String resourceType = myContext.getResourceType(resource);
899                                                nextWriteEntryRequestPartitionId =
900                                                                myRequestPartitionHelperService.determineCreatePartitionForRequest(
901                                                                                theRequestDetails, resource, resourceType);
902                                        }
903                                }
904                        }
905                }
906                return nextWriteEntryRequestPartitionId;
907        }
908
909        private boolean haveWriteOperationsHooks(RequestDetails theRequestDetails) {
910                IInterceptorBroadcaster compositeBroadcaster =
911                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
912                return compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE)
913                                || compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST);
914        }
915
916        private void callWriteOperationsHook(
917                        Pointcut thePointcut,
918                        RequestDetails theRequestDetails,
919                        TransactionDetails theTransactionDetails,
920                        TransactionWriteOperationsDetails theWriteOperationsDetails) {
921                IInterceptorBroadcaster compositeBroadcaster =
922                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
923                if (compositeBroadcaster.hasHooks(thePointcut)) {
924                        HookParams params = new HookParams()
925                                        .add(TransactionDetails.class, theTransactionDetails)
926                                        .add(TransactionWriteOperationsDetails.class, theWriteOperationsDetails);
927                        compositeBroadcaster.callHooks(thePointcut, params);
928                }
929        }
930
931        @SuppressWarnings("unchecked")
932        private TransactionWriteOperationsDetails buildWriteOperationsDetails(List<IBase> theEntries) {
933                TransactionWriteOperationsDetails writeOperationsDetails;
934                List<String> updateRequestUrls = new ArrayList<>();
935                List<String> conditionalCreateRequestUrls = new ArrayList<>();
936                // Extract
937                for (IBase nextEntry : theEntries) {
938                        String method = myVersionAdapter.getEntryRequestVerb(myContext, nextEntry);
939                        if ("PUT".equals(method)) {
940                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
941                                if (isNotBlank(requestUrl)) {
942                                        updateRequestUrls.add(requestUrl);
943                                }
944                        } else if ("POST".equals(method)) {
945                                String requestUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextEntry);
946                                if (isNotBlank(requestUrl) && requestUrl.contains("?")) {
947                                        conditionalCreateRequestUrls.add(requestUrl);
948                                }
949                        }
950                }
951
952                writeOperationsDetails = new TransactionWriteOperationsDetails();
953                writeOperationsDetails.setUpdateRequestUrls(updateRequestUrls);
954                writeOperationsDetails.setConditionalCreateRequestUrls(conditionalCreateRequestUrls);
955                return writeOperationsDetails;
956        }
957
958        private boolean isValidVerb(String theVerb) {
959                try {
960                        return org.hl7.fhir.r4.model.Bundle.HTTPVerb.fromCode(theVerb) != null;
961                } catch (FHIRException theE) {
962                        return false;
963                }
964        }
965
966        /**
967         * This method is called for nested bundles (e.g. if we received a transaction with an entry that
968         * was a GET search, this method is called on the bundle for the search result, that will be placed in the
969         * outer bundle). This method applies the _summary and _content parameters to the output of
970         * that bundle.
971         * <p>
972         * TODO: This isn't the most efficient way of doing this.. hopefully we can come up with something better in the future.
973         */
974        private IBaseResource filterNestedBundle(RequestDetails theRequestDetails, IBaseResource theResource) {
975                IParser p = myContext.newJsonParser();
976                RestfulServerUtils.configureResponseParser(theRequestDetails, p);
977                return p.parseResource(theResource.getClass(), p.encodeResourceToString(theResource));
978        }
979
980        protected void validateDependencies() {
981                Validate.notNull(myContext);
982                Validate.notNull(myTxManager);
983        }
984
985        private IIdType newIdType(String theValue) {
986                return myContext.getVersion().newIdType().setValue(theValue);
987        }
988
989        /**
990         * Searches for duplicate conditional creates and consolidates them.
991         */
992        @SuppressWarnings("unchecked")
993        private void consolidateDuplicateConditionals(
994                        RequestDetails theRequestDetails, String theActionName, List<IBase> theEntries) {
995                final Set<String> keysWithNoFullUrl = new HashSet<>();
996                final HashMap<String, String> keyToUuid = new HashMap<>();
997
998                for (int index = 0, originalIndex = 0; index < theEntries.size(); index++, originalIndex++) {
999                        IBase nextReqEntry = theEntries.get(index);
1000                        IBaseResource resource = myVersionAdapter.getResource(nextReqEntry);
1001                        if (resource != null) {
1002                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
1003                                String entryFullUrl = myVersionAdapter.getFullUrl(nextReqEntry);
1004                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextReqEntry);
1005                                String ifNoneExist = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
1006
1007                                // Conditional UPDATE
1008                                boolean consolidateEntryCandidate = false;
1009                                String conditionalUrl;
1010                                switch (verb) {
1011                                        case "PUT":
1012                                                conditionalUrl = requestUrl;
1013                                                if (isNotBlank(requestUrl)) {
1014                                                        int questionMarkIndex = requestUrl.indexOf('?');
1015                                                        if (questionMarkIndex >= 0 && requestUrl.length() > (questionMarkIndex + 1)) {
1016                                                                consolidateEntryCandidate = true;
1017                                                        }
1018                                                }
1019                                                break;
1020
1021                                                // Conditional CREATE
1022                                        case "POST":
1023                                                conditionalUrl = ifNoneExist;
1024                                                if (isNotBlank(ifNoneExist)) {
1025                                                        if (isBlank(entryFullUrl) || !entryFullUrl.equals(requestUrl)) {
1026                                                                consolidateEntryCandidate = true;
1027                                                        }
1028                                                }
1029                                                break;
1030
1031                                        default:
1032                                                continue;
1033                                }
1034
1035                                if (isNotBlank(conditionalUrl) && !conditionalUrl.contains("?")) {
1036                                        conditionalUrl = myContext.getResourceType(resource) + "?" + conditionalUrl;
1037                                }
1038
1039                                String key = verb + "|" + conditionalUrl;
1040                                if (consolidateEntryCandidate) {
1041                                        if (isBlank(entryFullUrl)) {
1042                                                if (isNotBlank(conditionalUrl)) {
1043                                                        if (!keysWithNoFullUrl.add(key)) {
1044                                                                throw new InvalidRequestException(Msg.code(2008) + "Unable to process " + theActionName
1045                                                                                + " - Request contains multiple anonymous entries (Bundle.entry.fullUrl not populated) with conditional URL: \""
1046                                                                                + UrlUtil.sanitizeUrlPart(conditionalUrl)
1047                                                                                + "\". Does transaction request contain duplicates?");
1048                                                        }
1049                                                }
1050                                        } else {
1051                                                if (!keyToUuid.containsKey(key)) {
1052                                                        keyToUuid.put(key, entryFullUrl);
1053                                                } else {
1054                                                        String msg = "Discarding transaction bundle entry " + originalIndex
1055                                                                        + " as it contained a duplicate conditional " + verb;
1056                                                        ourLog.info(msg);
1057                                                        // Interceptor broadcast: JPA_PERFTRACE_INFO
1058                                                        IInterceptorBroadcaster compositeBroadcaster =
1059                                                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(
1060                                                                                        myInterceptorBroadcaster, theRequestDetails);
1061                                                        if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_WARNING)) {
1062                                                                StorageProcessingMessage message = new StorageProcessingMessage().setMessage(msg);
1063                                                                HookParams params = new HookParams()
1064                                                                                .add(RequestDetails.class, theRequestDetails)
1065                                                                                .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
1066                                                                                .add(StorageProcessingMessage.class, message);
1067                                                                compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
1068                                                        }
1069
1070                                                        theEntries.remove(index);
1071                                                        index--;
1072                                                        String existingUuid = keyToUuid.get(key);
1073                                                        replaceReferencesInEntriesWithConsolidatedUUID(theEntries, entryFullUrl, existingUuid);
1074                                                }
1075                                        }
1076                                }
1077                        }
1078                }
1079        }
1080
1081        /**
1082         * Iterates over all entries, and if it finds any which have references which match the fullUrl of the entry that was consolidated out
1083         * replace them with our new consolidated UUID
1084         */
1085        private void replaceReferencesInEntriesWithConsolidatedUUID(
1086                        List<IBase> theEntries, String theEntryFullUrl, String existingUuid) {
1087                for (IBase nextEntry : theEntries) {
1088                        @SuppressWarnings("unchecked")
1089                        IBaseResource nextResource = myVersionAdapter.getResource(nextEntry);
1090                        if (nextResource != null) {
1091                                for (IBaseReference nextReference :
1092                                                myContext.newTerser().getAllPopulatedChildElementsOfType(nextResource, IBaseReference.class)) {
1093                                        // We're interested in any references directly to the placeholder ID, but also
1094                                        // references that have a resource target that has the placeholder ID.
1095                                        String nextReferenceId = nextReference.getReferenceElement().getValue();
1096                                        if (isBlank(nextReferenceId) && nextReference.getResource() != null) {
1097                                                nextReferenceId =
1098                                                                nextReference.getResource().getIdElement().getValue();
1099                                        }
1100                                        if (theEntryFullUrl.equals(nextReferenceId)) {
1101                                                nextReference.setReference(existingUuid);
1102                                                nextReference.setResource(null);
1103                                        }
1104                                }
1105                        }
1106                }
1107        }
1108
1109        /**
1110         * Retrieves the next resource id (IIdType) from the base resource and next request entry.
1111         *
1112         * @param theBaseResource - base resource
1113         * @param theNextReqEntry - next request entry
1114         * @param theAllIds       - set of all IIdType values
1115         * @param theVerb
1116         * @return
1117         */
1118        private IIdType getNextResourceIdFromBaseResource(
1119                        IBaseResource theBaseResource, IBase theNextReqEntry, Set<IIdType> theAllIds, String theVerb) {
1120                IIdType nextResourceId = null;
1121                if (theBaseResource != null) {
1122                        nextResourceId = theBaseResource.getIdElement();
1123
1124                        @SuppressWarnings("unchecked")
1125                        String fullUrl = myVersionAdapter.getFullUrl(theNextReqEntry);
1126                        if (isNotBlank(fullUrl)) {
1127                                IIdType fullUrlIdType = newIdType(fullUrl);
1128                                if (isPlaceholder(fullUrlIdType)) {
1129                                        nextResourceId = fullUrlIdType;
1130                                } else if (!nextResourceId.hasIdPart()) {
1131                                        nextResourceId = fullUrlIdType;
1132                                }
1133                        }
1134
1135                        if (nextResourceId.hasIdPart() && !isPlaceholder(nextResourceId)) {
1136                                int colonIndex = nextResourceId.getIdPart().indexOf(':');
1137                                if (colonIndex != -1) {
1138                                        if (INVALID_PLACEHOLDER_PATTERN
1139                                                        .matcher(nextResourceId.getIdPart())
1140                                                        .matches()) {
1141                                                throw new InvalidRequestException(
1142                                                                Msg.code(533) + "Invalid placeholder ID found: " + nextResourceId.getIdPart()
1143                                                                                + " - Must be of the form 'urn:uuid:[uuid]' or 'urn:oid:[oid]'");
1144                                        }
1145                                }
1146                        }
1147
1148                        if (nextResourceId.hasIdPart() && !nextResourceId.hasResourceType() && !isPlaceholder(nextResourceId)) {
1149                                nextResourceId = newIdType(toResourceName(theBaseResource.getClass()), nextResourceId.getIdPart());
1150                                theBaseResource.setId(nextResourceId);
1151                        }
1152
1153                        /*
1154                         * Ensure that the bundle doesn't have any duplicates, since this causes all kinds of weirdness
1155                         */
1156                        if (isPlaceholder(nextResourceId)) {
1157                                if (!theAllIds.add(nextResourceId)) {
1158                                        throw new InvalidRequestException(Msg.code(534)
1159                                                        + myContext
1160                                                                        .getLocalizer()
1161                                                                        .getMessage(
1162                                                                                        BaseStorageDao.class,
1163                                                                                        "transactionContainsMultipleWithDuplicateId",
1164                                                                                        nextResourceId));
1165                                }
1166                        } else if (nextResourceId.hasResourceType() && nextResourceId.hasIdPart() && !"POST".equals(theVerb)) {
1167                                IIdType nextId = nextResourceId.toUnqualifiedVersionless();
1168                                if (!theAllIds.add(nextId)) {
1169                                        throw new InvalidRequestException(Msg.code(535)
1170                                                        + myContext
1171                                                                        .getLocalizer()
1172                                                                        .getMessage(
1173                                                                                        BaseStorageDao.class,
1174                                                                                        "transactionContainsMultipleWithDuplicateId",
1175                                                                                        nextId));
1176                                }
1177                        }
1178                }
1179
1180                return nextResourceId;
1181        }
1182
1183        /**
1184         * After pre-hooks have been called
1185         */
1186        @SuppressWarnings({"unchecked", "rawtypes"})
1187        protected EntriesToProcessMap doTransactionWriteOperations(
1188                        final RequestDetails theRequest,
1189                        String theActionName,
1190                        TransactionDetails theTransactionDetails,
1191                        Set<IIdType> theAllIds,
1192                        IdSubstitutionMap theIdSubstitutions,
1193                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1194                        IBaseBundle theResponse,
1195                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
1196                        List<IBase> theEntries,
1197                        StopWatch theTransactionStopWatch) {
1198
1199                // During a transaction, we don't execute hooks, instead, we execute them all post-transaction.
1200                theTransactionDetails.beginAcceptingDeferredInterceptorBroadcasts(
1201                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED,
1202                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_UPDATED,
1203                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_DELETED);
1204                try {
1205                        Set<String> deletedResources = new HashSet<>();
1206                        DeleteConflictList deleteConflicts = new DeleteConflictList();
1207                        EntriesToProcessMap entriesToProcess = new EntriesToProcessMap();
1208                        Set<IIdType> nonUpdatedEntities = new HashSet<>();
1209                        Set<IBasePersistedResource> updatedEntities = new HashSet<>();
1210                        Map<String, IIdType> conditionalUrlToIdMap = new HashMap<>();
1211                        List<IBaseResource> updatedResources = new ArrayList<>();
1212                        Map<String, Class<? extends IBaseResource>> conditionalRequestUrls = new HashMap<>();
1213
1214                        /*
1215                         * Look for duplicate conditional creates and consolidate them
1216                         */
1217                        consolidateDuplicateConditionals(theRequest, theActionName, theEntries);
1218
1219                        /*
1220                         * Loop through the request and process any entries of type
1221                         * PUT, POST or DELETE
1222                         */
1223                        String previousVerb = null;
1224                        for (int i = 0; i < theEntries.size(); i++) {
1225                                if (i % 250 == 0) {
1226                                        ourLog.debug("Processed {} non-GET entries out of {} in transaction", i, theEntries.size());
1227                                }
1228
1229                                IBase nextReqEntry = theEntries.get(i);
1230                                IBaseResource res = myVersionAdapter.getResource(nextReqEntry);
1231
1232                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
1233                                if (previousVerb != null && !previousVerb.equals(verb)) {
1234                                        handleVerbChangeInTransactionWriteOperations();
1235                                }
1236                                previousVerb = verb;
1237
1238                                IIdType nextResourceId = getNextResourceIdFromBaseResource(res, nextReqEntry, theAllIds, verb);
1239
1240                                String resourceType = res != null ? myContext.getResourceType(res) : null;
1241                                Integer order = theOriginalRequestOrder.get(nextReqEntry);
1242                                IBase nextRespEntry =
1243                                                (IBase) myVersionAdapter.getEntries(theResponse).get(order);
1244
1245                                theTransactionStopWatch.startTask(
1246                                                "Bundle.entry[" + i + "]: " + verb + " " + defaultString(resourceType));
1247
1248                                if (res != null) {
1249                                        String previousResourceId = res.getIdElement().getValue();
1250                                        theTransactionDetails.addRollbackUndoAction(() -> res.setId(previousResourceId));
1251                                }
1252
1253                                switch (verb) {
1254                                        case "POST": {
1255                                                // CREATE
1256                                                /*
1257                                                 * To preserve existing functionality,
1258                                                 * we will only verify that the request url is
1259                                                 * valid if it's provided at all.
1260                                                 * Otherwise, we'll ignore it
1261                                                 */
1262                                                String url = myVersionAdapter.getEntryRequestUrl(nextReqEntry);
1263                                                if (isNotBlank(url)) {
1264                                                        extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1265                                                }
1266                                                validateResourcePresent(res, order, verb);
1267
1268                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
1269                                                res.setId((String) null);
1270
1271                                                DaoMethodOutcome outcome;
1272                                                String matchUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
1273                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1274                                                // create individual resource
1275                                                outcome = resourceDao.create(res, matchUrl, false, theRequest, theTransactionDetails);
1276                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1277                                                res.setId(outcome.getId());
1278
1279                                                if (nextResourceId != null) {
1280                                                        handleTransactionCreateOrUpdateOutcome(
1281                                                                        theIdSubstitutions,
1282                                                                        theIdToPersistedOutcome,
1283                                                                        nextResourceId,
1284                                                                        outcome,
1285                                                                        nextRespEntry,
1286                                                                        resourceType,
1287                                                                        res,
1288                                                                        theRequest);
1289                                                }
1290                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1291                                                theTransactionDetails.addResolvedResource(outcome.getId(), outcome::getResource);
1292                                                if (outcome.getCreated() == false) {
1293                                                        nonUpdatedEntities.add(outcome.getId());
1294                                                } else {
1295                                                        if (isNotBlank(matchUrl)) {
1296                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
1297                                                        }
1298                                                }
1299
1300                                                break;
1301                                        }
1302                                        case "DELETE": {
1303                                                // DELETE
1304                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1305                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1306                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
1307                                                int status = Constants.STATUS_HTTP_204_NO_CONTENT;
1308                                                if (parts.getResourceId() != null) {
1309                                                        IIdType deleteId = newIdType(parts.getResourceType(), parts.getResourceId());
1310                                                        if (!deletedResources.contains(deleteId.getValueAsString())) {
1311                                                                DaoMethodOutcome outcome =
1312                                                                                dao.delete(deleteId, deleteConflicts, theRequest, theTransactionDetails);
1313                                                                if (outcome.getEntity() != null) {
1314                                                                        deletedResources.add(deleteId.getValueAsString());
1315                                                                        entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1316                                                                }
1317                                                                myVersionAdapter.setResponseOutcome(nextRespEntry, outcome.getOperationOutcome());
1318                                                        }
1319                                                } else {
1320                                                        String matchUrl = parts.getResourceType() + '?' + parts.getParams();
1321                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1322                                                        DeleteMethodOutcome deleteOutcome =
1323                                                                        dao.deleteByUrl(matchUrl, deleteConflicts, theRequest, theTransactionDetails);
1324                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, deleteOutcome.getId());
1325                                                        List<? extends IBasePersistedResource> allDeleted = deleteOutcome.getDeletedEntities();
1326                                                        for (IBasePersistedResource deleted : allDeleted) {
1327                                                                deletedResources.add(deleted.getIdDt()
1328                                                                                .toUnqualifiedVersionless()
1329                                                                                .getValueAsString());
1330                                                        }
1331                                                        if (allDeleted.isEmpty()) {
1332                                                                status = Constants.STATUS_HTTP_204_NO_CONTENT;
1333                                                        }
1334
1335                                                        myVersionAdapter.setResponseOutcome(nextRespEntry, deleteOutcome.getOperationOutcome());
1336                                                }
1337
1338                                                myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(status));
1339
1340                                                break;
1341                                        }
1342                                        case "PUT": {
1343                                                // UPDATE
1344                                                validateResourcePresent(res, order, verb);
1345                                                @SuppressWarnings("rawtypes")
1346                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
1347
1348                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1349
1350                                                DaoMethodOutcome outcome;
1351                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1352                                                if (isNotBlank(parts.getResourceId())) {
1353                                                        String version = null;
1354                                                        if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) {
1355                                                                version = ParameterUtil.parseETagValue(
1356                                                                                myVersionAdapter.getEntryRequestIfMatch(nextReqEntry));
1357                                                        }
1358                                                        res.setId(newIdType(parts.getResourceType(), parts.getResourceId(), version));
1359                                                        outcome = resourceDao.update(res, null, false, false, theRequest, theTransactionDetails);
1360                                                } else {
1361                                                        if (!shouldConditionalUpdateMatchId(theTransactionDetails, res.getIdElement())) {
1362                                                                res.setId((String) null);
1363                                                        }
1364                                                        String matchUrl;
1365                                                        if (isNotBlank(parts.getParams())) {
1366                                                                matchUrl = parts.getResourceType() + '?' + parts.getParams();
1367                                                        } else {
1368                                                                matchUrl = parts.getResourceType();
1369                                                        }
1370                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1371                                                        outcome =
1372                                                                        resourceDao.update(res, matchUrl, false, false, theRequest, theTransactionDetails);
1373                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1374                                                        if (Boolean.TRUE.equals(outcome.getCreated())) {
1375                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
1376                                                        }
1377                                                }
1378
1379                                                if (outcome.getCreated() == Boolean.FALSE
1380                                                                || (outcome.getCreated() == Boolean.TRUE
1381                                                                                && outcome.getId().getVersionIdPartAsLong() > 1)) {
1382                                                        updatedEntities.add(outcome.getEntity());
1383                                                        if (outcome.getResource() != null) {
1384                                                                updatedResources.add(outcome.getResource());
1385                                                        }
1386                                                }
1387
1388                                                theTransactionDetails.addResolvedResource(outcome.getId(), outcome::getResource);
1389                                                handleTransactionCreateOrUpdateOutcome(
1390                                                                theIdSubstitutions,
1391                                                                theIdToPersistedOutcome,
1392                                                                nextResourceId,
1393                                                                outcome,
1394                                                                nextRespEntry,
1395                                                                resourceType,
1396                                                                res,
1397                                                                theRequest);
1398                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1399                                                break;
1400                                        }
1401                                        case "PATCH": {
1402                                                // PATCH
1403                                                validateResourcePresent(res, order, verb);
1404
1405                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1406                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1407
1408                                                String matchUrl = toMatchUrl(nextReqEntry);
1409                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1410                                                String patchBody = null;
1411                                                String contentType;
1412                                                IBaseParameters patchBodyParameters = null;
1413                                                PatchTypeEnum patchType = null;
1414
1415                                                if (res instanceof IBaseBinary) {
1416                                                        IBaseBinary binary = (IBaseBinary) res;
1417                                                        if (binary.getContent() != null && binary.getContent().length > 0) {
1418                                                                patchBody = toUtf8String(binary.getContent());
1419                                                        }
1420                                                        contentType = binary.getContentType();
1421                                                        patchType =
1422                                                                        PatchTypeEnum.forContentTypeOrThrowInvalidRequestException(myContext, contentType);
1423                                                        if (patchType == PatchTypeEnum.FHIR_PATCH_JSON
1424                                                                        || patchType == PatchTypeEnum.FHIR_PATCH_XML) {
1425                                                                String msg = myContext
1426                                                                                .getLocalizer()
1427                                                                                .getMessage(
1428                                                                                                BaseTransactionProcessor.class, "fhirPatchShouldNotUseBinaryResource");
1429                                                                throw new InvalidRequestException(Msg.code(536) + msg);
1430                                                        }
1431                                                } else if (res instanceof IBaseParameters) {
1432                                                        patchBodyParameters = (IBaseParameters) res;
1433                                                        patchType = PatchTypeEnum.FHIR_PATCH_JSON;
1434                                                }
1435
1436                                                if (patchBodyParameters == null) {
1437                                                        if (isBlank(patchBody)) {
1438                                                                String msg = myContext
1439                                                                                .getLocalizer()
1440                                                                                .getMessage(BaseTransactionProcessor.class, "missingPatchBody");
1441                                                                throw new InvalidRequestException(Msg.code(537) + msg);
1442                                                        }
1443                                                }
1444
1445                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
1446                                                IIdType patchId =
1447                                                                myContext.getVersion().newIdType(parts.getResourceType(), parts.getResourceId());
1448
1449                                                String conditionalUrl;
1450                                                if (isNull(patchId.getIdPart())) {
1451                                                        conditionalUrl = url;
1452                                                } else {
1453                                                        conditionalUrl = matchUrl;
1454                                                        String ifMatch = myVersionAdapter.getEntryRequestIfMatch(nextReqEntry);
1455                                                        if (isNotBlank(ifMatch)) {
1456                                                                patchId = UpdateMethodBinding.applyETagAsVersion(ifMatch, patchId);
1457                                                        }
1458                                                }
1459
1460                                                DaoMethodOutcome outcome = dao.patchInTransaction(
1461                                                                patchId,
1462                                                                conditionalUrl,
1463                                                                false,
1464                                                                patchType,
1465                                                                patchBody,
1466                                                                patchBodyParameters,
1467                                                                theRequest,
1468                                                                theTransactionDetails);
1469                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1470                                                updatedEntities.add(outcome.getEntity());
1471                                                if (outcome.getResource() != null) {
1472                                                        updatedResources.add(outcome.getResource());
1473                                                }
1474                                                if (nextResourceId != null) {
1475                                                        handleTransactionCreateOrUpdateOutcome(
1476                                                                        theIdSubstitutions,
1477                                                                        theIdToPersistedOutcome,
1478                                                                        nextResourceId,
1479                                                                        outcome,
1480                                                                        nextRespEntry,
1481                                                                        resourceType,
1482                                                                        res,
1483                                                                        theRequest);
1484                                                }
1485                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1486
1487                                                break;
1488                                        }
1489                                        case "GET":
1490                                                break;
1491                                        default:
1492                                                throw new InvalidRequestException(
1493                                                                Msg.code(538) + "Unable to handle verb in transaction: " + verb);
1494                                }
1495
1496                                theTransactionStopWatch.endCurrentTask();
1497                        }
1498
1499                        postTransactionProcess(theTransactionDetails);
1500
1501                        /*
1502                         * Make sure that there are no conflicts from deletions. E.g. we can't delete something
1503                         * if something else has a reference to it.. Unless the thing that has a reference to it
1504                         * was also deleted as a part of this transaction, which is why we check this now at the
1505                         * end.
1506                         */
1507                        checkForDeleteConflicts(deleteConflicts, deletedResources, updatedResources);
1508
1509                        theIdToPersistedOutcome.entrySet().forEach(idAndOutcome -> {
1510                                theTransactionDetails.addResolvedResourceId(
1511                                                idAndOutcome.getKey(), idAndOutcome.getValue().getPersistentId());
1512                        });
1513
1514                        /*
1515                         * Perform ID substitutions and then index each resource we have saved
1516                         */
1517
1518                        resolveReferencesThenSaveAndIndexResources(
1519                                        theRequest,
1520                                        theTransactionDetails,
1521                                        theIdSubstitutions,
1522                                        theIdToPersistedOutcome,
1523                                        theTransactionStopWatch,
1524                                        entriesToProcess,
1525                                        nonUpdatedEntities,
1526                                        updatedEntities);
1527
1528                        theTransactionStopWatch.endCurrentTask();
1529
1530                        // flush writes to db
1531                        theTransactionStopWatch.startTask("Flush writes to database");
1532
1533                        // flush the changes
1534                        flushSession(theIdToPersistedOutcome);
1535
1536                        theTransactionStopWatch.endCurrentTask();
1537
1538                        /*
1539                         * Double check we didn't allow any duplicates we shouldn't have
1540                         */
1541                        if (conditionalRequestUrls.size() > 0) {
1542                                theTransactionStopWatch.startTask("Check for conflicts in conditional resources");
1543                        }
1544                        if (!myStorageSettings.isMassIngestionMode()) {
1545                                validateNoDuplicates(
1546                                                theRequest,
1547                                                theTransactionDetails,
1548                                                theActionName,
1549                                                conditionalRequestUrls,
1550                                                theIdToPersistedOutcome.values());
1551                        }
1552
1553                        theTransactionStopWatch.endCurrentTask();
1554                        if (conditionalUrlToIdMap.size() > 0) {
1555                                theTransactionStopWatch.startTask(
1556                                                "Check that all conditionally created/updated entities actually match their conditionals.");
1557                        }
1558
1559                        if (!myStorageSettings.isMassIngestionMode()) {
1560                                validateAllInsertsMatchTheirConditionalUrls(theIdToPersistedOutcome, conditionalUrlToIdMap, theRequest);
1561                        }
1562                        theTransactionStopWatch.endCurrentTask();
1563
1564                        for (IIdType next : theAllIds) {
1565                                IIdType replacement = theIdSubstitutions.getForSource(next);
1566                                if (replacement != null && !replacement.equals(next)) {
1567                                        ourLog.debug(
1568                                                        "Placeholder resource ID \"{}\" was replaced with permanent ID \"{}\"", next, replacement);
1569                                }
1570                        }
1571
1572                        IInterceptorBroadcaster compositeBroadcaster =
1573                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequest);
1574                        ListMultimap<Pointcut, HookParams> deferredBroadcastEvents =
1575                                        theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1576                        for (Map.Entry<Pointcut, HookParams> nextEntry : deferredBroadcastEvents.entries()) {
1577                                Pointcut nextPointcut = nextEntry.getKey();
1578                                HookParams nextParams = nextEntry.getValue();
1579                                compositeBroadcaster.callHooks(nextPointcut, nextParams);
1580                        }
1581
1582                        DeferredInterceptorBroadcasts deferredInterceptorBroadcasts =
1583                                        new DeferredInterceptorBroadcasts(deferredBroadcastEvents);
1584                        HookParams params = new HookParams()
1585                                        .add(RequestDetails.class, theRequest)
1586                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
1587                                        .add(DeferredInterceptorBroadcasts.class, deferredInterceptorBroadcasts)
1588                                        .add(TransactionDetails.class, theTransactionDetails)
1589                                        .add(IBaseBundle.class, theResponse);
1590                        compositeBroadcaster.callHooks(Pointcut.STORAGE_TRANSACTION_PROCESSED, params);
1591
1592                        theTransactionDetails.deferredBroadcastProcessingFinished();
1593
1594                        // finishedCallingDeferredInterceptorBroadcasts
1595
1596                        return entriesToProcess;
1597
1598                } finally {
1599                        if (theTransactionDetails.isAcceptingDeferredInterceptorBroadcasts()) {
1600                                theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1601                        }
1602                }
1603        }
1604
1605        /**
1606         * Subclasses may override this in order to invoke specific operations when
1607         * we're finished handling all the write entries in the transaction bundle
1608         * with a given verb.
1609         */
1610        protected void handleVerbChangeInTransactionWriteOperations() {
1611                // nothing
1612        }
1613
1614        /**
1615         * Implement to handle post transaction processing
1616         */
1617        protected void postTransactionProcess(TransactionDetails theTransactionDetails) {
1618                // nothing
1619        }
1620
1621        /**
1622         * Check for if a resource id should be matched in a conditional update
1623         * If the FHIR version is older than R4, it follows the old specifications and does not match
1624         * If the resource id has been resolved, then it is an existing resource and does not need to be matched
1625         * If the resource id is local or a placeholder, the id is temporary and should not be matched
1626         */
1627        private boolean shouldConditionalUpdateMatchId(TransactionDetails theTransactionDetails, IIdType theId) {
1628                if (myContext.getVersion().getVersion().isOlderThan(FhirVersionEnum.R4)) {
1629                        return false;
1630                }
1631                if (theTransactionDetails.hasResolvedResourceId(theId)
1632                                && !theTransactionDetails.isResolvedResourceIdEmpty(theId)) {
1633                        return false;
1634                }
1635                if (theId != null && theId.getValue() != null) {
1636                        return !(theId.getValue().startsWith("urn:") || theId.getValue().startsWith("#"));
1637                }
1638                return true;
1639        }
1640
1641        private boolean shouldSwapBinaryToActualResource(
1642                        IBaseResource theResource, String theResourceType, IIdType theNextResourceId) {
1643                if ("Binary".equalsIgnoreCase(theResourceType)
1644                                && theNextResourceId.getResourceType() != null
1645                                && !theNextResourceId.getResourceType().equalsIgnoreCase("Binary")) {
1646                        return true;
1647                } else {
1648                        return false;
1649                }
1650        }
1651
1652        private void setConditionalUrlToBeValidatedLater(
1653                        Map<String, IIdType> theConditionalUrlToIdMap, String theMatchUrl, IIdType theId) {
1654                if (!StringUtils.isBlank(theMatchUrl)) {
1655                        theConditionalUrlToIdMap.put(theMatchUrl, theId);
1656                }
1657        }
1658
1659        /**
1660         * After transaction processing and resolution of indexes and references, we want to validate that the resources that were stored _actually_
1661         * match the conditional URLs that they were brought in on.
1662         */
1663        private void validateAllInsertsMatchTheirConditionalUrls(
1664                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1665                        Map<String, IIdType> conditionalUrlToIdMap,
1666                        RequestDetails theRequest) {
1667                conditionalUrlToIdMap.entrySet().stream()
1668                                .filter(entry -> entry.getKey() != null)
1669                                .forEach(entry -> {
1670                                        String matchUrl = entry.getKey();
1671                                        IIdType value = entry.getValue();
1672                                        DaoMethodOutcome daoMethodOutcome = theIdToPersistedOutcome.get(value);
1673                                        if (daoMethodOutcome != null
1674                                                        && !daoMethodOutcome.isNop()
1675                                                        && daoMethodOutcome.getResource() != null) {
1676                                                InMemoryMatchResult match =
1677                                                                mySearchParamMatcher.match(matchUrl, daoMethodOutcome.getResource(), theRequest);
1678                                                if (ourLog.isDebugEnabled()) {
1679                                                        ourLog.debug(
1680                                                                        "Checking conditional URL [{}] against resource with ID [{}]: Supported?:[{}], Matched?:[{}]",
1681                                                                        matchUrl,
1682                                                                        value,
1683                                                                        match.supported(),
1684                                                                        match.matched());
1685                                                }
1686                                                if (match.supported()) {
1687                                                        if (!match.matched()) {
1688                                                                throw new PreconditionFailedException(Msg.code(539) + "Invalid conditional URL \""
1689                                                                                + matchUrl + "\". The given resource is not matched by this URL.");
1690                                                        }
1691                                                }
1692                                        }
1693                                });
1694        }
1695
1696        /**
1697         * Checks for any delete conflicts.
1698         *
1699         * @param theDeleteConflicts  - set of delete conflicts
1700         * @param theDeletedResources - set of deleted resources
1701         * @param theUpdatedResources - list of updated resources
1702         */
1703        private void checkForDeleteConflicts(
1704                        DeleteConflictList theDeleteConflicts,
1705                        Set<String> theDeletedResources,
1706                        List<IBaseResource> theUpdatedResources) {
1707                for (Iterator<DeleteConflict> iter = theDeleteConflicts.iterator(); iter.hasNext(); ) {
1708                        DeleteConflict nextDeleteConflict = iter.next();
1709
1710                        /*
1711                         * If we have a conflict, it means we can't delete Resource/A because
1712                         * Resource/B has a reference to it. We'll ignore that conflict though
1713                         * if it turns out we're also deleting Resource/B in this transaction.
1714                         */
1715                        if (theDeletedResources.contains(
1716                                        nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue())) {
1717                                iter.remove();
1718                                continue;
1719                        }
1720
1721                        /*
1722                         * And then, this is kind of a last ditch check. It's also ok to delete
1723                         * Resource/A if Resource/B isn't being deleted, but it is being UPDATED
1724                         * in this transaction, and the updated version of it has no references
1725                         * to Resource/A any more.
1726                         */
1727                        String sourceId =
1728                                        nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue();
1729                        String targetId =
1730                                        nextDeleteConflict.getTargetId().toUnqualifiedVersionless().getValue();
1731                        Optional<IBaseResource> updatedSource = theUpdatedResources.stream()
1732                                        .filter(t -> sourceId.equals(
1733                                                        t.getIdElement().toUnqualifiedVersionless().getValue()))
1734                                        .findFirst();
1735                        if (updatedSource.isPresent()) {
1736                                List<ResourceReferenceInfo> referencesInSource =
1737                                                myContext.newTerser().getAllResourceReferences(updatedSource.get());
1738                                boolean sourceStillReferencesTarget = referencesInSource.stream()
1739                                                .anyMatch(t -> targetId.equals(t.getResourceReference()
1740                                                                .getReferenceElement()
1741                                                                .toUnqualifiedVersionless()
1742                                                                .getValue()));
1743                                if (!sourceStillReferencesTarget) {
1744                                        iter.remove();
1745                                }
1746                        }
1747                }
1748                DeleteConflictUtil.validateDeleteConflictsEmptyOrThrowException(myContext, theDeleteConflicts);
1749        }
1750
1751        /**
1752         * This method replaces any placeholder references in the
1753         * source transaction Bundle with their actual targets, then stores the resource contents and indexes
1754         * in the database. This is trickier than you'd think because of a couple of possibilities during the
1755         * save:
1756         * * There may be resources that have not changed (e.g. an update/PUT with a resource body identical
1757         * to what is already in the database)
1758         * * There may be resources with auto-versioned references, meaning we're replacing certain references
1759         * in the resource with a versioned references, referencing the current version at the time of the
1760         * transaction processing
1761         * * There may by auto-versioned references pointing to these unchanged targets
1762         * <p>
1763         * If we're not doing any auto-versioned references, we'll just iterate through all resources in the
1764         * transaction and save them one at a time.
1765         * <p>
1766         * However, if we have any auto-versioned references we do this in 2 passes: First the resources from the
1767         * transaction that don't have any auto-versioned references are stored. We do them first since there's
1768         * a chance they may be a NOP and we'll need to account for their version number not actually changing.
1769         * Then we do a second pass for any resources that have auto-versioned references. These happen in a separate
1770         * pass because it's too complex to try and insert the auto-versioned references and still
1771         * account for NOPs, so we block NOPs in that pass.
1772         */
1773        private void resolveReferencesThenSaveAndIndexResources(
1774                        RequestDetails theRequest,
1775                        TransactionDetails theTransactionDetails,
1776                        IdSubstitutionMap theIdSubstitutions,
1777                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1778                        StopWatch theTransactionStopWatch,
1779                        EntriesToProcessMap theEntriesToProcess,
1780                        Set<IIdType> theNonUpdatedEntities,
1781                        Set<IBasePersistedResource> theUpdatedEntities) {
1782                FhirTerser terser = myContext.newTerser();
1783                theTransactionStopWatch.startTask("Index " + theIdToPersistedOutcome.size() + " resources");
1784                IdentityHashMap<DaoMethodOutcome, Set<IBaseReference>> deferredIndexesForAutoVersioning = null;
1785                int i = 0;
1786                for (DaoMethodOutcome nextOutcome : theIdToPersistedOutcome.values()) {
1787
1788                        if (i++ % 250 == 0) {
1789                                ourLog.debug(
1790                                                "Have indexed {} entities out of {} in transaction",
1791                                                i,
1792                                                theIdToPersistedOutcome.values().size());
1793                        }
1794
1795                        if (nextOutcome.isNop()) {
1796                                continue;
1797                        }
1798
1799                        IBaseResource nextResource = nextOutcome.getResource();
1800                        if (nextResource == null) {
1801                                continue;
1802                        }
1803
1804                        Set<IBaseReference> referencesToAutoVersion =
1805                                        BaseStorageDao.extractReferencesToAutoVersion(myContext, myStorageSettings, nextResource);
1806                        Set<IBaseReference> referencesToKeepClientSuppliedVersion =
1807                                        BaseStorageDao.extractReferencesToAvoidReplacement(myContext, nextResource);
1808
1809                        if (referencesToAutoVersion.isEmpty()) {
1810                                // no references to autoversion - we can do the resolve and save now
1811                                resolveReferencesThenSaveAndIndexResource(
1812                                                theRequest,
1813                                                theTransactionDetails,
1814                                                theIdSubstitutions,
1815                                                theIdToPersistedOutcome,
1816                                                theEntriesToProcess,
1817                                                theNonUpdatedEntities,
1818                                                theUpdatedEntities,
1819                                                terser,
1820                                                nextOutcome,
1821                                                nextResource,
1822                                                referencesToAutoVersion, // this is empty
1823                                                referencesToKeepClientSuppliedVersion);
1824                        } else {
1825                                // we have autoversioned things to defer until later
1826                                if (deferredIndexesForAutoVersioning == null) {
1827                                        deferredIndexesForAutoVersioning = new IdentityHashMap<>();
1828                                }
1829                                deferredIndexesForAutoVersioning.put(nextOutcome, referencesToAutoVersion);
1830                        }
1831                }
1832
1833                // If we have any resources we'll be auto-versioning, index these next
1834                if (deferredIndexesForAutoVersioning != null) {
1835                        for (Map.Entry<DaoMethodOutcome, Set<IBaseReference>> nextEntry :
1836                                        deferredIndexesForAutoVersioning.entrySet()) {
1837                                DaoMethodOutcome nextOutcome = nextEntry.getKey();
1838                                Set<IBaseReference> referencesToAutoVersion = nextEntry.getValue();
1839                                IBaseResource nextResource = nextOutcome.getResource();
1840                                Set<IBaseReference> referencesToKeepClientSuppliedVersion =
1841                                                BaseStorageDao.extractReferencesToAvoidReplacement(myContext, nextResource);
1842
1843                                resolveReferencesThenSaveAndIndexResource(
1844                                                theRequest,
1845                                                theTransactionDetails,
1846                                                theIdSubstitutions,
1847                                                theIdToPersistedOutcome,
1848                                                theEntriesToProcess,
1849                                                theNonUpdatedEntities,
1850                                                theUpdatedEntities,
1851                                                terser,
1852                                                nextOutcome,
1853                                                nextResource,
1854                                                referencesToAutoVersion,
1855                                                referencesToKeepClientSuppliedVersion);
1856                        }
1857                }
1858        }
1859
1860        private void resolveReferencesThenSaveAndIndexResource(
1861                        RequestDetails theRequest,
1862                        TransactionDetails theTransactionDetails,
1863                        IdSubstitutionMap theIdSubstitutions,
1864                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1865                        EntriesToProcessMap theEntriesToProcess,
1866                        Set<IIdType> theNonUpdatedEntities,
1867                        Set<IBasePersistedResource> theUpdatedEntities,
1868                        FhirTerser theTerser,
1869                        DaoMethodOutcome theDaoMethodOutcome,
1870                        IBaseResource theResource,
1871                        Set<IBaseReference> theReferencesToAutoVersion,
1872                        Set<IBaseReference> theReferencesToKeepClientSuppliedVersion) {
1873                // References
1874                List<ResourceReferenceInfo> allRefs = theTerser.getAllResourceReferences(theResource);
1875                for (ResourceReferenceInfo nextRef : allRefs) {
1876                        IBaseReference resourceReference = nextRef.getResourceReference();
1877                        IIdType nextId = resourceReference.getReferenceElement();
1878                        IIdType newId = null;
1879                        if (!nextId.hasIdPart()) {
1880                                if (resourceReference.getResource() != null) {
1881                                        IIdType targetId = resourceReference.getResource().getIdElement();
1882                                        if (targetId.getValue() == null || targetId.getValue().startsWith("#")) {
1883                                                // This means it's a contained resource
1884                                                continue;
1885                                        } else if (theIdSubstitutions.containsTarget(targetId)) {
1886                                                newId = targetId;
1887                                        } else {
1888                                                throw new InternalErrorException(Msg.code(540)
1889                                                                + "References by resource with no reference ID are not supported in DAO layer");
1890                                        }
1891                                } else {
1892                                        continue;
1893                                }
1894                        }
1895                        if (newId != null || theIdSubstitutions.containsSource(nextId)) {
1896                                if (shouldReplaceResourceReference(
1897                                                theReferencesToAutoVersion, theReferencesToKeepClientSuppliedVersion, resourceReference)) {
1898                                        if (newId == null) {
1899                                                newId = theIdSubstitutions.getForSource(nextId);
1900                                        }
1901                                        if (newId != null) {
1902                                                ourLog.debug(" * Replacing resource ref {} with {}", nextId, newId);
1903                                                if (theReferencesToAutoVersion.contains(resourceReference)) {
1904                                                        replaceResourceReference(newId, resourceReference, theTransactionDetails);
1905                                                } else {
1906                                                        replaceResourceReference(newId.toVersionless(), resourceReference, theTransactionDetails);
1907                                                }
1908                                        }
1909                                }
1910                        } else if (nextId.getValue().startsWith("urn:")) {
1911                                throw new InvalidRequestException(
1912                                                Msg.code(541) + "Unable to satisfy placeholder ID " + nextId.getValue()
1913                                                                + " found in element named '" + nextRef.getName() + "' within resource of type: "
1914                                                                + theResource.getIdElement().getResourceType());
1915                        } else {
1916                                // get a map of
1917                                // existing ids -> PID (for resources that exist in the DB)
1918                                // should this be allPartitions?
1919                                ResourcePersistentIdMap resourceVersionMap = myResourceVersionSvc.getLatestVersionIdsForResourceIds(
1920                                                RequestPartitionId.allPartitions(),
1921                                                theReferencesToAutoVersion.stream()
1922                                                                .map(IBaseReference::getReferenceElement)
1923                                                                .collect(Collectors.toList()));
1924
1925                                for (IBaseReference baseRef : theReferencesToAutoVersion) {
1926                                        IIdType id = baseRef.getReferenceElement();
1927                                        if (!resourceVersionMap.containsKey(id)
1928                                                        && myStorageSettings.isAutoCreatePlaceholderReferenceTargets()) {
1929                                                // not in the db, but autocreateplaceholders is true
1930                                                // so the version we'll set is "1" (since it will be
1931                                                // created later)
1932                                                String newRef = id.withVersion("1").getValue();
1933                                                id.setValue(newRef);
1934                                        } else {
1935                                                // we will add the looked up info to the transaction
1936                                                // for later
1937                                                if (resourceVersionMap.containsKey(id)) {
1938                                                        theTransactionDetails.addResolvedResourceId(
1939                                                                        id, resourceVersionMap.getResourcePersistentId(id));
1940                                                }
1941                                        }
1942                                }
1943
1944                                if (theReferencesToAutoVersion.contains(resourceReference)) {
1945                                        DaoMethodOutcome outcome = theIdToPersistedOutcome.get(nextId);
1946
1947                                        if (outcome != null && !outcome.isNop() && !Boolean.TRUE.equals(outcome.getCreated())) {
1948                                                replaceResourceReference(nextId, resourceReference, theTransactionDetails);
1949                                        }
1950
1951                                        // if referenced resource is not in transaction but exists in the DB, resolving its version
1952                                        IResourcePersistentId persistedReferenceId = resourceVersionMap.getResourcePersistentId(nextId);
1953                                        if (outcome == null && persistedReferenceId != null && persistedReferenceId.getVersion() != null) {
1954                                                IIdType newReferenceId = nextId.withVersion(
1955                                                                persistedReferenceId.getVersion().toString());
1956                                                replaceResourceReference(newReferenceId, resourceReference, theTransactionDetails);
1957                                        }
1958                                }
1959                        }
1960                }
1961
1962                // URIs
1963                Class<? extends IPrimitiveType<?>> uriType = (Class<? extends IPrimitiveType<?>>)
1964                                myContext.getElementDefinition("uri").getImplementingClass();
1965                List<? extends IPrimitiveType<?>> allUris = theTerser.getAllPopulatedChildElementsOfType(theResource, uriType);
1966                for (IPrimitiveType<?> nextRef : allUris) {
1967                        if (nextRef instanceof IIdType) {
1968                                continue; // No substitution on the resource ID itself!
1969                        }
1970                        String nextUriString = nextRef.getValueAsString();
1971                        if (isNotBlank(nextUriString)) {
1972                                if (theIdSubstitutions.containsSource(nextUriString)) {
1973                                        IIdType newId = theIdSubstitutions.getForSource(nextUriString);
1974                                        ourLog.debug(" * Replacing resource ref {} with {}", nextUriString, newId);
1975
1976                                        String existingValue = nextRef.getValueAsString();
1977                                        theTransactionDetails.addRollbackUndoAction(() -> nextRef.setValueAsString(existingValue));
1978
1979                                        nextRef.setValueAsString(newId.toVersionless().getValue());
1980                                } else {
1981                                        ourLog.debug(" * Reference [{}] does not exist in bundle", nextUriString);
1982                                }
1983                        }
1984                }
1985
1986                IPrimitiveType<Date> deletedInstantOrNull = ResourceMetadataKeyEnum.DELETED_AT.get(theResource);
1987                Date deletedTimestampOrNull = deletedInstantOrNull != null ? deletedInstantOrNull.getValue() : null;
1988
1989                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(theResource.getClass());
1990                IJpaDao jpaDao = (IJpaDao) dao;
1991
1992                IBasePersistedResource updateOutcome = null;
1993                if (theUpdatedEntities.contains(theDaoMethodOutcome.getEntity())) {
1994                        boolean forceUpdateVersion = !theReferencesToAutoVersion.isEmpty();
1995                        String matchUrl = theDaoMethodOutcome.getMatchUrl();
1996                        RestOperationTypeEnum operationType = theDaoMethodOutcome.getOperationType();
1997                        DaoMethodOutcome daoMethodOutcome = jpaDao.updateInternal(
1998                                        theRequest,
1999                                        theResource,
2000                                        matchUrl,
2001                                        true,
2002                                        forceUpdateVersion,
2003                                        theDaoMethodOutcome.getEntity(),
2004                                        theResource.getIdElement(),
2005                                        theDaoMethodOutcome.getPreviousResource(),
2006                                        operationType,
2007                                        theTransactionDetails);
2008                        updateOutcome = daoMethodOutcome.getEntity();
2009                        theDaoMethodOutcome = daoMethodOutcome;
2010                } else if (!theNonUpdatedEntities.contains(theDaoMethodOutcome.getId())) {
2011                        updateOutcome = jpaDao.updateEntity(
2012                                        theRequest,
2013                                        theResource,
2014                                        theDaoMethodOutcome.getEntity(),
2015                                        deletedTimestampOrNull,
2016                                        true,
2017                                        false,
2018                                        theTransactionDetails,
2019                                        false,
2020                                        true);
2021                }
2022
2023                // Make sure we reflect the actual final version for the resource.
2024                if (updateOutcome != null) {
2025                        IIdType newId = updateOutcome.getIdDt();
2026
2027                        IIdType entryId = theEntriesToProcess.getIdWithVersionlessComparison(newId);
2028                        if (entryId != null && !StringUtils.equals(entryId.getValue(), newId.getValue())) {
2029                                entryId.setValue(newId.getValue());
2030                        }
2031
2032                        theDaoMethodOutcome.setId(newId);
2033
2034                        theIdSubstitutions.updateTargets(newId);
2035
2036                        // This will only be null if we're not intending to return an OO
2037                        IBaseOperationOutcome operationOutcome = theDaoMethodOutcome.getOperationOutcome();
2038                        if (operationOutcome != null) {
2039
2040                                List<IIdType> autoCreatedPlaceholders =
2041                                                theTransactionDetails.getAutoCreatedPlaceholderResourcesAndClear();
2042                                for (IIdType autoCreatedPlaceholder : autoCreatedPlaceholders) {
2043                                        BaseStorageDao.addIssueToOperationOutcomeForAutoCreatedPlaceholder(
2044                                                        myContext, autoCreatedPlaceholder, operationOutcome);
2045                                }
2046
2047                                IBase responseEntry = theEntriesToProcess.getResponseBundleEntryWithVersionlessComparison(newId);
2048                                myVersionAdapter.setResponseOutcome(responseEntry, operationOutcome);
2049                        }
2050                }
2051        }
2052
2053        /**
2054         * We should replace the references when
2055         * 1. It is not a reference we should keep the client-supplied version for as configured by `DontStripVersionsFromReferences` or
2056         * 2. It is a reference that has been identified for auto versioning or
2057         * 3. Is a placeholder reference
2058         *
2059         * @param theReferencesToAutoVersion               list of references identified for auto versioning
2060         * @param theReferencesToKeepClientSuppliedVersion list of references that we should not strip the version for
2061         * @param theResourceReference                     the resource reference
2062         * @return true if we should replace the resource reference, false if we should keep the client provided reference
2063         */
2064        private boolean shouldReplaceResourceReference(
2065                        Set<IBaseReference> theReferencesToAutoVersion,
2066                        Set<IBaseReference> theReferencesToKeepClientSuppliedVersion,
2067                        IBaseReference theResourceReference) {
2068                return (!theReferencesToKeepClientSuppliedVersion.contains(theResourceReference)
2069                                                && myContext.getParserOptions().isStripVersionsFromReferences())
2070                                || theReferencesToAutoVersion.contains(theResourceReference)
2071                                || isPlaceholder(theResourceReference.getReferenceElement());
2072        }
2073
2074        private void replaceResourceReference(
2075                        IIdType theReferenceId, IBaseReference theResourceReference, TransactionDetails theTransactionDetails) {
2076                addRollbackReferenceRestore(theTransactionDetails, theResourceReference);
2077                theResourceReference.setReference(theReferenceId.getValue());
2078                theResourceReference.setResource(null);
2079        }
2080
2081        private void addRollbackReferenceRestore(
2082                        TransactionDetails theTransactionDetails, IBaseReference resourceReference) {
2083                String existingValue = resourceReference.getReferenceElement().getValue();
2084                theTransactionDetails.addRollbackUndoAction(() -> resourceReference.setReference(existingValue));
2085        }
2086
2087        private void validateNoDuplicates(
2088                        RequestDetails theRequest,
2089                        TransactionDetails theTransactionDetails,
2090                        String theActionName,
2091                        Map<String, Class<? extends IBaseResource>> conditionalRequestUrls,
2092                        Collection<DaoMethodOutcome> thePersistedOutcomes) {
2093
2094                Map<ResourceTable, ResourceIndexedSearchParams> existingSearchParams =
2095                                theTransactionDetails.getOrCreateUserData(
2096                                                HapiTransactionService.XACT_USERDATA_KEY_EXISTING_SEARCH_PARAMS, Collections::emptyMap);
2097
2098                IdentityHashMap<IBaseResource, ResourceIndexedSearchParams> resourceToIndexedParams =
2099                                new IdentityHashMap<>(thePersistedOutcomes.size());
2100                thePersistedOutcomes.stream()
2101                                .filter(t -> !t.isNop())
2102                                // N.B. GGG: This validation never occurs for mongo, as nothing is a ResourceTable.
2103                                .filter(t -> t.getEntity() instanceof ResourceTable)
2104                                .filter(t -> t.getEntity().getDeleted() == null)
2105                                .filter(t -> t.getResource() != null)
2106                                .forEach(t -> {
2107                                        ResourceTable entity = (ResourceTable) t.getEntity();
2108                                        ResourceIndexedSearchParams params = existingSearchParams.get(entity);
2109                                        if (params == null) {
2110                                                params = ResourceIndexedSearchParams.withLists(entity);
2111                                        }
2112                                        resourceToIndexedParams.put(t.getResource(), params);
2113                                });
2114
2115                for (Map.Entry<String, Class<? extends IBaseResource>> nextEntry : conditionalRequestUrls.entrySet()) {
2116                        String matchUrl = nextEntry.getKey();
2117                        if (isNotBlank(matchUrl)) {
2118                                if (matchUrl.startsWith("?")
2119                                                || (!matchUrl.contains("?")
2120                                                                && UNQUALIFIED_MATCH_URL_START.matcher(matchUrl).find())) {
2121                                        StringBuilder b = new StringBuilder();
2122                                        b.append(myContext.getResourceType(nextEntry.getValue()));
2123                                        if (!matchUrl.startsWith("?")) {
2124                                                b.append("?");
2125                                        }
2126                                        b.append(matchUrl);
2127                                        matchUrl = b.toString();
2128                                }
2129
2130                                if (!myInMemoryResourceMatcher.canBeEvaluatedInMemory(matchUrl).supported()) {
2131                                        continue;
2132                                }
2133
2134                                int counter = 0;
2135                                for (Map.Entry<IBaseResource, ResourceIndexedSearchParams> entries :
2136                                                resourceToIndexedParams.entrySet()) {
2137                                        ResourceIndexedSearchParams indexedParams = entries.getValue();
2138                                        IBaseResource resource = entries.getKey();
2139
2140                                        String resourceType = myContext.getResourceType(resource);
2141                                        if (!matchUrl.startsWith(resourceType + "?")) {
2142                                                continue;
2143                                        }
2144
2145                                        if (myInMemoryResourceMatcher
2146                                                        .match(matchUrl, resource, indexedParams, theRequest)
2147                                                        .matched()) {
2148                                                counter++;
2149                                                if (counter > 1) {
2150                                                        throw new InvalidRequestException(Msg.code(542) + "Unable to process " + theActionName
2151                                                                        + " - Request would cause multiple resources to match URL: \"" + matchUrl
2152                                                                        + "\". Does transaction request contain duplicates?");
2153                                                }
2154                                        }
2155                                }
2156                        }
2157                }
2158        }
2159
2160        protected abstract void flushSession(Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome);
2161
2162        private void validateResourcePresent(IBaseResource theResource, Integer theOrder, String theVerb) {
2163                if (theResource == null) {
2164                        String msg = myContext
2165                                        .getLocalizer()
2166                                        .getMessage(BaseTransactionProcessor.class, "missingMandatoryResource", theVerb, theOrder);
2167                        throw new InvalidRequestException(Msg.code(543) + msg);
2168                }
2169        }
2170
2171        private IIdType newIdType(String theResourceType, String theResourceId, String theVersion) {
2172                org.hl7.fhir.r4.model.IdType id = new org.hl7.fhir.r4.model.IdType(theResourceType, theResourceId, theVersion);
2173                return myContext.getVersion().newIdType().setValue(id.getValue());
2174        }
2175
2176        private IIdType newIdType(String theToResourceName, String theIdPart) {
2177                return newIdType(theToResourceName, theIdPart, null);
2178        }
2179
2180        @VisibleForTesting
2181        public void setDaoRegistry(DaoRegistry theDaoRegistry) {
2182                myDaoRegistry = theDaoRegistry;
2183        }
2184
2185        private IFhirResourceDao getDaoOrThrowException(Class<? extends IBaseResource> theClass) {
2186                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDaoOrNull(theClass);
2187                if (dao == null) {
2188                        Set<String> types = new TreeSet<>(myDaoRegistry.getRegisteredDaoTypes());
2189                        String type = myContext.getResourceType(theClass);
2190                        String msg = myContext
2191                                        .getLocalizer()
2192                                        .getMessage(BaseTransactionProcessor.class, "unsupportedResourceType", type, types.toString());
2193                        throw new InvalidRequestException(Msg.code(544) + msg);
2194                }
2195                return dao;
2196        }
2197
2198        private String toResourceName(Class<? extends IBaseResource> theResourceType) {
2199                return myContext.getResourceType(theResourceType);
2200        }
2201
2202        public void setContext(FhirContext theContext) {
2203                myContext = theContext;
2204        }
2205
2206        /**
2207         * Extracts the transaction url from the entry and verifies it's:
2208         * * not null or blank
2209         * * is a relative url matching the resourceType it is about
2210         * <p>
2211         * Returns the transaction url (or throws an InvalidRequestException if url is not valid)
2212         */
2213        private String extractAndVerifyTransactionUrlForEntry(IBase theEntry, String theVerb) {
2214                String url = extractTransactionUrlOrThrowException(theEntry, theVerb);
2215
2216                if (!isValidResourceTypeUrl(url)) {
2217                        ourLog.debug("Invalid url. Should begin with a resource type: {}", url);
2218                        String msg =
2219                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, url);
2220                        throw new InvalidRequestException(Msg.code(2006) + msg);
2221                }
2222                return url;
2223        }
2224
2225        /**
2226         * Returns true if the provided url is a valid entry request.url.
2227         * <p>
2228         * This means:
2229         * a) not an absolute url (does not start with http/https)
2230         * b) starts with either a ResourceType or /ResourceType
2231         */
2232        private boolean isValidResourceTypeUrl(@Nonnull String theUrl) {
2233                if (UrlUtil.isAbsolute(theUrl)) {
2234                        return false;
2235                } else {
2236                        int queryStringIndex = theUrl.indexOf("?");
2237                        String url;
2238                        if (queryStringIndex > 0) {
2239                                url = theUrl.substring(0, theUrl.indexOf("?"));
2240                        } else {
2241                                url = theUrl;
2242                        }
2243                        String[] parts;
2244                        if (url.startsWith("/")) {
2245                                parts = url.substring(1).split("/");
2246                        } else {
2247                                parts = url.split("/");
2248                        }
2249                        Set<String> allResourceTypes = myContext.getResourceTypes();
2250
2251                        return allResourceTypes.contains(parts[0]);
2252                }
2253        }
2254
2255        /**
2256         * Extracts the transaction url from the entry and verifies that it is not null/blank
2257         * and returns it
2258         */
2259        private String extractTransactionUrlOrThrowException(IBase nextEntry, String verb) {
2260                String url = myVersionAdapter.getEntryRequestUrl(nextEntry);
2261                if (isBlank(url)) {
2262                        throw new InvalidRequestException(Msg.code(545)
2263                                        + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionMissingUrl", verb));
2264                }
2265                return url;
2266        }
2267
2268        private IFhirResourceDao<? extends IBaseResource> toDao(UrlUtil.UrlParts theParts, String theVerb, String theUrl) {
2269                RuntimeResourceDefinition resType;
2270                try {
2271                        resType = myContext.getResourceDefinition(theParts.getResourceType());
2272                } catch (DataFormatException e) {
2273                        String msg =
2274                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
2275                        throw new InvalidRequestException(Msg.code(546) + msg);
2276                }
2277                IFhirResourceDao<? extends IBaseResource> dao = null;
2278                if (resType != null) {
2279                        dao = myDaoRegistry.getResourceDao(resType.getImplementingClass());
2280                }
2281                if (dao == null) {
2282                        String msg =
2283                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
2284                        throw new InvalidRequestException(Msg.code(547) + msg);
2285                }
2286
2287                return dao;
2288        }
2289
2290        private String toMatchUrl(IBase theEntry) {
2291                String verb = myVersionAdapter.getEntryRequestVerb(myContext, theEntry);
2292                switch (defaultString(verb)) {
2293                        case POST:
2294                                return myVersionAdapter.getEntryIfNoneExist(theEntry);
2295                        case PUT:
2296                        case DELETE:
2297                        case PATCH:
2298                                String url = extractTransactionUrlOrThrowException(theEntry, verb);
2299                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
2300                                if (isBlank(parts.getResourceId())) {
2301                                        return parts.getResourceType() + '?' + parts.getParams();
2302                                }
2303                                return null;
2304                        default:
2305                                return null;
2306                }
2307        }
2308
2309        @VisibleForTesting
2310        public void setPartitionSettingsForUnitTest(PartitionSettings thePartitionSettings) {
2311                myPartitionSettings = thePartitionSettings;
2312        }
2313
2314        /**
2315         * Transaction Order, per the spec:
2316         * <p>
2317         * Process any DELETE interactions
2318         * Process any POST interactions
2319         * Process any PUT interactions
2320         * Process any PATCH interactions
2321         * Process any GET interactions
2322         */
2323        // @formatter:off
2324        public class TransactionSorter implements Comparator<IBase> {
2325
2326                private final Set<String> myPlaceholderIds;
2327
2328                public TransactionSorter(Set<String> thePlaceholderIds) {
2329                        myPlaceholderIds = thePlaceholderIds;
2330                }
2331
2332                @Override
2333                public int compare(IBase theO1, IBase theO2) {
2334                        int o1 = toOrder(theO1);
2335                        int o2 = toOrder(theO2);
2336
2337                        if (o1 == o2) {
2338                                String matchUrl1 = toMatchUrl(theO1);
2339                                String matchUrl2 = toMatchUrl(theO2);
2340                                if (isBlank(matchUrl1) && isBlank(matchUrl2)) {
2341                                        return 0;
2342                                }
2343                                if (isBlank(matchUrl1)) {
2344                                        return -1;
2345                                }
2346                                if (isBlank(matchUrl2)) {
2347                                        return 1;
2348                                }
2349
2350                                boolean match1containsSubstitutions = false;
2351                                boolean match2containsSubstitutions = false;
2352                                for (String nextPlaceholder : myPlaceholderIds) {
2353                                        if (matchUrl1.contains(nextPlaceholder)) {
2354                                                match1containsSubstitutions = true;
2355                                        }
2356                                        if (matchUrl2.contains(nextPlaceholder)) {
2357                                                match2containsSubstitutions = true;
2358                                        }
2359                                }
2360
2361                                if (match1containsSubstitutions && match2containsSubstitutions) {
2362                                        return 0;
2363                                }
2364                                if (!match1containsSubstitutions && !match2containsSubstitutions) {
2365                                        return 0;
2366                                }
2367                                if (match1containsSubstitutions) {
2368                                        return 1;
2369                                } else {
2370                                        return -1;
2371                                }
2372                        }
2373
2374                        return o1 - o2;
2375                }
2376
2377                private int toOrder(IBase theO1) {
2378                        int o1 = 0;
2379                        if (myVersionAdapter.getEntryRequestVerb(myContext, theO1) != null) {
2380                                switch (myVersionAdapter.getEntryRequestVerb(myContext, theO1)) {
2381                                        case "DELETE":
2382                                                o1 = 1;
2383                                                break;
2384                                        case "POST":
2385                                                o1 = 2;
2386                                                break;
2387                                        case "PUT":
2388                                                o1 = 3;
2389                                                break;
2390                                        case "PATCH":
2391                                                o1 = 4;
2392                                                break;
2393                                        case "GET":
2394                                                o1 = 5;
2395                                                break;
2396                                        default:
2397                                                o1 = 0;
2398                                                break;
2399                                }
2400                        }
2401                        return o1;
2402                }
2403        }
2404
2405        public class RetriableBundleTask implements Runnable {
2406
2407                private final CountDownLatch myCompletedLatch;
2408                private final RequestDetails myRequestDetails;
2409                private final IBase myNextReqEntry;
2410                private final Map<Integer, Object> myResponseMap;
2411                private final int myResponseOrder;
2412                private final boolean myNestedMode;
2413                private BaseServerResponseException myLastSeenException;
2414
2415                protected RetriableBundleTask(
2416                                CountDownLatch theCompletedLatch,
2417                                RequestDetails theRequestDetails,
2418                                Map<Integer, Object> theResponseMap,
2419                                int theResponseOrder,
2420                                IBase theNextReqEntry,
2421                                boolean theNestedMode) {
2422                        this.myCompletedLatch = theCompletedLatch;
2423                        this.myRequestDetails = theRequestDetails;
2424                        this.myNextReqEntry = theNextReqEntry;
2425                        this.myResponseMap = theResponseMap;
2426                        this.myResponseOrder = theResponseOrder;
2427                        this.myNestedMode = theNestedMode;
2428                        this.myLastSeenException = null;
2429                }
2430
2431                private void processBatchEntry() {
2432                        IBaseBundle subRequestBundle =
2433                                        myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode());
2434                        myVersionAdapter.addEntry(subRequestBundle, myNextReqEntry);
2435
2436                        IBaseBundle nextResponseBundle = processTransactionAsSubRequest(
2437                                        myRequestDetails, subRequestBundle, "Batch sub-request", myNestedMode);
2438
2439                        IBase subResponseEntry =
2440                                        (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
2441                        myResponseMap.put(myResponseOrder, subResponseEntry);
2442
2443                        /*
2444                         * If the individual entry didn't have a resource in its response, bring the sub-transaction's OperationOutcome across so the client can see it
2445                         */
2446                        if (myVersionAdapter.getResource(subResponseEntry) == null) {
2447                                IBase nextResponseBundleFirstEntry =
2448                                                (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
2449                                myResponseMap.put(myResponseOrder, nextResponseBundleFirstEntry);
2450                        }
2451                }
2452
2453                private boolean processBatchEntryWithRetry() {
2454                        int maxAttempts = 3;
2455                        for (int attempt = 1; ; attempt++) {
2456                                try {
2457                                        processBatchEntry();
2458                                        return true;
2459                                } catch (BaseServerResponseException e) {
2460                                        // If we catch a known and structured exception from HAPI, just fail.
2461                                        myLastSeenException = e;
2462                                        return false;
2463                                } catch (Throwable t) {
2464                                        myLastSeenException = new InternalErrorException(t);
2465                                        // If we have caught a non-tag-storage failure we are unfamiliar with, or we have exceeded max
2466                                        // attempts, exit.
2467                                        if (!DaoFailureUtil.isTagStorageFailure(t) || attempt >= maxAttempts) {
2468                                                ourLog.error("Failure during BATCH sub transaction processing", t);
2469                                                return false;
2470                                        }
2471                                }
2472                        }
2473                }
2474
2475                @Override
2476                public void run() {
2477                        boolean success = processBatchEntryWithRetry();
2478                        if (!success) {
2479                                populateResponseMapWithLastSeenException();
2480                        }
2481
2482                        // checking for the parallelism
2483                        ourLog.debug("processing batch for {} is completed", myVersionAdapter.getEntryRequestUrl(myNextReqEntry));
2484                        myCompletedLatch.countDown();
2485                }
2486
2487                private void populateResponseMapWithLastSeenException() {
2488                        ServerResponseExceptionHolder caughtEx = new ServerResponseExceptionHolder();
2489                        caughtEx.setException(myLastSeenException);
2490                        myResponseMap.put(myResponseOrder, caughtEx);
2491                }
2492        }
2493
2494        private static class ServerResponseExceptionHolder {
2495                private BaseServerResponseException myException;
2496
2497                public BaseServerResponseException getException() {
2498                        return myException;
2499                }
2500
2501                public void setException(BaseServerResponseException myException) {
2502                        this.myException = myException;
2503                }
2504        }
2505
2506        public static boolean isPlaceholder(IIdType theId) {
2507                if (theId != null && theId.getValue() != null) {
2508                        return theId.getValue().startsWith("urn:oid:") || theId.getValue().startsWith("urn:uuid:");
2509                }
2510                return false;
2511        }
2512
2513        private static String toStatusString(int theStatusCode) {
2514                return theStatusCode + " " + defaultString(Constants.HTTP_STATUS_NAMES.get(theStatusCode));
2515        }
2516
2517        /**
2518         * Given a match URL containing
2519         *
2520         * @param theIdSubstitutions
2521         * @param theMatchUrl
2522         * @return
2523         */
2524        public static String performIdSubstitutionsInMatchUrl(IdSubstitutionMap theIdSubstitutions, String theMatchUrl) {
2525                String matchUrl = theMatchUrl;
2526                if (isNotBlank(matchUrl) && !theIdSubstitutions.isEmpty()) {
2527                        int startIdx = 0;
2528                        while (startIdx != -1) {
2529
2530                                int endIdx = matchUrl.indexOf('&', startIdx + 1);
2531                                if (endIdx == -1) {
2532                                        endIdx = matchUrl.length();
2533                                }
2534
2535                                int equalsIdx = matchUrl.indexOf('=', startIdx + 1);
2536
2537                                int searchFrom;
2538                                if (equalsIdx == -1) {
2539                                        searchFrom = matchUrl.length();
2540                                } else if (equalsIdx >= endIdx) {
2541                                        // First equals we found is from a subsequent parameter
2542                                        searchFrom = matchUrl.length();
2543                                } else {
2544                                        String paramValue = matchUrl.substring(equalsIdx + 1, endIdx);
2545                                        boolean isUrn = isUrn(paramValue);
2546                                        boolean isUrnEscaped = !isUrn && isUrnEscaped(paramValue);
2547                                        if (isUrn || isUrnEscaped) {
2548                                                if (isUrnEscaped) {
2549                                                        paramValue = UrlUtil.unescape(paramValue);
2550                                                }
2551                                                IIdType replacement = theIdSubstitutions.getForSource(paramValue);
2552                                                if (replacement != null) {
2553                                                        String replacementValue;
2554                                                        if (replacement.hasVersionIdPart()) {
2555                                                                replacementValue = replacement.toVersionless().getValue();
2556                                                        } else {
2557                                                                replacementValue = replacement.getValue();
2558                                                        }
2559                                                        matchUrl = matchUrl.substring(0, equalsIdx + 1)
2560                                                                        + replacementValue
2561                                                                        + matchUrl.substring(endIdx);
2562                                                        searchFrom = equalsIdx + 1 + replacementValue.length();
2563                                                } else {
2564                                                        searchFrom = endIdx;
2565                                                }
2566                                        } else {
2567                                                searchFrom = endIdx;
2568                                        }
2569                                }
2570
2571                                if (searchFrom >= matchUrl.length()) {
2572                                        break;
2573                                }
2574
2575                                startIdx = matchUrl.indexOf('&', searchFrom);
2576                        }
2577                }
2578                return matchUrl;
2579        }
2580
2581        private static boolean isUrn(@Nonnull String theId) {
2582                return theId.startsWith(URN_PREFIX);
2583        }
2584
2585        private static boolean isUrnEscaped(@Nonnull String theId) {
2586                return theId.startsWith(URN_PREFIX_ESCAPED);
2587        }
2588}