001/*-
002 * #%L
003 * HAPI FHIR Storage api
004 * %%
005 * Copyright (C) 2014 - 2025 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.dao;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.FhirVersionEnum;
024import ca.uhn.fhir.context.RuntimeResourceDefinition;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.HookParams;
027import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
028import ca.uhn.fhir.interceptor.api.Pointcut;
029import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
030import ca.uhn.fhir.interceptor.model.RequestPartitionId;
031import ca.uhn.fhir.interceptor.model.TransactionWriteOperationsDetails;
032import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
033import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
034import ca.uhn.fhir.jpa.api.dao.IJpaDao;
035import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
036import ca.uhn.fhir.jpa.api.model.DeleteConflict;
037import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
038import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome;
039import ca.uhn.fhir.jpa.api.model.LazyDaoMethodOutcome;
040import ca.uhn.fhir.jpa.cache.IResourceVersionSvc;
041import ca.uhn.fhir.jpa.cache.ResourcePersistentIdMap;
042import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
043import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService;
044import ca.uhn.fhir.jpa.delete.DeleteConflictUtil;
045import ca.uhn.fhir.jpa.model.config.PartitionSettings;
046import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
047import ca.uhn.fhir.jpa.model.entity.ResourceTable;
048import ca.uhn.fhir.jpa.model.entity.StorageSettings;
049import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
050import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
051import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
052import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult;
053import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher;
054import ca.uhn.fhir.jpa.searchparam.matcher.SearchParamMatcher;
055import ca.uhn.fhir.jpa.util.TransactionSemanticsHeader;
056import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
057import ca.uhn.fhir.model.valueset.BundleEntryTransactionMethodEnum;
058import ca.uhn.fhir.parser.DataFormatException;
059import ca.uhn.fhir.parser.IParser;
060import ca.uhn.fhir.rest.api.Constants;
061import ca.uhn.fhir.rest.api.PatchTypeEnum;
062import ca.uhn.fhir.rest.api.PreferReturnEnum;
063import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
064import ca.uhn.fhir.rest.api.server.RequestDetails;
065import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
066import ca.uhn.fhir.rest.api.server.storage.DeferredInterceptorBroadcasts;
067import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
068import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
069import ca.uhn.fhir.rest.param.ParameterUtil;
070import ca.uhn.fhir.rest.server.RestfulServerUtils;
071import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException;
072import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
073import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
074import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
075import ca.uhn.fhir.rest.server.exceptions.NotModifiedException;
076import ca.uhn.fhir.rest.server.exceptions.PayloadTooLargeException;
077import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
078import ca.uhn.fhir.rest.server.method.BaseMethodBinding;
079import ca.uhn.fhir.rest.server.method.BaseResourceReturningMethodBinding;
080import ca.uhn.fhir.rest.server.method.UpdateMethodBinding;
081import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
082import ca.uhn.fhir.rest.server.servlet.ServletSubRequestDetails;
083import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
084import ca.uhn.fhir.rest.server.util.ServletRequestUtil;
085import ca.uhn.fhir.util.AsyncUtil;
086import ca.uhn.fhir.util.BundleUtil;
087import ca.uhn.fhir.util.ElementUtil;
088import ca.uhn.fhir.util.FhirTerser;
089import ca.uhn.fhir.util.ResourceReferenceInfo;
090import ca.uhn.fhir.util.StopWatch;
091import ca.uhn.fhir.util.UrlUtil;
092import com.google.common.annotations.VisibleForTesting;
093import com.google.common.collect.ArrayListMultimap;
094import com.google.common.collect.ListMultimap;
095import jakarta.annotation.Nonnull;
096import jakarta.annotation.Nullable;
097import org.apache.commons.lang3.RandomUtils;
098import org.apache.commons.lang3.StringUtils;
099import org.apache.commons.lang3.ThreadUtils;
100import org.apache.commons.lang3.Validate;
101import org.hl7.fhir.dstu3.model.Bundle;
102import org.hl7.fhir.exceptions.FHIRException;
103import org.hl7.fhir.instance.model.api.IBase;
104import org.hl7.fhir.instance.model.api.IBaseBinary;
105import org.hl7.fhir.instance.model.api.IBaseBundle;
106import org.hl7.fhir.instance.model.api.IBaseExtension;
107import org.hl7.fhir.instance.model.api.IBaseOperationOutcome;
108import org.hl7.fhir.instance.model.api.IBaseParameters;
109import org.hl7.fhir.instance.model.api.IBaseReference;
110import org.hl7.fhir.instance.model.api.IBaseResource;
111import org.hl7.fhir.instance.model.api.IIdType;
112import org.hl7.fhir.instance.model.api.IPrimitiveType;
113import org.hl7.fhir.r4.model.IdType;
114import org.slf4j.Logger;
115import org.slf4j.LoggerFactory;
116import org.springframework.beans.factory.annotation.Autowired;
117import org.springframework.core.task.TaskExecutor;
118import org.springframework.transaction.PlatformTransactionManager;
119import org.springframework.transaction.TransactionDefinition;
120import org.springframework.transaction.support.TransactionCallback;
121import org.springframework.transaction.support.TransactionTemplate;
122
123import java.time.Duration;
124import java.util.ArrayList;
125import java.util.Collection;
126import java.util.Collections;
127import java.util.Comparator;
128import java.util.Date;
129import java.util.HashMap;
130import java.util.HashSet;
131import java.util.IdentityHashMap;
132import java.util.Iterator;
133import java.util.LinkedHashMap;
134import java.util.LinkedHashSet;
135import java.util.List;
136import java.util.Map;
137import java.util.Optional;
138import java.util.Set;
139import java.util.TreeSet;
140import java.util.concurrent.ConcurrentHashMap;
141import java.util.concurrent.CountDownLatch;
142import java.util.concurrent.TimeUnit;
143import java.util.regex.Pattern;
144import java.util.stream.Collectors;
145
146import static ca.uhn.fhir.util.HapiExtensions.EXTENSION_TRANSACTION_ENTRY_PARTITION_IDS;
147import static ca.uhn.fhir.util.StringUtil.toUtf8String;
148import static java.util.Objects.isNull;
149import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
150import static org.apache.commons.lang3.ObjectUtils.getIfNull;
151import static org.apache.commons.lang3.StringUtils.defaultString;
152import static org.apache.commons.lang3.StringUtils.isBlank;
153import static org.apache.commons.lang3.StringUtils.isNotBlank;
154
155public abstract class BaseTransactionProcessor {
156
157        public static final String URN_PREFIX = "urn:";
158        public static final String URN_PREFIX_ESCAPED = UrlUtil.escapeUrlParam(URN_PREFIX);
159        public static final Pattern UNQUALIFIED_MATCH_URL_START = Pattern.compile("^[a-zA-Z0-9_-]+=");
160        public static final Pattern INVALID_PLACEHOLDER_PATTERN = Pattern.compile("[a-zA-Z]+:.*");
161
162        private static final Logger ourLog = LoggerFactory.getLogger(BaseTransactionProcessor.class);
163        private static final String POST = "POST";
164        private static final String PUT = "PUT";
165        private static final String DELETE = "DELETE";
166        private static final String PATCH = "PATCH";
167
168        @Autowired
169        private IRequestPartitionHelperSvc myRequestPartitionHelperService;
170
171        @Autowired
172        private PlatformTransactionManager myTxManager;
173
174        @Autowired
175        private FhirContext myContext;
176
177        @Autowired
178        @SuppressWarnings("rawtypes")
179        private ITransactionProcessorVersionAdapter myVersionAdapter;
180
181        @Autowired
182        private DaoRegistry myDaoRegistry;
183
184        @Autowired
185        private IInterceptorBroadcaster myInterceptorBroadcaster;
186
187        @Autowired
188        protected IHapiTransactionService myHapiTransactionService;
189
190        @Autowired
191        private StorageSettings myStorageSettings;
192
193        @Autowired
194        PartitionSettings myPartitionSettings;
195
196        @Autowired
197        private InMemoryResourceMatcher myInMemoryResourceMatcher;
198
199        @Autowired
200        private SearchParamMatcher mySearchParamMatcher;
201
202        @Autowired
203        private ThreadPoolFactory myThreadPoolFactory;
204
205        private TaskExecutor myExecutor;
206
207        @Autowired
208        private IResourceVersionSvc myResourceVersionSvc;
209
210        @VisibleForTesting
211        public void setStorageSettings(StorageSettings theStorageSettings) {
212                myStorageSettings = theStorageSettings;
213        }
214
215        public ITransactionProcessorVersionAdapter getVersionAdapter() {
216                return myVersionAdapter;
217        }
218
219        @VisibleForTesting
220        public void setVersionAdapter(ITransactionProcessorVersionAdapter theVersionAdapter) {
221                myVersionAdapter = theVersionAdapter;
222        }
223
224        private TaskExecutor getTaskExecutor() {
225                if (myExecutor == null) {
226                        myExecutor = myThreadPoolFactory.newThreadPool(
227                                        myStorageSettings.getBundleBatchPoolSize(),
228                                        myStorageSettings.getBundleBatchMaxPoolSize(),
229                                        "bundle-batch-");
230                }
231                return myExecutor;
232        }
233
234        public <BUNDLE extends IBaseBundle> BUNDLE transaction(
235                        RequestDetails theRequestDetails, BUNDLE theRequest, boolean theNestedMode) {
236                String actionName = "Transaction";
237
238                IInterceptorBroadcaster compositeBroadcaster =
239                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
240
241                // Interceptor call: STORAGE_TRANSACTION_PROCESSING
242                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING)) {
243                        HookParams params = new HookParams()
244                                        .add(RequestDetails.class, theRequestDetails)
245                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
246                                        .add(IBaseBundle.class, theRequest);
247                        compositeBroadcaster.callHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING, params);
248                }
249
250                IBaseBundle response;
251                // Interceptor call: STORAGE_TRANSACTION_PRE_PARTITION
252                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_PRE_PARTITION)) {
253                        response = new TransactionPartitionProcessor<BUNDLE>(
254                                                        this, myContext, theRequestDetails, theNestedMode, compositeBroadcaster, actionName)
255                                        .execute(theRequest);
256                } else {
257                        response = processTransactionAsSubRequest(theRequestDetails, theRequest, actionName, theNestedMode);
258                }
259
260                List<IBase> entries = myVersionAdapter.getEntries(response);
261                for (int i = 0; i < entries.size(); i++) {
262                        if (ElementUtil.isEmpty(entries.get(i))) {
263                                entries.remove(i);
264                                i--;
265                        }
266                }
267
268                return (BUNDLE) response;
269        }
270
271        public IBaseBundle collection(final RequestDetails theRequestDetails, IBaseBundle theRequest) {
272                String transactionType = myVersionAdapter.getBundleType(theRequest);
273
274                if (!org.hl7.fhir.r4.model.Bundle.BundleType.COLLECTION.toCode().equals(transactionType)) {
275                        throw new InvalidRequestException(
276                                        Msg.code(526) + "Can not process collection Bundle of type: " + transactionType);
277                }
278
279                ourLog.info(
280                                "Beginning storing collection with {} resources",
281                                myVersionAdapter.getEntries(theRequest).size());
282
283                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
284                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
285
286                IBaseBundle resp =
287                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
288
289                List<IBaseResource> resources = new ArrayList<>();
290                for (final Object nextRequestEntry : myVersionAdapter.getEntries(theRequest)) {
291                        IBaseResource resource = myVersionAdapter.getResource((IBase) nextRequestEntry);
292                        resources.add(resource);
293                }
294
295                IBaseBundle transactionBundle = myVersionAdapter.createBundle("transaction");
296                for (IBaseResource next : resources) {
297                        IBase entry = myVersionAdapter.addEntry(transactionBundle);
298                        myVersionAdapter.setResource(entry, next);
299                        myVersionAdapter.setRequestVerb(entry, "PUT");
300                        myVersionAdapter.setRequestUrl(
301                                        entry, next.getIdElement().toUnqualifiedVersionless().getValue());
302                }
303
304                transaction(theRequestDetails, transactionBundle, false);
305
306                return resp;
307        }
308
309        @SuppressWarnings("unchecked")
310        private void populateEntryWithOperationOutcome(BaseServerResponseException caughtEx, IBase nextEntry) {
311                myVersionAdapter.populateEntryWithOperationOutcome(caughtEx, nextEntry);
312        }
313
314        @SuppressWarnings("unchecked")
315        private void handleTransactionCreateOrUpdateOutcome(
316                        IdSubstitutionMap theIdSubstitutions,
317                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
318                        IIdType theNextResourceId,
319                        DaoMethodOutcome theOutcome,
320                        IBase theNewEntry,
321                        String theResourceType,
322                        IBaseResource theRes,
323                        RequestDetails theRequestDetails) {
324                IIdType newId = theOutcome.getId().toUnqualified();
325                IIdType resourceId =
326                                isPlaceholder(theNextResourceId) ? theNextResourceId : theNextResourceId.toUnqualifiedVersionless();
327                if (!newId.equals(resourceId)) {
328                        if (!theNextResourceId.isEmpty()) {
329                                theIdSubstitutions.put(resourceId, newId);
330                        }
331                        if (isPlaceholder(resourceId)) {
332                                /*
333                                 * The correct way for substitution IDs to be is to be with no resource type, but we'll accept the qualified kind too just to be lenient.
334                                 */
335                                IIdType id = myContext.getVersion().newIdType();
336                                id.setValue(theResourceType + '/' + resourceId.getValue());
337                                theIdSubstitutions.put(id, newId);
338                        }
339                }
340
341                populateIdToPersistedOutcomeMap(theIdToPersistedOutcome, newId, theOutcome);
342
343                if (shouldSwapBinaryToActualResource(theRes, theResourceType, theNextResourceId)) {
344                        theRes = theIdToPersistedOutcome.get(newId).getResource();
345                }
346
347                if (theOutcome.getCreated()) {
348                        myVersionAdapter.setResponseStatus(theNewEntry, toStatusString(Constants.STATUS_HTTP_201_CREATED));
349                } else {
350                        myVersionAdapter.setResponseStatus(theNewEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
351                }
352
353                Date lastModified = getLastModified(theRes);
354                myVersionAdapter.setResponseLastModified(theNewEntry, lastModified);
355
356                if (theOutcome.getOperationOutcome() != null) {
357                        myVersionAdapter.setResponseOutcome(theNewEntry, theOutcome.getOperationOutcome());
358                }
359
360                if (theRequestDetails != null) {
361                        String prefer = theRequestDetails.getHeader(Constants.HEADER_PREFER);
362                        PreferReturnEnum preferReturn =
363                                        RestfulServerUtils.parsePreferHeader(null, prefer).getReturn();
364                        if (preferReturn != null) {
365                                if (preferReturn == PreferReturnEnum.REPRESENTATION) {
366                                        if (theOutcome.getResource() != null) {
367                                                theOutcome.fireResourceViewCallbacks();
368                                                myVersionAdapter.setResource(theNewEntry, theOutcome.getResource());
369                                        }
370                                }
371                        }
372                }
373        }
374
375        /**
376         * Method which populates entry in idToPersistedOutcome.
377         * Will store whatever outcome is sent, unless the key already exists, then we only replace an instance if we find that the instance
378         * we are replacing with is non-lazy. This allows us to evaluate later more easily, as we _know_ we need access to these.
379         */
380        private void populateIdToPersistedOutcomeMap(
381                        Map<IIdType, DaoMethodOutcome> idToPersistedOutcome, IIdType newId, DaoMethodOutcome outcome) {
382                // Prefer real method outcomes over lazy ones.
383                if (idToPersistedOutcome.containsKey(newId)) {
384                        if (!(outcome instanceof LazyDaoMethodOutcome)) {
385                                idToPersistedOutcome.put(newId, outcome);
386                        }
387                } else {
388                        idToPersistedOutcome.put(newId, outcome);
389                }
390        }
391
392        private Date getLastModified(IBaseResource theRes) {
393                return theRes.getMeta().getLastUpdated();
394        }
395
396        IBaseBundle processTransactionAsSubRequest(
397                        RequestDetails theRequestDetails, IBaseBundle theRequest, String theActionName, boolean theNestedMode) {
398                return processTransactionAsSubRequest(
399                                theRequestDetails, new TransactionDetails(), theRequest, theActionName, theNestedMode);
400        }
401
402        IBaseBundle processTransactionAsSubRequest(
403                        RequestDetails theRequestDetails,
404                        TransactionDetails theTransactionDetails,
405                        IBaseBundle theRequest,
406                        String theActionName,
407                        boolean theNestedMode) {
408                BaseStorageDao.markRequestAsProcessingSubRequest(theRequestDetails);
409                try {
410                        return processTransaction(
411                                        theRequestDetails, theTransactionDetails, theRequest, theActionName, theNestedMode);
412                } finally {
413                        BaseStorageDao.clearRequestAsProcessingSubRequest(theRequestDetails);
414                }
415        }
416
417        @VisibleForTesting
418        public void setTxManager(PlatformTransactionManager theTxManager) {
419                myTxManager = theTxManager;
420        }
421
422        private IBaseBundle batch(final RequestDetails theRequestDetails, IBaseBundle theRequest, boolean theNestedMode) {
423                TransactionSemanticsHeader transactionSemantics = TransactionSemanticsHeader.DEFAULT;
424                if (theRequestDetails != null) {
425                        String transactionSemanticsString = theRequestDetails.getHeader("X-Transaction-Semantics");
426                        if (transactionSemanticsString != null) {
427                                transactionSemantics = TransactionSemanticsHeader.parse(transactionSemanticsString);
428                        }
429                }
430
431                int totalAttempts = defaultIfNull(transactionSemantics.getRetryCount(), 0) + 1;
432                boolean switchedToBatch = false;
433
434                int minRetryDelay = getIfNull(transactionSemantics.getMinRetryDelay(), 0);
435                int maxRetryDelay = getIfNull(transactionSemantics.getMaxRetryDelay(), 0);
436
437                // Don't let the user request a crazy delay, this could be used
438                // as a DOS attack
439                maxRetryDelay = Math.max(maxRetryDelay, minRetryDelay);
440                maxRetryDelay = Math.min(maxRetryDelay, 10000);
441                totalAttempts = Math.min(totalAttempts, 5);
442
443                IBaseBundle response;
444                for (int i = 1; ; i++) {
445                        try {
446                                if (i < totalAttempts && transactionSemantics.isTryBatchAsTransactionFirst()) {
447                                        BundleUtil.setBundleType(myContext, theRequest, "transaction");
448                                        response = processTransaction(
449                                                        theRequestDetails, new TransactionDetails(), theRequest, "Transaction", theNestedMode);
450                                } else {
451                                        BundleUtil.setBundleType(myContext, theRequest, "batch");
452                                        response = processBatch(theRequestDetails, theRequest, theNestedMode);
453                                }
454                                break;
455                        } catch (BaseServerResponseException e) {
456                                if (i >= totalAttempts || theNestedMode) {
457                                        throw e;
458                                }
459
460                                long delay = RandomUtils.insecure().randomLong(minRetryDelay, maxRetryDelay);
461                                String sleepMessage = "";
462                                if (delay > 0) {
463                                        sleepMessage = "Sleeping for " + delay + " ms. ";
464                                }
465
466                                String switchedToBatchMessage = "";
467                                if (switchedToBatch) {
468                                        switchedToBatchMessage = " (as batch)";
469                                }
470
471                                String switchingToBatchMessage = "";
472                                if (i + 1 == totalAttempts && transactionSemantics.isTryBatchAsTransactionFirst()) {
473                                        switchingToBatchMessage = "Performing final retry using batch semantics. ";
474                                        switchedToBatch = true;
475                                        BundleUtil.setBundleType(myContext, theRequest, "batch");
476                                }
477
478                                ourLog.warn(
479                                                "Transaction processing attempt{} {}/{} failed. {}{}",
480                                                switchedToBatchMessage,
481                                                i,
482                                                totalAttempts,
483                                                sleepMessage,
484                                                switchingToBatchMessage);
485
486                                if (delay > 0) {
487                                        ThreadUtils.sleepQuietly(Duration.ofMillis(delay));
488                                }
489                        }
490                }
491
492                return response;
493        }
494
495        private IBaseBundle processBatch(RequestDetails theRequestDetails, IBaseBundle theRequest, boolean theNestedMode) {
496                ourLog.info(
497                                "Beginning batch with {} resources",
498                                myVersionAdapter.getEntries(theRequest).size());
499
500                long start = System.currentTimeMillis();
501
502                IBaseBundle response =
503                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
504                Map<Integer, Object> responseMap = new ConcurrentHashMap<>();
505
506                List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest);
507                int requestEntriesSize = requestEntries.size();
508
509                // Now, run all non-gets sequentially, and all gets are submitted to the executor to run (potentially) in
510                // parallel
511                // The result is kept in the map to save the original position
512                List<RetriableBundleTask> getCalls = new ArrayList<>();
513                List<RetriableBundleTask> nonGetCalls = new ArrayList<>();
514
515                CountDownLatch completionLatch = new CountDownLatch(requestEntriesSize);
516                for (int i = 0; i < requestEntriesSize; i++) {
517                        IBase nextRequestEntry = requestEntries.get(i);
518                        RetriableBundleTask retriableBundleTask = new RetriableBundleTask(
519                                        completionLatch, theRequestDetails, responseMap, i, nextRequestEntry, theNestedMode);
520                        if (myVersionAdapter
521                                        .getEntryRequestVerb(myContext, nextRequestEntry)
522                                        .equalsIgnoreCase("GET")) {
523                                getCalls.add(retriableBundleTask);
524                        } else {
525                                nonGetCalls.add(retriableBundleTask);
526                        }
527                }
528                // Execute all non-gets on calling thread.
529                nonGetCalls.forEach(RetriableBundleTask::run);
530                // Execute all gets (potentially in a pool)
531                if (myStorageSettings.getBundleBatchPoolSize() == 1) {
532                        getCalls.forEach(RetriableBundleTask::run);
533                } else {
534                        getCalls.forEach(getCall -> getTaskExecutor().execute(getCall));
535                }
536
537                // waiting for all async tasks to be completed
538                AsyncUtil.awaitLatchAndIgnoreInterrupt(completionLatch, 300L, TimeUnit.SECONDS);
539
540                // Now, create the bundle response in original order
541                Object nextResponseEntry;
542                for (int i = 0; i < requestEntriesSize; i++) {
543
544                        nextResponseEntry = responseMap.get(i);
545                        if (nextResponseEntry instanceof ServerResponseExceptionHolder) {
546                                ServerResponseExceptionHolder caughtEx = (ServerResponseExceptionHolder) nextResponseEntry;
547                                if (caughtEx.getException() != null) {
548                                        IBase nextEntry = myVersionAdapter.addEntry(response);
549                                        populateEntryWithOperationOutcome(caughtEx.getException(), nextEntry);
550                                        myVersionAdapter.setResponseStatus(
551                                                        nextEntry, toStatusString(caughtEx.getException().getStatusCode()));
552                                }
553                        } else {
554                                myVersionAdapter.addEntry(response, (IBase) nextResponseEntry);
555                        }
556                }
557
558                long delay = System.currentTimeMillis() - start;
559                ourLog.info("Batch completed in {}ms", delay);
560
561                return response;
562        }
563
564        @VisibleForTesting
565        public void setHapiTransactionService(HapiTransactionService theHapiTransactionService) {
566                myHapiTransactionService = theHapiTransactionService;
567        }
568
569        private IBaseBundle processTransaction(
570                        final RequestDetails theRequestDetails,
571                        final TransactionDetails theTransactionDetails,
572                        final IBaseBundle theRequest,
573                        final String theActionName,
574                        boolean theNestedMode) {
575                validateDependencies();
576
577                String transactionType = myVersionAdapter.getBundleType(theRequest);
578
579                if (org.hl7.fhir.r4.model.Bundle.BundleType.BATCH.toCode().equals(transactionType)) {
580                        return batch(theRequestDetails, theRequest, theNestedMode);
581                }
582
583                if (transactionType == null) {
584                        String message = "Transaction Bundle did not specify valid Bundle.type, assuming "
585                                        + Bundle.BundleType.TRANSACTION.toCode();
586                        ourLog.warn(message);
587                        transactionType = org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode();
588                }
589                if (!org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode().equals(transactionType)) {
590                        throw new InvalidRequestException(
591                                        Msg.code(527) + "Unable to process transaction where incoming Bundle.type = " + transactionType);
592                }
593
594                // Make a copy of the list because it gets sorted below, and we don't want to
595                // modify the original Bundle
596                List<IBase> requestEntries = new ArrayList<>(myVersionAdapter.getEntries(theRequest));
597                int numberOfEntries = requestEntries.size();
598
599                if (myStorageSettings.getMaximumTransactionBundleSize() != null
600                                && numberOfEntries > myStorageSettings.getMaximumTransactionBundleSize()) {
601                        throw new PayloadTooLargeException(
602                                        Msg.code(528) + "Transaction Bundle Too large.  Transaction bundle contains " + numberOfEntries
603                                                        + " which exceedes the maximum permitted transaction bundle size of "
604                                                        + myStorageSettings.getMaximumTransactionBundleSize());
605                }
606
607                ourLog.debug("Beginning {} with {} resources", theActionName, numberOfEntries);
608
609                theTransactionDetails.setFhirTransaction(true);
610                final StopWatch transactionStopWatch = new StopWatch();
611
612                // Do all entries have a verb?
613                for (int i = 0; i < numberOfEntries; i++) {
614                        IBase nextReqEntry = requestEntries.get(i);
615                        String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
616                        if (verb == null || !isValidVerb(verb)) {
617                                throw new InvalidRequestException(Msg.code(529)
618                                                + myContext
619                                                                .getLocalizer()
620                                                                .getMessage(BaseStorageDao.class, "transactionEntryHasInvalidVerb", verb, i));
621                        }
622                }
623
624                /*
625                 * We want to execute the transaction request bundle elements in the order
626                 * specified by the FHIR specification (see TransactionSorter) so we save the
627                 * original order in the request, then sort it.
628                 *
629                 * Entries with a type of GET are removed from the bundle so that they
630                 * can be processed at the very end. We do this because the incoming resources
631                 * are saved in a two-phase way in order to deal with interdependencies, and
632                 * we want the GET processing to use the final indexing state
633                 */
634                final IBaseBundle response =
635                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTIONRESPONSE.toCode());
636                List<IBase> getEntries = new ArrayList<>();
637                final IdentityHashMap<IBase, Integer> originalRequestOrder = new IdentityHashMap<>();
638                for (int i = 0; i < requestEntries.size(); i++) {
639                        IBase requestEntry = requestEntries.get(i);
640
641                        originalRequestOrder.put(requestEntry, i);
642                        myVersionAdapter.addEntry(response);
643                        if (myVersionAdapter.getEntryRequestVerb(myContext, requestEntry).equals("GET")) {
644                                getEntries.add(requestEntry);
645                        }
646                }
647
648                /*
649                 * See FhirSystemDaoDstu3Test#testTransactionWithPlaceholderIdInMatchUrl
650                 * Basically if the resource has a match URL that references a placeholder,
651                 * we try to handle the resource with the placeholder first.
652                 */
653                Set<String> placeholderIds = new HashSet<>();
654                for (IBase nextEntry : requestEntries) {
655                        String fullUrl = myVersionAdapter.getFullUrl(nextEntry);
656                        if (isNotBlank(fullUrl) && fullUrl.startsWith(URN_PREFIX)) {
657                                placeholderIds.add(fullUrl);
658                        }
659                }
660                requestEntries.sort(new TransactionSorter(placeholderIds));
661
662                // perform all writes
663                prepareThenExecuteTransactionWriteOperations(
664                                theRequestDetails,
665                                theActionName,
666                                theTransactionDetails,
667                                transactionStopWatch,
668                                response,
669                                originalRequestOrder,
670                                requestEntries);
671
672                // perform all gets
673                // (we do these last so that the gets happen on the final state of the DB;
674                // see above note)
675                doTransactionReadOperations(
676                                theRequestDetails, response, getEntries, originalRequestOrder, transactionStopWatch, theNestedMode);
677
678                // Interceptor broadcast: JPA_PERFTRACE_INFO
679                IInterceptorBroadcaster compositeBroadcaster =
680                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
681                if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO)) {
682                        String taskDurations = transactionStopWatch.formatTaskDurations();
683                        StorageProcessingMessage message = new StorageProcessingMessage();
684                        message.setMessage("Transaction timing:\n" + taskDurations);
685                        HookParams params = new HookParams()
686                                        .add(RequestDetails.class, theRequestDetails)
687                                        .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
688                                        .add(StorageProcessingMessage.class, message);
689                        compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
690                }
691
692                return response;
693        }
694
695        @SuppressWarnings("unchecked")
696        private void doTransactionReadOperations(
697                        final RequestDetails theRequestDetails,
698                        IBaseBundle theResponse,
699                        List<IBase> theGetEntries,
700                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
701                        StopWatch theTransactionStopWatch,
702                        boolean theNestedMode) {
703                if (theGetEntries.size() > 0) {
704                        theTransactionStopWatch.startTask("Process " + theGetEntries.size() + " GET entries");
705
706                        /*
707                         * Loop through the request and process any entries of type GET
708                         */
709                        for (IBase nextReqEntry : theGetEntries) {
710                                if (theNestedMode) {
711                                        throw new InvalidRequestException(
712                                                        Msg.code(530) + "Can not invoke read operation on nested transaction");
713                                }
714
715                                if (!(theRequestDetails instanceof ServletRequestDetails)) {
716                                        throw new MethodNotAllowedException(
717                                                        Msg.code(531) + "Can not call transaction GET methods from this context");
718                                }
719
720                                ServletRequestDetails srd = (ServletRequestDetails) theRequestDetails;
721                                Integer originalOrder = theOriginalRequestOrder.get(nextReqEntry);
722                                IBase nextRespEntry =
723                                                (IBase) myVersionAdapter.getEntries(theResponse).get(originalOrder);
724
725                                ArrayListMultimap<String, String> paramValues = ArrayListMultimap.create();
726                                ServletSubRequestDetails requestDetailsForEntry =
727                                                createRequestDetailsForReadEntry(srd, nextReqEntry, paramValues);
728
729                                String url = requestDetailsForEntry.getRequestPath();
730
731                                BaseMethodBinding method = srd.getServer().determineResourceMethod(requestDetailsForEntry, url);
732                                if (method == null) {
733                                        throw new IllegalArgumentException(Msg.code(532) + "Unable to handle GET " + url);
734                                }
735
736                                Validate.isTrue(method instanceof BaseResourceReturningMethodBinding, "Unable to handle GET {}", url);
737                                try {
738                                        BaseResourceReturningMethodBinding methodBinding = (BaseResourceReturningMethodBinding) method;
739                                        requestDetailsForEntry.setRestOperationType(methodBinding.getRestOperationType());
740
741                                        IBaseResource resource = methodBinding.doInvokeServer(srd.getServer(), requestDetailsForEntry);
742                                        if (paramValues.containsKey(Constants.PARAM_SUMMARY)
743                                                        || paramValues.containsKey(Constants.PARAM_CONTENT)) {
744                                                resource = filterNestedBundle(requestDetailsForEntry, resource);
745                                        }
746                                        myVersionAdapter.setResource(nextRespEntry, resource);
747                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
748                                } catch (NotModifiedException e) {
749                                        myVersionAdapter.setResponseStatus(
750                                                        nextRespEntry, toStatusString(Constants.STATUS_HTTP_304_NOT_MODIFIED));
751                                } catch (BaseServerResponseException e) {
752                                        ourLog.info("Failure processing transaction GET {}: {}", url, e.toString());
753                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(e.getStatusCode()));
754                                        populateEntryWithOperationOutcome(e, nextRespEntry);
755                                }
756                        }
757                        theTransactionStopWatch.endCurrentTask();
758                }
759        }
760
761        /**
762         * Checks if the given request entry has the extension specified to override the partition ids to use when processing that entry.
763         * If the extension is present, this method will set the partition ids header in the given request details.
764         */
765        private void setRequestPartitionHeaderIfEntryHasTheExtension(IBase theReqEntry, RequestDetails theRequestDetails) {
766                Optional<IBaseExtension<?, ?>> partitionIdsExtensionOptional =
767                                myVersionAdapter.getEntryRequestExtensionByUrl(theReqEntry, EXTENSION_TRANSACTION_ENTRY_PARTITION_IDS);
768                if (partitionIdsExtensionOptional.isPresent()
769                                && partitionIdsExtensionOptional.get().getValue() instanceof IPrimitiveType<?>) {
770                        IPrimitiveType<?> valueAsPrimitiveType =
771                                        (IPrimitiveType<?>) partitionIdsExtensionOptional.get().getValue();
772                        String value = valueAsPrimitiveType.getValueAsString();
773                        theRequestDetails.setHeaders(Constants.HEADER_X_REQUEST_PARTITION_IDS, List.of(value));
774                }
775        }
776
777        /**
778         * Creates a new RequestDetails object based on the given RequestDetails. The new RequestDetails is to be used
779         * when processing a write entry.
780         * If the entry.request has the extension to override the partition ids, this method
781         * sets the partition ids header in the newly created request details with the values from extension.
782         * This allows using different partitions for different entries in the same transaction.
783         *
784         * @return the newly created request details
785         */
786        private RequestDetails createRequestDetailsForWriteEntry(
787                        RequestDetails theRequestDetails, IBase theEntry, String theUrl, String theVerb) {
788
789                if (theRequestDetails == null) {
790                        ourLog.warn(
791                                        "The RequestDetails passed in to the transaction is null. Cannot create a new RequestDetails for transaction entry.");
792                        return null;
793                }
794
795                RequestDetails newRequestDetails;
796                if (theRequestDetails instanceof ServletRequestDetails) {
797                        newRequestDetails = ServletRequestUtil.getServletSubRequestDetails(
798                                        (ServletRequestDetails) theRequestDetails, theUrl, theVerb, ArrayListMultimap.create());
799                } else if (theRequestDetails instanceof SystemRequestDetails) {
800                        newRequestDetails = new SystemRequestDetails((SystemRequestDetails) theRequestDetails);
801                } else {
802                        // RequestDetails is not a ServletRequestDetails or SystemRequestDetails, and I don't know how to properly
803                        // clone such a RequestDetails. Use the original RequestDetails without making any entry specific
804                        // modifications
805                        ourLog.warn(
806                                        "Cannot create a new RequestDetails for transaction entry out of the existing RequestDetails which is of type '{}'"
807                                                        + "Using the original RequestDetails for transaction entries without any entry specific modifications.",
808                                        theRequestDetails.getClass().getSimpleName());
809                        return theRequestDetails;
810                }
811                setRequestPartitionHeaderIfEntryHasTheExtension(theEntry, newRequestDetails);
812                return newRequestDetails;
813        }
814
815        /**
816         * Creates a new RequestDetails based on the given one. The returned RequestDetails is to be used when processing
817         * a GET entry of transaction.
818         * It sets the headers in the newly request details according to the information from entry.request if needed.
819         * Currently, GET entries only support ServletRequestDetails so it handles only that type.
820         */
821        private ServletSubRequestDetails createRequestDetailsForReadEntry(
822                        ServletRequestDetails theRequestDetails, IBase theEntry, ArrayListMultimap<String, String> theParamValues) {
823
824                final String verb = "GET";
825                String transactionUrl = extractTransactionUrlOrThrowException(theEntry, verb);
826
827                ServletSubRequestDetails subRequestDetails =
828                                ServletRequestUtil.getServletSubRequestDetails(theRequestDetails, transactionUrl, verb, theParamValues);
829
830                if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(theEntry))) {
831                        subRequestDetails.addHeader(Constants.HEADER_IF_MATCH, myVersionAdapter.getEntryRequestIfMatch(theEntry));
832                }
833                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneExist(theEntry))) {
834                        subRequestDetails.addHeader(
835                                        Constants.HEADER_IF_NONE_EXIST, myVersionAdapter.getEntryRequestIfNoneExist(theEntry));
836                }
837                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneMatch(theEntry))) {
838                        subRequestDetails.addHeader(
839                                        Constants.HEADER_IF_NONE_MATCH, myVersionAdapter.getEntryRequestIfNoneMatch(theEntry));
840                }
841
842                setRequestPartitionHeaderIfEntryHasTheExtension(theEntry, subRequestDetails);
843
844                return subRequestDetails;
845        }
846
847        /**
848         * All of the write operations in the transaction (PUT, POST, etc.. basically anything
849         * except GET) are performed in their own database transaction before we do the reads.
850         * We do this because the reads (specifically the searches) often spawn their own
851         * secondary database transaction and if we allow that within the primary
852         * database transaction we can end up with deadlocks if the server is under
853         * heavy load with lots of concurrent transactions using all available
854         * database connections.
855         */
856        @SuppressWarnings("unchecked")
857        private void prepareThenExecuteTransactionWriteOperations(
858                        RequestDetails theRequestDetails,
859                        String theActionName,
860                        TransactionDetails theTransactionDetails,
861                        StopWatch theTransactionStopWatch,
862                        IBaseBundle theResponse,
863                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
864                        List<IBase> theEntries) {
865
866                TransactionWriteOperationsDetails writeOperationsDetails = null;
867                if (haveWriteOperationsHooks(theRequestDetails)) {
868                        writeOperationsDetails = buildWriteOperationsDetails(theEntries);
869                        callWriteOperationsHook(
870                                        Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE,
871                                        theRequestDetails,
872                                        theTransactionDetails,
873                                        writeOperationsDetails);
874                }
875
876                TransactionCallback<EntriesToProcessMap> txCallback = status -> {
877                        final Set<IIdType> allIds = new LinkedHashSet<>();
878                        final IdSubstitutionMap idSubstitutions = new IdSubstitutionMap();
879                        final Map<IIdType, DaoMethodOutcome> idToPersistedOutcome = new LinkedHashMap<>();
880
881                        EntriesToProcessMap retVal = doTransactionWriteOperations(
882                                        theRequestDetails,
883                                        theActionName,
884                                        theTransactionDetails,
885                                        allIds,
886                                        idSubstitutions,
887                                        idToPersistedOutcome,
888                                        theResponse,
889                                        theOriginalRequestOrder,
890                                        theEntries,
891                                        theTransactionStopWatch);
892
893                        theTransactionStopWatch.startTask("Commit writes to database");
894                        return retVal;
895                };
896                EntriesToProcessMap entriesToProcess;
897
898                RequestPartitionId requestPartitionId =
899                                determineRequestPartitionIdForWriteEntries(theRequestDetails, theTransactionDetails, theEntries);
900
901                try {
902                        entriesToProcess = myHapiTransactionService
903                                        .withRequest(theRequestDetails)
904                                        .withRequestPartitionId(requestPartitionId)
905                                        .withTransactionDetails(theTransactionDetails)
906                                        .execute(txCallback);
907                } finally {
908                        if (haveWriteOperationsHooks(theRequestDetails)) {
909                                callWriteOperationsHook(
910                                                Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST,
911                                                theRequestDetails,
912                                                theTransactionDetails,
913                                                writeOperationsDetails);
914                        }
915                }
916
917                theTransactionStopWatch.endCurrentTask();
918
919                for (Map.Entry<IBase, IIdType> nextEntry : entriesToProcess.entrySet()) {
920                        String responseLocation = nextEntry.getValue().toUnqualified().getValue();
921                        String responseEtag = nextEntry.getValue().getVersionIdPart();
922                        myVersionAdapter.setResponseLocation(nextEntry.getKey(), responseLocation);
923                        myVersionAdapter.setResponseETag(nextEntry.getKey(), responseEtag);
924                }
925        }
926
927        /**
928         * This method looks at the FHIR actions being performed in a List of bundle entries,
929         * and determines the associated request partitions.
930         */
931        @Nullable
932        protected RequestPartitionId determineRequestPartitionIdForWriteEntries(
933                        RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, List<IBase> theEntries) {
934                if (!myPartitionSettings.isPartitioningEnabled()) {
935                        return RequestPartitionId.allPartitions();
936                }
937
938                return theEntries.stream()
939                                .map(e -> getEntryRequestPartitionId(theRequestDetails, theTransactionDetails, e))
940                                .reduce(null, (accumulator, nextPartition) -> {
941                                        if (accumulator == null) {
942                                                return nextPartition;
943                                        } else if (nextPartition == null) {
944                                                return accumulator;
945                                        } else if (myHapiTransactionService.isCompatiblePartition(accumulator, nextPartition)) {
946                                                return accumulator.mergeIds(nextPartition);
947                                        } else if (!nextPartition.isAllPartitions()) {
948                                                String msg = myContext
949                                                                .getLocalizer()
950                                                                .getMessage(
951                                                                                BaseTransactionProcessor.class, "multiplePartitionAccesses", theEntries.size());
952                                                throw new InvalidRequestException(Msg.code(2541) + msg);
953                                        } else {
954                                                return accumulator;
955                                        }
956                                });
957        }
958
959        @Nullable
960        private RequestPartitionId getEntryRequestPartitionId(
961                        RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, IBase theEntry) {
962
963                RequestPartitionId nextWriteEntryRequestPartitionId = null;
964                String verb = myVersionAdapter.getEntryRequestVerb(myContext, theEntry);
965                String url = extractTransactionUrlOrThrowException(theEntry, verb);
966                RequestDetails requestDetailsForEntry =
967                                createRequestDetailsForWriteEntry(theRequestDetails, theEntry, url, verb);
968                if (isNotBlank(verb)) {
969                        BundleEntryTransactionMethodEnum verbEnum = BundleEntryTransactionMethodEnum.valueOf(verb);
970                        switch (verbEnum) {
971                                case GET:
972                                        nextWriteEntryRequestPartitionId = null;
973                                        break;
974                                case DELETE: {
975                                        String requestUrl = myVersionAdapter.getEntryRequestUrl(theEntry);
976                                        if (isNotBlank(requestUrl)) {
977                                                IdType id = new IdType(requestUrl);
978                                                String resourceType = id.getResourceType();
979                                                ReadPartitionIdRequestDetails details =
980                                                                ReadPartitionIdRequestDetails.forDelete(resourceType, id);
981                                                nextWriteEntryRequestPartitionId =
982                                                                myRequestPartitionHelperService.determineReadPartitionForRequest(
983                                                                                requestDetailsForEntry, details);
984                                        }
985                                        break;
986                                }
987                                case PATCH: {
988                                        String requestUrl = myVersionAdapter.getEntryRequestUrl(theEntry);
989                                        if (isNotBlank(requestUrl)) {
990                                                IdType id = new IdType(requestUrl);
991                                                String resourceType = id.getResourceType();
992                                                ReadPartitionIdRequestDetails details =
993                                                                ReadPartitionIdRequestDetails.forPatch(resourceType, id);
994                                                nextWriteEntryRequestPartitionId =
995                                                                myRequestPartitionHelperService.determineReadPartitionForRequest(
996                                                                                requestDetailsForEntry, details);
997                                        }
998                                        break;
999                                }
1000                                case POST: {
1001                                        IBaseResource resource = myVersionAdapter.getResource(theEntry);
1002                                        String resourceType = myContext.getResourceType(resource);
1003                                        nextWriteEntryRequestPartitionId =
1004                                                        myRequestPartitionHelperService.determineCreatePartitionForRequest(
1005                                                                        requestDetailsForEntry, resource, resourceType);
1006                                        break;
1007                                }
1008                                case PUT: {
1009                                        IBaseResource resource = myVersionAdapter.getResource(theEntry);
1010                                        if (resource != null) {
1011                                                String resourceType = myContext.getResourceType(resource);
1012                                                String resourceId = null;
1013                                                if (resource.getIdElement().hasIdPart()) {
1014                                                        resourceId =
1015                                                                        resourceType + "/" + resource.getIdElement().getIdPart();
1016                                                }
1017                                                if (resourceId != null) {
1018                                                        nextWriteEntryRequestPartitionId = theTransactionDetails.getResolvedPartition(resourceId);
1019                                                }
1020                                                if (nextWriteEntryRequestPartitionId == null) {
1021                                                        nextWriteEntryRequestPartitionId =
1022                                                                        myRequestPartitionHelperService.determineCreatePartitionForRequest(
1023                                                                                        requestDetailsForEntry, resource, resourceType);
1024                                                        if (resourceId != null) {
1025                                                                theTransactionDetails.addResolvedPartition(
1026                                                                                resourceId, nextWriteEntryRequestPartitionId);
1027                                                        }
1028                                                }
1029                                        }
1030                                        break;
1031                                }
1032                        }
1033                }
1034                return nextWriteEntryRequestPartitionId;
1035        }
1036
1037        private boolean haveWriteOperationsHooks(RequestDetails theRequestDetails) {
1038                IInterceptorBroadcaster compositeBroadcaster =
1039                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
1040                return compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE)
1041                                || compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST);
1042        }
1043
1044        private void callWriteOperationsHook(
1045                        Pointcut thePointcut,
1046                        RequestDetails theRequestDetails,
1047                        TransactionDetails theTransactionDetails,
1048                        TransactionWriteOperationsDetails theWriteOperationsDetails) {
1049                IInterceptorBroadcaster compositeBroadcaster =
1050                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
1051                if (compositeBroadcaster.hasHooks(thePointcut)) {
1052                        HookParams params = new HookParams()
1053                                        .add(TransactionDetails.class, theTransactionDetails)
1054                                        .add(TransactionWriteOperationsDetails.class, theWriteOperationsDetails);
1055                        compositeBroadcaster.callHooks(thePointcut, params);
1056                }
1057        }
1058
1059        @SuppressWarnings("unchecked")
1060        private TransactionWriteOperationsDetails buildWriteOperationsDetails(List<IBase> theEntries) {
1061                TransactionWriteOperationsDetails writeOperationsDetails;
1062                List<String> updateRequestUrls = new ArrayList<>();
1063                List<String> conditionalCreateRequestUrls = new ArrayList<>();
1064                // Extract
1065                for (IBase nextEntry : theEntries) {
1066                        String method = myVersionAdapter.getEntryRequestVerb(myContext, nextEntry);
1067                        if ("PUT".equals(method)) {
1068                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
1069                                if (isNotBlank(requestUrl)) {
1070                                        updateRequestUrls.add(requestUrl);
1071                                }
1072                        } else if ("POST".equals(method)) {
1073                                String requestUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextEntry);
1074                                if (isNotBlank(requestUrl) && requestUrl.contains("?")) {
1075                                        conditionalCreateRequestUrls.add(requestUrl);
1076                                }
1077                        }
1078                }
1079
1080                writeOperationsDetails = new TransactionWriteOperationsDetails();
1081                writeOperationsDetails.setUpdateRequestUrls(updateRequestUrls);
1082                writeOperationsDetails.setConditionalCreateRequestUrls(conditionalCreateRequestUrls);
1083                return writeOperationsDetails;
1084        }
1085
1086        private boolean isValidVerb(String theVerb) {
1087                try {
1088                        return org.hl7.fhir.r4.model.Bundle.HTTPVerb.fromCode(theVerb) != null;
1089                } catch (FHIRException theE) {
1090                        return false;
1091                }
1092        }
1093
1094        /**
1095         * This method is called for nested bundles (e.g. if we received a transaction with an entry that
1096         * was a GET search, this method is called on the bundle for the search result, that will be placed in the
1097         * outer bundle). This method applies the _summary and _content parameters to the output of
1098         * that bundle.
1099         * <p>
1100         * TODO: This isn't the most efficient way of doing this.. hopefully we can come up with something better in the future.
1101         */
1102        private IBaseResource filterNestedBundle(RequestDetails theRequestDetails, IBaseResource theResource) {
1103                IParser p = myContext.newJsonParser();
1104                RestfulServerUtils.configureResponseParser(theRequestDetails, p);
1105                return p.parseResource(theResource.getClass(), p.encodeResourceToString(theResource));
1106        }
1107
1108        protected void validateDependencies() {
1109                Validate.notNull(myContext);
1110                Validate.notNull(myTxManager);
1111        }
1112
1113        private IIdType newIdType(String theValue) {
1114                return myContext.getVersion().newIdType().setValue(theValue);
1115        }
1116
1117        /**
1118         * Searches for duplicate conditional creates and consolidates them.
1119         */
1120        @SuppressWarnings("unchecked")
1121        private void consolidateDuplicateConditionals(
1122                        RequestDetails theRequestDetails, String theActionName, List<IBase> theEntries) {
1123                final Set<String> keysWithNoFullUrl = new HashSet<>();
1124                final HashMap<String, String> keyToUuid = new HashMap<>();
1125
1126                for (int index = 0, originalIndex = 0; index < theEntries.size(); index++, originalIndex++) {
1127                        IBase nextReqEntry = theEntries.get(index);
1128                        IBaseResource resource = myVersionAdapter.getResource(nextReqEntry);
1129                        if (resource != null) {
1130                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
1131                                String entryFullUrl = myVersionAdapter.getFullUrl(nextReqEntry);
1132                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextReqEntry);
1133                                String ifNoneExist = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
1134
1135                                // Conditional UPDATE
1136                                boolean consolidateEntryCandidate = false;
1137                                String conditionalUrl;
1138                                switch (verb) {
1139                                        case "PUT":
1140                                                conditionalUrl = requestUrl;
1141                                                if (isNotBlank(requestUrl)) {
1142                                                        int questionMarkIndex = requestUrl.indexOf('?');
1143                                                        if (questionMarkIndex >= 0 && requestUrl.length() > (questionMarkIndex + 1)) {
1144                                                                consolidateEntryCandidate = true;
1145                                                        }
1146                                                }
1147                                                break;
1148
1149                                                // Conditional CREATE
1150                                        case "POST":
1151                                                conditionalUrl = ifNoneExist;
1152                                                if (isNotBlank(ifNoneExist)) {
1153                                                        if (isBlank(entryFullUrl) || !entryFullUrl.equals(requestUrl)) {
1154                                                                consolidateEntryCandidate = true;
1155                                                        }
1156                                                }
1157                                                break;
1158
1159                                        default:
1160                                                continue;
1161                                }
1162
1163                                if (isNotBlank(conditionalUrl) && !conditionalUrl.contains("?")) {
1164                                        conditionalUrl = myContext.getResourceType(resource) + "?" + conditionalUrl;
1165                                }
1166
1167                                String key = verb + "|" + conditionalUrl;
1168                                if (consolidateEntryCandidate) {
1169                                        if (isBlank(entryFullUrl)) {
1170                                                if (isNotBlank(conditionalUrl)) {
1171                                                        if (!keysWithNoFullUrl.add(key)) {
1172                                                                throw new InvalidRequestException(Msg.code(2008) + "Unable to process " + theActionName
1173                                                                                + " - Request contains multiple anonymous entries (Bundle.entry.fullUrl not populated) with conditional URL: \""
1174                                                                                + UrlUtil.sanitizeUrlPart(conditionalUrl)
1175                                                                                + "\". Does transaction request contain duplicates?");
1176                                                        }
1177                                                }
1178                                        } else {
1179                                                if (!keyToUuid.containsKey(key)) {
1180                                                        keyToUuid.put(key, entryFullUrl);
1181                                                } else {
1182                                                        String msg = "Discarding transaction bundle entry " + originalIndex
1183                                                                        + " as it contained a duplicate conditional " + verb;
1184                                                        ourLog.info(msg);
1185                                                        // Interceptor broadcast: JPA_PERFTRACE_INFO
1186                                                        IInterceptorBroadcaster compositeBroadcaster =
1187                                                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(
1188                                                                                        myInterceptorBroadcaster, theRequestDetails);
1189                                                        if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_WARNING)) {
1190                                                                StorageProcessingMessage message = new StorageProcessingMessage().setMessage(msg);
1191                                                                HookParams params = new HookParams()
1192                                                                                .add(RequestDetails.class, theRequestDetails)
1193                                                                                .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
1194                                                                                .add(StorageProcessingMessage.class, message);
1195                                                                compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
1196                                                        }
1197
1198                                                        theEntries.remove(index);
1199                                                        index--;
1200                                                        String existingUuid = keyToUuid.get(key);
1201                                                        replaceReferencesInEntriesWithConsolidatedUUID(theEntries, entryFullUrl, existingUuid);
1202                                                }
1203                                        }
1204                                }
1205                        }
1206                }
1207        }
1208
1209        /**
1210         * Iterates over all entries, and if it finds any which have references which match the fullUrl of the entry that was consolidated out
1211         * replace them with our new consolidated UUID
1212         */
1213        private void replaceReferencesInEntriesWithConsolidatedUUID(
1214                        List<IBase> theEntries, String theEntryFullUrl, String existingUuid) {
1215                for (IBase nextEntry : theEntries) {
1216                        @SuppressWarnings("unchecked")
1217                        IBaseResource nextResource = myVersionAdapter.getResource(nextEntry);
1218                        if (nextResource != null) {
1219                                for (IBaseReference nextReference :
1220                                                myContext.newTerser().getAllPopulatedChildElementsOfType(nextResource, IBaseReference.class)) {
1221                                        // We're interested in any references directly to the placeholder ID, but also
1222                                        // references that have a resource target that has the placeholder ID.
1223                                        String nextReferenceId = nextReference.getReferenceElement().getValue();
1224                                        if (isBlank(nextReferenceId) && nextReference.getResource() != null) {
1225                                                nextReferenceId =
1226                                                                nextReference.getResource().getIdElement().getValue();
1227                                        }
1228                                        if (theEntryFullUrl.equals(nextReferenceId)) {
1229                                                nextReference.setReference(existingUuid);
1230                                                nextReference.setResource(null);
1231                                        }
1232                                }
1233                        }
1234                }
1235        }
1236
1237        /**
1238         * Retrieves the next resource id (IIdType) from the base resource and next request entry.
1239         *
1240         * @param theBaseResource - base resource
1241         * @param theNextReqEntry - next request entry
1242         * @param theAllIds       - set of all IIdType values
1243         * @param theVerb
1244         * @return
1245         */
1246        private IIdType getNextResourceIdFromBaseResource(
1247                        IBaseResource theBaseResource, IBase theNextReqEntry, Set<IIdType> theAllIds, String theVerb) {
1248                IIdType nextResourceId = null;
1249                if (theBaseResource != null) {
1250                        nextResourceId = theBaseResource.getIdElement();
1251
1252                        @SuppressWarnings("unchecked")
1253                        String fullUrl = myVersionAdapter.getFullUrl(theNextReqEntry);
1254                        if (isNotBlank(fullUrl)) {
1255                                IIdType fullUrlIdType = newIdType(fullUrl);
1256                                if (isPlaceholder(fullUrlIdType)) {
1257                                        nextResourceId = fullUrlIdType;
1258                                } else if (!nextResourceId.hasIdPart()) {
1259                                        nextResourceId = fullUrlIdType;
1260                                }
1261                        }
1262
1263                        if (nextResourceId.hasIdPart() && !isPlaceholder(nextResourceId)) {
1264                                int colonIndex = nextResourceId.getIdPart().indexOf(':');
1265                                if (colonIndex != -1) {
1266                                        if (INVALID_PLACEHOLDER_PATTERN
1267                                                        .matcher(nextResourceId.getIdPart())
1268                                                        .matches()) {
1269                                                throw new InvalidRequestException(
1270                                                                Msg.code(533) + "Invalid placeholder ID found: " + nextResourceId.getIdPart()
1271                                                                                + " - Must be of the form 'urn:uuid:[uuid]' or 'urn:oid:[oid]'");
1272                                        }
1273                                }
1274                        }
1275
1276                        if (nextResourceId.hasIdPart() && !nextResourceId.hasResourceType() && !isPlaceholder(nextResourceId)) {
1277                                nextResourceId = newIdType(toResourceName(theBaseResource.getClass()), nextResourceId.getIdPart());
1278                                theBaseResource.setId(nextResourceId);
1279                        }
1280
1281                        /*
1282                         * Ensure that the bundle doesn't have any duplicates, since this causes all kinds of weirdness
1283                         */
1284                        if (isPlaceholder(nextResourceId)) {
1285                                if (!theAllIds.add(nextResourceId)) {
1286                                        throw new InvalidRequestException(Msg.code(534)
1287                                                        + myContext
1288                                                                        .getLocalizer()
1289                                                                        .getMessage(
1290                                                                                        BaseStorageDao.class,
1291                                                                                        "transactionContainsMultipleWithDuplicateId",
1292                                                                                        nextResourceId));
1293                                }
1294                        } else if (nextResourceId.hasResourceType() && nextResourceId.hasIdPart() && !"POST".equals(theVerb)) {
1295                                IIdType nextId = nextResourceId.toUnqualifiedVersionless();
1296                                if (!theAllIds.add(nextId)) {
1297                                        throw new InvalidRequestException(Msg.code(535)
1298                                                        + myContext
1299                                                                        .getLocalizer()
1300                                                                        .getMessage(
1301                                                                                        BaseStorageDao.class,
1302                                                                                        "transactionContainsMultipleWithDuplicateId",
1303                                                                                        nextId));
1304                                }
1305                        }
1306                }
1307
1308                return nextResourceId;
1309        }
1310
1311        /**
1312         * After pre-hooks have been called
1313         */
1314        @SuppressWarnings({"unchecked", "rawtypes"})
1315        protected EntriesToProcessMap doTransactionWriteOperations(
1316                        final RequestDetails theRequest,
1317                        String theActionName,
1318                        TransactionDetails theTransactionDetails,
1319                        Set<IIdType> theAllIds,
1320                        IdSubstitutionMap theIdSubstitutions,
1321                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1322                        IBaseBundle theResponse,
1323                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
1324                        List<IBase> theEntries,
1325                        StopWatch theTransactionStopWatch) {
1326
1327                // During a transaction, we don't execute hooks, instead, we execute them all post-transaction.
1328                theTransactionDetails.beginAcceptingDeferredInterceptorBroadcasts(
1329                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED,
1330                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_UPDATED,
1331                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_DELETED);
1332                try {
1333                        Set<String> deletedResources = new HashSet<>();
1334                        DeleteConflictList deleteConflicts = new DeleteConflictList();
1335                        EntriesToProcessMap entriesToProcess = new EntriesToProcessMap();
1336                        Set<IIdType> nonUpdatedEntities = new HashSet<>();
1337                        Set<IBasePersistedResource> updatedEntities = new HashSet<>();
1338                        Map<String, IIdType> conditionalUrlToIdMap = new HashMap<>();
1339                        List<IBaseResource> updatedResources = new ArrayList<>();
1340                        Map<String, Class<? extends IBaseResource>> conditionalRequestUrls = new HashMap<>();
1341
1342                        /*
1343                         * Look for duplicate conditional creates and consolidate them
1344                         */
1345                        consolidateDuplicateConditionals(theRequest, theActionName, theEntries);
1346
1347                        /*
1348                         * Loop through the request and process any entries of type
1349                         * PUT, POST or DELETE
1350                         */
1351                        String previousVerb = null;
1352                        for (int i = 0; i < theEntries.size(); i++) {
1353                                if (i % 250 == 0) {
1354                                        ourLog.debug("Processed {} non-GET entries out of {} in transaction", i, theEntries.size());
1355                                }
1356
1357                                IBase nextReqEntry = theEntries.get(i);
1358
1359                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
1360
1361                                if (previousVerb != null && !previousVerb.equals(verb)) {
1362                                        handleVerbChangeInTransactionWriteOperations();
1363                                }
1364                                previousVerb = verb;
1365                                if ("GET".equals(verb)) {
1366                                        continue;
1367                                }
1368
1369                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1370                                RequestDetails requestDetailsForEntry =
1371                                                createRequestDetailsForWriteEntry(theRequest, nextReqEntry, url, verb);
1372
1373                                IBaseResource res = myVersionAdapter.getResource(nextReqEntry);
1374                                IIdType nextResourceId = getNextResourceIdFromBaseResource(res, nextReqEntry, theAllIds, verb);
1375
1376                                String resourceType = res != null ? myContext.getResourceType(res) : null;
1377                                Integer order = theOriginalRequestOrder.get(nextReqEntry);
1378                                IBase nextRespEntry =
1379                                                (IBase) myVersionAdapter.getEntries(theResponse).get(order);
1380
1381                                theTransactionStopWatch.startTask(
1382                                                "Bundle.entry[" + i + "]: " + verb + " " + defaultString(resourceType));
1383
1384                                if (res != null) {
1385                                        String previousResourceId = res.getIdElement().getValue();
1386                                        theTransactionDetails.addRollbackUndoAction(() -> res.setId(previousResourceId));
1387                                }
1388
1389                                switch (verb) {
1390                                        case "POST": {
1391                                                // CREATE
1392                                                validateResourcePresent(res, order, verb);
1393
1394                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
1395                                                res.setId((String) null);
1396
1397                                                DaoMethodOutcome outcome;
1398                                                String matchUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
1399                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1400                                                // create individual resource
1401                                                outcome =
1402                                                                resourceDao.create(res, matchUrl, false, requestDetailsForEntry, theTransactionDetails);
1403                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1404                                                res.setId(outcome.getId());
1405
1406                                                if (nextResourceId != null) {
1407                                                        handleTransactionCreateOrUpdateOutcome(
1408                                                                        theIdSubstitutions,
1409                                                                        theIdToPersistedOutcome,
1410                                                                        nextResourceId,
1411                                                                        outcome,
1412                                                                        nextRespEntry,
1413                                                                        resourceType,
1414                                                                        res,
1415                                                                        requestDetailsForEntry);
1416                                                }
1417                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1418                                                theTransactionDetails.addResolvedResource(outcome.getId(), outcome::getResource);
1419                                                if (outcome.getCreated() == false) {
1420                                                        nonUpdatedEntities.add(outcome.getId());
1421                                                } else {
1422                                                        if (isNotBlank(matchUrl)) {
1423                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
1424                                                        }
1425                                                }
1426
1427                                                break;
1428                                        }
1429                                        case "DELETE": {
1430                                                // DELETE
1431                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1432                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
1433                                                int status = Constants.STATUS_HTTP_204_NO_CONTENT;
1434                                                if (parts.getResourceId() != null) {
1435                                                        IIdType deleteId = newIdType(parts.getResourceType(), parts.getResourceId());
1436                                                        if (!deletedResources.contains(deleteId.getValueAsString())) {
1437                                                                DaoMethodOutcome outcome = dao.delete(
1438                                                                                deleteId, deleteConflicts, requestDetailsForEntry, theTransactionDetails);
1439                                                                if (outcome.getEntity() != null) {
1440                                                                        deletedResources.add(deleteId.getValueAsString());
1441                                                                        entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1442                                                                }
1443                                                                myVersionAdapter.setResponseOutcome(nextRespEntry, outcome.getOperationOutcome());
1444                                                        }
1445                                                } else {
1446                                                        String matchUrl = parts.getResourceType() + '?' + parts.getParams();
1447                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1448                                                        DeleteMethodOutcome deleteOutcome = dao.deleteByUrl(
1449                                                                        matchUrl, deleteConflicts, requestDetailsForEntry, theTransactionDetails);
1450                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, deleteOutcome.getId());
1451                                                        List<? extends IBasePersistedResource> allDeleted = deleteOutcome.getDeletedEntities();
1452                                                        for (IBasePersistedResource deleted : allDeleted) {
1453                                                                deletedResources.add(deleted.getIdDt()
1454                                                                                .toUnqualifiedVersionless()
1455                                                                                .getValueAsString());
1456                                                        }
1457                                                        if (allDeleted.isEmpty()) {
1458                                                                status = Constants.STATUS_HTTP_204_NO_CONTENT;
1459                                                        }
1460
1461                                                        myVersionAdapter.setResponseOutcome(nextRespEntry, deleteOutcome.getOperationOutcome());
1462                                                }
1463
1464                                                myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(status));
1465
1466                                                break;
1467                                        }
1468                                        case "PUT": {
1469                                                // UPDATE
1470                                                validateResourcePresent(res, order, verb);
1471                                                @SuppressWarnings("rawtypes")
1472                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
1473
1474                                                DaoMethodOutcome outcome;
1475                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1476                                                if (isNotBlank(parts.getResourceId())) {
1477                                                        String version = null;
1478                                                        if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) {
1479                                                                version = ParameterUtil.parseETagValue(
1480                                                                                myVersionAdapter.getEntryRequestIfMatch(nextReqEntry));
1481                                                        }
1482                                                        res.setId(newIdType(parts.getResourceType(), parts.getResourceId(), version));
1483                                                        outcome = resourceDao.update(
1484                                                                        res, null, false, false, requestDetailsForEntry, theTransactionDetails);
1485                                                } else {
1486                                                        if (!shouldConditionalUpdateMatchId(theTransactionDetails, res.getIdElement())) {
1487                                                                res.setId((String) null);
1488                                                        }
1489                                                        String matchUrl;
1490                                                        if (isNotBlank(parts.getParams())) {
1491                                                                matchUrl = parts.getResourceType() + '?' + parts.getParams();
1492                                                        } else {
1493                                                                matchUrl = parts.getResourceType();
1494                                                        }
1495                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1496                                                        outcome = resourceDao.update(
1497                                                                        res, matchUrl, false, false, requestDetailsForEntry, theTransactionDetails);
1498                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1499                                                        if (Boolean.TRUE.equals(outcome.getCreated())) {
1500                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
1501                                                        }
1502                                                }
1503
1504                                                if (outcome.getCreated() == Boolean.FALSE
1505                                                                || (outcome.getCreated() == Boolean.TRUE
1506                                                                                && outcome.getId().getVersionIdPartAsLong() > 1)) {
1507                                                        updatedEntities.add(outcome.getEntity());
1508                                                        if (outcome.getResource() != null) {
1509                                                                updatedResources.add(outcome.getResource());
1510                                                        }
1511                                                }
1512
1513                                                theTransactionDetails.addResolvedResource(outcome.getId(), outcome::getResource);
1514                                                handleTransactionCreateOrUpdateOutcome(
1515                                                                theIdSubstitutions,
1516                                                                theIdToPersistedOutcome,
1517                                                                nextResourceId,
1518                                                                outcome,
1519                                                                nextRespEntry,
1520                                                                resourceType,
1521                                                                res,
1522                                                                requestDetailsForEntry);
1523                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1524                                                break;
1525                                        }
1526                                        case "PATCH": {
1527                                                // PATCH
1528                                                validateResourcePresent(res, order, verb);
1529
1530                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1531
1532                                                String matchUrl = toMatchUrl(nextReqEntry);
1533                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1534                                                String patchBody = null;
1535                                                String contentType;
1536                                                IBaseParameters patchBodyParameters = null;
1537                                                PatchTypeEnum patchType = null;
1538
1539                                                if (res instanceof IBaseBinary) {
1540                                                        IBaseBinary binary = (IBaseBinary) res;
1541                                                        if (binary.getContent() != null && binary.getContent().length > 0) {
1542                                                                patchBody = toUtf8String(binary.getContent());
1543                                                        }
1544                                                        contentType = binary.getContentType();
1545                                                        patchType =
1546                                                                        PatchTypeEnum.forContentTypeOrThrowInvalidRequestException(myContext, contentType);
1547                                                        if (patchType == PatchTypeEnum.FHIR_PATCH_JSON
1548                                                                        || patchType == PatchTypeEnum.FHIR_PATCH_XML) {
1549                                                                String msg = myContext
1550                                                                                .getLocalizer()
1551                                                                                .getMessage(
1552                                                                                                BaseTransactionProcessor.class, "fhirPatchShouldNotUseBinaryResource");
1553                                                                throw new InvalidRequestException(Msg.code(536) + msg);
1554                                                        }
1555                                                } else if (res instanceof IBaseParameters) {
1556                                                        patchBodyParameters = (IBaseParameters) res;
1557                                                        patchType = PatchTypeEnum.FHIR_PATCH_JSON;
1558                                                }
1559
1560                                                if (patchBodyParameters == null) {
1561                                                        if (isBlank(patchBody)) {
1562                                                                String msg = myContext
1563                                                                                .getLocalizer()
1564                                                                                .getMessage(BaseTransactionProcessor.class, "missingPatchBody");
1565                                                                throw new InvalidRequestException(Msg.code(537) + msg);
1566                                                        }
1567                                                }
1568
1569                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
1570                                                IIdType patchId =
1571                                                                myContext.getVersion().newIdType(parts.getResourceType(), parts.getResourceId());
1572
1573                                                String conditionalUrl;
1574                                                if (isNull(patchId.getIdPart())) {
1575                                                        conditionalUrl = url;
1576                                                } else {
1577                                                        conditionalUrl = matchUrl;
1578                                                        String ifMatch = myVersionAdapter.getEntryRequestIfMatch(nextReqEntry);
1579                                                        if (isNotBlank(ifMatch)) {
1580                                                                patchId = UpdateMethodBinding.applyETagAsVersion(ifMatch, patchId);
1581                                                        }
1582                                                }
1583
1584                                                DaoMethodOutcome outcome = dao.patchInTransaction(
1585                                                                patchId,
1586                                                                conditionalUrl,
1587                                                                false,
1588                                                                patchType,
1589                                                                patchBody,
1590                                                                patchBodyParameters,
1591                                                                requestDetailsForEntry,
1592                                                                theTransactionDetails);
1593                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1594                                                updatedEntities.add(outcome.getEntity());
1595                                                if (outcome.getResource() != null) {
1596                                                        updatedResources.add(outcome.getResource());
1597                                                }
1598                                                if (nextResourceId != null) {
1599                                                        handleTransactionCreateOrUpdateOutcome(
1600                                                                        theIdSubstitutions,
1601                                                                        theIdToPersistedOutcome,
1602                                                                        nextResourceId,
1603                                                                        outcome,
1604                                                                        nextRespEntry,
1605                                                                        resourceType,
1606                                                                        res,
1607                                                                        requestDetailsForEntry);
1608                                                }
1609                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1610
1611                                                break;
1612                                        }
1613                                        case "GET":
1614                                                break;
1615                                        default:
1616                                                throw new InvalidRequestException(
1617                                                                Msg.code(538) + "Unable to handle verb in transaction: " + verb);
1618                                }
1619
1620                                theTransactionStopWatch.endCurrentTask();
1621                        }
1622
1623                        postTransactionProcess(theTransactionDetails);
1624
1625                        /*
1626                         * Make sure that there are no conflicts from deletions. E.g. we can't delete something
1627                         * if something else has a reference to it.. Unless the thing that has a reference to it
1628                         * was also deleted as a part of this transaction, which is why we check this now at the
1629                         * end.
1630                         */
1631                        checkForDeleteConflicts(deleteConflicts, deletedResources, updatedResources);
1632
1633                        theIdToPersistedOutcome.entrySet().forEach(idAndOutcome -> {
1634                                theTransactionDetails.addResolvedResourceId(
1635                                                idAndOutcome.getKey(), idAndOutcome.getValue().getPersistentId());
1636                        });
1637
1638                        /*
1639                         * Perform ID substitutions and then index each resource we have saved
1640                         */
1641
1642                        resolveReferencesThenSaveAndIndexResources(
1643                                        theRequest,
1644                                        theTransactionDetails,
1645                                        theIdSubstitutions,
1646                                        theIdToPersistedOutcome,
1647                                        theTransactionStopWatch,
1648                                        entriesToProcess,
1649                                        nonUpdatedEntities,
1650                                        updatedEntities);
1651
1652                        theTransactionStopWatch.endCurrentTask();
1653
1654                        // flush writes to db
1655                        theTransactionStopWatch.startTask("Flush writes to database");
1656
1657                        // flush the changes
1658                        flushSession(theIdToPersistedOutcome);
1659
1660                        theTransactionStopWatch.endCurrentTask();
1661
1662                        /*
1663                         * Double check we didn't allow any duplicates we shouldn't have
1664                         */
1665                        if (conditionalRequestUrls.size() > 0) {
1666                                theTransactionStopWatch.startTask("Check for conflicts in conditional resources");
1667                        }
1668                        if (!myStorageSettings.isMassIngestionMode()) {
1669                                validateNoDuplicates(
1670                                                theRequest,
1671                                                theTransactionDetails,
1672                                                theActionName,
1673                                                conditionalRequestUrls,
1674                                                theIdToPersistedOutcome.values());
1675                        }
1676
1677                        theTransactionStopWatch.endCurrentTask();
1678                        if (conditionalUrlToIdMap.size() > 0) {
1679                                theTransactionStopWatch.startTask(
1680                                                "Check that all conditionally created/updated entities actually match their conditionals.");
1681                        }
1682
1683                        if (!myStorageSettings.isMassIngestionMode()) {
1684                                validateAllInsertsMatchTheirConditionalUrls(theIdToPersistedOutcome, conditionalUrlToIdMap, theRequest);
1685                        }
1686                        theTransactionStopWatch.endCurrentTask();
1687
1688                        for (IIdType next : theAllIds) {
1689                                IIdType replacement = theIdSubstitutions.getForSource(next);
1690                                if (replacement != null && !replacement.equals(next)) {
1691                                        ourLog.debug(
1692                                                        "Placeholder resource ID \"{}\" was replaced with permanent ID \"{}\"", next, replacement);
1693                                }
1694                        }
1695
1696                        IInterceptorBroadcaster compositeBroadcaster =
1697                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequest);
1698                        ListMultimap<Pointcut, HookParams> deferredBroadcastEvents =
1699                                        theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1700                        for (Map.Entry<Pointcut, HookParams> nextEntry : deferredBroadcastEvents.entries()) {
1701                                Pointcut nextPointcut = nextEntry.getKey();
1702                                HookParams nextParams = nextEntry.getValue();
1703                                compositeBroadcaster.callHooks(nextPointcut, nextParams);
1704                        }
1705
1706                        DeferredInterceptorBroadcasts deferredInterceptorBroadcasts =
1707                                        new DeferredInterceptorBroadcasts(deferredBroadcastEvents);
1708                        HookParams params = new HookParams()
1709                                        .add(RequestDetails.class, theRequest)
1710                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
1711                                        .add(DeferredInterceptorBroadcasts.class, deferredInterceptorBroadcasts)
1712                                        .add(TransactionDetails.class, theTransactionDetails)
1713                                        .add(IBaseBundle.class, theResponse);
1714                        compositeBroadcaster.callHooks(Pointcut.STORAGE_TRANSACTION_PROCESSED, params);
1715
1716                        theTransactionDetails.deferredBroadcastProcessingFinished();
1717
1718                        // finishedCallingDeferredInterceptorBroadcasts
1719
1720                        return entriesToProcess;
1721
1722                } finally {
1723                        if (theTransactionDetails.isAcceptingDeferredInterceptorBroadcasts()) {
1724                                theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1725                        }
1726                }
1727        }
1728
1729        /**
1730         * Subclasses may override this in order to invoke specific operations when
1731         * we're finished handling all the write entries in the transaction bundle
1732         * with a given verb.
1733         */
1734        protected void handleVerbChangeInTransactionWriteOperations() {
1735                // nothing
1736        }
1737
1738        /**
1739         * Implement to handle post transaction processing
1740         */
1741        protected void postTransactionProcess(TransactionDetails theTransactionDetails) {
1742                // nothing
1743        }
1744
1745        /**
1746         * Check for if a resource id should be matched in a conditional update
1747         * If the FHIR version is older than R4, it follows the old specifications and does not match
1748         * If the resource id has been resolved, then it is an existing resource and does not need to be matched
1749         * If the resource id is local or a placeholder, the id is temporary and should not be matched
1750         */
1751        private boolean shouldConditionalUpdateMatchId(TransactionDetails theTransactionDetails, IIdType theId) {
1752                if (myContext.getVersion().getVersion().isOlderThan(FhirVersionEnum.R4)) {
1753                        return false;
1754                }
1755                if (theTransactionDetails.hasResolvedResourceId(theId)
1756                                && !theTransactionDetails.isResolvedResourceIdEmpty(theId)) {
1757                        return false;
1758                }
1759                if (theId != null && theId.getValue() != null) {
1760                        return !(theId.getValue().startsWith("urn:") || theId.getValue().startsWith("#"));
1761                }
1762                return true;
1763        }
1764
1765        private boolean shouldSwapBinaryToActualResource(
1766                        IBaseResource theResource, String theResourceType, IIdType theNextResourceId) {
1767                if ("Binary".equalsIgnoreCase(theResourceType)
1768                                && theNextResourceId.getResourceType() != null
1769                                && !theNextResourceId.getResourceType().equalsIgnoreCase("Binary")) {
1770                        return true;
1771                } else {
1772                        return false;
1773                }
1774        }
1775
1776        private void setConditionalUrlToBeValidatedLater(
1777                        Map<String, IIdType> theConditionalUrlToIdMap, String theMatchUrl, IIdType theId) {
1778                if (!isBlank(theMatchUrl)) {
1779                        theConditionalUrlToIdMap.put(theMatchUrl, theId);
1780                }
1781        }
1782
1783        /**
1784         * After transaction processing and resolution of indexes and references, we want to validate that the resources that were stored _actually_
1785         * match the conditional URLs that they were brought in on.
1786         */
1787        private void validateAllInsertsMatchTheirConditionalUrls(
1788                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1789                        Map<String, IIdType> conditionalUrlToIdMap,
1790                        RequestDetails theRequest) {
1791                conditionalUrlToIdMap.entrySet().stream()
1792                                .filter(entry -> entry.getKey() != null)
1793                                .forEach(entry -> {
1794                                        String matchUrl = entry.getKey();
1795                                        IIdType value = entry.getValue();
1796                                        DaoMethodOutcome daoMethodOutcome = theIdToPersistedOutcome.get(value);
1797                                        if (daoMethodOutcome != null
1798                                                        && !daoMethodOutcome.isNop()
1799                                                        && daoMethodOutcome.getResource() != null) {
1800                                                InMemoryMatchResult match =
1801                                                                mySearchParamMatcher.match(matchUrl, daoMethodOutcome.getResource(), theRequest);
1802                                                if (ourLog.isDebugEnabled()) {
1803                                                        ourLog.debug(
1804                                                                        "Checking conditional URL [{}] against resource with ID [{}]: Supported?:[{}], Matched?:[{}]",
1805                                                                        matchUrl,
1806                                                                        value,
1807                                                                        match.supported(),
1808                                                                        match.matched());
1809                                                }
1810                                                if (match.supported()) {
1811                                                        if (!match.matched()) {
1812                                                                String msg = myContext
1813                                                                                .getLocalizer()
1814                                                                                .getMessage(
1815                                                                                                BaseTransactionProcessor.class,
1816                                                                                                "invalidConditionalUrlResourceDoesntMatch",
1817                                                                                                matchUrl);
1818                                                                throw new PreconditionFailedException(Msg.code(539) + msg);
1819                                                        }
1820                                                }
1821                                        }
1822                                });
1823        }
1824
1825        /**
1826         * Checks for any delete conflicts.
1827         *
1828         * @param theDeleteConflicts  - set of delete conflicts
1829         * @param theDeletedResources - set of deleted resources
1830         * @param theUpdatedResources - list of updated resources
1831         */
1832        private void checkForDeleteConflicts(
1833                        DeleteConflictList theDeleteConflicts,
1834                        Set<String> theDeletedResources,
1835                        List<IBaseResource> theUpdatedResources) {
1836                for (Iterator<DeleteConflict> iter = theDeleteConflicts.iterator(); iter.hasNext(); ) {
1837                        DeleteConflict nextDeleteConflict = iter.next();
1838
1839                        /*
1840                         * If we have a conflict, it means we can't delete Resource/A because
1841                         * Resource/B has a reference to it. We'll ignore that conflict though
1842                         * if it turns out we're also deleting Resource/B in this transaction.
1843                         */
1844                        if (theDeletedResources.contains(
1845                                        nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue())) {
1846                                iter.remove();
1847                                continue;
1848                        }
1849
1850                        /*
1851                         * And then, this is kind of a last ditch check. It's also ok to delete
1852                         * Resource/A if Resource/B isn't being deleted, but it is being UPDATED
1853                         * in this transaction, and the updated version of it has no references
1854                         * to Resource/A any more.
1855                         */
1856                        String sourceId =
1857                                        nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue();
1858                        String targetId =
1859                                        nextDeleteConflict.getTargetId().toUnqualifiedVersionless().getValue();
1860                        Optional<IBaseResource> updatedSource = theUpdatedResources.stream()
1861                                        .filter(t -> sourceId.equals(
1862                                                        t.getIdElement().toUnqualifiedVersionless().getValue()))
1863                                        .findFirst();
1864                        if (updatedSource.isPresent()) {
1865                                List<ResourceReferenceInfo> referencesInSource =
1866                                                myContext.newTerser().getAllResourceReferences(updatedSource.get());
1867                                boolean sourceStillReferencesTarget = referencesInSource.stream()
1868                                                .anyMatch(t -> targetId.equals(t.getResourceReference()
1869                                                                .getReferenceElement()
1870                                                                .toUnqualifiedVersionless()
1871                                                                .getValue()));
1872                                if (!sourceStillReferencesTarget) {
1873                                        iter.remove();
1874                                }
1875                        }
1876                }
1877                DeleteConflictUtil.validateDeleteConflictsEmptyOrThrowException(myContext, theDeleteConflicts);
1878        }
1879
1880        /**
1881         * This method replaces any placeholder references in the
1882         * source transaction Bundle with their actual targets, then stores the resource contents and indexes
1883         * in the database. This is trickier than you'd think because of a couple of possibilities during the
1884         * save:
1885         * * There may be resources that have not changed (e.g. an update/PUT with a resource body identical
1886         * to what is already in the database)
1887         * * There may be resources with auto-versioned references, meaning we're replacing certain references
1888         * in the resource with a versioned references, referencing the current version at the time of the
1889         * transaction processing
1890         * * There may by auto-versioned references pointing to these unchanged targets
1891         * <p>
1892         * If we're not doing any auto-versioned references, we'll just iterate through all resources in the
1893         * transaction and save them one at a time.
1894         * <p>
1895         * However, if we have any auto-versioned references we do this in 2 passes: First the resources from the
1896         * transaction that don't have any auto-versioned references are stored. We do them first since there's
1897         * a chance they may be a NOP and we'll need to account for their version number not actually changing.
1898         * Then we do a second pass for any resources that have auto-versioned references. These happen in a separate
1899         * pass because it's too complex to try and insert the auto-versioned references and still
1900         * account for NOPs, so we block NOPs in that pass.
1901         */
1902        private void resolveReferencesThenSaveAndIndexResources(
1903                        RequestDetails theRequest,
1904                        TransactionDetails theTransactionDetails,
1905                        IdSubstitutionMap theIdSubstitutions,
1906                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1907                        StopWatch theTransactionStopWatch,
1908                        EntriesToProcessMap theEntriesToProcess,
1909                        Set<IIdType> theNonUpdatedEntities,
1910                        Set<IBasePersistedResource> theUpdatedEntities) {
1911                FhirTerser terser = myContext.newTerser();
1912                theTransactionStopWatch.startTask("Index " + theIdToPersistedOutcome.size() + " resources");
1913                IdentityHashMap<DaoMethodOutcome, Set<IBaseReference>> deferredIndexesForAutoVersioning = null;
1914                int i = 0;
1915                for (DaoMethodOutcome nextOutcome : theIdToPersistedOutcome.values()) {
1916
1917                        if (i++ % 250 == 0) {
1918                                ourLog.debug(
1919                                                "Have indexed {} entities out of {} in transaction",
1920                                                i,
1921                                                theIdToPersistedOutcome.values().size());
1922                        }
1923
1924                        if (nextOutcome.isNop()) {
1925                                continue;
1926                        }
1927
1928                        IBaseResource nextResource = nextOutcome.getResource();
1929                        if (nextResource == null) {
1930                                continue;
1931                        }
1932
1933                        Set<IBaseReference> referencesToAutoVersion =
1934                                        BaseStorageDao.extractReferencesToAutoVersion(myContext, myStorageSettings, nextResource);
1935                        Set<IBaseReference> referencesToKeepClientSuppliedVersion =
1936                                        BaseStorageDao.extractReferencesToAvoidReplacement(myContext, nextResource);
1937
1938                        if (referencesToAutoVersion.isEmpty()) {
1939                                // no references to autoversion - we can do the resolve and save now
1940                                resolveReferencesThenSaveAndIndexResource(
1941                                                theRequest,
1942                                                theTransactionDetails,
1943                                                theIdSubstitutions,
1944                                                theIdToPersistedOutcome,
1945                                                theEntriesToProcess,
1946                                                theNonUpdatedEntities,
1947                                                theUpdatedEntities,
1948                                                terser,
1949                                                nextOutcome,
1950                                                nextResource,
1951                                                referencesToAutoVersion, // this is empty
1952                                                referencesToKeepClientSuppliedVersion);
1953                        } else {
1954                                // we have autoversioned things to defer until later
1955                                if (deferredIndexesForAutoVersioning == null) {
1956                                        deferredIndexesForAutoVersioning = new IdentityHashMap<>();
1957                                }
1958                                deferredIndexesForAutoVersioning.put(nextOutcome, referencesToAutoVersion);
1959                        }
1960                }
1961
1962                // If we have any resources we'll be auto-versioning, index these next
1963                if (deferredIndexesForAutoVersioning != null) {
1964                        for (Map.Entry<DaoMethodOutcome, Set<IBaseReference>> nextEntry :
1965                                        deferredIndexesForAutoVersioning.entrySet()) {
1966                                DaoMethodOutcome nextOutcome = nextEntry.getKey();
1967                                Set<IBaseReference> referencesToAutoVersion = nextEntry.getValue();
1968                                IBaseResource nextResource = nextOutcome.getResource();
1969                                Set<IBaseReference> referencesToKeepClientSuppliedVersion =
1970                                                BaseStorageDao.extractReferencesToAvoidReplacement(myContext, nextResource);
1971
1972                                resolveReferencesThenSaveAndIndexResource(
1973                                                theRequest,
1974                                                theTransactionDetails,
1975                                                theIdSubstitutions,
1976                                                theIdToPersistedOutcome,
1977                                                theEntriesToProcess,
1978                                                theNonUpdatedEntities,
1979                                                theUpdatedEntities,
1980                                                terser,
1981                                                nextOutcome,
1982                                                nextResource,
1983                                                referencesToAutoVersion,
1984                                                referencesToKeepClientSuppliedVersion);
1985                        }
1986                }
1987        }
1988
1989        private void resolveReferencesThenSaveAndIndexResource(
1990                        RequestDetails theRequest,
1991                        TransactionDetails theTransactionDetails,
1992                        IdSubstitutionMap theIdSubstitutions,
1993                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1994                        EntriesToProcessMap theEntriesToProcess,
1995                        Set<IIdType> theNonUpdatedEntities,
1996                        Set<IBasePersistedResource> theUpdatedEntities,
1997                        FhirTerser theTerser,
1998                        DaoMethodOutcome theDaoMethodOutcome,
1999                        IBaseResource theResource,
2000                        Set<IBaseReference> theReferencesToAutoVersion,
2001                        Set<IBaseReference> theReferencesToKeepClientSuppliedVersion) {
2002                // References
2003                List<ResourceReferenceInfo> allRefs = theTerser.getAllResourceReferences(theResource);
2004                for (ResourceReferenceInfo nextRef : allRefs) {
2005                        IBaseReference resourceReference = nextRef.getResourceReference();
2006                        IIdType nextId = resourceReference.getReferenceElement();
2007                        IIdType newId = null;
2008                        if (!nextId.hasIdPart()) {
2009                                if (resourceReference.getResource() != null) {
2010                                        IIdType targetId = resourceReference.getResource().getIdElement();
2011                                        if (targetId.getValue() == null) {
2012                                                // This means it's a contained resource
2013                                                continue;
2014                                        } else if (theIdSubstitutions.containsTarget(targetId)) {
2015                                                newId = targetId;
2016                                        } else {
2017                                                throw new InternalErrorException(Msg.code(540)
2018                                                                + "References by resource with no reference ID are not supported in DAO layer");
2019                                        }
2020                                } else {
2021                                        continue;
2022                                }
2023                        }
2024                        if (newId != null || theIdSubstitutions.containsSource(nextId)) {
2025                                if (shouldReplaceResourceReference(
2026                                                theReferencesToAutoVersion, theReferencesToKeepClientSuppliedVersion, resourceReference)) {
2027                                        if (newId == null) {
2028                                                newId = theIdSubstitutions.getForSource(nextId);
2029                                        }
2030                                        if (newId != null) {
2031                                                ourLog.debug(" * Replacing resource ref {} with {}", nextId, newId);
2032                                                if (theReferencesToAutoVersion.contains(resourceReference)) {
2033                                                        replaceResourceReference(newId, resourceReference, theTransactionDetails);
2034                                                } else {
2035                                                        replaceResourceReference(newId.toVersionless(), resourceReference, theTransactionDetails);
2036                                                }
2037                                        }
2038                                }
2039                        } else if (nextId.getValue().startsWith("urn:")) {
2040                                throw new InvalidRequestException(
2041                                                Msg.code(541) + "Unable to satisfy placeholder ID " + nextId.getValue()
2042                                                                + " found in element named '" + nextRef.getName() + "' within resource of type: "
2043                                                                + theResource.getIdElement().getResourceType());
2044                        } else {
2045                                // get a map of
2046                                // existing ids -> PID (for resources that exist in the DB)
2047                                // should this be allPartitions?
2048                                ResourcePersistentIdMap resourceVersionMap = myResourceVersionSvc.getLatestVersionIdsForResourceIds(
2049                                                RequestPartitionId.allPartitions(),
2050                                                theReferencesToAutoVersion.stream()
2051                                                                .map(IBaseReference::getReferenceElement)
2052                                                                .collect(Collectors.toList()));
2053
2054                                for (IBaseReference baseRef : theReferencesToAutoVersion) {
2055                                        IIdType id = baseRef.getReferenceElement();
2056                                        if (!resourceVersionMap.containsKey(id)
2057                                                        && myStorageSettings.isAutoCreatePlaceholderReferenceTargets()) {
2058                                                // not in the db, but autocreateplaceholders is true
2059                                                // so the version we'll set is "1" (since it will be
2060                                                // created later)
2061                                                String newRef = id.withVersion("1").getValue();
2062                                                id.setValue(newRef);
2063                                        } else {
2064                                                // we will add the looked up info to the transaction
2065                                                // for later
2066                                                if (resourceVersionMap.containsKey(id)) {
2067                                                        theTransactionDetails.addResolvedResourceId(
2068                                                                        id, resourceVersionMap.getResourcePersistentId(id));
2069                                                }
2070                                        }
2071                                }
2072
2073                                if (theReferencesToAutoVersion.contains(resourceReference)) {
2074                                        DaoMethodOutcome outcome = theIdToPersistedOutcome.get(nextId);
2075
2076                                        if (outcome != null && !outcome.isNop() && !Boolean.TRUE.equals(outcome.getCreated())) {
2077                                                replaceResourceReference(nextId, resourceReference, theTransactionDetails);
2078                                        }
2079
2080                                        // if referenced resource is not in transaction but exists in the DB, resolving its version
2081                                        IResourcePersistentId persistedReferenceId = resourceVersionMap.getResourcePersistentId(nextId);
2082                                        if (outcome == null && persistedReferenceId != null && persistedReferenceId.getVersion() != null) {
2083                                                IIdType newReferenceId = nextId.withVersion(
2084                                                                persistedReferenceId.getVersion().toString());
2085                                                replaceResourceReference(newReferenceId, resourceReference, theTransactionDetails);
2086                                        }
2087                                }
2088                        }
2089                }
2090
2091                // URIs
2092                Class<? extends IPrimitiveType<?>> uriType = (Class<? extends IPrimitiveType<?>>)
2093                                myContext.getElementDefinition("uri").getImplementingClass();
2094                List<? extends IPrimitiveType<?>> allUris = theTerser.getAllPopulatedChildElementsOfType(theResource, uriType);
2095                for (IPrimitiveType<?> nextRef : allUris) {
2096                        if (nextRef instanceof IIdType) {
2097                                continue; // No substitution on the resource ID itself!
2098                        }
2099                        String nextUriString = nextRef.getValueAsString();
2100                        if (isNotBlank(nextUriString)) {
2101                                if (theIdSubstitutions.containsSource(nextUriString)) {
2102                                        IIdType newId = theIdSubstitutions.getForSource(nextUriString);
2103                                        ourLog.debug(" * Replacing resource ref {} with {}", nextUriString, newId);
2104
2105                                        String existingValue = nextRef.getValueAsString();
2106                                        theTransactionDetails.addRollbackUndoAction(() -> nextRef.setValueAsString(existingValue));
2107
2108                                        nextRef.setValueAsString(newId.toVersionless().getValue());
2109                                } else {
2110                                        ourLog.debug(" * Reference [{}] does not exist in bundle", nextUriString);
2111                                }
2112                        }
2113                }
2114
2115                IPrimitiveType<Date> deletedInstantOrNull = ResourceMetadataKeyEnum.DELETED_AT.get(theResource);
2116                Date deletedTimestampOrNull = deletedInstantOrNull != null ? deletedInstantOrNull.getValue() : null;
2117
2118                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(theResource.getClass());
2119                IJpaDao jpaDao = (IJpaDao) dao;
2120
2121                IBasePersistedResource updateOutcome = null;
2122                if (theUpdatedEntities.contains(theDaoMethodOutcome.getEntity())) {
2123                        boolean forceUpdateVersion = !theReferencesToAutoVersion.isEmpty();
2124                        String matchUrl = theDaoMethodOutcome.getMatchUrl();
2125                        RestOperationTypeEnum operationType = theDaoMethodOutcome.getOperationType();
2126                        DaoMethodOutcome daoMethodOutcome = jpaDao.updateInternal(
2127                                        theRequest,
2128                                        theResource,
2129                                        matchUrl,
2130                                        true,
2131                                        forceUpdateVersion,
2132                                        theDaoMethodOutcome.getEntity(),
2133                                        theResource.getIdElement(),
2134                                        theDaoMethodOutcome.getPreviousResource(),
2135                                        operationType,
2136                                        theTransactionDetails);
2137                        updateOutcome = daoMethodOutcome.getEntity();
2138                        theDaoMethodOutcome = daoMethodOutcome;
2139                } else if (!theNonUpdatedEntities.contains(theDaoMethodOutcome.getId())) {
2140                        updateOutcome = jpaDao.updateEntity(
2141                                        theRequest,
2142                                        theResource,
2143                                        theDaoMethodOutcome.getEntity(),
2144                                        deletedTimestampOrNull,
2145                                        true,
2146                                        false,
2147                                        theTransactionDetails,
2148                                        false,
2149                                        true);
2150                }
2151
2152                // Make sure we reflect the actual final version for the resource.
2153                if (updateOutcome != null) {
2154                        IIdType newId = updateOutcome.getIdDt();
2155
2156                        IIdType entryId = theEntriesToProcess.getIdWithVersionlessComparison(newId);
2157                        if (entryId != null && !StringUtils.equals(entryId.getValue(), newId.getValue())) {
2158                                entryId.setValue(newId.getValue());
2159                        }
2160
2161                        theDaoMethodOutcome.setId(newId);
2162
2163                        theIdSubstitutions.updateTargets(newId);
2164
2165                        // This will only be null if we're not intending to return an OO
2166                        IBaseOperationOutcome operationOutcome = theDaoMethodOutcome.getOperationOutcome();
2167                        if (operationOutcome != null) {
2168
2169                                List<IIdType> autoCreatedPlaceholders =
2170                                                theTransactionDetails.getAutoCreatedPlaceholderResourcesAndClear();
2171                                for (IIdType autoCreatedPlaceholder : autoCreatedPlaceholders) {
2172                                        BaseStorageDao.addIssueToOperationOutcomeForAutoCreatedPlaceholder(
2173                                                        myContext, autoCreatedPlaceholder, operationOutcome);
2174                                }
2175
2176                                IBase responseEntry = theEntriesToProcess.getResponseBundleEntryWithVersionlessComparison(newId);
2177                                myVersionAdapter.setResponseOutcome(responseEntry, operationOutcome);
2178                        }
2179                }
2180        }
2181
2182        /**
2183         * We should replace the references when
2184         * 1. It is not a reference we should keep the client-supplied version for as configured by `DontStripVersionsFromReferences` or
2185         * 2. It is a reference that has been identified for auto versioning or
2186         * 3. Is a placeholder reference
2187         *
2188         * @param theReferencesToAutoVersion               list of references identified for auto versioning
2189         * @param theReferencesToKeepClientSuppliedVersion list of references that we should not strip the version for
2190         * @param theResourceReference                     the resource reference
2191         * @return true if we should replace the resource reference, false if we should keep the client provided reference
2192         */
2193        private boolean shouldReplaceResourceReference(
2194                        Set<IBaseReference> theReferencesToAutoVersion,
2195                        Set<IBaseReference> theReferencesToKeepClientSuppliedVersion,
2196                        IBaseReference theResourceReference) {
2197                return (!theReferencesToKeepClientSuppliedVersion.contains(theResourceReference)
2198                                                && myContext.getParserOptions().isStripVersionsFromReferences())
2199                                || theReferencesToAutoVersion.contains(theResourceReference)
2200                                || isPlaceholder(theResourceReference.getReferenceElement());
2201        }
2202
2203        private void replaceResourceReference(
2204                        IIdType theReferenceId, IBaseReference theResourceReference, TransactionDetails theTransactionDetails) {
2205                addRollbackReferenceRestore(theTransactionDetails, theResourceReference);
2206                theResourceReference.setReference(theReferenceId.getValue());
2207                theResourceReference.setResource(null);
2208        }
2209
2210        private void addRollbackReferenceRestore(
2211                        TransactionDetails theTransactionDetails, IBaseReference resourceReference) {
2212                String existingValue = resourceReference.getReferenceElement().getValue();
2213                theTransactionDetails.addRollbackUndoAction(() -> resourceReference.setReference(existingValue));
2214        }
2215
2216        private void validateNoDuplicates(
2217                        RequestDetails theRequest,
2218                        TransactionDetails theTransactionDetails,
2219                        String theActionName,
2220                        Map<String, Class<? extends IBaseResource>> conditionalRequestUrls,
2221                        Collection<DaoMethodOutcome> thePersistedOutcomes) {
2222
2223                Map<ResourceTable, ResourceIndexedSearchParams> existingSearchParams =
2224                                theTransactionDetails.getOrCreateUserData(
2225                                                HapiTransactionService.XACT_USERDATA_KEY_EXISTING_SEARCH_PARAMS, Collections::emptyMap);
2226
2227                IdentityHashMap<IBaseResource, ResourceIndexedSearchParams> resourceToIndexedParams =
2228                                new IdentityHashMap<>(thePersistedOutcomes.size());
2229                thePersistedOutcomes.stream()
2230                                .filter(t -> !t.isNop())
2231                                // N.B. GGG: This validation never occurs for mongo, as nothing is a ResourceTable.
2232                                .filter(t -> t.getEntity() instanceof ResourceTable)
2233                                .filter(t -> t.getEntity().getDeleted() == null)
2234                                .filter(t -> t.getResource() != null)
2235                                .forEach(t -> {
2236                                        ResourceTable entity = (ResourceTable) t.getEntity();
2237                                        ResourceIndexedSearchParams params = existingSearchParams.get(entity);
2238                                        if (params == null) {
2239                                                params = ResourceIndexedSearchParams.withLists(entity);
2240                                        }
2241                                        resourceToIndexedParams.put(t.getResource(), params);
2242                                });
2243
2244                for (Map.Entry<String, Class<? extends IBaseResource>> nextEntry : conditionalRequestUrls.entrySet()) {
2245                        String matchUrl = nextEntry.getKey();
2246                        if (isNotBlank(matchUrl)) {
2247                                if (matchUrl.startsWith("?")
2248                                                || (!matchUrl.contains("?")
2249                                                                && UNQUALIFIED_MATCH_URL_START.matcher(matchUrl).find())) {
2250                                        StringBuilder b = new StringBuilder();
2251                                        b.append(myContext.getResourceType(nextEntry.getValue()));
2252                                        if (!matchUrl.startsWith("?")) {
2253                                                b.append("?");
2254                                        }
2255                                        b.append(matchUrl);
2256                                        matchUrl = b.toString();
2257                                }
2258
2259                                if (!myInMemoryResourceMatcher.canBeEvaluatedInMemory(matchUrl).supported()) {
2260                                        continue;
2261                                }
2262
2263                                int counter = 0;
2264                                for (Map.Entry<IBaseResource, ResourceIndexedSearchParams> entries :
2265                                                resourceToIndexedParams.entrySet()) {
2266                                        ResourceIndexedSearchParams indexedParams = entries.getValue();
2267                                        IBaseResource resource = entries.getKey();
2268
2269                                        String resourceType = myContext.getResourceType(resource);
2270                                        if (!matchUrl.startsWith(resourceType + "?")) {
2271                                                continue;
2272                                        }
2273
2274                                        if (myInMemoryResourceMatcher
2275                                                        .match(matchUrl, resource, indexedParams, theRequest)
2276                                                        .matched()) {
2277                                                counter++;
2278                                                if (counter > 1) {
2279                                                        throw new InvalidRequestException(Msg.code(542) + "Unable to process " + theActionName
2280                                                                        + " - Request would cause multiple resources to match URL: \"" + matchUrl
2281                                                                        + "\". Does transaction request contain duplicates?");
2282                                                }
2283                                        }
2284                                }
2285                        }
2286                }
2287        }
2288
2289        protected abstract void flushSession(Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome);
2290
2291        private void validateResourcePresent(IBaseResource theResource, Integer theOrder, String theVerb) {
2292                if (theResource == null) {
2293                        String msg = myContext
2294                                        .getLocalizer()
2295                                        .getMessage(BaseTransactionProcessor.class, "missingMandatoryResource", theVerb, theOrder);
2296                        throw new InvalidRequestException(Msg.code(543) + msg);
2297                }
2298        }
2299
2300        private IIdType newIdType(String theResourceType, String theResourceId, String theVersion) {
2301                IdType id = new IdType(theResourceType, theResourceId, theVersion);
2302                return myContext.getVersion().newIdType().setValue(id.getValue());
2303        }
2304
2305        private IIdType newIdType(String theToResourceName, String theIdPart) {
2306                return newIdType(theToResourceName, theIdPart, null);
2307        }
2308
2309        @VisibleForTesting
2310        public void setDaoRegistry(DaoRegistry theDaoRegistry) {
2311                myDaoRegistry = theDaoRegistry;
2312        }
2313
2314        private IFhirResourceDao getDaoOrThrowException(Class<? extends IBaseResource> theClass) {
2315                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDaoOrNull(theClass);
2316                if (dao == null) {
2317                        Set<String> types = new TreeSet<>(myDaoRegistry.getRegisteredDaoTypes());
2318                        String type = myContext.getResourceType(theClass);
2319                        String msg = myContext
2320                                        .getLocalizer()
2321                                        .getMessage(BaseTransactionProcessor.class, "unsupportedResourceType", type, types.toString());
2322                        throw new InvalidRequestException(Msg.code(544) + msg);
2323                }
2324                return dao;
2325        }
2326
2327        private String toResourceName(Class<? extends IBaseResource> theResourceType) {
2328                return myContext.getResourceType(theResourceType);
2329        }
2330
2331        public void setContext(FhirContext theContext) {
2332                myContext = theContext;
2333        }
2334
2335        /**
2336         * Extracts the transaction url from the entry and verifies it's:
2337         * <li>not null or blank (unless it is a POST), and</li>
2338         * <li>is a relative url matching the resourceType it is about</li>
2339         * <p>
2340         * For POST requests, the url is allowed to be blank to preserve the existing behavior.
2341         * <p>
2342         * Returns the transaction url (or throws an InvalidRequestException if url is missing or not valid)
2343         */
2344        private String extractAndVerifyTransactionUrlForEntry(IBase theEntry, String theVerb) {
2345                String url = extractTransactionUrlOrThrowException(theEntry, theVerb);
2346                if (url.isEmpty() || isValidResourceTypeUrl(url)) {
2347                        // for POST requests, the url is allowed to be blank, which is checked in
2348                        // extractTransactionUrlOrThrowException and an empty string is returned in that case.
2349                        // That is why empty url is allowed here.
2350                        return url;
2351                }
2352
2353                ourLog.debug("Invalid url. Should begin with a resource type: {}", url);
2354                String msg = myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, url);
2355                throw new InvalidRequestException(Msg.code(2006) + msg);
2356        }
2357
2358        /**
2359         * Returns true if the provided url is a valid entry request.url.
2360         * <p>
2361         * This means:
2362         * a) not an absolute url (does not start with http/https)
2363         * b) starts with either a ResourceType or /ResourceType
2364         */
2365        private boolean isValidResourceTypeUrl(@Nonnull String theUrl) {
2366                if (UrlUtil.isAbsolute(theUrl)) {
2367                        return false;
2368                } else {
2369                        int queryStringIndex = theUrl.indexOf("?");
2370                        String url;
2371                        if (queryStringIndex > 0) {
2372                                url = theUrl.substring(0, theUrl.indexOf("?"));
2373                        } else {
2374                                url = theUrl;
2375                        }
2376                        String[] parts;
2377                        if (url.startsWith("/")) {
2378                                parts = url.substring(1).split("/");
2379                        } else {
2380                                parts = url.split("/");
2381                        }
2382                        Set<String> allResourceTypes = myContext.getResourceTypes();
2383
2384                        return allResourceTypes.contains(parts[0]);
2385                }
2386        }
2387
2388        /**
2389         * Extracts the transaction url from the entry and verifies that it is not null/blank, unless it is a POST,
2390         * and returns it. For POST requests allows null or blank values to keep the existing behaviour and returns
2391         * an empty string in that case.
2392         */
2393        private String extractTransactionUrlOrThrowException(IBase nextEntry, String verb) {
2394                String url = myVersionAdapter.getEntryRequestUrl(nextEntry);
2395                if ("POST".equals(verb) && isBlank(url)) {
2396                        // allow blanks for POST to keep the existing behaviour
2397                        // returning empty string instead of null to not get NPE later
2398                        return "";
2399                }
2400
2401                if (isBlank(url)) {
2402                        throw new InvalidRequestException(Msg.code(545)
2403                                        + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionMissingUrl", verb));
2404                }
2405                return url;
2406        }
2407
2408        private IFhirResourceDao<? extends IBaseResource> toDao(UrlUtil.UrlParts theParts, String theVerb, String theUrl) {
2409                RuntimeResourceDefinition resType;
2410                try {
2411                        resType = myContext.getResourceDefinition(theParts.getResourceType());
2412                } catch (DataFormatException e) {
2413                        String msg =
2414                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
2415                        throw new InvalidRequestException(Msg.code(546) + msg);
2416                }
2417                IFhirResourceDao<? extends IBaseResource> dao = null;
2418                if (resType != null) {
2419                        dao = myDaoRegistry.getResourceDao(resType.getImplementingClass());
2420                }
2421                if (dao == null) {
2422                        String msg =
2423                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
2424                        throw new InvalidRequestException(Msg.code(547) + msg);
2425                }
2426
2427                return dao;
2428        }
2429
2430        private String toMatchUrl(IBase theEntry) {
2431                String verb = myVersionAdapter.getEntryRequestVerb(myContext, theEntry);
2432                switch (defaultString(verb)) {
2433                        case POST:
2434                                return myVersionAdapter.getEntryIfNoneExist(theEntry);
2435                        case PUT:
2436                        case DELETE:
2437                        case PATCH:
2438                                String url = extractTransactionUrlOrThrowException(theEntry, verb);
2439                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
2440                                if (isBlank(parts.getResourceId())) {
2441                                        return parts.getResourceType() + '?' + parts.getParams();
2442                                }
2443                                return null;
2444                        default:
2445                                return null;
2446                }
2447        }
2448
2449        @VisibleForTesting
2450        public void setInterceptorBroadcasterForUnitTest(IInterceptorBroadcaster theInterceptorBroadcaster) {
2451                myInterceptorBroadcaster = theInterceptorBroadcaster;
2452        }
2453
2454        @VisibleForTesting
2455        public void setPartitionSettingsForUnitTest(PartitionSettings thePartitionSettings) {
2456                myPartitionSettings = thePartitionSettings;
2457        }
2458
2459        /**
2460         * Transaction Order, per the spec:
2461         * <p>
2462         * Process any DELETE interactions
2463         * Process any POST interactions
2464         * Process any PUT interactions
2465         * Process any PATCH interactions
2466         * Process any GET interactions
2467         */
2468        // @formatter:off
2469        public class TransactionSorter implements Comparator<IBase> {
2470
2471                private final Set<String> myPlaceholderIds;
2472
2473                public TransactionSorter(Set<String> thePlaceholderIds) {
2474                        myPlaceholderIds = thePlaceholderIds;
2475                }
2476
2477                @Override
2478                public int compare(IBase theO1, IBase theO2) {
2479                        int o1 = toOrder(theO1);
2480                        int o2 = toOrder(theO2);
2481
2482                        if (o1 == o2) {
2483                                String matchUrl1 = toMatchUrl(theO1);
2484                                String matchUrl2 = toMatchUrl(theO2);
2485                                if (isBlank(matchUrl1) && isBlank(matchUrl2)) {
2486                                        return 0;
2487                                }
2488                                if (isBlank(matchUrl1)) {
2489                                        return -1;
2490                                }
2491                                if (isBlank(matchUrl2)) {
2492                                        return 1;
2493                                }
2494
2495                                boolean match1containsSubstitutions = false;
2496                                boolean match2containsSubstitutions = false;
2497                                for (String nextPlaceholder : myPlaceholderIds) {
2498                                        if (matchUrl1.contains(nextPlaceholder)) {
2499                                                match1containsSubstitutions = true;
2500                                        }
2501                                        if (matchUrl2.contains(nextPlaceholder)) {
2502                                                match2containsSubstitutions = true;
2503                                        }
2504                                }
2505
2506                                if (match1containsSubstitutions && match2containsSubstitutions) {
2507                                        return 0;
2508                                }
2509                                if (!match1containsSubstitutions && !match2containsSubstitutions) {
2510                                        return 0;
2511                                }
2512                                if (match1containsSubstitutions) {
2513                                        return 1;
2514                                } else {
2515                                        return -1;
2516                                }
2517                        }
2518
2519                        return o1 - o2;
2520                }
2521
2522                private int toOrder(IBase theO1) {
2523                        int o1 = 0;
2524                        if (myVersionAdapter.getEntryRequestVerb(myContext, theO1) != null) {
2525                                switch (myVersionAdapter.getEntryRequestVerb(myContext, theO1)) {
2526                                        case "DELETE":
2527                                                o1 = 1;
2528                                                break;
2529                                        case "POST":
2530                                                o1 = 2;
2531                                                break;
2532                                        case "PUT":
2533                                                o1 = 3;
2534                                                break;
2535                                        case "PATCH":
2536                                                o1 = 4;
2537                                                break;
2538                                        case "GET":
2539                                                o1 = 5;
2540                                                break;
2541                                        default:
2542                                                o1 = 0;
2543                                                break;
2544                                }
2545                        }
2546                        return o1;
2547                }
2548        }
2549
2550        public class RetriableBundleTask implements Runnable {
2551
2552                private final CountDownLatch myCompletedLatch;
2553                private final RequestDetails myRequestDetails;
2554                private final IBase myNextReqEntry;
2555                private final Map<Integer, Object> myResponseMap;
2556                private final int myResponseOrder;
2557                private final boolean myNestedMode;
2558                private BaseServerResponseException myLastSeenException;
2559
2560                protected RetriableBundleTask(
2561                                CountDownLatch theCompletedLatch,
2562                                RequestDetails theRequestDetails,
2563                                Map<Integer, Object> theResponseMap,
2564                                int theResponseOrder,
2565                                IBase theNextReqEntry,
2566                                boolean theNestedMode) {
2567                        this.myCompletedLatch = theCompletedLatch;
2568                        this.myRequestDetails = theRequestDetails;
2569                        this.myNextReqEntry = theNextReqEntry;
2570                        this.myResponseMap = theResponseMap;
2571                        this.myResponseOrder = theResponseOrder;
2572                        this.myNestedMode = theNestedMode;
2573                        this.myLastSeenException = null;
2574                }
2575
2576                private void processBatchEntry() {
2577                        IBaseBundle subRequestBundle =
2578                                        myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode());
2579                        myVersionAdapter.addEntry(subRequestBundle, myNextReqEntry);
2580
2581                        IInterceptorBroadcaster compositeBroadcaster =
2582                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, myRequestDetails);
2583
2584                        // Interceptor call: STORAGE_TRANSACTION_PROCESSING
2585                        if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING)) {
2586                                HookParams params = new HookParams()
2587                                                .add(RequestDetails.class, myRequestDetails)
2588                                                .addIfMatchesType(ServletRequestDetails.class, myRequestDetails)
2589                                                .add(IBaseBundle.class, subRequestBundle);
2590                                compositeBroadcaster.callHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING, params);
2591                        }
2592
2593                        IBaseBundle nextResponseBundle = processTransactionAsSubRequest(
2594                                        myRequestDetails, subRequestBundle, "Batch sub-request", myNestedMode);
2595
2596                        IBase subResponseEntry =
2597                                        (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
2598                        myResponseMap.put(myResponseOrder, subResponseEntry);
2599
2600                        /*
2601                         * If the individual entry didn't have a resource in its response, bring the sub-transaction's OperationOutcome across so the client can see it
2602                         */
2603                        if (myVersionAdapter.getResource(subResponseEntry) == null) {
2604                                IBase nextResponseBundleFirstEntry =
2605                                                (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
2606                                myResponseMap.put(myResponseOrder, nextResponseBundleFirstEntry);
2607                        }
2608                }
2609
2610                private boolean processBatchEntryWithRetry() {
2611                        int maxAttempts = 3;
2612                        for (int attempt = 1; ; attempt++) {
2613                                try {
2614                                        processBatchEntry();
2615                                        return true;
2616                                } catch (BaseServerResponseException e) {
2617                                        // If we catch a known and structured exception from HAPI, just fail.
2618                                        myLastSeenException = e;
2619                                        return false;
2620                                } catch (Throwable t) {
2621                                        myLastSeenException = new InternalErrorException(t);
2622                                        // If we have caught a non-tag-storage failure we are unfamiliar with, or we have exceeded max
2623                                        // attempts, exit.
2624                                        if (!DaoFailureUtil.isTagStorageFailure(t) || attempt >= maxAttempts) {
2625                                                ourLog.error("Failure during BATCH sub transaction processing", t);
2626                                                return false;
2627                                        }
2628                                }
2629                        }
2630                }
2631
2632                @Override
2633                public void run() {
2634                        boolean success = processBatchEntryWithRetry();
2635                        if (!success) {
2636                                populateResponseMapWithLastSeenException();
2637                        }
2638
2639                        // checking for the parallelism
2640                        ourLog.debug("processing batch for {} is completed", myVersionAdapter.getEntryRequestUrl(myNextReqEntry));
2641                        myCompletedLatch.countDown();
2642                }
2643
2644                private void populateResponseMapWithLastSeenException() {
2645                        ServerResponseExceptionHolder caughtEx = new ServerResponseExceptionHolder();
2646                        caughtEx.setException(myLastSeenException);
2647                        myResponseMap.put(myResponseOrder, caughtEx);
2648                }
2649        }
2650
2651        private static class ServerResponseExceptionHolder {
2652                private BaseServerResponseException myException;
2653
2654                public BaseServerResponseException getException() {
2655                        return myException;
2656                }
2657
2658                public void setException(BaseServerResponseException myException) {
2659                        this.myException = myException;
2660                }
2661        }
2662
2663        public static boolean isPlaceholder(IIdType theId) {
2664                if (theId != null && theId.getValue() != null) {
2665                        return theId.getValue().startsWith("urn:oid:") || theId.getValue().startsWith("urn:uuid:");
2666                }
2667                return false;
2668        }
2669
2670        public static String toStatusString(int theStatusCode) {
2671                return theStatusCode + " " + defaultString(Constants.HTTP_STATUS_NAMES.get(theStatusCode));
2672        }
2673
2674        /**
2675         * Given a match URL containing
2676         *
2677         * @param theIdSubstitutions
2678         * @param theMatchUrl
2679         * @return
2680         */
2681        public static String performIdSubstitutionsInMatchUrl(IdSubstitutionMap theIdSubstitutions, String theMatchUrl) {
2682                String matchUrl = theMatchUrl;
2683                if (isNotBlank(matchUrl) && !theIdSubstitutions.isEmpty()) {
2684                        int startIdx = 0;
2685                        while (startIdx != -1) {
2686
2687                                int endIdx = matchUrl.indexOf('&', startIdx + 1);
2688                                if (endIdx == -1) {
2689                                        endIdx = matchUrl.length();
2690                                }
2691
2692                                int equalsIdx = matchUrl.indexOf('=', startIdx + 1);
2693
2694                                int searchFrom;
2695                                if (equalsIdx == -1) {
2696                                        searchFrom = matchUrl.length();
2697                                } else if (equalsIdx >= endIdx) {
2698                                        // First equals we found is from a subsequent parameter
2699                                        searchFrom = matchUrl.length();
2700                                } else {
2701                                        String paramValue = matchUrl.substring(equalsIdx + 1, endIdx);
2702                                        boolean isUrn = isUrn(paramValue);
2703                                        boolean isUrnEscaped = !isUrn && isUrnEscaped(paramValue);
2704                                        if (isUrn || isUrnEscaped) {
2705                                                if (isUrnEscaped) {
2706                                                        paramValue = UrlUtil.unescape(paramValue);
2707                                                }
2708                                                IIdType replacement = theIdSubstitutions.getForSource(paramValue);
2709                                                if (replacement != null) {
2710                                                        String replacementValue;
2711                                                        if (replacement.hasVersionIdPart()) {
2712                                                                replacementValue = replacement.toVersionless().getValue();
2713                                                        } else {
2714                                                                replacementValue = replacement.getValue();
2715                                                        }
2716                                                        matchUrl = matchUrl.substring(0, equalsIdx + 1)
2717                                                                        + replacementValue
2718                                                                        + matchUrl.substring(endIdx);
2719                                                        searchFrom = equalsIdx + 1 + replacementValue.length();
2720                                                } else {
2721                                                        searchFrom = endIdx;
2722                                                }
2723                                        } else {
2724                                                searchFrom = endIdx;
2725                                        }
2726                                }
2727
2728                                if (searchFrom >= matchUrl.length()) {
2729                                        break;
2730                                }
2731
2732                                startIdx = matchUrl.indexOf('&', searchFrom);
2733                        }
2734                }
2735                return matchUrl;
2736        }
2737
2738        private static boolean isUrn(@Nonnull String theId) {
2739                return theId.startsWith(URN_PREFIX);
2740        }
2741
2742        private static boolean isUrnEscaped(@Nonnull String theId) {
2743                return theId.startsWith(URN_PREFIX_ESCAPED);
2744        }
2745}