Source code

001/*-
002 * #%L
003 * HAPI FHIR Storage api
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.dao;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.FhirVersionEnum;
024import ca.uhn.fhir.context.RuntimeResourceDefinition;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.HookParams;
027import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
028import ca.uhn.fhir.interceptor.api.Pointcut;
029import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
030import ca.uhn.fhir.interceptor.model.RequestPartitionId;
031import ca.uhn.fhir.interceptor.model.TransactionWriteOperationsDetails;
032import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
033import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
034import ca.uhn.fhir.jpa.api.dao.IJpaDao;
035import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
036import ca.uhn.fhir.jpa.api.model.DeleteConflict;
037import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
038import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome;
039import ca.uhn.fhir.jpa.api.model.LazyDaoMethodOutcome;
040import ca.uhn.fhir.jpa.cache.IResourceVersionSvc;
041import ca.uhn.fhir.jpa.cache.ResourcePersistentIdMap;
042import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
043import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService;
044import ca.uhn.fhir.jpa.delete.DeleteConflictUtil;
045import ca.uhn.fhir.jpa.model.config.PartitionSettings;
046import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
047import ca.uhn.fhir.jpa.model.entity.ResourceTable;
048import ca.uhn.fhir.jpa.model.entity.StorageSettings;
049import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
050import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
051import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
052import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult;
053import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher;
054import ca.uhn.fhir.jpa.searchparam.matcher.SearchParamMatcher;
055import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
056import ca.uhn.fhir.model.valueset.BundleEntryTransactionMethodEnum;
057import ca.uhn.fhir.parser.DataFormatException;
058import ca.uhn.fhir.parser.IParser;
059import ca.uhn.fhir.rest.api.Constants;
060import ca.uhn.fhir.rest.api.PatchTypeEnum;
061import ca.uhn.fhir.rest.api.PreferReturnEnum;
062import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
063import ca.uhn.fhir.rest.api.server.RequestDetails;
064import ca.uhn.fhir.rest.api.server.storage.DeferredInterceptorBroadcasts;
065import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
066import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
067import ca.uhn.fhir.rest.param.ParameterUtil;
068import ca.uhn.fhir.rest.server.RestfulServerUtils;
069import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException;
070import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
071import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
072import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
073import ca.uhn.fhir.rest.server.exceptions.NotModifiedException;
074import ca.uhn.fhir.rest.server.exceptions.PayloadTooLargeException;
075import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
076import ca.uhn.fhir.rest.server.method.BaseMethodBinding;
077import ca.uhn.fhir.rest.server.method.BaseResourceReturningMethodBinding;
078import ca.uhn.fhir.rest.server.method.UpdateMethodBinding;
079import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
080import ca.uhn.fhir.rest.server.servlet.ServletSubRequestDetails;
081import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
082import ca.uhn.fhir.rest.server.util.ServletRequestUtil;
083import ca.uhn.fhir.util.AsyncUtil;
084import ca.uhn.fhir.util.ElementUtil;
085import ca.uhn.fhir.util.FhirTerser;
086import ca.uhn.fhir.util.ResourceReferenceInfo;
087import ca.uhn.fhir.util.StopWatch;
088import ca.uhn.fhir.util.UrlUtil;
089import com.google.common.annotations.VisibleForTesting;
090import com.google.common.collect.ArrayListMultimap;
091import com.google.common.collect.ListMultimap;
092import jakarta.annotation.Nonnull;
093import jakarta.annotation.Nullable;
094import org.apache.commons.lang3.StringUtils;
095import org.apache.commons.lang3.Validate;
096import org.hl7.fhir.dstu3.model.Bundle;
097import org.hl7.fhir.exceptions.FHIRException;
098import org.hl7.fhir.instance.model.api.IBase;
099import org.hl7.fhir.instance.model.api.IBaseBinary;
100import org.hl7.fhir.instance.model.api.IBaseBundle;
101import org.hl7.fhir.instance.model.api.IBaseParameters;
102import org.hl7.fhir.instance.model.api.IBaseReference;
103import org.hl7.fhir.instance.model.api.IBaseResource;
104import org.hl7.fhir.instance.model.api.IIdType;
105import org.hl7.fhir.instance.model.api.IPrimitiveType;
106import org.hl7.fhir.r4.model.IdType;
107import org.slf4j.Logger;
108import org.slf4j.LoggerFactory;
109import org.springframework.beans.factory.annotation.Autowired;
110import org.springframework.core.task.TaskExecutor;
111import org.springframework.transaction.PlatformTransactionManager;
112import org.springframework.transaction.TransactionDefinition;
113import org.springframework.transaction.support.TransactionCallback;
114import org.springframework.transaction.support.TransactionTemplate;
115
116import java.util.ArrayList;
117import java.util.Collection;
118import java.util.Comparator;
119import java.util.Date;
120import java.util.HashMap;
121import java.util.HashSet;
122import java.util.IdentityHashMap;
123import java.util.Iterator;
124import java.util.LinkedHashSet;
125import java.util.List;
126import java.util.Map;
127import java.util.Optional;
128import java.util.Set;
129import java.util.TreeSet;
130import java.util.concurrent.ConcurrentHashMap;
131import java.util.concurrent.CountDownLatch;
132import java.util.concurrent.TimeUnit;
133import java.util.regex.Pattern;
134import java.util.stream.Collectors;
135
136import static ca.uhn.fhir.util.StringUtil.toUtf8String;
137import static java.util.Objects.isNull;
138import static org.apache.commons.lang3.StringUtils.defaultString;
139import static org.apache.commons.lang3.StringUtils.isBlank;
140import static org.apache.commons.lang3.StringUtils.isNotBlank;
141
142public abstract class BaseTransactionProcessor {
143
144        public static final String URN_PREFIX = "urn:";
145        public static final String URN_PREFIX_ESCAPED = UrlUtil.escapeUrlParam(URN_PREFIX);
146        public static final Pattern UNQUALIFIED_MATCH_URL_START = Pattern.compile("^[a-zA-Z0-9_-]+=");
147        public static final Pattern INVALID_PLACEHOLDER_PATTERN = Pattern.compile("[a-zA-Z]+:.*");
148        private static final Logger ourLog = LoggerFactory.getLogger(BaseTransactionProcessor.class);
149
150        @Autowired
151        private IRequestPartitionHelperSvc myRequestPartitionHelperService;
152
153        @Autowired
154        private PlatformTransactionManager myTxManager;
155
156        @Autowired
157        private FhirContext myContext;
158
159        @Autowired
160        @SuppressWarnings("rawtypes")
161        private ITransactionProcessorVersionAdapter myVersionAdapter;
162
163        @Autowired
164        private DaoRegistry myDaoRegistry;
165
166        @Autowired
167        private IInterceptorBroadcaster myInterceptorBroadcaster;
168
169        @Autowired
170        private IHapiTransactionService myHapiTransactionService;
171
172        @Autowired
173        private StorageSettings myStorageSettings;
174
175        @Autowired
176        PartitionSettings myPartitionSettings;
177
178        @Autowired
179        private InMemoryResourceMatcher myInMemoryResourceMatcher;
180
181        @Autowired
182        private SearchParamMatcher mySearchParamMatcher;
183
184        @Autowired
185        private ThreadPoolFactory myThreadPoolFactory;
186
187        private TaskExecutor myExecutor;
188
189        @Autowired
190        private IResourceVersionSvc myResourceVersionSvc;
191
192        @VisibleForTesting
193        public void setStorageSettings(StorageSettings theStorageSettings) {
194                myStorageSettings = theStorageSettings;
195        }
196
197        public ITransactionProcessorVersionAdapter getVersionAdapter() {
198                return myVersionAdapter;
199        }
200
201        @VisibleForTesting
202        public void setVersionAdapter(ITransactionProcessorVersionAdapter theVersionAdapter) {
203                myVersionAdapter = theVersionAdapter;
204        }
205
206        private TaskExecutor getTaskExecutor() {
207                if (myExecutor == null) {
208                        myExecutor = myThreadPoolFactory.newThreadPool(
209                                        myStorageSettings.getBundleBatchPoolSize(),
210                                        myStorageSettings.getBundleBatchMaxPoolSize(),
211                                        "bundle-batch-");
212                }
213                return myExecutor;
214        }
215
216        public <BUNDLE extends IBaseBundle> BUNDLE transaction(
217                        RequestDetails theRequestDetails, BUNDLE theRequest, boolean theNestedMode) {
218                String actionName = "Transaction";
219                IBaseBundle response = processTransactionAsSubRequest(theRequestDetails, theRequest, actionName, theNestedMode);
220
221                List<IBase> entries = myVersionAdapter.getEntries(response);
222                for (int i = 0; i < entries.size(); i++) {
223                        if (ElementUtil.isEmpty(entries.get(i))) {
224                                entries.remove(i);
225                                i--;
226                        }
227                }
228
229                return (BUNDLE) response;
230        }
231
232        public IBaseBundle collection(final RequestDetails theRequestDetails, IBaseBundle theRequest) {
233                String transactionType = myVersionAdapter.getBundleType(theRequest);
234
235                if (!org.hl7.fhir.r4.model.Bundle.BundleType.COLLECTION.toCode().equals(transactionType)) {
236                        throw new InvalidRequestException(
237                                        Msg.code(526) + "Can not process collection Bundle of type: " + transactionType);
238                }
239
240                ourLog.info(
241                                "Beginning storing collection with {} resources",
242                                myVersionAdapter.getEntries(theRequest).size());
243
244                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
245                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
246
247                IBaseBundle resp =
248                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
249
250                List<IBaseResource> resources = new ArrayList<>();
251                for (final Object nextRequestEntry : myVersionAdapter.getEntries(theRequest)) {
252                        IBaseResource resource = myVersionAdapter.getResource((IBase) nextRequestEntry);
253                        resources.add(resource);
254                }
255
256                IBaseBundle transactionBundle = myVersionAdapter.createBundle("transaction");
257                for (IBaseResource next : resources) {
258                        IBase entry = myVersionAdapter.addEntry(transactionBundle);
259                        myVersionAdapter.setResource(entry, next);
260                        myVersionAdapter.setRequestVerb(entry, "PUT");
261                        myVersionAdapter.setRequestUrl(
262                                        entry, next.getIdElement().toUnqualifiedVersionless().getValue());
263                }
264
265                transaction(theRequestDetails, transactionBundle, false);
266
267                return resp;
268        }
269
270        @SuppressWarnings("unchecked")
271        private void populateEntryWithOperationOutcome(BaseServerResponseException caughtEx, IBase nextEntry) {
272                myVersionAdapter.populateEntryWithOperationOutcome(caughtEx, nextEntry);
273        }
274
275        @SuppressWarnings("unchecked")
276        private void handleTransactionCreateOrUpdateOutcome(
277                        IdSubstitutionMap idSubstitutions,
278                        Map<IIdType, DaoMethodOutcome> idToPersistedOutcome,
279                        IIdType nextResourceId,
280                        DaoMethodOutcome outcome,
281                        IBase newEntry,
282                        String theResourceType,
283                        IBaseResource theRes,
284                        RequestDetails theRequestDetails) {
285                IIdType newId = outcome.getId().toUnqualified();
286                IIdType resourceId = isPlaceholder(nextResourceId) ? nextResourceId : nextResourceId.toUnqualifiedVersionless();
287                if (newId.equals(resourceId) == false) {
288                        if (!nextResourceId.isEmpty()) {
289                                idSubstitutions.put(resourceId, newId);
290                        }
291                        if (isPlaceholder(resourceId)) {
292                                /*
293                                 * The correct way for substitution IDs to be is to be with no resource type, but we'll accept the qualified kind too just to be lenient.
294                                 */
295                                IIdType id = myContext.getVersion().newIdType();
296                                id.setValue(theResourceType + '/' + resourceId.getValue());
297                                idSubstitutions.put(id, newId);
298                        }
299                }
300
301                populateIdToPersistedOutcomeMap(idToPersistedOutcome, newId, outcome);
302
303                if (shouldSwapBinaryToActualResource(theRes, theResourceType, nextResourceId)) {
304                        theRes = idToPersistedOutcome.get(newId).getResource();
305                }
306
307                if (outcome.getCreated()) {
308                        myVersionAdapter.setResponseStatus(newEntry, toStatusString(Constants.STATUS_HTTP_201_CREATED));
309                } else {
310                        myVersionAdapter.setResponseStatus(newEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
311                }
312
313                Date lastModified = getLastModified(theRes);
314                myVersionAdapter.setResponseLastModified(newEntry, lastModified);
315
316                if (outcome.getOperationOutcome() != null) {
317                        myVersionAdapter.setResponseOutcome(newEntry, outcome.getOperationOutcome());
318                }
319
320                if (theRequestDetails != null) {
321                        String prefer = theRequestDetails.getHeader(Constants.HEADER_PREFER);
322                        PreferReturnEnum preferReturn =
323                                        RestfulServerUtils.parsePreferHeader(null, prefer).getReturn();
324                        if (preferReturn != null) {
325                                if (preferReturn == PreferReturnEnum.REPRESENTATION) {
326                                        if (outcome.getResource() != null) {
327                                                outcome.fireResourceViewCallbacks();
328                                                myVersionAdapter.setResource(newEntry, outcome.getResource());
329                                        }
330                                }
331                        }
332                }
333        }
334
335        /**
336         * Method which populates entry in idToPersistedOutcome.
337         * Will store whatever outcome is sent, unless the key already exists, then we only replace an instance if we find that the instance
338         * we are replacing with is non-lazy. This allows us to evaluate later more easily, as we _know_ we need access to these.
339         */
340        private void populateIdToPersistedOutcomeMap(
341                        Map<IIdType, DaoMethodOutcome> idToPersistedOutcome, IIdType newId, DaoMethodOutcome outcome) {
342                // Prefer real method outcomes over lazy ones.
343                if (idToPersistedOutcome.containsKey(newId)) {
344                        if (!(outcome instanceof LazyDaoMethodOutcome)) {
345                                idToPersistedOutcome.put(newId, outcome);
346                        }
347                } else {
348                        idToPersistedOutcome.put(newId, outcome);
349                }
350        }
351
352        private Date getLastModified(IBaseResource theRes) {
353                return theRes.getMeta().getLastUpdated();
354        }
355
356        private IBaseBundle processTransactionAsSubRequest(
357                        RequestDetails theRequestDetails, IBaseBundle theRequest, String theActionName, boolean theNestedMode) {
358                BaseStorageDao.markRequestAsProcessingSubRequest(theRequestDetails);
359                try {
360
361                        // Interceptor call: STORAGE_TRANSACTION_PROCESSING
362                        IInterceptorBroadcaster compositeBroadcaster = CompositeInterceptorBroadcaster.newCompositeBroadcaster(
363                                        myInterceptorBroadcaster, theRequestDetails);
364                        if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING)) {
365                                HookParams params = new HookParams()
366                                                .add(RequestDetails.class, theRequestDetails)
367                                                .addIfMatchesType(ServletRequestDetails.class, theRequest)
368                                                .add(IBaseBundle.class, theRequest);
369                                compositeBroadcaster.callHooks(Pointcut.STORAGE_TRANSACTION_PROCESSING, params);
370                        }
371
372                        return processTransaction(theRequestDetails, theRequest, theActionName, theNestedMode);
373                } finally {
374                        BaseStorageDao.clearRequestAsProcessingSubRequest(theRequestDetails);
375                }
376        }
377
378        @VisibleForTesting
379        public void setTxManager(PlatformTransactionManager theTxManager) {
380                myTxManager = theTxManager;
381        }
382
383        private IBaseBundle batch(final RequestDetails theRequestDetails, IBaseBundle theRequest, boolean theNestedMode) {
384                ourLog.info(
385                                "Beginning batch with {} resources",
386                                myVersionAdapter.getEntries(theRequest).size());
387
388                long start = System.currentTimeMillis();
389
390                IBaseBundle response =
391                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
392                Map<Integer, Object> responseMap = new ConcurrentHashMap<>();
393
394                List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest);
395                int requestEntriesSize = requestEntries.size();
396
397                // Now, run all non-gets sequentially, and all gets are submitted to the executor to run (potentially) in
398                // parallel
399                // The result is kept in the map to save the original position
400                List<RetriableBundleTask> getCalls = new ArrayList<>();
401                List<RetriableBundleTask> nonGetCalls = new ArrayList<>();
402
403                CountDownLatch completionLatch = new CountDownLatch(requestEntriesSize);
404                for (int i = 0; i < requestEntriesSize; i++) {
405                        IBase nextRequestEntry = requestEntries.get(i);
406                        RetriableBundleTask retriableBundleTask = new RetriableBundleTask(
407                                        completionLatch, theRequestDetails, responseMap, i, nextRequestEntry, theNestedMode);
408                        if (myVersionAdapter
409                                        .getEntryRequestVerb(myContext, nextRequestEntry)
410                                        .equalsIgnoreCase("GET")) {
411                                getCalls.add(retriableBundleTask);
412                        } else {
413                                nonGetCalls.add(retriableBundleTask);
414                        }
415                }
416                // Execute all non-gets on calling thread.
417                nonGetCalls.forEach(RetriableBundleTask::run);
418                // Execute all gets (potentially in a pool)
419                if (myStorageSettings.getBundleBatchPoolSize() == 1) {
420                        getCalls.forEach(RetriableBundleTask::run);
421                } else {
422                        getCalls.forEach(getCall -> getTaskExecutor().execute(getCall));
423                }
424
425                // waiting for all async tasks to be completed
426                AsyncUtil.awaitLatchAndIgnoreInterrupt(completionLatch, 300L, TimeUnit.SECONDS);
427
428                // Now, create the bundle response in original order
429                Object nextResponseEntry;
430                for (int i = 0; i < requestEntriesSize; i++) {
431
432                        nextResponseEntry = responseMap.get(i);
433                        if (nextResponseEntry instanceof ServerResponseExceptionHolder) {
434                                ServerResponseExceptionHolder caughtEx = (ServerResponseExceptionHolder) nextResponseEntry;
435                                if (caughtEx.getException() != null) {
436                                        IBase nextEntry = myVersionAdapter.addEntry(response);
437                                        populateEntryWithOperationOutcome(caughtEx.getException(), nextEntry);
438                                        myVersionAdapter.setResponseStatus(
439                                                        nextEntry, toStatusString(caughtEx.getException().getStatusCode()));
440                                }
441                        } else {
442                                myVersionAdapter.addEntry(response, (IBase) nextResponseEntry);
443                        }
444                }
445
446                long delay = System.currentTimeMillis() - start;
447                ourLog.info("Batch completed in {}ms", delay);
448
449                return response;
450        }
451
452        @VisibleForTesting
453        public void setHapiTransactionService(HapiTransactionService theHapiTransactionService) {
454                myHapiTransactionService = theHapiTransactionService;
455        }
456
457        private IBaseBundle processTransaction(
458                        final RequestDetails theRequestDetails,
459                        final IBaseBundle theRequest,
460                        final String theActionName,
461                        boolean theNestedMode) {
462                validateDependencies();
463
464                String transactionType = myVersionAdapter.getBundleType(theRequest);
465
466                if (org.hl7.fhir.r4.model.Bundle.BundleType.BATCH.toCode().equals(transactionType)) {
467                        return batch(theRequestDetails, theRequest, theNestedMode);
468                }
469
470                if (transactionType == null) {
471                        String message = "Transaction Bundle did not specify valid Bundle.type, assuming "
472                                        + Bundle.BundleType.TRANSACTION.toCode();
473                        ourLog.warn(message);
474                        transactionType = org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode();
475                }
476                if (!org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode().equals(transactionType)) {
477                        throw new InvalidRequestException(
478                                        Msg.code(527) + "Unable to process transaction where incoming Bundle.type = " + transactionType);
479                }
480
481                List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest);
482                int numberOfEntries = requestEntries.size();
483
484                if (myStorageSettings.getMaximumTransactionBundleSize() != null
485                                && numberOfEntries > myStorageSettings.getMaximumTransactionBundleSize()) {
486                        throw new PayloadTooLargeException(
487                                        Msg.code(528) + "Transaction Bundle Too large.  Transaction bundle contains " + numberOfEntries
488                                                        + " which exceedes the maximum permitted transaction bundle size of "
489                                                        + myStorageSettings.getMaximumTransactionBundleSize());
490                }
491
492                ourLog.debug("Beginning {} with {} resources", theActionName, numberOfEntries);
493
494                final TransactionDetails transactionDetails = new TransactionDetails();
495                transactionDetails.setFhirTransaction(true);
496                final StopWatch transactionStopWatch = new StopWatch();
497
498                // Do all entries have a verb?
499                for (int i = 0; i < numberOfEntries; i++) {
500                        IBase nextReqEntry = requestEntries.get(i);
501                        String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
502                        if (verb == null || !isValidVerb(verb)) {
503                                throw new InvalidRequestException(Msg.code(529)
504                                                + myContext
505                                                                .getLocalizer()
506                                                                .getMessage(BaseStorageDao.class, "transactionEntryHasInvalidVerb", verb, i));
507                        }
508                }
509
510                /*
511                 * We want to execute the transaction request bundle elements in the order
512                 * specified by the FHIR specification (see TransactionSorter) so we save the
513                 * original order in the request, then sort it.
514                 *
515                 * Entries with a type of GET are removed from the bundle so that they
516                 * can be processed at the very end. We do this because the incoming resources
517                 * are saved in a two-phase way in order to deal with interdependencies, and
518                 * we want the GET processing to use the final indexing state
519                 */
520                final IBaseBundle response =
521                                myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTIONRESPONSE.toCode());
522                List<IBase> getEntries = new ArrayList<>();
523                final IdentityHashMap<IBase, Integer> originalRequestOrder = new IdentityHashMap<>();
524                for (int i = 0; i < requestEntries.size(); i++) {
525                        IBase requestEntry = requestEntries.get(i);
526                        originalRequestOrder.put(requestEntry, i);
527                        myVersionAdapter.addEntry(response);
528                        if (myVersionAdapter.getEntryRequestVerb(myContext, requestEntry).equals("GET")) {
529                                getEntries.add(requestEntry);
530                        }
531                }
532
533                /*
534                 * See FhirSystemDaoDstu3Test#testTransactionWithPlaceholderIdInMatchUrl
535                 * Basically if the resource has a match URL that references a placeholder,
536                 * we try to handle the resource with the placeholder first.
537                 */
538                Set<String> placeholderIds = new HashSet<>();
539                for (IBase nextEntry : requestEntries) {
540                        String fullUrl = myVersionAdapter.getFullUrl(nextEntry);
541                        if (isNotBlank(fullUrl) && fullUrl.startsWith(URN_PREFIX)) {
542                                placeholderIds.add(fullUrl);
543                        }
544                }
545                requestEntries.sort(new TransactionSorter(placeholderIds));
546
547                // perform all writes
548                prepareThenExecuteTransactionWriteOperations(
549                                theRequestDetails,
550                                theActionName,
551                                transactionDetails,
552                                transactionStopWatch,
553                                response,
554                                originalRequestOrder,
555                                requestEntries);
556
557                // perform all gets
558                // (we do these last so that the gets happen on the final state of the DB;
559                // see above note)
560                doTransactionReadOperations(
561                                theRequestDetails, response, getEntries, originalRequestOrder, transactionStopWatch, theNestedMode);
562
563                // Interceptor broadcast: JPA_PERFTRACE_INFO
564                IInterceptorBroadcaster compositeBroadcaster =
565                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
566                if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO)) {
567                        String taskDurations = transactionStopWatch.formatTaskDurations();
568                        StorageProcessingMessage message = new StorageProcessingMessage();
569                        message.setMessage("Transaction timing:\n" + taskDurations);
570                        HookParams params = new HookParams()
571                                        .add(RequestDetails.class, theRequestDetails)
572                                        .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
573                                        .add(StorageProcessingMessage.class, message);
574                        compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
575                }
576
577                return response;
578        }
579
580        @SuppressWarnings("unchecked")
581        private void doTransactionReadOperations(
582                        final RequestDetails theRequestDetails,
583                        IBaseBundle theResponse,
584                        List<IBase> theGetEntries,
585                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
586                        StopWatch theTransactionStopWatch,
587                        boolean theNestedMode) {
588                if (theGetEntries.size() > 0) {
589                        theTransactionStopWatch.startTask("Process " + theGetEntries.size() + " GET entries");
590
591                        /*
592                         * Loop through the request and process any entries of type GET
593                         */
594                        for (IBase nextReqEntry : theGetEntries) {
595                                if (theNestedMode) {
596                                        throw new InvalidRequestException(
597                                                        Msg.code(530) + "Can not invoke read operation on nested transaction");
598                                }
599
600                                if (!(theRequestDetails instanceof ServletRequestDetails)) {
601                                        throw new MethodNotAllowedException(
602                                                        Msg.code(531) + "Can not call transaction GET methods from this context");
603                                }
604
605                                ServletRequestDetails srd = (ServletRequestDetails) theRequestDetails;
606                                Integer originalOrder = theOriginalRequestOrder.get(nextReqEntry);
607                                IBase nextRespEntry =
608                                                (IBase) myVersionAdapter.getEntries(theResponse).get(originalOrder);
609
610                                ArrayListMultimap<String, String> paramValues = ArrayListMultimap.create();
611
612                                String transactionUrl = extractTransactionUrlOrThrowException(nextReqEntry, "GET");
613
614                                ServletSubRequestDetails requestDetails =
615                                                ServletRequestUtil.getServletSubRequestDetails(srd, transactionUrl, paramValues);
616
617                                String url = requestDetails.getRequestPath();
618
619                                BaseMethodBinding method = srd.getServer().determineResourceMethod(requestDetails, url);
620                                if (method == null) {
621                                        throw new IllegalArgumentException(Msg.code(532) + "Unable to handle GET " + url);
622                                }
623
624                                if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) {
625                                        requestDetails.addHeader(
626                                                        Constants.HEADER_IF_MATCH, myVersionAdapter.getEntryRequestIfMatch(nextReqEntry));
627                                }
628                                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry))) {
629                                        requestDetails.addHeader(
630                                                        Constants.HEADER_IF_NONE_EXIST, myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry));
631                                }
632                                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry))) {
633                                        requestDetails.addHeader(
634                                                        Constants.HEADER_IF_NONE_MATCH, myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry));
635                                }
636
637                                Validate.isTrue(method instanceof BaseResourceReturningMethodBinding, "Unable to handle GET {}", url);
638                                try {
639                                        BaseResourceReturningMethodBinding methodBinding = (BaseResourceReturningMethodBinding) method;
640                                        requestDetails.setRestOperationType(methodBinding.getRestOperationType());
641
642                                        IBaseResource resource = methodBinding.doInvokeServer(srd.getServer(), requestDetails);
643                                        if (paramValues.containsKey(Constants.PARAM_SUMMARY)
644                                                        || paramValues.containsKey(Constants.PARAM_CONTENT)) {
645                                                resource = filterNestedBundle(requestDetails, resource);
646                                        }
647                                        myVersionAdapter.setResource(nextRespEntry, resource);
648                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
649                                } catch (NotModifiedException e) {
650                                        myVersionAdapter.setResponseStatus(
651                                                        nextRespEntry, toStatusString(Constants.STATUS_HTTP_304_NOT_MODIFIED));
652                                } catch (BaseServerResponseException e) {
653                                        ourLog.info("Failure processing transaction GET {}: {}", url, e.toString());
654                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(e.getStatusCode()));
655                                        populateEntryWithOperationOutcome(e, nextRespEntry);
656                                }
657                        }
658                        theTransactionStopWatch.endCurrentTask();
659                }
660        }
661
662        /**
663         * All of the write operations in the transaction (PUT, POST, etc.. basically anything
664         * except GET) are performed in their own database transaction before we do the reads.
665         * We do this because the reads (specifically the searches) often spawn their own
666         * secondary database transaction and if we allow that within the primary
667         * database transaction we can end up with deadlocks if the server is under
668         * heavy load with lots of concurrent transactions using all available
669         * database connections.
670         */
671        @SuppressWarnings("unchecked")
672        private void prepareThenExecuteTransactionWriteOperations(
673                        RequestDetails theRequestDetails,
674                        String theActionName,
675                        TransactionDetails theTransactionDetails,
676                        StopWatch theTransactionStopWatch,
677                        IBaseBundle theResponse,
678                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
679                        List<IBase> theEntries) {
680
681                TransactionWriteOperationsDetails writeOperationsDetails = null;
682                if (haveWriteOperationsHooks(theRequestDetails)) {
683                        writeOperationsDetails = buildWriteOperationsDetails(theEntries);
684                        callWriteOperationsHook(
685                                        Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE,
686                                        theRequestDetails,
687                                        theTransactionDetails,
688                                        writeOperationsDetails);
689                }
690
691                TransactionCallback<EntriesToProcessMap> txCallback = status -> {
692                        final Set<IIdType> allIds = new LinkedHashSet<>();
693                        final IdSubstitutionMap idSubstitutions = new IdSubstitutionMap();
694                        final Map<IIdType, DaoMethodOutcome> idToPersistedOutcome = new HashMap<>();
695
696                        EntriesToProcessMap retVal = doTransactionWriteOperations(
697                                        theRequestDetails,
698                                        theActionName,
699                                        theTransactionDetails,
700                                        allIds,
701                                        idSubstitutions,
702                                        idToPersistedOutcome,
703                                        theResponse,
704                                        theOriginalRequestOrder,
705                                        theEntries,
706                                        theTransactionStopWatch);
707
708                        theTransactionStopWatch.startTask("Commit writes to database");
709                        return retVal;
710                };
711                EntriesToProcessMap entriesToProcess;
712
713                RequestPartitionId requestPartitionId =
714                                determineRequestPartitionIdForWriteEntries(theRequestDetails, theEntries);
715
716                try {
717                        entriesToProcess = myHapiTransactionService
718                                        .withRequest(theRequestDetails)
719                                        .withRequestPartitionId(requestPartitionId)
720                                        .withTransactionDetails(theTransactionDetails)
721                                        .execute(txCallback);
722                } finally {
723                        if (haveWriteOperationsHooks(theRequestDetails)) {
724                                callWriteOperationsHook(
725                                                Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST,
726                                                theRequestDetails,
727                                                theTransactionDetails,
728                                                writeOperationsDetails);
729                        }
730                }
731
732                theTransactionStopWatch.endCurrentTask();
733
734                for (Map.Entry<IBase, IIdType> nextEntry : entriesToProcess.entrySet()) {
735                        String responseLocation = nextEntry.getValue().toUnqualified().getValue();
736                        String responseEtag = nextEntry.getValue().getVersionIdPart();
737                        myVersionAdapter.setResponseLocation(nextEntry.getKey(), responseLocation);
738                        myVersionAdapter.setResponseETag(nextEntry.getKey(), responseEtag);
739                }
740        }
741
742        /**
743         * This method looks at the FHIR actions being performed in a List of bundle entries,
744         * and determines the associated request partitions.
745         */
746        @Nullable
747        protected RequestPartitionId determineRequestPartitionIdForWriteEntries(
748                        RequestDetails theRequestDetails, List<IBase> theEntries) {
749                if (!myPartitionSettings.isPartitioningEnabled()) {
750                        return RequestPartitionId.allPartitions();
751                }
752
753                return theEntries.stream()
754                                .map(e -> getEntryRequestPartitionId(theRequestDetails, e))
755                                .reduce(null, (accumulator, nextPartition) -> {
756                                        if (accumulator == null) {
757                                                return nextPartition;
758                                        } else if (nextPartition == null) {
759                                                return accumulator;
760                                        } else if (myHapiTransactionService.isCompatiblePartition(accumulator, nextPartition)) {
761                                                return accumulator.mergeIds(nextPartition);
762                                        } else {
763                                                String msg = myContext
764                                                                .getLocalizer()
765                                                                .getMessage(
766                                                                                BaseTransactionProcessor.class, "multiplePartitionAccesses", theEntries.size());
767                                                throw new InvalidRequestException(Msg.code(2541) + msg);
768                                        }
769                                });
770        }
771
772        @Nullable
773        private RequestPartitionId getEntryRequestPartitionId(RequestDetails theRequestDetails, IBase nextEntry) {
774                RequestPartitionId nextWriteEntryRequestPartitionId = null;
775                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextEntry);
776                if (isNotBlank(verb)) {
777                        BundleEntryTransactionMethodEnum verbEnum = BundleEntryTransactionMethodEnum.valueOf(verb);
778                        switch (verbEnum) {
779                                case GET:
780                                        nextWriteEntryRequestPartitionId = null;
781                                        break;
782                                case DELETE: {
783                                        String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
784                                        if (isNotBlank(requestUrl)) {
785                                                IdType id = new IdType(requestUrl);
786                                                String resourceType = id.getResourceType();
787                                                ReadPartitionIdRequestDetails details =
788                                                                ReadPartitionIdRequestDetails.forDelete(resourceType, id);
789                                                nextWriteEntryRequestPartitionId =
790                                                                myRequestPartitionHelperService.determineReadPartitionForRequest(
791                                                                                theRequestDetails, details);
792                                        }
793                                        break;
794                                }
795                                case PATCH: {
796                                        String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
797                                        if (isNotBlank(requestUrl)) {
798                                                IdType id = new IdType(requestUrl);
799                                                String resourceType = id.getResourceType();
800                                                ReadPartitionIdRequestDetails details =
801                                                                ReadPartitionIdRequestDetails.forPatch(resourceType, id);
802                                                nextWriteEntryRequestPartitionId =
803                                                                myRequestPartitionHelperService.determineReadPartitionForRequest(
804                                                                                theRequestDetails, details);
805                                        }
806                                        break;
807                                }
808                                case POST:
809                                case PUT: {
810                                        IBaseResource resource = myVersionAdapter.getResource(nextEntry);
811                                        if (resource != null) {
812                                                String resourceType = myContext.getResourceType(resource);
813                                                nextWriteEntryRequestPartitionId =
814                                                                myRequestPartitionHelperService.determineCreatePartitionForRequest(
815                                                                                theRequestDetails, resource, resourceType);
816                                        }
817                                }
818                        }
819                }
820                return nextWriteEntryRequestPartitionId;
821        }
822
823        private boolean haveWriteOperationsHooks(RequestDetails theRequestDetails) {
824                IInterceptorBroadcaster compositeBroadcaster =
825                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
826                return compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE)
827                                || compositeBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST);
828        }
829
830        private void callWriteOperationsHook(
831                        Pointcut thePointcut,
832                        RequestDetails theRequestDetails,
833                        TransactionDetails theTransactionDetails,
834                        TransactionWriteOperationsDetails theWriteOperationsDetails) {
835                IInterceptorBroadcaster compositeBroadcaster =
836                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequestDetails);
837                if (compositeBroadcaster.hasHooks(thePointcut)) {
838                        HookParams params = new HookParams()
839                                        .add(TransactionDetails.class, theTransactionDetails)
840                                        .add(TransactionWriteOperationsDetails.class, theWriteOperationsDetails);
841                        compositeBroadcaster.callHooks(thePointcut, params);
842                }
843        }
844
845        @SuppressWarnings("unchecked")
846        private TransactionWriteOperationsDetails buildWriteOperationsDetails(List<IBase> theEntries) {
847                TransactionWriteOperationsDetails writeOperationsDetails;
848                List<String> updateRequestUrls = new ArrayList<>();
849                List<String> conditionalCreateRequestUrls = new ArrayList<>();
850                // Extract
851                for (IBase nextEntry : theEntries) {
852                        String method = myVersionAdapter.getEntryRequestVerb(myContext, nextEntry);
853                        if ("PUT".equals(method)) {
854                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
855                                if (isNotBlank(requestUrl)) {
856                                        updateRequestUrls.add(requestUrl);
857                                }
858                        } else if ("POST".equals(method)) {
859                                String requestUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextEntry);
860                                if (isNotBlank(requestUrl) && requestUrl.contains("?")) {
861                                        conditionalCreateRequestUrls.add(requestUrl);
862                                }
863                        }
864                }
865
866                writeOperationsDetails = new TransactionWriteOperationsDetails();
867                writeOperationsDetails.setUpdateRequestUrls(updateRequestUrls);
868                writeOperationsDetails.setConditionalCreateRequestUrls(conditionalCreateRequestUrls);
869                return writeOperationsDetails;
870        }
871
872        private boolean isValidVerb(String theVerb) {
873                try {
874                        return org.hl7.fhir.r4.model.Bundle.HTTPVerb.fromCode(theVerb) != null;
875                } catch (FHIRException theE) {
876                        return false;
877                }
878        }
879
880        /**
881         * This method is called for nested bundles (e.g. if we received a transaction with an entry that
882         * was a GET search, this method is called on the bundle for the search result, that will be placed in the
883         * outer bundle). This method applies the _summary and _content parameters to the output of
884         * that bundle.
885         * <p>
886         * TODO: This isn't the most efficient way of doing this.. hopefully we can come up with something better in the future.
887         */
888        private IBaseResource filterNestedBundle(RequestDetails theRequestDetails, IBaseResource theResource) {
889                IParser p = myContext.newJsonParser();
890                RestfulServerUtils.configureResponseParser(theRequestDetails, p);
891                return p.parseResource(theResource.getClass(), p.encodeResourceToString(theResource));
892        }
893
894        protected void validateDependencies() {
895                Validate.notNull(myContext);
896                Validate.notNull(myTxManager);
897        }
898
899        private IIdType newIdType(String theValue) {
900                return myContext.getVersion().newIdType().setValue(theValue);
901        }
902
903        /**
904         * Searches for duplicate conditional creates and consolidates them.
905         */
906        @SuppressWarnings("unchecked")
907        private void consolidateDuplicateConditionals(
908                        RequestDetails theRequestDetails, String theActionName, List<IBase> theEntries) {
909                final Set<String> keysWithNoFullUrl = new HashSet<>();
910                final HashMap<String, String> keyToUuid = new HashMap<>();
911
912                for (int index = 0, originalIndex = 0; index < theEntries.size(); index++, originalIndex++) {
913                        IBase nextReqEntry = theEntries.get(index);
914                        IBaseResource resource = myVersionAdapter.getResource(nextReqEntry);
915                        if (resource != null) {
916                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
917                                String entryFullUrl = myVersionAdapter.getFullUrl(nextReqEntry);
918                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextReqEntry);
919                                String ifNoneExist = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
920
921                                // Conditional UPDATE
922                                boolean consolidateEntryCandidate = false;
923                                String conditionalUrl;
924                                switch (verb) {
925                                        case "PUT":
926                                                conditionalUrl = requestUrl;
927                                                if (isNotBlank(requestUrl)) {
928                                                        int questionMarkIndex = requestUrl.indexOf('?');
929                                                        if (questionMarkIndex >= 0 && requestUrl.length() > (questionMarkIndex + 1)) {
930                                                                consolidateEntryCandidate = true;
931                                                        }
932                                                }
933                                                break;
934
935                                                // Conditional CREATE
936                                        case "POST":
937                                                conditionalUrl = ifNoneExist;
938                                                if (isNotBlank(ifNoneExist)) {
939                                                        if (isBlank(entryFullUrl) || !entryFullUrl.equals(requestUrl)) {
940                                                                consolidateEntryCandidate = true;
941                                                        }
942                                                }
943                                                break;
944
945                                        default:
946                                                continue;
947                                }
948
949                                if (isNotBlank(conditionalUrl) && !conditionalUrl.contains("?")) {
950                                        conditionalUrl = myContext.getResourceType(resource) + "?" + conditionalUrl;
951                                }
952
953                                String key = verb + "|" + conditionalUrl;
954                                if (consolidateEntryCandidate) {
955                                        if (isBlank(entryFullUrl)) {
956                                                if (isNotBlank(conditionalUrl)) {
957                                                        if (!keysWithNoFullUrl.add(key)) {
958                                                                throw new InvalidRequestException(Msg.code(2008) + "Unable to process " + theActionName
959                                                                                + " - Request contains multiple anonymous entries (Bundle.entry.fullUrl not populated) with conditional URL: \""
960                                                                                + UrlUtil.sanitizeUrlPart(conditionalUrl)
961                                                                                + "\". Does transaction request contain duplicates?");
962                                                        }
963                                                }
964                                        } else {
965                                                if (!keyToUuid.containsKey(key)) {
966                                                        keyToUuid.put(key, entryFullUrl);
967                                                } else {
968                                                        String msg = "Discarding transaction bundle entry " + originalIndex
969                                                                        + " as it contained a duplicate conditional " + verb;
970                                                        ourLog.info(msg);
971                                                        // Interceptor broadcast: JPA_PERFTRACE_INFO
972                                                        IInterceptorBroadcaster compositeBroadcaster =
973                                                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(
974                                                                                        myInterceptorBroadcaster, theRequestDetails);
975                                                        if (compositeBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_WARNING)) {
976                                                                StorageProcessingMessage message = new StorageProcessingMessage().setMessage(msg);
977                                                                HookParams params = new HookParams()
978                                                                                .add(RequestDetails.class, theRequestDetails)
979                                                                                .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
980                                                                                .add(StorageProcessingMessage.class, message);
981                                                                compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_INFO, params);
982                                                        }
983
984                                                        theEntries.remove(index);
985                                                        index--;
986                                                        String existingUuid = keyToUuid.get(key);
987                                                        replaceReferencesInEntriesWithConsolidatedUUID(theEntries, entryFullUrl, existingUuid);
988                                                }
989                                        }
990                                }
991                        }
992                }
993        }
994
995        /**
996         * Iterates over all entries, and if it finds any which have references which match the fullUrl of the entry that was consolidated out
997         * replace them with our new consolidated UUID
998         */
999        private void replaceReferencesInEntriesWithConsolidatedUUID(
1000                        List<IBase> theEntries, String theEntryFullUrl, String existingUuid) {
1001                for (IBase nextEntry : theEntries) {
1002                        @SuppressWarnings("unchecked")
1003                        IBaseResource nextResource = myVersionAdapter.getResource(nextEntry);
1004                        if (nextResource != null) {
1005                                for (IBaseReference nextReference :
1006                                                myContext.newTerser().getAllPopulatedChildElementsOfType(nextResource, IBaseReference.class)) {
1007                                        // We're interested in any references directly to the placeholder ID, but also
1008                                        // references that have a resource target that has the placeholder ID.
1009                                        String nextReferenceId = nextReference.getReferenceElement().getValue();
1010                                        if (isBlank(nextReferenceId) && nextReference.getResource() != null) {
1011                                                nextReferenceId =
1012                                                                nextReference.getResource().getIdElement().getValue();
1013                                        }
1014                                        if (theEntryFullUrl.equals(nextReferenceId)) {
1015                                                nextReference.setReference(existingUuid);
1016                                                nextReference.setResource(null);
1017                                        }
1018                                }
1019                        }
1020                }
1021        }
1022
1023        /**
1024         * Retrieves the next resource id (IIdType) from the base resource and next request entry.
1025         *
1026         * @param theBaseResource - base resource
1027         * @param theNextReqEntry - next request entry
1028         * @param theAllIds       - set of all IIdType values
1029         * @return
1030         */
1031        private IIdType getNextResourceIdFromBaseResource(
1032                        IBaseResource theBaseResource, IBase theNextReqEntry, Set<IIdType> theAllIds) {
1033                IIdType nextResourceId = null;
1034                if (theBaseResource != null) {
1035                        nextResourceId = theBaseResource.getIdElement();
1036
1037                        @SuppressWarnings("unchecked")
1038                        String fullUrl = myVersionAdapter.getFullUrl(theNextReqEntry);
1039                        if (isNotBlank(fullUrl)) {
1040                                IIdType fullUrlIdType = newIdType(fullUrl);
1041                                if (isPlaceholder(fullUrlIdType)) {
1042                                        nextResourceId = fullUrlIdType;
1043                                } else if (!nextResourceId.hasIdPart()) {
1044                                        nextResourceId = fullUrlIdType;
1045                                }
1046                        }
1047
1048                        if (nextResourceId.hasIdPart() && !isPlaceholder(nextResourceId)) {
1049                                int colonIndex = nextResourceId.getIdPart().indexOf(':');
1050                                if (colonIndex != -1) {
1051                                        if (INVALID_PLACEHOLDER_PATTERN
1052                                                        .matcher(nextResourceId.getIdPart())
1053                                                        .matches()) {
1054                                                throw new InvalidRequestException(
1055                                                                Msg.code(533) + "Invalid placeholder ID found: " + nextResourceId.getIdPart()
1056                                                                                + " - Must be of the form 'urn:uuid:[uuid]' or 'urn:oid:[oid]'");
1057                                        }
1058                                }
1059                        }
1060
1061                        if (nextResourceId.hasIdPart() && !nextResourceId.hasResourceType() && !isPlaceholder(nextResourceId)) {
1062                                nextResourceId = newIdType(toResourceName(theBaseResource.getClass()), nextResourceId.getIdPart());
1063                                theBaseResource.setId(nextResourceId);
1064                        }
1065
1066                        /*
1067                         * Ensure that the bundle doesn't have any duplicates, since this causes all kinds of weirdness
1068                         */
1069                        if (isPlaceholder(nextResourceId)) {
1070                                if (!theAllIds.add(nextResourceId)) {
1071                                        throw new InvalidRequestException(Msg.code(534)
1072                                                        + myContext
1073                                                                        .getLocalizer()
1074                                                                        .getMessage(
1075                                                                                        BaseStorageDao.class,
1076                                                                                        "transactionContainsMultipleWithDuplicateId",
1077                                                                                        nextResourceId));
1078                                }
1079                        } else if (nextResourceId.hasResourceType() && nextResourceId.hasIdPart()) {
1080                                IIdType nextId = nextResourceId.toUnqualifiedVersionless();
1081                                if (!theAllIds.add(nextId)) {
1082                                        throw new InvalidRequestException(Msg.code(535)
1083                                                        + myContext
1084                                                                        .getLocalizer()
1085                                                                        .getMessage(
1086                                                                                        BaseStorageDao.class,
1087                                                                                        "transactionContainsMultipleWithDuplicateId",
1088                                                                                        nextId));
1089                                }
1090                        }
1091                }
1092
1093                return nextResourceId;
1094        }
1095
1096        /**
1097         * After pre-hooks have been called
1098         */
1099        @SuppressWarnings({"unchecked", "rawtypes"})
1100        protected EntriesToProcessMap doTransactionWriteOperations(
1101                        final RequestDetails theRequest,
1102                        String theActionName,
1103                        TransactionDetails theTransactionDetails,
1104                        Set<IIdType> theAllIds,
1105                        IdSubstitutionMap theIdSubstitutions,
1106                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1107                        IBaseBundle theResponse,
1108                        IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
1109                        List<IBase> theEntries,
1110                        StopWatch theTransactionStopWatch) {
1111
1112                // During a transaction, we don't execute hooks, instead, we execute them all post-transaction.
1113                theTransactionDetails.beginAcceptingDeferredInterceptorBroadcasts(
1114                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED,
1115                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_UPDATED,
1116                                Pointcut.STORAGE_PRECOMMIT_RESOURCE_DELETED);
1117                try {
1118                        Set<String> deletedResources = new HashSet<>();
1119                        DeleteConflictList deleteConflicts = new DeleteConflictList();
1120                        EntriesToProcessMap entriesToProcess = new EntriesToProcessMap();
1121                        Set<IIdType> nonUpdatedEntities = new HashSet<>();
1122                        Set<IBasePersistedResource> updatedEntities = new HashSet<>();
1123                        Map<String, IIdType> conditionalUrlToIdMap = new HashMap<>();
1124                        List<IBaseResource> updatedResources = new ArrayList<>();
1125                        Map<String, Class<? extends IBaseResource>> conditionalRequestUrls = new HashMap<>();
1126
1127                        /*
1128                         * Look for duplicate conditional creates and consolidate them
1129                         */
1130                        consolidateDuplicateConditionals(theRequest, theActionName, theEntries);
1131
1132                        /*
1133                         * Loop through the request and process any entries of type
1134                         * PUT, POST or DELETE
1135                         */
1136                        String previousVerb = null;
1137                        for (int i = 0; i < theEntries.size(); i++) {
1138                                if (i % 250 == 0) {
1139                                        ourLog.debug("Processed {} non-GET entries out of {} in transaction", i, theEntries.size());
1140                                }
1141
1142                                IBase nextReqEntry = theEntries.get(i);
1143                                IBaseResource res = myVersionAdapter.getResource(nextReqEntry);
1144
1145                                IIdType nextResourceId = getNextResourceIdFromBaseResource(res, nextReqEntry, theAllIds);
1146
1147                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
1148                                if (previousVerb != null && !previousVerb.equals(verb)) {
1149                                        handleVerbChangeInTransactionWriteOperations();
1150                                }
1151                                previousVerb = verb;
1152
1153                                String resourceType = res != null ? myContext.getResourceType(res) : null;
1154                                Integer order = theOriginalRequestOrder.get(nextReqEntry);
1155                                IBase nextRespEntry =
1156                                                (IBase) myVersionAdapter.getEntries(theResponse).get(order);
1157
1158                                theTransactionStopWatch.startTask(
1159                                                "Bundle.entry[" + i + "]: " + verb + " " + defaultString(resourceType));
1160
1161                                if (res != null) {
1162                                        String previousResourceId = res.getIdElement().getValue();
1163                                        theTransactionDetails.addRollbackUndoAction(() -> res.setId(previousResourceId));
1164                                }
1165
1166                                switch (verb) {
1167                                        case "POST": {
1168                                                // CREATE
1169                                                /*
1170                                                 * To preserve existing functionality,
1171                                                 * we will only verify that the request url is
1172                                                 * valid if it's provided at all.
1173                                                 * Otherwise, we'll ignore it
1174                                                 */
1175                                                String url = myVersionAdapter.getEntryRequestUrl(nextReqEntry);
1176                                                if (isNotBlank(url)) {
1177                                                        extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1178                                                }
1179                                                validateResourcePresent(res, order, verb);
1180
1181                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
1182                                                res.setId((String) null);
1183
1184                                                DaoMethodOutcome outcome;
1185                                                String matchUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
1186                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1187                                                // create individual resource
1188                                                outcome = resourceDao.create(res, matchUrl, false, theRequest, theTransactionDetails);
1189                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1190                                                res.setId(outcome.getId());
1191
1192                                                if (nextResourceId != null) {
1193                                                        handleTransactionCreateOrUpdateOutcome(
1194                                                                        theIdSubstitutions,
1195                                                                        theIdToPersistedOutcome,
1196                                                                        nextResourceId,
1197                                                                        outcome,
1198                                                                        nextRespEntry,
1199                                                                        resourceType,
1200                                                                        res,
1201                                                                        theRequest);
1202                                                }
1203                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1204                                                theTransactionDetails.addResolvedResource(outcome.getId(), outcome::getResource);
1205                                                if (outcome.getCreated() == false) {
1206                                                        nonUpdatedEntities.add(outcome.getId());
1207                                                } else {
1208                                                        if (isNotBlank(matchUrl)) {
1209                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
1210                                                        }
1211                                                }
1212
1213                                                break;
1214                                        }
1215                                        case "DELETE": {
1216                                                // DELETE
1217                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1218                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1219                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
1220                                                int status = Constants.STATUS_HTTP_204_NO_CONTENT;
1221                                                if (parts.getResourceId() != null) {
1222                                                        IIdType deleteId = newIdType(parts.getResourceType(), parts.getResourceId());
1223                                                        if (!deletedResources.contains(deleteId.getValueAsString())) {
1224                                                                DaoMethodOutcome outcome =
1225                                                                                dao.delete(deleteId, deleteConflicts, theRequest, theTransactionDetails);
1226                                                                if (outcome.getEntity() != null) {
1227                                                                        deletedResources.add(deleteId.getValueAsString());
1228                                                                        entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1229                                                                }
1230                                                                myVersionAdapter.setResponseOutcome(nextRespEntry, outcome.getOperationOutcome());
1231                                                        }
1232                                                } else {
1233                                                        String matchUrl = parts.getResourceType() + '?' + parts.getParams();
1234                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1235                                                        DeleteMethodOutcome deleteOutcome =
1236                                                                        dao.deleteByUrl(matchUrl, deleteConflicts, theRequest, theTransactionDetails);
1237                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, deleteOutcome.getId());
1238                                                        List<? extends IBasePersistedResource> allDeleted = deleteOutcome.getDeletedEntities();
1239                                                        for (IBasePersistedResource deleted : allDeleted) {
1240                                                                deletedResources.add(deleted.getIdDt()
1241                                                                                .toUnqualifiedVersionless()
1242                                                                                .getValueAsString());
1243                                                        }
1244                                                        if (allDeleted.isEmpty()) {
1245                                                                status = Constants.STATUS_HTTP_204_NO_CONTENT;
1246                                                        }
1247
1248                                                        myVersionAdapter.setResponseOutcome(nextRespEntry, deleteOutcome.getOperationOutcome());
1249                                                }
1250
1251                                                myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(status));
1252
1253                                                break;
1254                                        }
1255                                        case "PUT": {
1256                                                // UPDATE
1257                                                validateResourcePresent(res, order, verb);
1258                                                @SuppressWarnings("rawtypes")
1259                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
1260
1261                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1262
1263                                                DaoMethodOutcome outcome;
1264                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1265                                                if (isNotBlank(parts.getResourceId())) {
1266                                                        String version = null;
1267                                                        if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) {
1268                                                                version = ParameterUtil.parseETagValue(
1269                                                                                myVersionAdapter.getEntryRequestIfMatch(nextReqEntry));
1270                                                        }
1271                                                        res.setId(newIdType(parts.getResourceType(), parts.getResourceId(), version));
1272                                                        outcome = resourceDao.update(res, null, false, false, theRequest, theTransactionDetails);
1273                                                } else {
1274                                                        if (!shouldConditionalUpdateMatchId(theTransactionDetails, res.getIdElement())) {
1275                                                                res.setId((String) null);
1276                                                        }
1277                                                        String matchUrl;
1278                                                        if (isNotBlank(parts.getParams())) {
1279                                                                matchUrl = parts.getResourceType() + '?' + parts.getParams();
1280                                                        } else {
1281                                                                matchUrl = parts.getResourceType();
1282                                                        }
1283                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1284                                                        outcome =
1285                                                                        resourceDao.update(res, matchUrl, false, false, theRequest, theTransactionDetails);
1286                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1287                                                        if (Boolean.TRUE.equals(outcome.getCreated())) {
1288                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
1289                                                        }
1290                                                }
1291
1292                                                if (outcome.getCreated() == Boolean.FALSE
1293                                                                || (outcome.getCreated() == Boolean.TRUE
1294                                                                                && outcome.getId().getVersionIdPartAsLong() > 1)) {
1295                                                        updatedEntities.add(outcome.getEntity());
1296                                                        if (outcome.getResource() != null) {
1297                                                                updatedResources.add(outcome.getResource());
1298                                                        }
1299                                                }
1300
1301                                                theTransactionDetails.addResolvedResource(outcome.getId(), outcome::getResource);
1302                                                handleTransactionCreateOrUpdateOutcome(
1303                                                                theIdSubstitutions,
1304                                                                theIdToPersistedOutcome,
1305                                                                nextResourceId,
1306                                                                outcome,
1307                                                                nextRespEntry,
1308                                                                resourceType,
1309                                                                res,
1310                                                                theRequest);
1311                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1312                                                break;
1313                                        }
1314                                        case "PATCH": {
1315                                                // PATCH
1316                                                validateResourcePresent(res, order, verb);
1317
1318                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1319                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1320
1321                                                String matchUrl = toMatchUrl(nextReqEntry);
1322                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1323                                                String patchBody = null;
1324                                                String contentType;
1325                                                IBaseParameters patchBodyParameters = null;
1326                                                PatchTypeEnum patchType = null;
1327
1328                                                if (res instanceof IBaseBinary) {
1329                                                        IBaseBinary binary = (IBaseBinary) res;
1330                                                        if (binary.getContent() != null && binary.getContent().length > 0) {
1331                                                                patchBody = toUtf8String(binary.getContent());
1332                                                        }
1333                                                        contentType = binary.getContentType();
1334                                                        patchType =
1335                                                                        PatchTypeEnum.forContentTypeOrThrowInvalidRequestException(myContext, contentType);
1336                                                        if (patchType == PatchTypeEnum.FHIR_PATCH_JSON
1337                                                                        || patchType == PatchTypeEnum.FHIR_PATCH_XML) {
1338                                                                String msg = myContext
1339                                                                                .getLocalizer()
1340                                                                                .getMessage(
1341                                                                                                BaseTransactionProcessor.class, "fhirPatchShouldNotUseBinaryResource");
1342                                                                throw new InvalidRequestException(Msg.code(536) + msg);
1343                                                        }
1344                                                } else if (res instanceof IBaseParameters) {
1345                                                        patchBodyParameters = (IBaseParameters) res;
1346                                                        patchType = PatchTypeEnum.FHIR_PATCH_JSON;
1347                                                }
1348
1349                                                if (patchBodyParameters == null) {
1350                                                        if (isBlank(patchBody)) {
1351                                                                String msg = myContext
1352                                                                                .getLocalizer()
1353                                                                                .getMessage(BaseTransactionProcessor.class, "missingPatchBody");
1354                                                                throw new InvalidRequestException(Msg.code(537) + msg);
1355                                                        }
1356                                                }
1357
1358                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
1359                                                IIdType patchId =
1360                                                                myContext.getVersion().newIdType(parts.getResourceType(), parts.getResourceId());
1361
1362                                                String conditionalUrl;
1363                                                if (isNull(patchId.getIdPart())) {
1364                                                        conditionalUrl = url;
1365                                                } else {
1366                                                        conditionalUrl = matchUrl;
1367                                                        String ifMatch = myVersionAdapter.getEntryRequestIfMatch(nextReqEntry);
1368                                                        if (isNotBlank(ifMatch)) {
1369                                                                patchId = UpdateMethodBinding.applyETagAsVersion(ifMatch, patchId);
1370                                                        }
1371                                                }
1372
1373                                                DaoMethodOutcome outcome = dao.patchInTransaction(
1374                                                                patchId,
1375                                                                conditionalUrl,
1376                                                                false,
1377                                                                patchType,
1378                                                                patchBody,
1379                                                                patchBodyParameters,
1380                                                                theRequest,
1381                                                                theTransactionDetails);
1382                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1383                                                updatedEntities.add(outcome.getEntity());
1384                                                if (outcome.getResource() != null) {
1385                                                        updatedResources.add(outcome.getResource());
1386                                                }
1387                                                if (nextResourceId != null) {
1388                                                        handleTransactionCreateOrUpdateOutcome(
1389                                                                        theIdSubstitutions,
1390                                                                        theIdToPersistedOutcome,
1391                                                                        nextResourceId,
1392                                                                        outcome,
1393                                                                        nextRespEntry,
1394                                                                        resourceType,
1395                                                                        res,
1396                                                                        theRequest);
1397                                                }
1398                                                entriesToProcess.put(nextRespEntry, outcome.getId(), nextRespEntry);
1399
1400                                                break;
1401                                        }
1402                                        case "GET":
1403                                                break;
1404                                        default:
1405                                                throw new InvalidRequestException(
1406                                                                Msg.code(538) + "Unable to handle verb in transaction: " + verb);
1407                                }
1408
1409                                theTransactionStopWatch.endCurrentTask();
1410                        }
1411
1412                        postTransactionProcess(theTransactionDetails);
1413
1414                        /*
1415                         * Make sure that there are no conflicts from deletions. E.g. we can't delete something
1416                         * if something else has a reference to it.. Unless the thing that has a reference to it
1417                         * was also deleted as a part of this transaction, which is why we check this now at the
1418                         * end.
1419                         */
1420                        checkForDeleteConflicts(deleteConflicts, deletedResources, updatedResources);
1421
1422                        theIdToPersistedOutcome.entrySet().forEach(idAndOutcome -> {
1423                                theTransactionDetails.addResolvedResourceId(
1424                                                idAndOutcome.getKey(), idAndOutcome.getValue().getPersistentId());
1425                        });
1426
1427                        /*
1428                         * Perform ID substitutions and then index each resource we have saved
1429                         */
1430
1431                        resolveReferencesThenSaveAndIndexResources(
1432                                        theRequest,
1433                                        theTransactionDetails,
1434                                        theIdSubstitutions,
1435                                        theIdToPersistedOutcome,
1436                                        theTransactionStopWatch,
1437                                        entriesToProcess,
1438                                        nonUpdatedEntities,
1439                                        updatedEntities);
1440
1441                        theTransactionStopWatch.endCurrentTask();
1442
1443                        // flush writes to db
1444                        theTransactionStopWatch.startTask("Flush writes to database");
1445
1446                        // flush the changes
1447                        flushSession(theIdToPersistedOutcome);
1448
1449                        theTransactionStopWatch.endCurrentTask();
1450
1451                        /*
1452                         * Double check we didn't allow any duplicates we shouldn't have
1453                         */
1454                        if (conditionalRequestUrls.size() > 0) {
1455                                theTransactionStopWatch.startTask("Check for conflicts in conditional resources");
1456                        }
1457                        if (!myStorageSettings.isMassIngestionMode()) {
1458                                validateNoDuplicates(
1459                                                theRequest, theActionName, conditionalRequestUrls, theIdToPersistedOutcome.values());
1460                        }
1461
1462                        theTransactionStopWatch.endCurrentTask();
1463                        if (conditionalUrlToIdMap.size() > 0) {
1464                                theTransactionStopWatch.startTask(
1465                                                "Check that all conditionally created/updated entities actually match their conditionals.");
1466                        }
1467
1468                        if (!myStorageSettings.isMassIngestionMode()) {
1469                                validateAllInsertsMatchTheirConditionalUrls(theIdToPersistedOutcome, conditionalUrlToIdMap, theRequest);
1470                        }
1471                        theTransactionStopWatch.endCurrentTask();
1472
1473                        for (IIdType next : theAllIds) {
1474                                IIdType replacement = theIdSubstitutions.getForSource(next);
1475                                if (replacement != null && !replacement.equals(next)) {
1476                                        ourLog.debug(
1477                                                        "Placeholder resource ID \"{}\" was replaced with permanent ID \"{}\"", next, replacement);
1478                                }
1479                        }
1480
1481                        IInterceptorBroadcaster compositeBroadcaster =
1482                                        CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequest);
1483                        ListMultimap<Pointcut, HookParams> deferredBroadcastEvents =
1484                                        theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1485                        for (Map.Entry<Pointcut, HookParams> nextEntry : deferredBroadcastEvents.entries()) {
1486                                Pointcut nextPointcut = nextEntry.getKey();
1487                                HookParams nextParams = nextEntry.getValue();
1488                                compositeBroadcaster.callHooks(nextPointcut, nextParams);
1489                        }
1490
1491                        DeferredInterceptorBroadcasts deferredInterceptorBroadcasts =
1492                                        new DeferredInterceptorBroadcasts(deferredBroadcastEvents);
1493                        HookParams params = new HookParams()
1494                                        .add(RequestDetails.class, theRequest)
1495                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
1496                                        .add(DeferredInterceptorBroadcasts.class, deferredInterceptorBroadcasts)
1497                                        .add(TransactionDetails.class, theTransactionDetails)
1498                                        .add(IBaseBundle.class, theResponse);
1499                        compositeBroadcaster.callHooks(Pointcut.STORAGE_TRANSACTION_PROCESSED, params);
1500
1501                        theTransactionDetails.deferredBroadcastProcessingFinished();
1502
1503                        // finishedCallingDeferredInterceptorBroadcasts
1504
1505                        return entriesToProcess;
1506
1507                } finally {
1508                        if (theTransactionDetails.isAcceptingDeferredInterceptorBroadcasts()) {
1509                                theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1510                        }
1511                }
1512        }
1513
1514        /**
1515         * Subclasses may override this in order to invoke specific operations when
1516         * we're finished handling all the write entries in the transaction bundle
1517         * with a given verb.
1518         */
1519        protected void handleVerbChangeInTransactionWriteOperations() {
1520                // nothing
1521        }
1522
1523        /**
1524         * Implement to handle post transaction processing
1525         */
1526        protected void postTransactionProcess(TransactionDetails theTransactionDetails) {
1527                // nothing
1528        }
1529
1530        /**
1531         * Check for if a resource id should be matched in a conditional update
1532         * If the FHIR version is older than R4, it follows the old specifications and does not match
1533         * If the resource id has been resolved, then it is an existing resource and does not need to be matched
1534         * If the resource id is local or a placeholder, the id is temporary and should not be matched
1535         */
1536        private boolean shouldConditionalUpdateMatchId(TransactionDetails theTransactionDetails, IIdType theId) {
1537                if (myContext.getVersion().getVersion().isOlderThan(FhirVersionEnum.R4)) {
1538                        return false;
1539                }
1540                if (theTransactionDetails.hasResolvedResourceId(theId)
1541                                && !theTransactionDetails.isResolvedResourceIdEmpty(theId)) {
1542                        return false;
1543                }
1544                if (theId != null && theId.getValue() != null) {
1545                        return !(theId.getValue().startsWith("urn:") || theId.getValue().startsWith("#"));
1546                }
1547                return true;
1548        }
1549
1550        private boolean shouldSwapBinaryToActualResource(
1551                        IBaseResource theResource, String theResourceType, IIdType theNextResourceId) {
1552                if ("Binary".equalsIgnoreCase(theResourceType)
1553                                && theNextResourceId.getResourceType() != null
1554                                && !theNextResourceId.getResourceType().equalsIgnoreCase("Binary")) {
1555                        return true;
1556                } else {
1557                        return false;
1558                }
1559        }
1560
1561        private void setConditionalUrlToBeValidatedLater(
1562                        Map<String, IIdType> theConditionalUrlToIdMap, String theMatchUrl, IIdType theId) {
1563                if (!StringUtils.isBlank(theMatchUrl)) {
1564                        theConditionalUrlToIdMap.put(theMatchUrl, theId);
1565                }
1566        }
1567
1568        /**
1569         * After transaction processing and resolution of indexes and references, we want to validate that the resources that were stored _actually_
1570         * match the conditional URLs that they were brought in on.
1571         */
1572        private void validateAllInsertsMatchTheirConditionalUrls(
1573                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1574                        Map<String, IIdType> conditionalUrlToIdMap,
1575                        RequestDetails theRequest) {
1576                conditionalUrlToIdMap.entrySet().stream()
1577                                .filter(entry -> entry.getKey() != null)
1578                                .forEach(entry -> {
1579                                        String matchUrl = entry.getKey();
1580                                        IIdType value = entry.getValue();
1581                                        DaoMethodOutcome daoMethodOutcome = theIdToPersistedOutcome.get(value);
1582                                        if (daoMethodOutcome != null
1583                                                        && !daoMethodOutcome.isNop()
1584                                                        && daoMethodOutcome.getResource() != null) {
1585                                                InMemoryMatchResult match =
1586                                                                mySearchParamMatcher.match(matchUrl, daoMethodOutcome.getResource(), theRequest);
1587                                                if (ourLog.isDebugEnabled()) {
1588                                                        ourLog.debug(
1589                                                                        "Checking conditional URL [{}] against resource with ID [{}]: Supported?:[{}], Matched?:[{}]",
1590                                                                        matchUrl,
1591                                                                        value,
1592                                                                        match.supported(),
1593                                                                        match.matched());
1594                                                }
1595                                                if (match.supported()) {
1596                                                        if (!match.matched()) {
1597                                                                throw new PreconditionFailedException(Msg.code(539) + "Invalid conditional URL \""
1598                                                                                + matchUrl + "\". The given resource is not matched by this URL.");
1599                                                        }
1600                                                }
1601                                        }
1602                                });
1603        }
1604
1605        /**
1606         * Checks for any delete conflicts.
1607         *
1608         * @param theDeleteConflicts  - set of delete conflicts
1609         * @param theDeletedResources - set of deleted resources
1610         * @param theUpdatedResources - list of updated resources
1611         */
1612        private void checkForDeleteConflicts(
1613                        DeleteConflictList theDeleteConflicts,
1614                        Set<String> theDeletedResources,
1615                        List<IBaseResource> theUpdatedResources) {
1616                for (Iterator<DeleteConflict> iter = theDeleteConflicts.iterator(); iter.hasNext(); ) {
1617                        DeleteConflict nextDeleteConflict = iter.next();
1618
1619                        /*
1620                         * If we have a conflict, it means we can't delete Resource/A because
1621                         * Resource/B has a reference to it. We'll ignore that conflict though
1622                         * if it turns out we're also deleting Resource/B in this transaction.
1623                         */
1624                        if (theDeletedResources.contains(
1625                                        nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue())) {
1626                                iter.remove();
1627                                continue;
1628                        }
1629
1630                        /*
1631                         * And then, this is kind of a last ditch check. It's also ok to delete
1632                         * Resource/A if Resource/B isn't being deleted, but it is being UPDATED
1633                         * in this transaction, and the updated version of it has no references
1634                         * to Resource/A any more.
1635                         */
1636                        String sourceId =
1637                                        nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue();
1638                        String targetId =
1639                                        nextDeleteConflict.getTargetId().toUnqualifiedVersionless().getValue();
1640                        Optional<IBaseResource> updatedSource = theUpdatedResources.stream()
1641                                        .filter(t -> sourceId.equals(
1642                                                        t.getIdElement().toUnqualifiedVersionless().getValue()))
1643                                        .findFirst();
1644                        if (updatedSource.isPresent()) {
1645                                List<ResourceReferenceInfo> referencesInSource =
1646                                                myContext.newTerser().getAllResourceReferences(updatedSource.get());
1647                                boolean sourceStillReferencesTarget = referencesInSource.stream()
1648                                                .anyMatch(t -> targetId.equals(t.getResourceReference()
1649                                                                .getReferenceElement()
1650                                                                .toUnqualifiedVersionless()
1651                                                                .getValue()));
1652                                if (!sourceStillReferencesTarget) {
1653                                        iter.remove();
1654                                }
1655                        }
1656                }
1657                DeleteConflictUtil.validateDeleteConflictsEmptyOrThrowException(myContext, theDeleteConflicts);
1658        }
1659
1660        /**
1661         * This method replaces any placeholder references in the
1662         * source transaction Bundle with their actual targets, then stores the resource contents and indexes
1663         * in the database. This is trickier than you'd think because of a couple of possibilities during the
1664         * save:
1665         * * There may be resources that have not changed (e.g. an update/PUT with a resource body identical
1666         * to what is already in the database)
1667         * * There may be resources with auto-versioned references, meaning we're replacing certain references
1668         * in the resource with a versioned references, referencing the current version at the time of the
1669         * transaction processing
1670         * * There may by auto-versioned references pointing to these unchanged targets
1671         * <p>
1672         * If we're not doing any auto-versioned references, we'll just iterate through all resources in the
1673         * transaction and save them one at a time.
1674         * <p>
1675         * However, if we have any auto-versioned references we do this in 2 passes: First the resources from the
1676         * transaction that don't have any auto-versioned references are stored. We do them first since there's
1677         * a chance they may be a NOP and we'll need to account for their version number not actually changing.
1678         * Then we do a second pass for any resources that have auto-versioned references. These happen in a separate
1679         * pass because it's too complex to try and insert the auto-versioned references and still
1680         * account for NOPs, so we block NOPs in that pass.
1681         */
1682        private void resolveReferencesThenSaveAndIndexResources(
1683                        RequestDetails theRequest,
1684                        TransactionDetails theTransactionDetails,
1685                        IdSubstitutionMap theIdSubstitutions,
1686                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1687                        StopWatch theTransactionStopWatch,
1688                        EntriesToProcessMap entriesToProcess,
1689                        Set<IIdType> nonUpdatedEntities,
1690                        Set<IBasePersistedResource> updatedEntities) {
1691                FhirTerser terser = myContext.newTerser();
1692                theTransactionStopWatch.startTask("Index " + theIdToPersistedOutcome.size() + " resources");
1693                IdentityHashMap<DaoMethodOutcome, Set<IBaseReference>> deferredIndexesForAutoVersioning = null;
1694                int i = 0;
1695                for (DaoMethodOutcome nextOutcome : theIdToPersistedOutcome.values()) {
1696
1697                        if (i++ % 250 == 0) {
1698                                ourLog.debug(
1699                                                "Have indexed {} entities out of {} in transaction",
1700                                                i,
1701                                                theIdToPersistedOutcome.values().size());
1702                        }
1703
1704                        if (nextOutcome.isNop()) {
1705                                continue;
1706                        }
1707
1708                        IBaseResource nextResource = nextOutcome.getResource();
1709                        if (nextResource == null) {
1710                                continue;
1711                        }
1712
1713                        Set<IBaseReference> referencesToAutoVersion =
1714                                        BaseStorageDao.extractReferencesToAutoVersion(myContext, myStorageSettings, nextResource);
1715                        if (referencesToAutoVersion.isEmpty()) {
1716                                // no references to autoversion - we can do the resolve and save now
1717                                resolveReferencesThenSaveAndIndexResource(
1718                                                theRequest,
1719                                                theTransactionDetails,
1720                                                theIdSubstitutions,
1721                                                theIdToPersistedOutcome,
1722                                                entriesToProcess,
1723                                                nonUpdatedEntities,
1724                                                updatedEntities,
1725                                                terser,
1726                                                nextOutcome,
1727                                                nextResource,
1728                                                referencesToAutoVersion); // this is empty
1729                        } else {
1730                                // we have autoversioned things to defer until later
1731                                if (deferredIndexesForAutoVersioning == null) {
1732                                        deferredIndexesForAutoVersioning = new IdentityHashMap<>();
1733                                }
1734                                deferredIndexesForAutoVersioning.put(nextOutcome, referencesToAutoVersion);
1735                        }
1736                }
1737
1738                // If we have any resources we'll be auto-versioning, index these next
1739                if (deferredIndexesForAutoVersioning != null) {
1740                        for (Map.Entry<DaoMethodOutcome, Set<IBaseReference>> nextEntry :
1741                                        deferredIndexesForAutoVersioning.entrySet()) {
1742                                DaoMethodOutcome nextOutcome = nextEntry.getKey();
1743                                Set<IBaseReference> referencesToAutoVersion = nextEntry.getValue();
1744                                IBaseResource nextResource = nextOutcome.getResource();
1745
1746                                resolveReferencesThenSaveAndIndexResource(
1747                                                theRequest,
1748                                                theTransactionDetails,
1749                                                theIdSubstitutions,
1750                                                theIdToPersistedOutcome,
1751                                                entriesToProcess,
1752                                                nonUpdatedEntities,
1753                                                updatedEntities,
1754                                                terser,
1755                                                nextOutcome,
1756                                                nextResource,
1757                                                referencesToAutoVersion);
1758                        }
1759                }
1760        }
1761
1762        private void resolveReferencesThenSaveAndIndexResource(
1763                        RequestDetails theRequest,
1764                        TransactionDetails theTransactionDetails,
1765                        IdSubstitutionMap theIdSubstitutions,
1766                        Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1767                        EntriesToProcessMap entriesToProcess,
1768                        Set<IIdType> nonUpdatedEntities,
1769                        Set<IBasePersistedResource> updatedEntities,
1770                        FhirTerser terser,
1771                        DaoMethodOutcome theDaoMethodOutcome,
1772                        IBaseResource theResource,
1773                        Set<IBaseReference> theReferencesToAutoVersion) {
1774                // References
1775                List<ResourceReferenceInfo> allRefs = terser.getAllResourceReferences(theResource);
1776                for (ResourceReferenceInfo nextRef : allRefs) {
1777                        IBaseReference resourceReference = nextRef.getResourceReference();
1778                        IIdType nextId = resourceReference.getReferenceElement();
1779                        IIdType newId = null;
1780                        if (!nextId.hasIdPart()) {
1781                                if (resourceReference.getResource() != null) {
1782                                        IIdType targetId = resourceReference.getResource().getIdElement();
1783                                        if (targetId.getValue() == null || targetId.getValue().startsWith("#")) {
1784                                                // This means it's a contained resource
1785                                                continue;
1786                                        } else if (theIdSubstitutions.containsTarget(targetId)) {
1787                                                newId = targetId;
1788                                        } else {
1789                                                throw new InternalErrorException(Msg.code(540)
1790                                                                + "References by resource with no reference ID are not supported in DAO layer");
1791                                        }
1792                                } else {
1793                                        continue;
1794                                }
1795                        }
1796                        if (newId != null || theIdSubstitutions.containsSource(nextId)) {
1797                                if (newId == null) {
1798                                        newId = theIdSubstitutions.getForSource(nextId);
1799                                }
1800                                if (newId != null) {
1801                                        ourLog.debug(" * Replacing resource ref {} with {}", nextId, newId);
1802
1803                                        if (theReferencesToAutoVersion.contains(resourceReference)) {
1804                                                replaceResourceReference(newId, resourceReference, theTransactionDetails);
1805                                        } else {
1806                                                replaceResourceReference(newId.toVersionless(), resourceReference, theTransactionDetails);
1807                                        }
1808                                }
1809                        } else if (nextId.getValue().startsWith("urn:")) {
1810                                throw new InvalidRequestException(
1811                                                Msg.code(541) + "Unable to satisfy placeholder ID " + nextId.getValue()
1812                                                                + " found in element named '" + nextRef.getName() + "' within resource of type: "
1813                                                                + theResource.getIdElement().getResourceType());
1814                        } else {
1815                                // get a map of
1816                                // existing ids -> PID (for resources that exist in the DB)
1817                                // should this be allPartitions?
1818                                ResourcePersistentIdMap resourceVersionMap = myResourceVersionSvc.getLatestVersionIdsForResourceIds(
1819                                                RequestPartitionId.allPartitions(),
1820                                                theReferencesToAutoVersion.stream()
1821                                                                .map(IBaseReference::getReferenceElement)
1822                                                                .collect(Collectors.toList()));
1823
1824                                for (IBaseReference baseRef : theReferencesToAutoVersion) {
1825                                        IIdType id = baseRef.getReferenceElement();
1826                                        if (!resourceVersionMap.containsKey(id)
1827                                                        && myStorageSettings.isAutoCreatePlaceholderReferenceTargets()) {
1828                                                // not in the db, but autocreateplaceholders is true
1829                                                // so the version we'll set is "1" (since it will be
1830                                                // created later)
1831                                                String newRef = id.withVersion("1").getValue();
1832                                                id.setValue(newRef);
1833                                        } else {
1834                                                // we will add the looked up info to the transaction
1835                                                // for later
1836                                                theTransactionDetails.addResolvedResourceId(id, resourceVersionMap.getResourcePersistentId(id));
1837                                        }
1838                                }
1839
1840                                if (theReferencesToAutoVersion.contains(resourceReference)) {
1841                                        DaoMethodOutcome outcome = theIdToPersistedOutcome.get(nextId);
1842
1843                                        if (outcome != null && !outcome.isNop() && !Boolean.TRUE.equals(outcome.getCreated())) {
1844                                                replaceResourceReference(nextId, resourceReference, theTransactionDetails);
1845                                        }
1846
1847                                        // if referenced resource is not in transaction but exists in the DB, resolving its version
1848                                        IResourcePersistentId persistedReferenceId = resourceVersionMap.getResourcePersistentId(nextId);
1849                                        if (outcome == null && persistedReferenceId != null && persistedReferenceId.getVersion() != null) {
1850                                                IIdType newReferenceId = nextId.withVersion(
1851                                                                persistedReferenceId.getVersion().toString());
1852                                                replaceResourceReference(newReferenceId, resourceReference, theTransactionDetails);
1853                                        }
1854                                }
1855                        }
1856                }
1857
1858                // URIs
1859                Class<? extends IPrimitiveType<?>> uriType = (Class<? extends IPrimitiveType<?>>)
1860                                myContext.getElementDefinition("uri").getImplementingClass();
1861                List<? extends IPrimitiveType<?>> allUris = terser.getAllPopulatedChildElementsOfType(theResource, uriType);
1862                for (IPrimitiveType<?> nextRef : allUris) {
1863                        if (nextRef instanceof IIdType) {
1864                                continue; // No substitution on the resource ID itself!
1865                        }
1866                        String nextUriString = nextRef.getValueAsString();
1867                        if (isNotBlank(nextUriString)) {
1868                                if (theIdSubstitutions.containsSource(nextUriString)) {
1869                                        IIdType newId = theIdSubstitutions.getForSource(nextUriString);
1870                                        ourLog.debug(" * Replacing resource ref {} with {}", nextUriString, newId);
1871
1872                                        String existingValue = nextRef.getValueAsString();
1873                                        theTransactionDetails.addRollbackUndoAction(() -> nextRef.setValueAsString(existingValue));
1874
1875                                        nextRef.setValueAsString(newId.toVersionless().getValue());
1876                                } else {
1877                                        ourLog.debug(" * Reference [{}] does not exist in bundle", nextUriString);
1878                                }
1879                        }
1880                }
1881
1882                IPrimitiveType<Date> deletedInstantOrNull = ResourceMetadataKeyEnum.DELETED_AT.get(theResource);
1883                Date deletedTimestampOrNull = deletedInstantOrNull != null ? deletedInstantOrNull.getValue() : null;
1884
1885                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(theResource.getClass());
1886                IJpaDao jpaDao = (IJpaDao) dao;
1887
1888                IBasePersistedResource updateOutcome = null;
1889                if (updatedEntities.contains(theDaoMethodOutcome.getEntity())) {
1890                        boolean forceUpdateVersion = !theReferencesToAutoVersion.isEmpty();
1891                        String matchUrl = theDaoMethodOutcome.getMatchUrl();
1892                        RestOperationTypeEnum operationType = theDaoMethodOutcome.getOperationType();
1893                        DaoMethodOutcome daoMethodOutcome = jpaDao.updateInternal(
1894                                        theRequest,
1895                                        theResource,
1896                                        matchUrl,
1897                                        true,
1898                                        forceUpdateVersion,
1899                                        theDaoMethodOutcome.getEntity(),
1900                                        theResource.getIdElement(),
1901                                        theDaoMethodOutcome.getPreviousResource(),
1902                                        operationType,
1903                                        theTransactionDetails);
1904                        updateOutcome = daoMethodOutcome.getEntity();
1905                        theDaoMethodOutcome = daoMethodOutcome;
1906                } else if (!nonUpdatedEntities.contains(theDaoMethodOutcome.getId())) {
1907                        updateOutcome = jpaDao.updateEntity(
1908                                        theRequest,
1909                                        theResource,
1910                                        theDaoMethodOutcome.getEntity(),
1911                                        deletedTimestampOrNull,
1912                                        true,
1913                                        false,
1914                                        theTransactionDetails,
1915                                        false,
1916                                        true);
1917                }
1918
1919                // Make sure we reflect the actual final version for the resource.
1920                if (updateOutcome != null) {
1921                        IIdType newId = updateOutcome.getIdDt();
1922
1923                        IIdType entryId = entriesToProcess.getIdWithVersionlessComparison(newId);
1924                        if (entryId != null && !StringUtils.equals(entryId.getValue(), newId.getValue())) {
1925                                entryId.setValue(newId.getValue());
1926                        }
1927
1928                        theDaoMethodOutcome.setId(newId);
1929
1930                        theIdSubstitutions.updateTargets(newId);
1931
1932                        if (theDaoMethodOutcome.getOperationOutcome() != null) {
1933                                IBase responseEntry = entriesToProcess.getResponseBundleEntryWithVersionlessComparison(newId);
1934                                myVersionAdapter.setResponseOutcome(responseEntry, theDaoMethodOutcome.getOperationOutcome());
1935                        }
1936                }
1937        }
1938
1939        private void replaceResourceReference(
1940                        IIdType theReferenceId, IBaseReference theResourceReference, TransactionDetails theTransactionDetails) {
1941                addRollbackReferenceRestore(theTransactionDetails, theResourceReference);
1942                theResourceReference.setReference(theReferenceId.getValue());
1943                theResourceReference.setResource(null);
1944        }
1945
1946        private void addRollbackReferenceRestore(
1947                        TransactionDetails theTransactionDetails, IBaseReference resourceReference) {
1948                String existingValue = resourceReference.getReferenceElement().getValue();
1949                theTransactionDetails.addRollbackUndoAction(() -> resourceReference.setReference(existingValue));
1950        }
1951
1952        private void validateNoDuplicates(
1953                        RequestDetails theRequest,
1954                        String theActionName,
1955                        Map<String, Class<? extends IBaseResource>> conditionalRequestUrls,
1956                        Collection<DaoMethodOutcome> thePersistedOutcomes) {
1957
1958                IdentityHashMap<IBaseResource, ResourceIndexedSearchParams> resourceToIndexedParams =
1959                                new IdentityHashMap<>(thePersistedOutcomes.size());
1960                thePersistedOutcomes.stream()
1961                                .filter(t -> !t.isNop())
1962                                .filter(t -> t.getEntity()
1963                                                instanceof ResourceTable) // N.B. GGG: This validation never occurs for mongo, as nothing is a
1964                                // ResourceTable.
1965                                .filter(t -> t.getEntity().getDeleted() == null)
1966                                .filter(t -> t.getResource() != null)
1967                                .forEach(t -> resourceToIndexedParams.put(
1968                                                t.getResource(), ResourceIndexedSearchParams.withLists((ResourceTable) t.getEntity())));
1969
1970                for (Map.Entry<String, Class<? extends IBaseResource>> nextEntry : conditionalRequestUrls.entrySet()) {
1971                        String matchUrl = nextEntry.getKey();
1972                        if (isNotBlank(matchUrl)) {
1973                                if (matchUrl.startsWith("?")
1974                                                || (!matchUrl.contains("?")
1975                                                                && UNQUALIFIED_MATCH_URL_START.matcher(matchUrl).find())) {
1976                                        StringBuilder b = new StringBuilder();
1977                                        b.append(myContext.getResourceType(nextEntry.getValue()));
1978                                        if (!matchUrl.startsWith("?")) {
1979                                                b.append("?");
1980                                        }
1981                                        b.append(matchUrl);
1982                                        matchUrl = b.toString();
1983                                }
1984
1985                                if (!myInMemoryResourceMatcher.canBeEvaluatedInMemory(matchUrl).supported()) {
1986                                        continue;
1987                                }
1988
1989                                int counter = 0;
1990                                for (Map.Entry<IBaseResource, ResourceIndexedSearchParams> entries :
1991                                                resourceToIndexedParams.entrySet()) {
1992                                        ResourceIndexedSearchParams indexedParams = entries.getValue();
1993                                        IBaseResource resource = entries.getKey();
1994
1995                                        String resourceType = myContext.getResourceType(resource);
1996                                        if (!matchUrl.startsWith(resourceType + "?")) {
1997                                                continue;
1998                                        }
1999
2000                                        if (myInMemoryResourceMatcher
2001                                                        .match(matchUrl, resource, indexedParams, theRequest)
2002                                                        .matched()) {
2003                                                counter++;
2004                                                if (counter > 1) {
2005                                                        throw new InvalidRequestException(Msg.code(542) + "Unable to process " + theActionName
2006                                                                        + " - Request would cause multiple resources to match URL: \"" + matchUrl
2007                                                                        + "\". Does transaction request contain duplicates?");
2008                                                }
2009                                        }
2010                                }
2011                        }
2012                }
2013        }
2014
2015        protected abstract void flushSession(Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome);
2016
2017        private void validateResourcePresent(IBaseResource theResource, Integer theOrder, String theVerb) {
2018                if (theResource == null) {
2019                        String msg = myContext
2020                                        .getLocalizer()
2021                                        .getMessage(BaseTransactionProcessor.class, "missingMandatoryResource", theVerb, theOrder);
2022                        throw new InvalidRequestException(Msg.code(543) + msg);
2023                }
2024        }
2025
2026        private IIdType newIdType(String theResourceType, String theResourceId, String theVersion) {
2027                org.hl7.fhir.r4.model.IdType id = new org.hl7.fhir.r4.model.IdType(theResourceType, theResourceId, theVersion);
2028                return myContext.getVersion().newIdType().setValue(id.getValue());
2029        }
2030
2031        private IIdType newIdType(String theToResourceName, String theIdPart) {
2032                return newIdType(theToResourceName, theIdPart, null);
2033        }
2034
2035        @VisibleForTesting
2036        public void setDaoRegistry(DaoRegistry theDaoRegistry) {
2037                myDaoRegistry = theDaoRegistry;
2038        }
2039
2040        private IFhirResourceDao getDaoOrThrowException(Class<? extends IBaseResource> theClass) {
2041                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDaoOrNull(theClass);
2042                if (dao == null) {
2043                        Set<String> types = new TreeSet<>(myDaoRegistry.getRegisteredDaoTypes());
2044                        String type = myContext.getResourceType(theClass);
2045                        String msg = myContext
2046                                        .getLocalizer()
2047                                        .getMessage(BaseTransactionProcessor.class, "unsupportedResourceType", type, types.toString());
2048                        throw new InvalidRequestException(Msg.code(544) + msg);
2049                }
2050                return dao;
2051        }
2052
2053        private String toResourceName(Class<? extends IBaseResource> theResourceType) {
2054                return myContext.getResourceType(theResourceType);
2055        }
2056
2057        public void setContext(FhirContext theContext) {
2058                myContext = theContext;
2059        }
2060
2061        /**
2062         * Extracts the transaction url from the entry and verifies it's:
2063         * * not null or blank
2064         * * is a relative url matching the resourceType it is about
2065         * <p>
2066         * Returns the transaction url (or throws an InvalidRequestException if url is not valid)
2067         */
2068        private String extractAndVerifyTransactionUrlForEntry(IBase theEntry, String theVerb) {
2069                String url = extractTransactionUrlOrThrowException(theEntry, theVerb);
2070
2071                if (!isValidResourceTypeUrl(url)) {
2072                        ourLog.debug("Invalid url. Should begin with a resource type: {}", url);
2073                        String msg =
2074                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, url);
2075                        throw new InvalidRequestException(Msg.code(2006) + msg);
2076                }
2077                return url;
2078        }
2079
2080        /**
2081         * Returns true if the provided url is a valid entry request.url.
2082         * <p>
2083         * This means:
2084         * a) not an absolute url (does not start with http/https)
2085         * b) starts with either a ResourceType or /ResourceType
2086         */
2087        private boolean isValidResourceTypeUrl(@Nonnull String theUrl) {
2088                if (UrlUtil.isAbsolute(theUrl)) {
2089                        return false;
2090                } else {
2091                        int queryStringIndex = theUrl.indexOf("?");
2092                        String url;
2093                        if (queryStringIndex > 0) {
2094                                url = theUrl.substring(0, theUrl.indexOf("?"));
2095                        } else {
2096                                url = theUrl;
2097                        }
2098                        String[] parts;
2099                        if (url.startsWith("/")) {
2100                                parts = url.substring(1).split("/");
2101                        } else {
2102                                parts = url.split("/");
2103                        }
2104                        Set<String> allResourceTypes = myContext.getResourceTypes();
2105
2106                        return allResourceTypes.contains(parts[0]);
2107                }
2108        }
2109
2110        /**
2111         * Extracts the transaction url from the entry and verifies that it is not null/blank
2112         * and returns it
2113         */
2114        private String extractTransactionUrlOrThrowException(IBase nextEntry, String verb) {
2115                String url = myVersionAdapter.getEntryRequestUrl(nextEntry);
2116                if (isBlank(url)) {
2117                        throw new InvalidRequestException(Msg.code(545)
2118                                        + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionMissingUrl", verb));
2119                }
2120                return url;
2121        }
2122
2123        private IFhirResourceDao<? extends IBaseResource> toDao(UrlUtil.UrlParts theParts, String theVerb, String theUrl) {
2124                RuntimeResourceDefinition resType;
2125                try {
2126                        resType = myContext.getResourceDefinition(theParts.getResourceType());
2127                } catch (DataFormatException e) {
2128                        String msg =
2129                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
2130                        throw new InvalidRequestException(Msg.code(546) + msg);
2131                }
2132                IFhirResourceDao<? extends IBaseResource> dao = null;
2133                if (resType != null) {
2134                        dao = myDaoRegistry.getResourceDao(resType.getImplementingClass());
2135                }
2136                if (dao == null) {
2137                        String msg =
2138                                        myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
2139                        throw new InvalidRequestException(Msg.code(547) + msg);
2140                }
2141
2142                return dao;
2143        }
2144
2145        private String toMatchUrl(IBase theEntry) {
2146                String verb = myVersionAdapter.getEntryRequestVerb(myContext, theEntry);
2147                switch (defaultString(verb)) {
2148                        case "POST":
2149                                return myVersionAdapter.getEntryIfNoneExist(theEntry);
2150                        case "PUT":
2151                        case "DELETE":
2152                        case "PATCH":
2153                                String url = extractTransactionUrlOrThrowException(theEntry, verb);
2154                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
2155                                if (isBlank(parts.getResourceId())) {
2156                                        return parts.getResourceType() + '?' + parts.getParams();
2157                                }
2158                                return null;
2159                        default:
2160                                return null;
2161                }
2162        }
2163
2164        @VisibleForTesting
2165        public void setPartitionSettingsForUnitTest(PartitionSettings thePartitionSettings) {
2166                myPartitionSettings = thePartitionSettings;
2167        }
2168
2169        /**
2170         * Transaction Order, per the spec:
2171         * <p>
2172         * Process any DELETE interactions
2173         * Process any POST interactions
2174         * Process any PUT interactions
2175         * Process any PATCH interactions
2176         * Process any GET interactions
2177         */
2178        // @formatter:off
2179        public class TransactionSorter implements Comparator<IBase> {
2180
2181                private final Set<String> myPlaceholderIds;
2182
2183                public TransactionSorter(Set<String> thePlaceholderIds) {
2184                        myPlaceholderIds = thePlaceholderIds;
2185                }
2186
2187                @Override
2188                public int compare(IBase theO1, IBase theO2) {
2189                        int o1 = toOrder(theO1);
2190                        int o2 = toOrder(theO2);
2191
2192                        if (o1 == o2) {
2193                                String matchUrl1 = toMatchUrl(theO1);
2194                                String matchUrl2 = toMatchUrl(theO2);
2195                                if (isBlank(matchUrl1) && isBlank(matchUrl2)) {
2196                                        return 0;
2197                                }
2198                                if (isBlank(matchUrl1)) {
2199                                        return -1;
2200                                }
2201                                if (isBlank(matchUrl2)) {
2202                                        return 1;
2203                                }
2204
2205                                boolean match1containsSubstitutions = false;
2206                                boolean match2containsSubstitutions = false;
2207                                for (String nextPlaceholder : myPlaceholderIds) {
2208                                        if (matchUrl1.contains(nextPlaceholder)) {
2209                                                match1containsSubstitutions = true;
2210                                        }
2211                                        if (matchUrl2.contains(nextPlaceholder)) {
2212                                                match2containsSubstitutions = true;
2213                                        }
2214                                }
2215
2216                                if (match1containsSubstitutions && match2containsSubstitutions) {
2217                                        return 0;
2218                                }
2219                                if (!match1containsSubstitutions && !match2containsSubstitutions) {
2220                                        return 0;
2221                                }
2222                                if (match1containsSubstitutions) {
2223                                        return 1;
2224                                } else {
2225                                        return -1;
2226                                }
2227                        }
2228
2229                        return o1 - o2;
2230                }
2231
2232                private int toOrder(IBase theO1) {
2233                        int o1 = 0;
2234                        if (myVersionAdapter.getEntryRequestVerb(myContext, theO1) != null) {
2235                                switch (myVersionAdapter.getEntryRequestVerb(myContext, theO1)) {
2236                                        case "DELETE":
2237                                                o1 = 1;
2238                                                break;
2239                                        case "POST":
2240                                                o1 = 2;
2241                                                break;
2242                                        case "PUT":
2243                                                o1 = 3;
2244                                                break;
2245                                        case "PATCH":
2246                                                o1 = 4;
2247                                                break;
2248                                        case "GET":
2249                                                o1 = 5;
2250                                                break;
2251                                        default:
2252                                                o1 = 0;
2253                                                break;
2254                                }
2255                        }
2256                        return o1;
2257                }
2258        }
2259
2260        public class RetriableBundleTask implements Runnable {
2261
2262                private final CountDownLatch myCompletedLatch;
2263                private final RequestDetails myRequestDetails;
2264                private final IBase myNextReqEntry;
2265                private final Map<Integer, Object> myResponseMap;
2266                private final int myResponseOrder;
2267                private final boolean myNestedMode;
2268                private BaseServerResponseException myLastSeenException;
2269
2270                protected RetriableBundleTask(
2271                                CountDownLatch theCompletedLatch,
2272                                RequestDetails theRequestDetails,
2273                                Map<Integer, Object> theResponseMap,
2274                                int theResponseOrder,
2275                                IBase theNextReqEntry,
2276                                boolean theNestedMode) {
2277                        this.myCompletedLatch = theCompletedLatch;
2278                        this.myRequestDetails = theRequestDetails;
2279                        this.myNextReqEntry = theNextReqEntry;
2280                        this.myResponseMap = theResponseMap;
2281                        this.myResponseOrder = theResponseOrder;
2282                        this.myNestedMode = theNestedMode;
2283                        this.myLastSeenException = null;
2284                }
2285
2286                private void processBatchEntry() {
2287                        IBaseBundle subRequestBundle =
2288                                        myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode());
2289                        myVersionAdapter.addEntry(subRequestBundle, myNextReqEntry);
2290
2291                        IBaseBundle nextResponseBundle = processTransactionAsSubRequest(
2292                                        myRequestDetails, subRequestBundle, "Batch sub-request", myNestedMode);
2293
2294                        IBase subResponseEntry =
2295                                        (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
2296                        myResponseMap.put(myResponseOrder, subResponseEntry);
2297
2298                        /*
2299                         * If the individual entry didn't have a resource in its response, bring the sub-transaction's OperationOutcome across so the client can see it
2300                         */
2301                        if (myVersionAdapter.getResource(subResponseEntry) == null) {
2302                                IBase nextResponseBundleFirstEntry =
2303                                                (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
2304                                myResponseMap.put(myResponseOrder, nextResponseBundleFirstEntry);
2305                        }
2306                }
2307
2308                private boolean processBatchEntryWithRetry() {
2309                        int maxAttempts = 3;
2310                        for (int attempt = 1; ; attempt++) {
2311                                try {
2312                                        processBatchEntry();
2313                                        return true;
2314                                } catch (BaseServerResponseException e) {
2315                                        // If we catch a known and structured exception from HAPI, just fail.
2316                                        myLastSeenException = e;
2317                                        return false;
2318                                } catch (Throwable t) {
2319                                        myLastSeenException = new InternalErrorException(t);
2320                                        // If we have caught a non-tag-storage failure we are unfamiliar with, or we have exceeded max
2321                                        // attempts, exit.
2322                                        if (!DaoFailureUtil.isTagStorageFailure(t) || attempt >= maxAttempts) {
2323                                                ourLog.error("Failure during BATCH sub transaction processing", t);
2324                                                return false;
2325                                        }
2326                                }
2327                        }
2328                }
2329
2330                @Override
2331                public void run() {
2332                        boolean success = processBatchEntryWithRetry();
2333                        if (!success) {
2334                                populateResponseMapWithLastSeenException();
2335                        }
2336
2337                        // checking for the parallelism
2338                        ourLog.debug("processing batch for {} is completed", myVersionAdapter.getEntryRequestUrl(myNextReqEntry));
2339                        myCompletedLatch.countDown();
2340                }
2341
2342                private void populateResponseMapWithLastSeenException() {
2343                        ServerResponseExceptionHolder caughtEx = new ServerResponseExceptionHolder();
2344                        caughtEx.setException(myLastSeenException);
2345                        myResponseMap.put(myResponseOrder, caughtEx);
2346                }
2347        }
2348
2349        private static class ServerResponseExceptionHolder {
2350                private BaseServerResponseException myException;
2351
2352                public BaseServerResponseException getException() {
2353                        return myException;
2354                }
2355
2356                public void setException(BaseServerResponseException myException) {
2357                        this.myException = myException;
2358                }
2359        }
2360
2361        public static boolean isPlaceholder(IIdType theId) {
2362                if (theId != null && theId.getValue() != null) {
2363                        return theId.getValue().startsWith("urn:oid:") || theId.getValue().startsWith("urn:uuid:");
2364                }
2365                return false;
2366        }
2367
2368        private static String toStatusString(int theStatusCode) {
2369                return theStatusCode + " " + defaultString(Constants.HTTP_STATUS_NAMES.get(theStatusCode));
2370        }
2371
2372        /**
2373         * Given a match URL containing
2374         *
2375         * @param theIdSubstitutions
2376         * @param theMatchUrl
2377         * @return
2378         */
2379        public static String performIdSubstitutionsInMatchUrl(IdSubstitutionMap theIdSubstitutions, String theMatchUrl) {
2380                String matchUrl = theMatchUrl;
2381                if (isNotBlank(matchUrl) && !theIdSubstitutions.isEmpty()) {
2382                        int startIdx = 0;
2383                        while (startIdx != -1) {
2384
2385                                int endIdx = matchUrl.indexOf('&', startIdx + 1);
2386                                if (endIdx == -1) {
2387                                        endIdx = matchUrl.length();
2388                                }
2389
2390                                int equalsIdx = matchUrl.indexOf('=', startIdx + 1);
2391
2392                                int searchFrom;
2393                                if (equalsIdx == -1) {
2394                                        searchFrom = matchUrl.length();
2395                                } else if (equalsIdx >= endIdx) {
2396                                        // First equals we found is from a subsequent parameter
2397                                        searchFrom = matchUrl.length();
2398                                } else {
2399                                        String paramValue = matchUrl.substring(equalsIdx + 1, endIdx);
2400                                        boolean isUrn = isUrn(paramValue);
2401                                        boolean isUrnEscaped = !isUrn && isUrnEscaped(paramValue);
2402                                        if (isUrn || isUrnEscaped) {
2403                                                if (isUrnEscaped) {
2404                                                        paramValue = UrlUtil.unescape(paramValue);
2405                                                }
2406                                                IIdType replacement = theIdSubstitutions.getForSource(paramValue);
2407                                                if (replacement != null) {
2408                                                        String replacementValue;
2409                                                        if (replacement.hasVersionIdPart()) {
2410                                                                replacementValue = replacement.toVersionless().getValue();
2411                                                        } else {
2412                                                                replacementValue = replacement.getValue();
2413                                                        }
2414                                                        matchUrl = matchUrl.substring(0, equalsIdx + 1)
2415                                                                        + replacementValue
2416                                                                        + matchUrl.substring(endIdx);
2417                                                        searchFrom = equalsIdx + 1 + replacementValue.length();
2418                                                } else {
2419                                                        searchFrom = endIdx;
2420                                                }
2421                                        } else {
2422                                                searchFrom = endIdx;
2423                                        }
2424                                }
2425
2426                                if (searchFrom >= matchUrl.length()) {
2427                                        break;
2428                                }
2429
2430                                startIdx = matchUrl.indexOf('&', searchFrom);
2431                        }
2432                }
2433                return matchUrl;
2434        }
2435
2436        private static boolean isUrn(@Nonnull String theId) {
2437                return theId.startsWith(URN_PREFIX);
2438        }
2439
2440        private static boolean isUrnEscaped(@Nonnull String theId) {
2441                return theId.startsWith(URN_PREFIX_ESCAPED);
2442        }
2443}