001package ca.uhn.fhir.jpa.dao;
002
003/*-
004 * #%L
005 * HAPI FHIR Storage api
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.FhirContext;
024import ca.uhn.fhir.context.RuntimeResourceDefinition;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.HookParams;
027import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
028import ca.uhn.fhir.interceptor.api.Pointcut;
029import ca.uhn.fhir.interceptor.model.RequestPartitionId;
030import ca.uhn.fhir.interceptor.model.TransactionWriteOperationsDetails;
031import ca.uhn.fhir.jpa.api.config.DaoConfig;
032import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
033import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
034import ca.uhn.fhir.jpa.api.dao.IJpaDao;
035import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome;
036import ca.uhn.fhir.jpa.api.model.DeleteConflict;
037import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
038import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome;
039import ca.uhn.fhir.jpa.api.model.LazyDaoMethodOutcome;
040import ca.uhn.fhir.jpa.cache.IResourceVersionSvc;
041import ca.uhn.fhir.jpa.cache.ResourcePersistentIdMap;
042import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
043import ca.uhn.fhir.jpa.delete.DeleteConflictUtil;
044import ca.uhn.fhir.jpa.model.cross.IBasePersistedResource;
045import ca.uhn.fhir.jpa.model.entity.ModelConfig;
046import ca.uhn.fhir.jpa.model.entity.ResourceTable;
047import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
048import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams;
049import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult;
050import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher;
051import ca.uhn.fhir.jpa.searchparam.matcher.SearchParamMatcher;
052import ca.uhn.fhir.model.api.IResource;
053import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
054import ca.uhn.fhir.parser.DataFormatException;
055import ca.uhn.fhir.parser.IParser;
056import ca.uhn.fhir.rest.api.Constants;
057import ca.uhn.fhir.rest.api.PatchTypeEnum;
058import ca.uhn.fhir.rest.api.PreferReturnEnum;
059import ca.uhn.fhir.rest.api.RestOperationTypeEnum;
060import ca.uhn.fhir.rest.api.server.RequestDetails;
061import ca.uhn.fhir.rest.api.server.storage.DeferredInterceptorBroadcasts;
062import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
063import ca.uhn.fhir.rest.param.ParameterUtil;
064import ca.uhn.fhir.rest.server.RestfulServerUtils;
065import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException;
066import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
067import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
068import ca.uhn.fhir.rest.server.exceptions.MethodNotAllowedException;
069import ca.uhn.fhir.rest.server.exceptions.NotModifiedException;
070import ca.uhn.fhir.rest.server.exceptions.PayloadTooLargeException;
071import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
072import ca.uhn.fhir.rest.server.interceptor.IServerInterceptor;
073import ca.uhn.fhir.rest.server.method.BaseMethodBinding;
074import ca.uhn.fhir.rest.server.method.BaseResourceReturningMethodBinding;
075import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
076import ca.uhn.fhir.rest.server.servlet.ServletSubRequestDetails;
077import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
078import ca.uhn.fhir.rest.server.util.ServletRequestUtil;
079import ca.uhn.fhir.util.AsyncUtil;
080import ca.uhn.fhir.util.ElementUtil;
081import ca.uhn.fhir.util.FhirTerser;
082import ca.uhn.fhir.util.ResourceReferenceInfo;
083import ca.uhn.fhir.util.StopWatch;
084import ca.uhn.fhir.util.ThreadPoolUtil;
085import ca.uhn.fhir.util.UrlUtil;
086import com.google.common.annotations.VisibleForTesting;
087import com.google.common.collect.ArrayListMultimap;
088import com.google.common.collect.ListMultimap;
089import org.apache.commons.lang3.StringUtils;
090import org.apache.commons.lang3.Validate;
091import org.hl7.fhir.dstu3.model.Bundle;
092import org.hl7.fhir.exceptions.FHIRException;
093import org.hl7.fhir.instance.model.api.IAnyResource;
094import org.hl7.fhir.instance.model.api.IBase;
095import org.hl7.fhir.instance.model.api.IBaseBinary;
096import org.hl7.fhir.instance.model.api.IBaseBundle;
097import org.hl7.fhir.instance.model.api.IBaseParameters;
098import org.hl7.fhir.instance.model.api.IBaseReference;
099import org.hl7.fhir.instance.model.api.IBaseResource;
100import org.hl7.fhir.instance.model.api.IIdType;
101import org.hl7.fhir.instance.model.api.IPrimitiveType;
102import org.slf4j.Logger;
103import org.slf4j.LoggerFactory;
104import org.springframework.beans.factory.annotation.Autowired;
105import org.springframework.core.task.SyncTaskExecutor;
106import org.springframework.core.task.TaskExecutor;
107import org.springframework.transaction.PlatformTransactionManager;
108import org.springframework.transaction.TransactionDefinition;
109import org.springframework.transaction.support.TransactionCallback;
110import org.springframework.transaction.support.TransactionTemplate;
111
112import javax.annotation.Nonnull;
113import javax.annotation.PostConstruct;
114import java.util.ArrayList;
115import java.util.Collection;
116import java.util.Comparator;
117import java.util.Date;
118import java.util.HashMap;
119import java.util.HashSet;
120import java.util.IdentityHashMap;
121import java.util.Iterator;
122import java.util.LinkedHashSet;
123import java.util.List;
124import java.util.Map;
125import java.util.Optional;
126import java.util.Set;
127import java.util.TreeSet;
128import java.util.concurrent.ConcurrentHashMap;
129import java.util.concurrent.CountDownLatch;
130import java.util.concurrent.TimeUnit;
131import java.util.regex.Pattern;
132import java.util.stream.Collectors;
133
134import static ca.uhn.fhir.util.StringUtil.toUtf8String;
135import static java.util.Objects.isNull;
136import static org.apache.commons.lang3.StringUtils.defaultString;
137import static org.apache.commons.lang3.StringUtils.isBlank;
138import static org.apache.commons.lang3.StringUtils.isNotBlank;
139
140public abstract class BaseTransactionProcessor {
141
142        public static final String URN_PREFIX = "urn:";
143        public static final String URN_PREFIX_ESCAPED = UrlUtil.escapeUrlParam(URN_PREFIX);
144        public static final Pattern UNQUALIFIED_MATCH_URL_START = Pattern.compile("^[a-zA-Z0-9_]+=");
145        private static final Logger ourLog = LoggerFactory.getLogger(BaseTransactionProcessor.class);
146        public static final Pattern INVALID_PLACEHOLDER_PATTERN = Pattern.compile("[a-zA-Z]+:.*");
147        private BaseStorageDao myDao;
148        @Autowired
149        private PlatformTransactionManager myTxManager;
150        @Autowired
151        private FhirContext myContext;
152        @Autowired
153        private ITransactionProcessorVersionAdapter myVersionAdapter;
154        @Autowired
155        private DaoRegistry myDaoRegistry;
156        @Autowired
157        private IInterceptorBroadcaster myInterceptorBroadcaster;
158        @Autowired
159        private HapiTransactionService myHapiTransactionService;
160        @Autowired
161        private DaoConfig myDaoConfig;
162        @Autowired
163        private ModelConfig myModelConfig;
164        @Autowired
165        private InMemoryResourceMatcher myInMemoryResourceMatcher;
166        @Autowired
167        private SearchParamMatcher mySearchParamMatcher;
168
169        private TaskExecutor myExecutor;
170
171        @Autowired
172        private IResourceVersionSvc myResourceVersionSvc;
173
174        @VisibleForTesting
175        public void setDaoConfig(DaoConfig theDaoConfig) {
176                myDaoConfig = theDaoConfig;
177        }
178
179        public ITransactionProcessorVersionAdapter getVersionAdapter() {
180                return myVersionAdapter;
181        }
182
183        @VisibleForTesting
184        public void setVersionAdapter(ITransactionProcessorVersionAdapter theVersionAdapter) {
185                myVersionAdapter = theVersionAdapter;
186        }
187
188        @PostConstruct
189        public void start() {
190                ourLog.trace("Starting transaction processor");
191        }
192
193        private TaskExecutor getTaskExecutor() {
194                if (myExecutor == null) {
195                        if (myDaoConfig.getBundleBatchPoolSize() > 1) {
196                                myExecutor = ThreadPoolUtil.newThreadPool(myDaoConfig.getBundleBatchPoolSize(), myDaoConfig.getBundleBatchMaxPoolSize(), "bundle-batch-");
197                        } else {
198                                SyncTaskExecutor executor = new SyncTaskExecutor();
199                                myExecutor = executor;
200                        }
201                }
202                return myExecutor;
203        }
204
205        public <BUNDLE extends IBaseBundle> BUNDLE transaction(RequestDetails theRequestDetails, BUNDLE theRequest, boolean theNestedMode) {
206                if (theRequestDetails != null && theRequestDetails.getServer() != null && myDao != null) {
207                        IServerInterceptor.ActionRequestDetails requestDetails = new IServerInterceptor.ActionRequestDetails(theRequestDetails, theRequest, "Bundle", null);
208                        myDao.notifyInterceptors(RestOperationTypeEnum.TRANSACTION, requestDetails);
209                }
210
211                String actionName = "Transaction";
212                IBaseBundle response = processTransactionAsSubRequest(theRequestDetails, theRequest, actionName, theNestedMode);
213
214                List<IBase> entries = myVersionAdapter.getEntries(response);
215                for (int i = 0; i < entries.size(); i++) {
216                        if (ElementUtil.isEmpty(entries.get(i))) {
217                                entries.remove(i);
218                                i--;
219                        }
220                }
221
222                return (BUNDLE) response;
223        }
224
225        public IBaseBundle collection(final RequestDetails theRequestDetails, IBaseBundle theRequest) {
226                String transactionType = myVersionAdapter.getBundleType(theRequest);
227
228                if (!org.hl7.fhir.r4.model.Bundle.BundleType.COLLECTION.toCode().equals(transactionType)) {
229                        throw new InvalidRequestException(Msg.code(526) + "Can not process collection Bundle of type: " + transactionType);
230                }
231
232                ourLog.info("Beginning storing collection with {} resources", myVersionAdapter.getEntries(theRequest).size());
233
234                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
235                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
236
237                IBaseBundle resp = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
238
239                List<IBaseResource> resources = new ArrayList<>();
240                for (final Object nextRequestEntry : myVersionAdapter.getEntries(theRequest)) {
241                        IBaseResource resource = myVersionAdapter.getResource((IBase) nextRequestEntry);
242                        resources.add(resource);
243                }
244
245                IBaseBundle transactionBundle = myVersionAdapter.createBundle("transaction");
246                for (IBaseResource next : resources) {
247                        IBase entry = myVersionAdapter.addEntry(transactionBundle);
248                        myVersionAdapter.setResource(entry, next);
249                        myVersionAdapter.setRequestVerb(entry, "PUT");
250                        myVersionAdapter.setRequestUrl(entry, next.getIdElement().toUnqualifiedVersionless().getValue());
251                }
252
253                transaction(theRequestDetails, transactionBundle, false);
254
255                return resp;
256        }
257
258        private void populateEntryWithOperationOutcome(BaseServerResponseException caughtEx, IBase nextEntry) {
259                myVersionAdapter.populateEntryWithOperationOutcome(caughtEx, nextEntry);
260        }
261
262        private void handleTransactionCreateOrUpdateOutcome(IdSubstitutionMap idSubstitutions, Map<IIdType, DaoMethodOutcome> idToPersistedOutcome,
263                                                                                                                                                 IIdType nextResourceId, DaoMethodOutcome outcome,
264                                                                                                                                                 IBase newEntry, String theResourceType,
265                                                                                                                                                 IBaseResource theRes, RequestDetails theRequestDetails) {
266                IIdType newId = outcome.getId().toUnqualified();
267                IIdType resourceId = isPlaceholder(nextResourceId) ? nextResourceId : nextResourceId.toUnqualifiedVersionless();
268                if (newId.equals(resourceId) == false) {
269                        if (!nextResourceId.isEmpty()) {
270                                idSubstitutions.put(resourceId, newId);
271                        }
272                        if (isPlaceholder(resourceId)) {
273                                /*
274                                 * The correct way for substitution IDs to be is to be with no resource type, but we'll accept the qualified kind too just to be lenient.
275                                 */
276                                IIdType id = myContext.getVersion().newIdType();
277                                id.setValue(theResourceType + '/' + resourceId.getValue());
278                                idSubstitutions.put(id, newId);
279                        }
280                }
281
282                populateIdToPersistedOutcomeMap(idToPersistedOutcome, newId, outcome);
283
284                if(shouldSwapBinaryToActualResource(theRes, theResourceType, nextResourceId)) {
285                        theRes = idToPersistedOutcome.get(newId).getResource();
286                        theResourceType = idToPersistedOutcome.get(newId).getResource().fhirType();
287                }
288
289                if (outcome.getCreated()) {
290                        myVersionAdapter.setResponseStatus(newEntry, toStatusString(Constants.STATUS_HTTP_201_CREATED));
291                } else {
292                        myVersionAdapter.setResponseStatus(newEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
293                }
294                Date lastModified = getLastModified(theRes);
295                myVersionAdapter.setResponseLastModified(newEntry, lastModified);
296
297                if (theRequestDetails != null) {
298                        String prefer = theRequestDetails.getHeader(Constants.HEADER_PREFER);
299                        PreferReturnEnum preferReturn = RestfulServerUtils.parsePreferHeader(null, prefer).getReturn();
300                        if (preferReturn != null) {
301                                if (preferReturn == PreferReturnEnum.REPRESENTATION) {
302                                        if (outcome.getResource() != null) {
303                                                outcome.fireResourceViewCallbacks();
304                                                myVersionAdapter.setResource(newEntry, outcome.getResource());
305                                        }
306                                }
307                        }
308                }
309
310        }
311
312        /**
313         * Method which populates entry in idToPersistedOutcome.
314         * Will store whatever outcome is sent, unless the key already exists, then we only replace an instance if we find that the instance
315         * we are replacing with is non-lazy. This allows us to evaluate later more easily, as we _know_ we need access to these.
316         */
317        private void populateIdToPersistedOutcomeMap(Map<IIdType, DaoMethodOutcome> idToPersistedOutcome, IIdType newId, DaoMethodOutcome outcome) {
318                //Prefer real method outcomes over lazy ones.
319                if (idToPersistedOutcome.containsKey(newId)) {
320                        if (!(outcome instanceof LazyDaoMethodOutcome)) {
321                                idToPersistedOutcome.put(newId, outcome);
322                        }
323                } else {
324                        idToPersistedOutcome.put(newId, outcome);
325                }
326        }
327
328        private Date getLastModified(IBaseResource theRes) {
329                return theRes.getMeta().getLastUpdated();
330        }
331
332        public void setDao(BaseStorageDao theDao) {
333                myDao = theDao;
334        }
335
336        private IBaseBundle processTransactionAsSubRequest(RequestDetails theRequestDetails, IBaseBundle theRequest, String theActionName, boolean theNestedMode) {
337                BaseStorageDao.markRequestAsProcessingSubRequest(theRequestDetails);
338                try {
339                        return processTransaction(theRequestDetails, theRequest, theActionName, theNestedMode);
340                } finally {
341                        BaseStorageDao.clearRequestAsProcessingSubRequest(theRequestDetails);
342                }
343        }
344
345        @VisibleForTesting
346        public void setTxManager(PlatformTransactionManager theTxManager) {
347                myTxManager = theTxManager;
348        }
349
350        private IBaseBundle batch(final RequestDetails theRequestDetails, IBaseBundle theRequest, boolean theNestedMode) {
351                ourLog.info("Beginning batch with {} resources", myVersionAdapter.getEntries(theRequest).size());
352
353                long start = System.currentTimeMillis();
354
355                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
356                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
357
358                IBaseBundle response = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.BATCHRESPONSE.toCode());
359                Map<Integer, Object> responseMap = new ConcurrentHashMap<>();
360
361                List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest);
362                int requestEntriesSize = requestEntries.size();
363
364                // Now, run all non-gets sequentially, and all gets are submitted to the executor to run (potentially) in parallel
365                // The result is kept in the map to save the original position
366                List<RetriableBundleTask> getCalls = new ArrayList<>();
367                List<RetriableBundleTask> nonGetCalls = new ArrayList<>();
368
369                CountDownLatch completionLatch = new CountDownLatch(requestEntriesSize);
370                for (int i = 0; i < requestEntriesSize; i++) {
371                        IBase nextRequestEntry = requestEntries.get(i);
372                        RetriableBundleTask retriableBundleTask = new RetriableBundleTask(completionLatch, theRequestDetails, responseMap, i, nextRequestEntry, theNestedMode);
373                        if (myVersionAdapter.getEntryRequestVerb(myContext, nextRequestEntry).equalsIgnoreCase("GET")) {
374                                getCalls.add(retriableBundleTask);
375                        } else {
376                                nonGetCalls.add(retriableBundleTask);
377                        }
378                }
379                //Execute all non-gets on calling thread.
380                nonGetCalls.forEach(RetriableBundleTask::run);
381                //Execute all gets (potentially in a pool)
382                getCalls.forEach(getCall -> getTaskExecutor().execute(getCall));
383
384                // waiting for all async tasks to be completed
385                AsyncUtil.awaitLatchAndIgnoreInterrupt(completionLatch, 300L, TimeUnit.SECONDS);
386
387                // Now, create the bundle response in original order
388                Object nextResponseEntry;
389                for (int i = 0; i < requestEntriesSize; i++) {
390
391                        nextResponseEntry = responseMap.get(i);
392                        if (nextResponseEntry instanceof BaseServerResponseExceptionHolder) {
393                                BaseServerResponseExceptionHolder caughtEx = (BaseServerResponseExceptionHolder) nextResponseEntry;
394                                if (caughtEx.getException() != null) {
395                                        IBase nextEntry = myVersionAdapter.addEntry(response);
396                                        populateEntryWithOperationOutcome(caughtEx.getException(), nextEntry);
397                                        myVersionAdapter.setResponseStatus(nextEntry, toStatusString(caughtEx.getException().getStatusCode()));
398                                }
399                        } else {
400                                myVersionAdapter.addEntry(response, (IBase) nextResponseEntry);
401                        }
402                }
403
404                long delay = System.currentTimeMillis() - start;
405                ourLog.info("Batch completed in {}ms", delay);
406
407                return response;
408        }
409
410        @VisibleForTesting
411        public void setHapiTransactionService(HapiTransactionService theHapiTransactionService) {
412                myHapiTransactionService = theHapiTransactionService;
413        }
414
415        private IBaseBundle processTransaction(final RequestDetails theRequestDetails, final IBaseBundle theRequest,
416                                                                                                                final String theActionName, boolean theNestedMode) {
417                validateDependencies();
418
419                String transactionType = myVersionAdapter.getBundleType(theRequest);
420
421                if (org.hl7.fhir.r4.model.Bundle.BundleType.BATCH.toCode().equals(transactionType)) {
422                        return batch(theRequestDetails, theRequest, theNestedMode);
423                }
424
425                if (transactionType == null) {
426                        String message = "Transaction Bundle did not specify valid Bundle.type, assuming " + Bundle.BundleType.TRANSACTION.toCode();
427                        ourLog.warn(message);
428                        transactionType = org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode();
429                }
430                if (!org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode().equals(transactionType)) {
431                        throw new InvalidRequestException(Msg.code(527) + "Unable to process transaction where incoming Bundle.type = " + transactionType);
432                }
433
434                List<IBase> requestEntries = myVersionAdapter.getEntries(theRequest);
435                int numberOfEntries = requestEntries.size();
436
437                if (myDaoConfig.getMaximumTransactionBundleSize() != null && numberOfEntries > myDaoConfig.getMaximumTransactionBundleSize()) {
438                        throw new PayloadTooLargeException(Msg.code(528) + "Transaction Bundle Too large.  Transaction bundle contains " +
439                                numberOfEntries +
440                                " which exceedes the maximum permitted transaction bundle size of " + myDaoConfig.getMaximumTransactionBundleSize());
441                }
442
443                ourLog.debug("Beginning {} with {} resources", theActionName, numberOfEntries);
444
445                final TransactionDetails transactionDetails = new TransactionDetails();
446                final StopWatch transactionStopWatch = new StopWatch();
447
448                // Do all entries have a verb?
449                for (int i = 0; i < numberOfEntries; i++) {
450                        IBase nextReqEntry = requestEntries.get(i);
451                        String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
452                        if (verb == null || !isValidVerb(verb)) {
453                                throw new InvalidRequestException(Msg.code(529) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionEntryHasInvalidVerb", verb, i));
454                        }
455                }
456
457                /*
458                 * We want to execute the transaction request bundle elements in the order
459                 * specified by the FHIR specification (see TransactionSorter) so we save the
460                 * original order in the request, then sort it.
461                 *
462                 * Entries with a type of GET are removed from the bundle so that they
463                 * can be processed at the very end. We do this because the incoming resources
464                 * are saved in a two-phase way in order to deal with interdependencies, and
465                 * we want the GET processing to use the final indexing state
466                 */
467                final IBaseBundle response = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTIONRESPONSE.toCode());
468                List<IBase> getEntries = new ArrayList<>();
469                final IdentityHashMap<IBase, Integer> originalRequestOrder = new IdentityHashMap<>();
470                for (int i = 0; i < requestEntries.size(); i++) {
471                        IBase requestEntry = requestEntries.get(i);
472                        originalRequestOrder.put(requestEntry, i);
473                        myVersionAdapter.addEntry(response);
474                        if (myVersionAdapter.getEntryRequestVerb(myContext, requestEntry).equals("GET")) {
475                                getEntries.add(requestEntry);
476                        }
477                }
478
479                /*
480                 * See FhirSystemDaoDstu3Test#testTransactionWithPlaceholderIdInMatchUrl
481                 * Basically if the resource has a match URL that references a placeholder,
482                 * we try to handle the resource with the placeholder first.
483                 */
484                Set<String> placeholderIds = new HashSet<>();
485                for (IBase nextEntry : requestEntries) {
486                        String fullUrl = myVersionAdapter.getFullUrl(nextEntry);
487                        if (isNotBlank(fullUrl) && fullUrl.startsWith(URN_PREFIX)) {
488                                placeholderIds.add(fullUrl);
489                        }
490                }
491                requestEntries.sort(new TransactionSorter(placeholderIds));
492
493                // perform all writes
494                prepareThenExecuteTransactionWriteOperations(theRequestDetails, theActionName,
495                        transactionDetails, transactionStopWatch,
496                        response, originalRequestOrder, requestEntries);
497
498                // perform all gets
499                // (we do these last so that the gets happen on the final state of the DB;
500                // see above note)
501                doTransactionReadOperations(theRequestDetails, response,
502                        getEntries, originalRequestOrder,
503                        transactionStopWatch, theNestedMode);
504
505                // Interceptor broadcast: JPA_PERFTRACE_INFO
506                if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO, myInterceptorBroadcaster, theRequestDetails)) {
507                        String taskDurations = transactionStopWatch.formatTaskDurations();
508                        StorageProcessingMessage message = new StorageProcessingMessage();
509                        message.setMessage("Transaction timing:\n" + taskDurations);
510                        HookParams params = new HookParams()
511                                .add(RequestDetails.class, theRequestDetails)
512                                .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
513                                .add(StorageProcessingMessage.class, message);
514                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_INFO, params);
515                }
516
517                return response;
518        }
519
520        private void doTransactionReadOperations(final RequestDetails theRequestDetails, IBaseBundle theResponse,
521                                                                                                                  List<IBase> theGetEntries, IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
522                                                                                                                  StopWatch theTransactionStopWatch, boolean theNestedMode) {
523                if (theGetEntries.size() > 0) {
524                        theTransactionStopWatch.startTask("Process " + theGetEntries.size() + " GET entries");
525
526                        /*
527                         * Loop through the request and process any entries of type GET
528                         */
529                        for (IBase nextReqEntry : theGetEntries) {
530                                if (theNestedMode) {
531                                        throw new InvalidRequestException(Msg.code(530) + "Can not invoke read operation on nested transaction");
532                                }
533
534                                if (!(theRequestDetails instanceof ServletRequestDetails)) {
535                                        throw new MethodNotAllowedException(Msg.code(531) + "Can not call transaction GET methods from this context");
536                                }
537
538                                ServletRequestDetails srd = (ServletRequestDetails) theRequestDetails;
539                                Integer originalOrder = theOriginalRequestOrder.get(nextReqEntry);
540                                IBase nextRespEntry = (IBase) myVersionAdapter.getEntries(theResponse).get(originalOrder);
541
542                                ArrayListMultimap<String, String> paramValues = ArrayListMultimap.create();
543
544                                String transactionUrl = extractTransactionUrlOrThrowException(nextReqEntry, "GET");
545
546                                ServletSubRequestDetails requestDetails = ServletRequestUtil.getServletSubRequestDetails(srd, transactionUrl, paramValues);
547
548                                String url = requestDetails.getRequestPath();
549
550                                BaseMethodBinding<?> method = srd.getServer().determineResourceMethod(requestDetails, url);
551                                if (method == null) {
552                                        throw new IllegalArgumentException(Msg.code(532) + "Unable to handle GET " + url);
553                                }
554
555                                if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) {
556                                        requestDetails.addHeader(Constants.HEADER_IF_MATCH, myVersionAdapter.getEntryRequestIfMatch(nextReqEntry));
557                                }
558                                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry))) {
559                                        requestDetails.addHeader(Constants.HEADER_IF_NONE_EXIST, myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry));
560                                }
561                                if (isNotBlank(myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry))) {
562                                        requestDetails.addHeader(Constants.HEADER_IF_NONE_MATCH, myVersionAdapter.getEntryRequestIfNoneMatch(nextReqEntry));
563                                }
564
565                                Validate.isTrue(method instanceof BaseResourceReturningMethodBinding, "Unable to handle GET {}", url);
566                                try {
567                                        BaseResourceReturningMethodBinding methodBinding = (BaseResourceReturningMethodBinding) method;
568                                        requestDetails.setRestOperationType(methodBinding.getRestOperationType());
569
570                                        IBaseResource resource = methodBinding.doInvokeServer(srd.getServer(), requestDetails);
571                                        if (paramValues.containsKey(Constants.PARAM_SUMMARY) || paramValues.containsKey(Constants.PARAM_CONTENT)) {
572                                                resource = filterNestedBundle(requestDetails, resource);
573                                        }
574                                        myVersionAdapter.setResource(nextRespEntry, resource);
575                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(Constants.STATUS_HTTP_200_OK));
576                                } catch (NotModifiedException e) {
577                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(Constants.STATUS_HTTP_304_NOT_MODIFIED));
578                                } catch (BaseServerResponseException e) {
579                                        ourLog.info("Failure processing transaction GET {}: {}", url, e.toString());
580                                        myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(e.getStatusCode()));
581                                        populateEntryWithOperationOutcome(e, nextRespEntry);
582                                }
583                        }
584                        theTransactionStopWatch.endCurrentTask();
585                }
586        }
587
588        /**
589         * All of the write operations in the transaction (PUT, POST, etc.. basically anything
590         * except GET) are performed in their own database transaction before we do the reads.
591         * We do this because the reads (specifically the searches) often spawn their own
592         * secondary database transaction and if we allow that within the primary
593         * database transaction we can end up with deadlocks if the server is under
594         * heavy load with lots of concurrent transactions using all available
595         * database connections.
596         */
597        private void prepareThenExecuteTransactionWriteOperations(RequestDetails theRequestDetails, String theActionName,
598                                                                                                                                                                 TransactionDetails theTransactionDetails, StopWatch theTransactionStopWatch,
599                                                                                                                                                                 IBaseBundle theResponse, IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
600                                                                                                                                                                 List<IBase> theEntries) {
601
602                TransactionWriteOperationsDetails writeOperationsDetails = null;
603                if (haveWriteOperationsHooks(theRequestDetails)) {
604                        writeOperationsDetails = buildWriteOperationsDetails(theEntries);
605                        callWriteOperationsHook(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE, theRequestDetails, theTransactionDetails, writeOperationsDetails);
606                }
607
608                TransactionCallback<EntriesToProcessMap> txCallback = status -> {
609                        final Set<IIdType> allIds = new LinkedHashSet<>();
610                        final IdSubstitutionMap idSubstitutions = new IdSubstitutionMap();
611                        final Map<IIdType, DaoMethodOutcome> idToPersistedOutcome = new HashMap<>();
612
613                        EntriesToProcessMap retVal = doTransactionWriteOperations(theRequestDetails, theActionName,
614                                theTransactionDetails, allIds,
615                                idSubstitutions, idToPersistedOutcome,
616                                theResponse, theOriginalRequestOrder,
617                                theEntries, theTransactionStopWatch);
618
619                        theTransactionStopWatch.startTask("Commit writes to database");
620                        return retVal;
621                };
622                EntriesToProcessMap entriesToProcess;
623
624                try {
625                        entriesToProcess = myHapiTransactionService.execute(theRequestDetails, theTransactionDetails, txCallback);
626                } finally {
627                        if (haveWriteOperationsHooks(theRequestDetails)) {
628                                callWriteOperationsHook(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST, theRequestDetails, theTransactionDetails, writeOperationsDetails);
629                        }
630                }
631
632                theTransactionStopWatch.endCurrentTask();
633
634                for (Map.Entry<IBase, IIdType> nextEntry : entriesToProcess.entrySet()) {
635                        String responseLocation = nextEntry.getValue().toUnqualified().getValue();
636                        String responseEtag = nextEntry.getValue().getVersionIdPart();
637                        myVersionAdapter.setResponseLocation(nextEntry.getKey(), responseLocation);
638                        myVersionAdapter.setResponseETag(nextEntry.getKey(), responseEtag);
639                }
640        }
641
642        private boolean haveWriteOperationsHooks(RequestDetails theRequestDetails) {
643                return CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_PRE, myInterceptorBroadcaster, theRequestDetails) ||
644                        CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_TRANSACTION_WRITE_OPERATIONS_POST, myInterceptorBroadcaster, theRequestDetails);
645        }
646
647        private void callWriteOperationsHook(Pointcut thePointcut, RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, TransactionWriteOperationsDetails theWriteOperationsDetails) {
648                HookParams params = new HookParams()
649                        .add(TransactionDetails.class, theTransactionDetails)
650                        .add(TransactionWriteOperationsDetails.class, theWriteOperationsDetails);
651                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, thePointcut, params);
652        }
653
654        private TransactionWriteOperationsDetails buildWriteOperationsDetails(List<IBase> theEntries) {
655                TransactionWriteOperationsDetails writeOperationsDetails;
656                List<String> updateRequestUrls = new ArrayList<>();
657                List<String> conditionalCreateRequestUrls = new ArrayList<>();
658                //Extract
659                for (IBase nextEntry : theEntries) {
660                        String method = myVersionAdapter.getEntryRequestVerb(myContext, nextEntry);
661                        if ("PUT".equals(method)) {
662                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextEntry);
663                                if (isNotBlank(requestUrl)) {
664                                        updateRequestUrls.add(requestUrl);
665                                }
666                        } else if ("POST".equals(method)) {
667                                String requestUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextEntry);
668                                if (isNotBlank(requestUrl) && requestUrl.contains("?")) {
669                                        conditionalCreateRequestUrls.add(requestUrl);
670                                }
671                        }
672                }
673
674                writeOperationsDetails = new TransactionWriteOperationsDetails();
675                writeOperationsDetails.setUpdateRequestUrls(updateRequestUrls);
676                writeOperationsDetails.setConditionalCreateRequestUrls(conditionalCreateRequestUrls);
677                return writeOperationsDetails;
678        }
679
680        private boolean isValidVerb(String theVerb) {
681                try {
682                        return org.hl7.fhir.r4.model.Bundle.HTTPVerb.fromCode(theVerb) != null;
683                } catch (FHIRException theE) {
684                        return false;
685                }
686        }
687
688        /**
689         * This method is called for nested bundles (e.g. if we received a transaction with an entry that
690         * was a GET search, this method is called on the bundle for the search result, that will be placed in the
691         * outer bundle). This method applies the _summary and _content parameters to the output of
692         * that bundle.
693         * <p>
694         * TODO: This isn't the most efficient way of doing this.. hopefully we can come up with something better in the future.
695         */
696        private IBaseResource filterNestedBundle(RequestDetails theRequestDetails, IBaseResource theResource) {
697                IParser p = myContext.newJsonParser();
698                RestfulServerUtils.configureResponseParser(theRequestDetails, p);
699                return p.parseResource(theResource.getClass(), p.encodeResourceToString(theResource));
700        }
701
702        protected void validateDependencies() {
703                Validate.notNull(myContext);
704                Validate.notNull(myTxManager);
705        }
706
707        private IIdType newIdType(String theValue) {
708                return myContext.getVersion().newIdType().setValue(theValue);
709        }
710
711        @VisibleForTesting
712        public void setModelConfig(ModelConfig theModelConfig) {
713                myModelConfig = theModelConfig;
714        }
715
716        /**
717         * Searches for duplicate conditional creates and consolidates them.
718         */
719        private void consolidateDuplicateConditionals(RequestDetails theRequestDetails, String theActionName, List<IBase> theEntries) {
720                final Set<String> keysWithNoFullUrl = new HashSet<>();
721                final HashMap<String, String> keyToUuid = new HashMap<>();
722
723                for (int index = 0, originalIndex = 0; index < theEntries.size(); index++, originalIndex++) {
724                        IBase nextReqEntry = theEntries.get(index);
725                        IBaseResource resource = myVersionAdapter.getResource(nextReqEntry);
726                        if (resource != null) {
727                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
728                                String entryFullUrl = myVersionAdapter.getFullUrl(nextReqEntry);
729                                String requestUrl = myVersionAdapter.getEntryRequestUrl(nextReqEntry);
730                                String ifNoneExist = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
731
732                                // Conditional UPDATE
733                                boolean consolidateEntryCandidate = false;
734                                String conditionalUrl;
735                                switch (verb) {
736                                        case "PUT":
737                                                conditionalUrl = requestUrl;
738                                                if (isNotBlank(requestUrl)) {
739                                                        int questionMarkIndex = requestUrl.indexOf('?');
740                                                        if (questionMarkIndex >= 0 && requestUrl.length() > (questionMarkIndex + 1)) {
741                                                                consolidateEntryCandidate = true;
742                                                        }
743                                                }
744                                                break;
745
746                                        // Conditional CREATE
747                                        case "POST":
748                                                conditionalUrl = ifNoneExist;
749                                                if (isNotBlank(ifNoneExist)) {
750                                                        if (isBlank(entryFullUrl) || !entryFullUrl.equals(requestUrl)) {
751                                                                consolidateEntryCandidate = true;
752                                                        }
753                                                }
754                                                break;
755
756                                        default:
757                                                continue;
758                                }
759
760                                if (isNotBlank(conditionalUrl) && !conditionalUrl.contains("?")) {
761                                        conditionalUrl = myContext.getResourceType(resource) + "?" + conditionalUrl;
762                                }
763
764                                String key = verb + "|" + conditionalUrl;
765                                if (consolidateEntryCandidate) {
766                                        if (isBlank(entryFullUrl)) {
767                                                if (isNotBlank(conditionalUrl)) {
768                                                        if (!keysWithNoFullUrl.add(key)) {
769                                                                throw new InvalidRequestException(
770                                                                        Msg.code(2008) + "Unable to process " + theActionName + " - Request contains multiple anonymous entries (Bundle.entry.fullUrl not populated) with conditional URL: \"" + UrlUtil.sanitizeUrlPart(conditionalUrl) + "\". Does transaction request contain duplicates?");
771                                                        }
772                                                }
773                                        } else {
774                                                if (!keyToUuid.containsKey(key)) {
775                                                        keyToUuid.put(key, entryFullUrl);
776                                                } else {
777                                                        String msg = "Discarding transaction bundle entry " + originalIndex + " as it contained a duplicate conditional " + verb;
778                                                        ourLog.info(msg);
779                                                        // Interceptor broadcast: JPA_PERFTRACE_INFO
780                                                        if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_WARNING, myInterceptorBroadcaster, theRequestDetails)) {
781                                                                StorageProcessingMessage message = new StorageProcessingMessage().setMessage(msg);
782                                                                HookParams params = new HookParams()
783                                                                        .add(RequestDetails.class, theRequestDetails)
784                                                                        .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
785                                                                        .add(StorageProcessingMessage.class, message);
786                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_INFO, params);
787                                                        }
788
789                                                        theEntries.remove(index);
790                                                        index--;
791                                                        String existingUuid = keyToUuid.get(key);
792                                                        replaceReferencesInEntriesWithConsolidatedUUID(theEntries, entryFullUrl, existingUuid);
793                                                }
794                                        }
795                                }
796                        }
797                }
798        }
799
800        /**
801         * Iterates over all entries, and if it finds any which have references which match the fullUrl of the entry that was consolidated out
802         * replace them with our new consolidated UUID
803         */
804        private void replaceReferencesInEntriesWithConsolidatedUUID(List<IBase> theEntries, String theEntryFullUrl, String existingUuid) {
805                for (IBase nextEntry : theEntries) {
806                        IBaseResource nextResource = myVersionAdapter.getResource(nextEntry);
807                        for (IBaseReference nextReference : myContext.newTerser().getAllPopulatedChildElementsOfType(nextResource, IBaseReference.class)) {
808                                // We're interested in any references directly to the placeholder ID, but also
809                                // references that have a resource target that has the placeholder ID.
810                                String nextReferenceId = nextReference.getReferenceElement().getValue();
811                                if (isBlank(nextReferenceId) && nextReference.getResource() != null) {
812                                        nextReferenceId = nextReference.getResource().getIdElement().getValue();
813                                }
814                                if (theEntryFullUrl.equals(nextReferenceId)) {
815                                        nextReference.setReference(existingUuid);
816                                        nextReference.setResource(null);
817                                }
818                        }
819                }
820        }
821
822        /**
823         * Retrieves the next resource id (IIdType) from the base resource and next request entry.
824         *
825         * @param theBaseResource - base resource
826         * @param theNextReqEntry - next request entry
827         * @param theAllIds       - set of all IIdType values
828         * @return
829         */
830        private IIdType getNextResourceIdFromBaseResource(IBaseResource theBaseResource,
831                                                                                                                                          IBase theNextReqEntry,
832                                                                                                                                          Set<IIdType> theAllIds) {
833                IIdType nextResourceId = null;
834                if (theBaseResource != null) {
835                        nextResourceId = theBaseResource.getIdElement();
836
837                        String fullUrl = myVersionAdapter.getFullUrl(theNextReqEntry);
838                        if (isNotBlank(fullUrl)) {
839                                IIdType fullUrlIdType = newIdType(fullUrl);
840                                if (isPlaceholder(fullUrlIdType)) {
841                                        nextResourceId = fullUrlIdType;
842                                } else if (!nextResourceId.hasIdPart()) {
843                                        nextResourceId = fullUrlIdType;
844                                }
845                        }
846
847                        if (nextResourceId.hasIdPart() && !isPlaceholder(nextResourceId)) {
848                                int colonIndex = nextResourceId.getIdPart().indexOf(':');
849                                if (colonIndex != -1) {
850                                        if (INVALID_PLACEHOLDER_PATTERN.matcher(nextResourceId.getIdPart()).matches()) {
851                                                throw new InvalidRequestException(Msg.code(533) + "Invalid placeholder ID found: " + nextResourceId.getIdPart() + " - Must be of the form 'urn:uuid:[uuid]' or 'urn:oid:[oid]'");
852                                        }
853                                }
854                        }
855
856                        if (nextResourceId.hasIdPart() && !nextResourceId.hasResourceType() && !isPlaceholder(nextResourceId)) {
857                                nextResourceId = newIdType(toResourceName(theBaseResource.getClass()), nextResourceId.getIdPart());
858                                theBaseResource.setId(nextResourceId);
859                        }
860
861                        /*
862                         * Ensure that the bundle doesn't have any duplicates, since this causes all kinds of weirdness
863                         */
864                        if (isPlaceholder(nextResourceId)) {
865                                if (!theAllIds.add(nextResourceId)) {
866                                        throw new InvalidRequestException(Msg.code(534) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionContainsMultipleWithDuplicateId", nextResourceId));
867                                }
868                        } else if (nextResourceId.hasResourceType() && nextResourceId.hasIdPart()) {
869                                IIdType nextId = nextResourceId.toUnqualifiedVersionless();
870                                if (!theAllIds.add(nextId)) {
871                                        throw new InvalidRequestException(Msg.code(535) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionContainsMultipleWithDuplicateId", nextId));
872                                }
873                        }
874
875                }
876
877                return nextResourceId;
878        }
879
880        /**
881         * After pre-hooks have been called
882         */
883        protected EntriesToProcessMap doTransactionWriteOperations(final RequestDetails theRequest, String theActionName,
884                                                                                                                                                                  TransactionDetails theTransactionDetails, Set<IIdType> theAllIds,
885                                                                                                                                                                  IdSubstitutionMap theIdSubstitutions, Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
886                                                                                                                                                                  IBaseBundle theResponse, IdentityHashMap<IBase, Integer> theOriginalRequestOrder,
887                                                                                                                                                                  List<IBase> theEntries, StopWatch theTransactionStopWatch) {
888
889                // During a transaction, we don't execute hooks, instead, we execute them all post-transaction.
890                theTransactionDetails.beginAcceptingDeferredInterceptorBroadcasts(
891                        Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED,
892                        Pointcut.STORAGE_PRECOMMIT_RESOURCE_UPDATED,
893                        Pointcut.STORAGE_PRECOMMIT_RESOURCE_DELETED
894                );
895                try {
896                        Set<String> deletedResources = new HashSet<>();
897                        DeleteConflictList deleteConflicts = new DeleteConflictList();
898                        EntriesToProcessMap entriesToProcess = new EntriesToProcessMap();
899                        Set<IIdType> nonUpdatedEntities = new HashSet<>();
900                        Set<IBasePersistedResource> updatedEntities = new HashSet<>();
901                        Map<String, IIdType> conditionalUrlToIdMap = new HashMap<>();
902                        List<IBaseResource> updatedResources = new ArrayList<>();
903                        Map<String, Class<? extends IBaseResource>> conditionalRequestUrls = new HashMap<>();
904
905                        /*
906                         * Look for duplicate conditional creates and consolidate them
907                         */
908                        consolidateDuplicateConditionals(theRequest, theActionName, theEntries);
909
910                        /*
911                         * Loop through the request and process any entries of type
912                         * PUT, POST or DELETE
913                         */
914                        for (int i = 0; i < theEntries.size(); i++) {
915                                if (i % 250 == 0) {
916                                        ourLog.debug("Processed {} non-GET entries out of {} in transaction", i, theEntries.size());
917                                }
918
919                                IBase nextReqEntry = theEntries.get(i);
920                                IBaseResource res = myVersionAdapter.getResource(nextReqEntry);
921                                IIdType nextResourceId = getNextResourceIdFromBaseResource(res, nextReqEntry, theAllIds);
922
923                                String verb = myVersionAdapter.getEntryRequestVerb(myContext, nextReqEntry);
924                                String resourceType = res != null ? myContext.getResourceType(res) : null;
925                                Integer order = theOriginalRequestOrder.get(nextReqEntry);
926                                IBase nextRespEntry = (IBase) myVersionAdapter.getEntries(theResponse).get(order);
927
928                                theTransactionStopWatch.startTask("Bundle.entry[" + i + "]: " + verb + " " + defaultString(resourceType));
929
930                                switch (verb) {
931                                        case "POST": {
932                                                // CREATE
933                                                /*
934                                                 * To preserve existing functionality,
935                                                 * we will only verify that the request url is
936                                                 * valid if it's provided at all.
937                                                 * Otherwise, we'll ignore it
938                                                 */
939                                                String url = myVersionAdapter.getEntryRequestUrl(nextReqEntry);
940                                                if (isNotBlank(url)) {
941                                                        extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
942                                                }
943                                                validateResourcePresent(res, order, verb);
944                                                @SuppressWarnings("rawtypes")
945                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
946                                                res.setId((String) null);
947                                                DaoMethodOutcome outcome;
948                                                String matchUrl = myVersionAdapter.getEntryRequestIfNoneExist(nextReqEntry);
949                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
950                                                outcome = resourceDao.create(res, matchUrl, false, theTransactionDetails, theRequest);
951                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
952                                                res.setId(outcome.getId());
953                                                if (nextResourceId != null) {
954                                                        handleTransactionCreateOrUpdateOutcome(theIdSubstitutions, theIdToPersistedOutcome, nextResourceId, outcome, nextRespEntry, resourceType, res, theRequest);
955                                                }
956                                                entriesToProcess.put(nextRespEntry, outcome.getId());
957                                                if (outcome.getCreated() == false) {
958                                                        nonUpdatedEntities.add(outcome.getId());
959                                                } else {
960                                                        if (isNotBlank(matchUrl)) {
961                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
962                                                        }
963                                                }
964
965                                                break;
966                                        }
967                                        case "DELETE": {
968                                                // DELETE
969                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
970                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
971                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
972                                                int status = Constants.STATUS_HTTP_204_NO_CONTENT;
973                                                if (parts.getResourceId() != null) {
974                                                        IIdType deleteId = newIdType(parts.getResourceType(), parts.getResourceId());
975                                                        if (!deletedResources.contains(deleteId.getValueAsString())) {
976                                                                DaoMethodOutcome outcome = dao.delete(deleteId, deleteConflicts, theRequest, theTransactionDetails);
977                                                                if (outcome.getEntity() != null) {
978                                                                        deletedResources.add(deleteId.getValueAsString());
979                                                                        entriesToProcess.put(nextRespEntry, outcome.getId());
980                                                                }
981                                                        }
982                                                } else {
983                                                        String matchUrl = parts.getResourceType() + '?' + parts.getParams();
984                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
985                                                        DeleteMethodOutcome deleteOutcome = dao.deleteByUrl(matchUrl, deleteConflicts, theRequest);
986                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, deleteOutcome.getId());
987                                                        List<ResourceTable> allDeleted = deleteOutcome.getDeletedEntities();
988                                                        for (ResourceTable deleted : allDeleted) {
989                                                                deletedResources.add(deleted.getIdDt().toUnqualifiedVersionless().getValueAsString());
990                                                        }
991                                                        if (allDeleted.isEmpty()) {
992                                                                status = Constants.STATUS_HTTP_204_NO_CONTENT;
993                                                        }
994
995                                                        myVersionAdapter.setResponseOutcome(nextRespEntry, deleteOutcome.getOperationOutcome());
996                                                }
997
998                                                myVersionAdapter.setResponseStatus(nextRespEntry, toStatusString(status));
999
1000                                                break;
1001                                        }
1002                                        case "PUT": {
1003                                                // UPDATE
1004                                                validateResourcePresent(res, order, verb);
1005                                                @SuppressWarnings("rawtypes")
1006                                                IFhirResourceDao resourceDao = getDaoOrThrowException(res.getClass());
1007
1008                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1009
1010                                                DaoMethodOutcome outcome;
1011                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1012                                                if (isNotBlank(parts.getResourceId())) {
1013                                                        String version = null;
1014                                                        if (isNotBlank(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry))) {
1015                                                                version = ParameterUtil.parseETagValue(myVersionAdapter.getEntryRequestIfMatch(nextReqEntry));
1016                                                        }
1017                                                        res.setId(newIdType(parts.getResourceType(), parts.getResourceId(), version));
1018                                                        outcome = resourceDao.update(res, null, false, false, theRequest, theTransactionDetails);
1019                                                } else {
1020                                                        res.setId((String) null);
1021                                                        String matchUrl;
1022                                                        if (isNotBlank(parts.getParams())) {
1023                                                                matchUrl = parts.getResourceType() + '?' + parts.getParams();
1024                                                        } else {
1025                                                                matchUrl = parts.getResourceType();
1026                                                        }
1027                                                        matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1028                                                        outcome = resourceDao.update(res, matchUrl, false, false, theRequest, theTransactionDetails);
1029                                                        setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1030                                                        if (Boolean.TRUE.equals(outcome.getCreated())) {
1031                                                                conditionalRequestUrls.put(matchUrl, res.getClass());
1032                                                        }
1033                                                }
1034
1035                                                if (outcome.getCreated() == Boolean.FALSE
1036                                                        || (outcome.getCreated() == Boolean.TRUE && outcome.getId().getVersionIdPartAsLong() > 1)) {
1037                                                        updatedEntities.add(outcome.getEntity());
1038                                                        if (outcome.getResource() != null) {
1039                                                                updatedResources.add(outcome.getResource());
1040                                                        }
1041                                                }
1042
1043                                                handleTransactionCreateOrUpdateOutcome(theIdSubstitutions, theIdToPersistedOutcome, nextResourceId,
1044                                                        outcome, nextRespEntry, resourceType, res, theRequest);
1045                                                entriesToProcess.put(nextRespEntry, outcome.getId());
1046                                                break;
1047                                        }
1048                                        case "PATCH": {
1049                                                // PATCH
1050                                                validateResourcePresent(res, order, verb);
1051
1052                                                String url = extractAndVerifyTransactionUrlForEntry(nextReqEntry, verb);
1053                                                UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1054
1055                                                String matchUrl = toMatchUrl(nextReqEntry);
1056                                                matchUrl = performIdSubstitutionsInMatchUrl(theIdSubstitutions, matchUrl);
1057                                                String patchBody = null;
1058                                                String contentType;
1059                                                IBaseParameters patchBodyParameters = null;
1060                                                PatchTypeEnum patchType = null;
1061
1062                                                if (res instanceof IBaseBinary) {
1063                                                        IBaseBinary binary = (IBaseBinary) res;
1064                                                        if (binary.getContent() != null && binary.getContent().length > 0) {
1065                                                                patchBody = toUtf8String(binary.getContent());
1066                                                        }
1067                                                        contentType = binary.getContentType();
1068                                                        patchType = PatchTypeEnum.forContentTypeOrThrowInvalidRequestException(myContext, contentType);
1069                                                        if (patchType == PatchTypeEnum.FHIR_PATCH_JSON || patchType == PatchTypeEnum.FHIR_PATCH_XML) {
1070                                                                String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "fhirPatchShouldNotUseBinaryResource");
1071                                                                throw new InvalidRequestException(Msg.code(536) + msg);
1072                                                        }
1073                                                } else if (res instanceof IBaseParameters) {
1074                                                        patchBodyParameters = (IBaseParameters) res;
1075                                                        patchType = PatchTypeEnum.FHIR_PATCH_JSON;
1076                                                }
1077
1078                                                if (patchBodyParameters == null) {
1079                                                        if (isBlank(patchBody)) {
1080                                                                String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "missingPatchBody");
1081                                                                throw new InvalidRequestException(Msg.code(537) + msg);
1082                                                        }
1083                                                }
1084
1085                                                IFhirResourceDao<? extends IBaseResource> dao = toDao(parts, verb, url);
1086                                                IIdType patchId = myContext.getVersion().newIdType().setValue(parts.getResourceId());
1087
1088                                                String conditionalUrl = isNull(patchId.getIdPart()) ? url : matchUrl;
1089
1090                                                DaoMethodOutcome outcome = dao.patch(patchId, conditionalUrl, patchType, patchBody, patchBodyParameters, theRequest);
1091                                                setConditionalUrlToBeValidatedLater(conditionalUrlToIdMap, matchUrl, outcome.getId());
1092                                                updatedEntities.add(outcome.getEntity());
1093                                                if (outcome.getResource() != null) {
1094                                                        updatedResources.add(outcome.getResource());
1095                                                }
1096                                                if (nextResourceId != null) {
1097                                                        handleTransactionCreateOrUpdateOutcome(theIdSubstitutions, theIdToPersistedOutcome, nextResourceId, outcome, nextRespEntry, resourceType, res, theRequest);
1098                                                }
1099                                                entriesToProcess.put(nextRespEntry, outcome.getId());
1100
1101                                                break;
1102                                        }
1103                                        case "GET":
1104                                                break;
1105                                        default:
1106                                                throw new InvalidRequestException(Msg.code(538) + "Unable to handle verb in transaction: " + verb);
1107
1108                                }
1109
1110                                theTransactionStopWatch.endCurrentTask();
1111                        }
1112
1113                        /*
1114                         * Make sure that there are no conflicts from deletions. E.g. we can't delete something
1115                         * if something else has a reference to it.. Unless the thing that has a reference to it
1116                         * was also deleted as a part of this transaction, which is why we check this now at the
1117                         * end.
1118                         */
1119                        checkForDeleteConflicts(deleteConflicts, deletedResources, updatedResources);
1120
1121                        theIdToPersistedOutcome.entrySet().forEach(idAndOutcome -> {
1122                                theTransactionDetails.addResolvedResourceId(idAndOutcome.getKey(), idAndOutcome.getValue().getPersistentId());
1123                        });
1124
1125                        /*
1126                         * Perform ID substitutions and then index each resource we have saved
1127                         */
1128
1129                        resolveReferencesThenSaveAndIndexResources(theRequest, theTransactionDetails,
1130                                theIdSubstitutions, theIdToPersistedOutcome,
1131                                theTransactionStopWatch, entriesToProcess,
1132                                nonUpdatedEntities, updatedEntities);
1133
1134                        theTransactionStopWatch.endCurrentTask();
1135
1136                        // flush writes to db
1137                        theTransactionStopWatch.startTask("Flush writes to database");
1138
1139                        flushSession(theIdToPersistedOutcome);
1140
1141                        theTransactionStopWatch.endCurrentTask();
1142
1143                        /*
1144                         * Double check we didn't allow any duplicates we shouldn't have
1145                         */
1146                        if (conditionalRequestUrls.size() > 0) {
1147                                theTransactionStopWatch.startTask("Check for conflicts in conditional resources");
1148                        }
1149                        if (!myDaoConfig.isMassIngestionMode()) {
1150                                validateNoDuplicates(theRequest, theActionName, conditionalRequestUrls, theIdToPersistedOutcome.values());
1151                        }
1152
1153                        theTransactionStopWatch.endCurrentTask();
1154                        if (conditionalUrlToIdMap.size() > 0) {
1155                                theTransactionStopWatch.startTask("Check that all conditionally created/updated entities actually match their conditionals.");
1156                        }
1157
1158                        if (!myDaoConfig.isMassIngestionMode()) {
1159                                validateAllInsertsMatchTheirConditionalUrls(theIdToPersistedOutcome, conditionalUrlToIdMap, theRequest);
1160                        }
1161                        theTransactionStopWatch.endCurrentTask();
1162
1163                        for (IIdType next : theAllIds) {
1164                                IIdType replacement = theIdSubstitutions.getForSource(next);
1165                                if (replacement != null && !replacement.equals(next)) {
1166                                        ourLog.debug("Placeholder resource ID \"{}\" was replaced with permanent ID \"{}\"", next, replacement);
1167                                }
1168                        }
1169
1170                        ListMultimap<Pointcut, HookParams> deferredBroadcastEvents = theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1171                        for (Map.Entry<Pointcut, HookParams> nextEntry : deferredBroadcastEvents.entries()) {
1172                                Pointcut nextPointcut = nextEntry.getKey();
1173                                HookParams nextParams = nextEntry.getValue();
1174                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, nextPointcut, nextParams);
1175                        }
1176
1177                        DeferredInterceptorBroadcasts deferredInterceptorBroadcasts = new DeferredInterceptorBroadcasts(deferredBroadcastEvents);
1178                        HookParams params = new HookParams()
1179                                .add(RequestDetails.class, theRequest)
1180                                .addIfMatchesType(ServletRequestDetails.class, theRequest)
1181                                .add(DeferredInterceptorBroadcasts.class, deferredInterceptorBroadcasts)
1182                                .add(TransactionDetails.class, theTransactionDetails)
1183                                .add(IBaseBundle.class, theResponse);
1184                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_TRANSACTION_PROCESSED, params);
1185
1186                        theTransactionDetails.deferredBroadcastProcessingFinished();
1187
1188                        //finishedCallingDeferredInterceptorBroadcasts
1189
1190                        return entriesToProcess;
1191
1192                } finally {
1193                        if (theTransactionDetails.isAcceptingDeferredInterceptorBroadcasts()) {
1194                                theTransactionDetails.endAcceptingDeferredInterceptorBroadcasts();
1195                        }
1196                }
1197        }
1198
1199        private boolean shouldSwapBinaryToActualResource(IBaseResource theResource, String theResourceType, IIdType theNextResourceId) {
1200                if ("Binary".equalsIgnoreCase(theResourceType) && theNextResourceId.getResourceType() != null && !theNextResourceId.getResourceType().equalsIgnoreCase("Binary")) {
1201                        return true;
1202                } else {
1203                        return false;
1204                }
1205        }
1206
1207        private void setConditionalUrlToBeValidatedLater(Map<String, IIdType> theConditionalUrlToIdMap, String theMatchUrl, IIdType theId) {
1208                if (!StringUtils.isBlank(theMatchUrl)) {
1209                        theConditionalUrlToIdMap.put(theMatchUrl, theId);
1210                }
1211        }
1212
1213        /**
1214         * After transaction processing and resolution of indexes and references, we want to validate that the resources that were stored _actually_
1215         * match the conditional URLs that they were brought in on.
1216         */
1217        private void validateAllInsertsMatchTheirConditionalUrls(Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome, Map<String, IIdType> conditionalUrlToIdMap, RequestDetails theRequest) {
1218                conditionalUrlToIdMap.entrySet().stream()
1219                        .filter(entry -> entry.getKey() != null)
1220                        .forEach(entry -> {
1221                                String matchUrl = entry.getKey();
1222                                IIdType value = entry.getValue();
1223                                DaoMethodOutcome daoMethodOutcome = theIdToPersistedOutcome.get(value);
1224                                if (daoMethodOutcome != null && !daoMethodOutcome.isNop() && daoMethodOutcome.getResource() != null) {
1225                                        InMemoryMatchResult match = mySearchParamMatcher.match(matchUrl, daoMethodOutcome.getResource(), theRequest);
1226                                        if (ourLog.isDebugEnabled()) {
1227                                                ourLog.debug("Checking conditional URL [{}] against resource with ID [{}]: Supported?:[{}], Matched?:[{}]", matchUrl, value, match.supported(), match.matched());
1228                                        }
1229                                        if (match.supported()) {
1230                                                if (!match.matched()) {
1231                                                        throw new PreconditionFailedException(Msg.code(539) + "Invalid conditional URL \"" + matchUrl + "\". The given resource is not matched by this URL.");
1232                                                }
1233                                                ;
1234                                        }
1235                                }
1236                        });
1237        }
1238
1239        /**
1240         * Checks for any delete conflicts.
1241         *
1242         * @param theDeleteConflicts  - set of delete conflicts
1243         * @param theDeletedResources - set of deleted resources
1244         * @param theUpdatedResources - list of updated resources
1245         */
1246        private void checkForDeleteConflicts(DeleteConflictList theDeleteConflicts,
1247                                                                                                         Set<String> theDeletedResources,
1248                                                                                                         List<IBaseResource> theUpdatedResources) {
1249                for (Iterator<DeleteConflict> iter = theDeleteConflicts.iterator(); iter.hasNext(); ) {
1250                        DeleteConflict nextDeleteConflict = iter.next();
1251
1252                        /*
1253                         * If we have a conflict, it means we can't delete Resource/A because
1254                         * Resource/B has a reference to it. We'll ignore that conflict though
1255                         * if it turns out we're also deleting Resource/B in this transaction.
1256                         */
1257                        if (theDeletedResources.contains(nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue())) {
1258                                iter.remove();
1259                                continue;
1260                        }
1261
1262                        /*
1263                         * And then, this is kind of a last ditch check. It's also ok to delete
1264                         * Resource/A if Resource/B isn't being deleted, but it is being UPDATED
1265                         * in this transaction, and the updated version of it has no references
1266                         * to Resource/A any more.
1267                         */
1268                        String sourceId = nextDeleteConflict.getSourceId().toUnqualifiedVersionless().getValue();
1269                        String targetId = nextDeleteConflict.getTargetId().toUnqualifiedVersionless().getValue();
1270                        Optional<IBaseResource> updatedSource = theUpdatedResources
1271                                .stream()
1272                                .filter(t -> sourceId.equals(t.getIdElement().toUnqualifiedVersionless().getValue()))
1273                                .findFirst();
1274                        if (updatedSource.isPresent()) {
1275                                List<ResourceReferenceInfo> referencesInSource = myContext.newTerser().getAllResourceReferences(updatedSource.get());
1276                                boolean sourceStillReferencesTarget = referencesInSource
1277                                        .stream()
1278                                        .anyMatch(t -> targetId.equals(t.getResourceReference().getReferenceElement().toUnqualifiedVersionless().getValue()));
1279                                if (!sourceStillReferencesTarget) {
1280                                        iter.remove();
1281                                }
1282                        }
1283                }
1284                DeleteConflictUtil.validateDeleteConflictsEmptyOrThrowException(myContext, theDeleteConflicts);
1285        }
1286
1287        /**
1288         * This method replaces any placeholder references in the
1289         * source transaction Bundle with their actual targets, then stores the resource contents and indexes
1290         * in the database. This is trickier than you'd think because of a couple of possibilities during the
1291         * save:
1292         * * There may be resources that have not changed (e.g. an update/PUT with a resource body identical
1293         * to what is already in the database)
1294         * * There may be resources with auto-versioned references, meaning we're replacing certain references
1295         * in the resource with a versioned references, referencing the current version at the time of the
1296         * transaction processing
1297         * * There may by auto-versioned references pointing to these unchanged targets
1298         * <p>
1299         * If we're not doing any auto-versioned references, we'll just iterate through all resources in the
1300         * transaction and save them one at a time.
1301         * <p>
1302         * However, if we have any auto-versioned references we do this in 2 passes: First the resources from the
1303         * transaction that don't have any auto-versioned references are stored. We do them first since there's
1304         * a chance they may be a NOP and we'll need to account for their version number not actually changing.
1305         * Then we do a second pass for any resources that have auto-versioned references. These happen in a separate
1306         * pass because it's too complex to try and insert the auto-versioned references and still
1307         * account for NOPs, so we block NOPs in that pass.
1308         */
1309        private void resolveReferencesThenSaveAndIndexResources(RequestDetails theRequest, TransactionDetails theTransactionDetails,
1310                                                                                                                                                          IdSubstitutionMap theIdSubstitutions, Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1311                                                                                                                                                          StopWatch theTransactionStopWatch, EntriesToProcessMap entriesToProcess,
1312                                                                                                                                                          Set<IIdType> nonUpdatedEntities, Set<IBasePersistedResource> updatedEntities) {
1313                FhirTerser terser = myContext.newTerser();
1314                theTransactionStopWatch.startTask("Index " + theIdToPersistedOutcome.size() + " resources");
1315                IdentityHashMap<DaoMethodOutcome, Set<IBaseReference>> deferredIndexesForAutoVersioning = null;
1316                int i = 0;
1317                for (DaoMethodOutcome nextOutcome : theIdToPersistedOutcome.values()) {
1318
1319                        if (i++ % 250 == 0) {
1320                                ourLog.debug("Have indexed {} entities out of {} in transaction", i, theIdToPersistedOutcome.values().size());
1321                        }
1322
1323                        if (nextOutcome.isNop()) {
1324                                continue;
1325                        }
1326
1327                        IBaseResource nextResource = nextOutcome.getResource();
1328                        if (nextResource == null) {
1329                                continue;
1330                        }
1331
1332                        Set<IBaseReference> referencesToAutoVersion = BaseStorageDao.extractReferencesToAutoVersion(myContext, myModelConfig, nextResource);
1333                        if (referencesToAutoVersion.isEmpty()) {
1334                                // no references to autoversion - we can do the resolve and save now
1335                                resolveReferencesThenSaveAndIndexResource(theRequest, theTransactionDetails,
1336                                        theIdSubstitutions, theIdToPersistedOutcome,
1337                                        entriesToProcess, nonUpdatedEntities,
1338                                        updatedEntities, terser,
1339                                        nextOutcome, nextResource,
1340                                        referencesToAutoVersion); // this is empty
1341                        } else {
1342                                // we have autoversioned things to defer until later
1343                                if (deferredIndexesForAutoVersioning == null) {
1344                                        deferredIndexesForAutoVersioning = new IdentityHashMap<>();
1345                                }
1346                                deferredIndexesForAutoVersioning.put(nextOutcome, referencesToAutoVersion);
1347                        }
1348                }
1349
1350                // If we have any resources we'll be auto-versioning, index these next
1351                if (deferredIndexesForAutoVersioning != null) {
1352                        for (Map.Entry<DaoMethodOutcome, Set<IBaseReference>> nextEntry : deferredIndexesForAutoVersioning.entrySet()) {
1353                                DaoMethodOutcome nextOutcome = nextEntry.getKey();
1354                                Set<IBaseReference> referencesToAutoVersion = nextEntry.getValue();
1355                                IBaseResource nextResource = nextOutcome.getResource();
1356
1357
1358                                resolveReferencesThenSaveAndIndexResource(theRequest, theTransactionDetails,
1359                                        theIdSubstitutions, theIdToPersistedOutcome,
1360                                        entriesToProcess, nonUpdatedEntities,
1361                                        updatedEntities, terser,
1362                                        nextOutcome, nextResource,
1363                                        referencesToAutoVersion);
1364                        }
1365                }
1366        }
1367
1368        private void resolveReferencesThenSaveAndIndexResource(RequestDetails theRequest, TransactionDetails theTransactionDetails,
1369                                                                                                                                                         IdSubstitutionMap theIdSubstitutions, Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome,
1370                                                                                                                                                         EntriesToProcessMap entriesToProcess, Set<IIdType> nonUpdatedEntities,
1371                                                                                                                                                         Set<IBasePersistedResource> updatedEntities, FhirTerser terser,
1372                                                                                                                                                         DaoMethodOutcome nextOutcome, IBaseResource nextResource,
1373                                                                                                                                                         Set<IBaseReference> theReferencesToAutoVersion) {
1374                // References
1375                List<ResourceReferenceInfo> allRefs = terser.getAllResourceReferences(nextResource);
1376                for (ResourceReferenceInfo nextRef : allRefs) {
1377                        IBaseReference resourceReference = nextRef.getResourceReference();
1378                        IIdType nextId = resourceReference.getReferenceElement();
1379                        IIdType newId = null;
1380                        if (!nextId.hasIdPart()) {
1381                                if (resourceReference.getResource() != null) {
1382                                        IIdType targetId = resourceReference.getResource().getIdElement();
1383                                        if (targetId.getValue() == null || targetId.getValue().startsWith("#")) {
1384                                                // This means it's a contained resource
1385                                                continue;
1386                                        } else if (theIdSubstitutions.containsTarget(targetId)) {
1387                                                newId = targetId;
1388                                        } else {
1389                                                throw new InternalErrorException(Msg.code(540) + "References by resource with no reference ID are not supported in DAO layer");
1390                                        }
1391                                } else {
1392                                        continue;
1393                                }
1394                        }
1395                        if (newId != null || theIdSubstitutions.containsSource(nextId)) {
1396                                if (newId == null) {
1397                                        newId = theIdSubstitutions.getForSource(nextId);
1398                                }
1399                                if (newId != null) {
1400                                        ourLog.debug(" * Replacing resource ref {} with {}", nextId, newId);
1401
1402                                        addRollbackReferenceRestore(theTransactionDetails, resourceReference);
1403                                        if (theReferencesToAutoVersion.contains(resourceReference)) {
1404                                                resourceReference.setReference(newId.getValue());
1405                                                resourceReference.setResource(null);
1406                                        } else {
1407                                                resourceReference.setReference(newId.toVersionless().getValue());
1408                                                resourceReference.setResource(null);
1409                                        }
1410                                }
1411                        } else if (nextId.getValue().startsWith("urn:")) {
1412                                throw new InvalidRequestException(Msg.code(541) + "Unable to satisfy placeholder ID " + nextId.getValue() + " found in element named '" + nextRef.getName() + "' within resource of type: " + nextResource.getIdElement().getResourceType());
1413                        } else {
1414                                // get a map of
1415                                // existing ids -> PID (for resources that exist in the DB)
1416                                // should this be allPartitions?
1417                                ResourcePersistentIdMap resourceVersionMap = myResourceVersionSvc.getLatestVersionIdsForResourceIds(RequestPartitionId.allPartitions(),
1418                                        theReferencesToAutoVersion.stream()
1419                                                .map(IBaseReference::getReferenceElement).collect(Collectors.toList()));
1420
1421                                for (IBaseReference baseRef : theReferencesToAutoVersion) {
1422                                        IIdType id = baseRef.getReferenceElement();
1423                                        if (!resourceVersionMap.containsKey(id)
1424                                                && myDaoConfig.isAutoCreatePlaceholderReferenceTargets()) {
1425                                                // not in the db, but autocreateplaceholders is true
1426                                                // so the version we'll set is "1" (since it will be
1427                                                // created later)
1428                                                String newRef = id.withVersion("1").getValue();
1429                                                id.setValue(newRef);
1430                                        } else {
1431                                                // we will add the looked up info to the transaction
1432                                                // for later
1433                                                theTransactionDetails.addResolvedResourceId(id,
1434                                                        resourceVersionMap.getResourcePersistentId(id));
1435                                        }
1436                                }
1437
1438                                if (theReferencesToAutoVersion.contains(resourceReference)) {
1439                                        DaoMethodOutcome outcome = theIdToPersistedOutcome.get(nextId);
1440
1441                                        if (outcome != null && !outcome.isNop() && !Boolean.TRUE.equals(outcome.getCreated())) {
1442                                                addRollbackReferenceRestore(theTransactionDetails, resourceReference);
1443                                                resourceReference.setReference(nextId.getValue());
1444                                                resourceReference.setResource(null);
1445                                        }
1446                                }
1447                        }
1448                }
1449
1450                // URIs
1451                Class<? extends IPrimitiveType<?>> uriType = (Class<? extends IPrimitiveType<?>>) myContext.getElementDefinition("uri").getImplementingClass();
1452                List<? extends IPrimitiveType<?>> allUris = terser.getAllPopulatedChildElementsOfType(nextResource, uriType);
1453                for (IPrimitiveType<?> nextRef : allUris) {
1454                        if (nextRef instanceof IIdType) {
1455                                continue; // No substitution on the resource ID itself!
1456                        }
1457                        String nextUriString = nextRef.getValueAsString();
1458                        if (theIdSubstitutions.containsSource(nextUriString)) {
1459                                IIdType newId = theIdSubstitutions.getForSource(nextUriString);
1460                                ourLog.debug(" * Replacing resource ref {} with {}", nextUriString, newId);
1461
1462                                String existingValue = nextRef.getValueAsString();
1463                                theTransactionDetails.addRollbackUndoAction(() -> nextRef.setValueAsString(existingValue));
1464
1465                                nextRef.setValueAsString(newId.toVersionless().getValue());
1466                        } else {
1467                                ourLog.debug(" * Reference [{}] does not exist in bundle", nextUriString);
1468                        }
1469                }
1470
1471                IPrimitiveType<Date> deletedInstantOrNull;
1472                if (nextResource instanceof IAnyResource) {
1473                        deletedInstantOrNull = ResourceMetadataKeyEnum.DELETED_AT.get((IAnyResource) nextResource);
1474                } else {
1475                        deletedInstantOrNull = ResourceMetadataKeyEnum.DELETED_AT.get((IResource) nextResource);
1476                }
1477                Date deletedTimestampOrNull = deletedInstantOrNull != null ? deletedInstantOrNull.getValue() : null;
1478
1479                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(nextResource.getClass());
1480                IJpaDao jpaDao = (IJpaDao) dao;
1481
1482                IBasePersistedResource updateOutcome = null;
1483                if (updatedEntities.contains(nextOutcome.getEntity())) {
1484                        boolean forceUpdateVersion = !theReferencesToAutoVersion.isEmpty();
1485
1486                        updateOutcome = jpaDao.updateInternal(theRequest, nextResource, true, forceUpdateVersion, nextOutcome.getEntity(), nextResource.getIdElement(), nextOutcome.getPreviousResource(), theTransactionDetails);
1487                } else if (!nonUpdatedEntities.contains(nextOutcome.getId())) {
1488                        updateOutcome = jpaDao.updateEntity(theRequest, nextResource, nextOutcome.getEntity(), deletedTimestampOrNull, true, false, theTransactionDetails, false, true);
1489                }
1490
1491                // Make sure we reflect the actual final version for the resource.
1492                if (updateOutcome != null) {
1493                        IIdType newId = updateOutcome.getIdDt();
1494
1495                        IIdType entryId = entriesToProcess.getIdWithVersionlessComparison(newId);
1496                        if (entryId != null && !StringUtils.equals(entryId.getValue(), newId.getValue())) {
1497                                entryId.setValue(newId.getValue());
1498                        }
1499
1500                        nextOutcome.setId(newId);
1501
1502                        IIdType target = theIdSubstitutions.getForSource(newId);
1503                        if (target != null) {
1504                                target.setValue(newId.getValue());
1505                        }
1506
1507                }
1508        }
1509
1510        private void addRollbackReferenceRestore(TransactionDetails theTransactionDetails, IBaseReference resourceReference) {
1511                String existingValue = resourceReference.getReferenceElement().getValue();
1512                theTransactionDetails.addRollbackUndoAction(() -> resourceReference.setReference(existingValue));
1513        }
1514
1515        private void validateNoDuplicates(RequestDetails theRequest, String theActionName, Map<String, Class<? extends IBaseResource>> conditionalRequestUrls, Collection<DaoMethodOutcome> thePersistedOutcomes) {
1516
1517                IdentityHashMap<IBaseResource, ResourceIndexedSearchParams> resourceToIndexedParams = new IdentityHashMap<>(thePersistedOutcomes.size());
1518                thePersistedOutcomes
1519                        .stream()
1520                        .filter(t -> !t.isNop())
1521                        .filter(t -> t.getEntity() instanceof ResourceTable)//N.B. GGG: This validation never occurs for mongo, as nothing is a ResourceTable.
1522                        .filter(t -> t.getEntity().getDeleted() == null)
1523                        .filter(t -> t.getResource() != null)
1524                        .forEach(t -> resourceToIndexedParams.put(t.getResource(), new ResourceIndexedSearchParams((ResourceTable) t.getEntity())));
1525
1526                for (Map.Entry<String, Class<? extends IBaseResource>> nextEntry : conditionalRequestUrls.entrySet()) {
1527                        String matchUrl = nextEntry.getKey();
1528                        if (isNotBlank(matchUrl)) {
1529                                if (matchUrl.startsWith("?") || (!matchUrl.contains("?") && UNQUALIFIED_MATCH_URL_START.matcher(matchUrl).find())) {
1530                                        StringBuilder b = new StringBuilder();
1531                                        b.append(myContext.getResourceType(nextEntry.getValue()));
1532                                        if (!matchUrl.startsWith("?")) {
1533                                                b.append("?");
1534                                        }
1535                                        b.append(matchUrl);
1536                                        matchUrl = b.toString();
1537                                }
1538
1539                                if (!myInMemoryResourceMatcher.canBeEvaluatedInMemory(matchUrl).supported()) {
1540                                        continue;
1541                                }
1542
1543                                int counter = 0;
1544                                for (Map.Entry<IBaseResource, ResourceIndexedSearchParams> entries : resourceToIndexedParams.entrySet()) {
1545                                        ResourceIndexedSearchParams indexedParams = entries.getValue();
1546                                        IBaseResource resource = entries.getKey();
1547
1548                                        String resourceType = myContext.getResourceType(resource);
1549                                        if (!matchUrl.startsWith(resourceType + "?")) {
1550                                                continue;
1551                                        }
1552
1553                                        if (myInMemoryResourceMatcher.match(matchUrl, resource, indexedParams).matched()) {
1554                                                counter++;
1555                                                if (counter > 1) {
1556                                                        throw new InvalidRequestException(Msg.code(542) + "Unable to process " + theActionName + " - Request would cause multiple resources to match URL: \"" + matchUrl + "\". Does transaction request contain duplicates?");
1557                                                }
1558                                        }
1559                                }
1560                        }
1561                }
1562        }
1563
1564        protected abstract void flushSession(Map<IIdType, DaoMethodOutcome> theIdToPersistedOutcome);
1565
1566        private void validateResourcePresent(IBaseResource theResource, Integer theOrder, String theVerb) {
1567                if (theResource == null) {
1568                        String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "missingMandatoryResource", theVerb, theOrder);
1569                        throw new InvalidRequestException(Msg.code(543) + msg);
1570                }
1571        }
1572
1573        private IIdType newIdType(String theResourceType, String theResourceId, String theVersion) {
1574                org.hl7.fhir.r4.model.IdType id = new org.hl7.fhir.r4.model.IdType(theResourceType, theResourceId, theVersion);
1575                return myContext.getVersion().newIdType().setValue(id.getValue());
1576        }
1577
1578        private IIdType newIdType(String theToResourceName, String theIdPart) {
1579                return newIdType(theToResourceName, theIdPart, null);
1580        }
1581
1582        @VisibleForTesting
1583        public void setDaoRegistry(DaoRegistry theDaoRegistry) {
1584                myDaoRegistry = theDaoRegistry;
1585        }
1586
1587        private IFhirResourceDao getDaoOrThrowException(Class<? extends IBaseResource> theClass) {
1588                IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDaoOrNull(theClass);
1589                if (dao == null) {
1590                        Set<String> types = new TreeSet<>(myDaoRegistry.getRegisteredDaoTypes());
1591                        String type = myContext.getResourceType(theClass);
1592                        String msg = myContext.getLocalizer().getMessage(BaseTransactionProcessor.class, "unsupportedResourceType", type, types.toString());
1593                        throw new InvalidRequestException(Msg.code(544) + msg);
1594                }
1595                return dao;
1596        }
1597
1598        private String toResourceName(Class<? extends IBaseResource> theResourceType) {
1599                return myContext.getResourceType(theResourceType);
1600        }
1601
1602        public void setContext(FhirContext theContext) {
1603                myContext = theContext;
1604        }
1605
1606        /**
1607         * Extracts the transaction url from the entry and verifies it's:
1608         * * not null or bloack
1609         * * is a relative url matching the resourceType it is about
1610         *
1611         * Returns the transaction url (or throws an InvalidRequestException if url is not valid)
1612         */
1613        private String extractAndVerifyTransactionUrlForEntry(IBase theEntry, String theVerb) {
1614                String url = extractTransactionUrlOrThrowException(theEntry, theVerb);
1615
1616                if (!isValidResourceTypeUrl(url)) {
1617                        ourLog.debug("Invalid url. Should begin with a resource type: {}", url);
1618                        String msg = myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, url);
1619                        throw new InvalidRequestException(Msg.code(2006) + msg);
1620                }
1621                return url;
1622        }
1623
1624        /**
1625         * Returns true if the provided url is a valid entry request.url.
1626         *
1627         * This means:
1628         * a) not an absolute url (does not start with http/https)
1629         * b) starts with either a ResourceType or /ResourceType
1630         */
1631        private boolean isValidResourceTypeUrl(@Nonnull String theUrl) {
1632                if (UrlUtil.isAbsolute(theUrl)) {
1633                        return false;
1634                } else {
1635                        int queryStringIndex = theUrl.indexOf("?");
1636                        String url;
1637                        if (queryStringIndex > 0) {
1638                                url = theUrl.substring(0, theUrl.indexOf("?"));
1639                        } else {
1640                                url = theUrl;
1641                        }
1642                        String[] parts;
1643                        if (url.startsWith("/")) {
1644                                parts = url.substring(1).split("/");
1645                        } else {
1646                                parts = url.split("/");
1647                        }
1648                        Set<String> allResourceTypes = myContext.getResourceTypes();
1649
1650                        return allResourceTypes.contains(parts[0]);
1651                }
1652        }
1653
1654        /**
1655         * Extracts the transaction url from the entry and verifies that it is not null/blank
1656         * and returns it
1657         */
1658        private String extractTransactionUrlOrThrowException(IBase nextEntry, String verb) {
1659                String url = myVersionAdapter.getEntryRequestUrl(nextEntry);
1660                if (isBlank(url)) {
1661                        throw new InvalidRequestException(Msg.code(545) + myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionMissingUrl", verb));
1662                }
1663                return url;
1664        }
1665
1666        private IFhirResourceDao<? extends IBaseResource> toDao(UrlUtil.UrlParts theParts, String theVerb, String theUrl) {
1667                RuntimeResourceDefinition resType;
1668                try {
1669                        resType = myContext.getResourceDefinition(theParts.getResourceType());
1670                } catch (DataFormatException e) {
1671                        String msg = myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
1672                        throw new InvalidRequestException(Msg.code(546) + msg);
1673                }
1674                IFhirResourceDao<? extends IBaseResource> dao = null;
1675                if (resType != null) {
1676                        dao = myDaoRegistry.getResourceDao(resType.getImplementingClass());
1677                }
1678                if (dao == null) {
1679                        String msg = myContext.getLocalizer().getMessage(BaseStorageDao.class, "transactionInvalidUrl", theVerb, theUrl);
1680                        throw new InvalidRequestException(Msg.code(547) + msg);
1681                }
1682
1683                return dao;
1684        }
1685
1686        private String toMatchUrl(IBase theEntry) {
1687                String verb = myVersionAdapter.getEntryRequestVerb(myContext, theEntry);
1688                if (verb.equals("POST")) {
1689                        return myVersionAdapter.getEntryIfNoneExist(theEntry);
1690                }
1691                if (verb.equals("PATCH")) {
1692                        return myVersionAdapter.getEntryRequestIfMatch(theEntry);
1693                }
1694                if (verb.equals("PUT") || verb.equals("DELETE")) {
1695                        String url = extractTransactionUrlOrThrowException(theEntry, verb);
1696                        UrlUtil.UrlParts parts = UrlUtil.parseUrl(url);
1697                        if (isBlank(parts.getResourceId())) {
1698                                return parts.getResourceType() + '?' + parts.getParams();
1699                        }
1700                }
1701                return null;
1702        }
1703
1704        /**
1705         * Transaction Order, per the spec:
1706         * <p>
1707         * Process any DELETE interactions
1708         * Process any POST interactions
1709         * Process any PUT interactions
1710         * Process any PATCH interactions
1711         * Process any GET interactions
1712         */
1713        //@formatter:off
1714        public class TransactionSorter implements Comparator<IBase> {
1715
1716                private final Set<String> myPlaceholderIds;
1717
1718                public TransactionSorter(Set<String> thePlaceholderIds) {
1719                        myPlaceholderIds = thePlaceholderIds;
1720                }
1721
1722                @Override
1723                public int compare(IBase theO1, IBase theO2) {
1724                        int o1 = toOrder(theO1);
1725                        int o2 = toOrder(theO2);
1726
1727                        if (o1 == o2) {
1728                                String matchUrl1 = toMatchUrl(theO1);
1729                                String matchUrl2 = toMatchUrl(theO2);
1730                                if (isBlank(matchUrl1) && isBlank(matchUrl2)) {
1731                                        return 0;
1732                                }
1733                                if (isBlank(matchUrl1)) {
1734                                        return -1;
1735                                }
1736                                if (isBlank(matchUrl2)) {
1737                                        return 1;
1738                                }
1739
1740                                boolean match1containsSubstitutions = false;
1741                                boolean match2containsSubstitutions = false;
1742                                for (String nextPlaceholder : myPlaceholderIds) {
1743                                        if (matchUrl1.contains(nextPlaceholder)) {
1744                                                match1containsSubstitutions = true;
1745                                        }
1746                                        if (matchUrl2.contains(nextPlaceholder)) {
1747                                                match2containsSubstitutions = true;
1748                                        }
1749                                }
1750
1751                                if (match1containsSubstitutions && match2containsSubstitutions) {
1752                                        return 0;
1753                                }
1754                                if (!match1containsSubstitutions && !match2containsSubstitutions) {
1755                                        return 0;
1756                                }
1757                                if (match1containsSubstitutions) {
1758                                        return 1;
1759                                } else {
1760                                        return -1;
1761                                }
1762                        }
1763
1764                        return o1 - o2;
1765                }
1766
1767                private int toOrder(IBase theO1) {
1768                        int o1 = 0;
1769                        if (myVersionAdapter.getEntryRequestVerb(myContext, theO1) != null) {
1770                                switch (myVersionAdapter.getEntryRequestVerb(myContext, theO1)) {
1771                                        case "DELETE":
1772                                                o1 = 1;
1773                                                break;
1774                                        case "POST":
1775                                                o1 = 2;
1776                                                break;
1777                                        case "PUT":
1778                                                o1 = 3;
1779                                                break;
1780                                        case "PATCH":
1781                                                o1 = 4;
1782                                                break;
1783                                        case "GET":
1784                                                o1 = 5;
1785                                                break;
1786                                        default:
1787                                                o1 = 0;
1788                                                break;
1789                                }
1790                        }
1791                        return o1;
1792                }
1793
1794        }
1795
1796        public class RetriableBundleTask implements Runnable {
1797
1798                private final CountDownLatch myCompletedLatch;
1799                private final RequestDetails myRequestDetails;
1800                private final IBase myNextReqEntry;
1801                private final Map<Integer, Object> myResponseMap;
1802                private final int myResponseOrder;
1803                private final boolean myNestedMode;
1804                private BaseServerResponseException myLastSeenException;
1805
1806                protected RetriableBundleTask(CountDownLatch theCompletedLatch, RequestDetails theRequestDetails, Map<Integer, Object> theResponseMap, int theResponseOrder, IBase theNextReqEntry, boolean theNestedMode) {
1807                        this.myCompletedLatch = theCompletedLatch;
1808                        this.myRequestDetails = theRequestDetails;
1809                        this.myNextReqEntry = theNextReqEntry;
1810                        this.myResponseMap = theResponseMap;
1811                        this.myResponseOrder = theResponseOrder;
1812                        this.myNestedMode = theNestedMode;
1813                        this.myLastSeenException = null;
1814                }
1815
1816                private void processBatchEntry() {
1817                        IBaseBundle subRequestBundle = myVersionAdapter.createBundle(org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION.toCode());
1818                        myVersionAdapter.addEntry(subRequestBundle, myNextReqEntry);
1819
1820                        IBaseBundle nextResponseBundle = processTransactionAsSubRequest(myRequestDetails, subRequestBundle, "Batch sub-request", myNestedMode);
1821
1822                        IBase subResponseEntry = (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
1823                        myResponseMap.put(myResponseOrder, subResponseEntry);
1824
1825                        /*
1826                         * If the individual entry didn't have a resource in its response, bring the sub-transaction's OperationOutcome across so the client can see it
1827                         */
1828                        if (myVersionAdapter.getResource(subResponseEntry) == null) {
1829                                IBase nextResponseBundleFirstEntry = (IBase) myVersionAdapter.getEntries(nextResponseBundle).get(0);
1830                                myResponseMap.put(myResponseOrder, nextResponseBundleFirstEntry);
1831                        }
1832                }
1833
1834                private boolean processBatchEntryWithRetry() {
1835                        int maxAttempts = 3;
1836                        for (int attempt = 1; ; attempt++) {
1837                                try {
1838                                        processBatchEntry();
1839                                        return true;
1840                                } catch (BaseServerResponseException e) {
1841                                        //If we catch a known and structured exception from HAPI, just fail.
1842                                        myLastSeenException = e;
1843                                        return false;
1844                                } catch (Throwable t) {
1845                                        myLastSeenException = new InternalErrorException(t);
1846                                        //If we have caught a non-tag-storage failure we are unfamiliar with, or we have exceeded max attempts, exit.
1847                                        if (!DaoFailureUtil.isTagStorageFailure(t) || attempt >= maxAttempts) {
1848                                                ourLog.error("Failure during BATCH sub transaction processing", t);
1849                                                return false;
1850                                        }
1851                                }
1852                        }
1853                }
1854
1855                @Override
1856                public void run() {
1857                        boolean success = processBatchEntryWithRetry();
1858                        if (!success) {
1859                                populateResponseMapWithLastSeenException();
1860                        }
1861
1862                        // checking for the parallelism
1863                        ourLog.debug("processing batch for {} is completed", myVersionAdapter.getEntryRequestUrl(myNextReqEntry));
1864                        myCompletedLatch.countDown();
1865                }
1866
1867                private void populateResponseMapWithLastSeenException() {
1868                        BaseServerResponseExceptionHolder caughtEx = new BaseServerResponseExceptionHolder();
1869                        caughtEx.setException(myLastSeenException);
1870                        myResponseMap.put(myResponseOrder, caughtEx);
1871                }
1872
1873        }
1874
1875        private static class BaseServerResponseExceptionHolder {
1876                private BaseServerResponseException myException;
1877
1878                public BaseServerResponseException getException() {
1879                        return myException;
1880                }
1881
1882                public void setException(BaseServerResponseException myException) {
1883                        this.myException = myException;
1884                }
1885        }
1886
1887        public static boolean isPlaceholder(IIdType theId) {
1888                if (theId != null && theId.getValue() != null) {
1889                        return theId.getValue().startsWith("urn:oid:") || theId.getValue().startsWith("urn:uuid:");
1890                }
1891                return false;
1892        }
1893
1894        private static String toStatusString(int theStatusCode) {
1895                return theStatusCode + " " + defaultString(Constants.HTTP_STATUS_NAMES.get(theStatusCode));
1896        }
1897
1898        /**
1899         * Given a match URL containing
1900         *
1901         * @param theIdSubstitutions
1902         * @param theMatchUrl
1903         * @return
1904         */
1905        public static String performIdSubstitutionsInMatchUrl(IdSubstitutionMap theIdSubstitutions, String theMatchUrl) {
1906                String matchUrl = theMatchUrl;
1907                if (isNotBlank(matchUrl) && !theIdSubstitutions.isEmpty()) {
1908
1909                        int startIdx = matchUrl.indexOf('?');
1910                        while (startIdx != -1) {
1911
1912                                int endIdx = matchUrl.indexOf('&', startIdx + 1);
1913                                if (endIdx == -1) {
1914                                        endIdx = matchUrl.length();
1915                                }
1916
1917                                int equalsIdx = matchUrl.indexOf('=', startIdx + 1);
1918
1919                                int searchFrom;
1920                                if (equalsIdx == -1) {
1921                                        searchFrom = matchUrl.length();
1922                                } else if (equalsIdx >= endIdx) {
1923                                        // First equals we found is from a subsequent parameter
1924                                        searchFrom = matchUrl.length();
1925                                } else {
1926                                        String paramValue = matchUrl.substring(equalsIdx + 1, endIdx);
1927                                        boolean isUrn = isUrn(paramValue);
1928                                        boolean isUrnEscaped = !isUrn && isUrnEscaped(paramValue);
1929                                        if (isUrn || isUrnEscaped) {
1930                                                if (isUrnEscaped) {
1931                                                        paramValue = UrlUtil.unescape(paramValue);
1932                                                }
1933                                                IIdType replacement = theIdSubstitutions.getForSource(paramValue);
1934                                                if (replacement != null) {
1935                                                        String replacementValue;
1936                                                        if (replacement.hasVersionIdPart()) {
1937                                                                replacementValue = replacement.toVersionless().getValue();
1938                                                        } else {
1939                                                                replacementValue = replacement.getValue();
1940                                                        }
1941                                                        matchUrl = matchUrl.substring(0, equalsIdx + 1) + replacementValue + matchUrl.substring(endIdx);
1942                                                        searchFrom = equalsIdx + 1 + replacementValue.length();
1943                                                } else {
1944                                                        searchFrom = endIdx;
1945                                                }
1946                                        } else {
1947                                                searchFrom = endIdx;
1948                                        }
1949                                }
1950
1951                                if (searchFrom >= matchUrl.length()) {
1952                                        break;
1953                                }
1954
1955                                startIdx = matchUrl.indexOf('&', searchFrom);
1956                        }
1957
1958                }
1959                return matchUrl;
1960        }
1961
1962        private static boolean isUrn(@Nonnull String theId) {
1963                return theId.startsWith(URN_PREFIX);
1964        }
1965
1966        private static boolean isUrnEscaped(@Nonnull String theId) {
1967                return theId.startsWith(URN_PREFIX_ESCAPED);
1968        }
1969}