001/*-
002 * #%L
003 * HAPI FHIR - Master Data Management
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.mdm.interceptor;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.i18n.Msg;
024import ca.uhn.fhir.interceptor.api.Hook;
025import ca.uhn.fhir.interceptor.api.Pointcut;
026import ca.uhn.fhir.interceptor.model.RequestPartitionId;
027import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
028import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
029import ca.uhn.fhir.jpa.api.model.DeleteConflictList;
030import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc;
031import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
032import ca.uhn.fhir.jpa.api.svc.IMdmClearHelperSvc;
033import ca.uhn.fhir.jpa.dao.expunge.IExpungeEverythingService;
034import ca.uhn.fhir.mdm.api.IMdmLink;
035import ca.uhn.fhir.mdm.api.IMdmLinkUpdaterSvc;
036import ca.uhn.fhir.mdm.api.IMdmSettings;
037import ca.uhn.fhir.mdm.api.IMdmSubmitSvc;
038import ca.uhn.fhir.mdm.api.MdmConstants;
039import ca.uhn.fhir.mdm.api.MdmMatchResultEnum;
040import ca.uhn.fhir.mdm.dao.IMdmLinkDao;
041import ca.uhn.fhir.mdm.model.CanonicalEID;
042import ca.uhn.fhir.mdm.model.MdmCreateOrUpdateParams;
043import ca.uhn.fhir.mdm.model.MdmTransactionContext;
044import ca.uhn.fhir.mdm.svc.MdmLinkDeleteSvc;
045import ca.uhn.fhir.mdm.util.EIDHelper;
046import ca.uhn.fhir.mdm.util.MdmResourceUtil;
047import ca.uhn.fhir.rest.api.server.RequestDetails;
048import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
049import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
050import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
051import ca.uhn.fhir.rest.server.TransactionLogMessages;
052import ca.uhn.fhir.rest.server.exceptions.ForbiddenOperationException;
053import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
054import org.hl7.fhir.instance.model.api.IAnyResource;
055import org.hl7.fhir.instance.model.api.IBaseResource;
056import org.hl7.fhir.instance.model.api.IIdType;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059import org.springframework.beans.factory.annotation.Autowired;
060import org.springframework.stereotype.Service;
061
062import java.util.ArrayList;
063import java.util.Collections;
064import java.util.HashSet;
065import java.util.List;
066import java.util.Map;
067import java.util.Set;
068import java.util.concurrent.ConcurrentHashMap;
069import java.util.concurrent.atomic.AtomicInteger;
070import java.util.stream.Collectors;
071
072import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.MATCH;
073import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.NO_MATCH;
074import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.POSSIBLE_MATCH;
075
076@SuppressWarnings("rawtypes")
077@Service
078public class MdmStorageInterceptor implements IMdmStorageInterceptor {
079
080        private static final String GOLDEN_RESOURCES_TO_DELETE = "GR_TO_DELETE";
081
082        private static final Logger ourLog = LoggerFactory.getLogger(MdmStorageInterceptor.class);
083
084        // Used to bypass trying to remove mdm links associated to a resource when running mdm-clear batch job, which
085        // deletes all links beforehand, and impacts performance for no action
086        private static final ThreadLocal<Boolean> ourLinksDeletedBeforehand = ThreadLocal.withInitial(() -> Boolean.FALSE);
087
088        @Autowired
089        private IMdmClearHelperSvc<? extends IResourcePersistentId<?>> myIMdmClearHelperSvc;
090
091        @Autowired
092        private IExpungeEverythingService myExpungeEverythingService;
093
094        @Autowired
095        private MdmLinkDeleteSvc myMdmLinkDeleteSvc;
096
097        @Autowired
098        private FhirContext myFhirContext;
099
100        @Autowired
101        private EIDHelper myEIDHelper;
102
103        @Autowired
104        private IMdmSettings myMdmSettings;
105
106        @Autowired
107        private IIdHelperService myIdHelperSvc;
108
109        @Autowired
110        private IMdmLinkDao myMdmLinkDao;
111
112        @Autowired
113        private IMdmSubmitSvc myMdmSubmitSvc;
114
115        @Autowired
116        private DaoRegistry myDaoRegistry;
117
118        @Autowired
119        private IMdmLinkUpdaterSvc mdmLinkUpdaterSvc;
120
121        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED)
122        public void blockManualResourceManipulationOnCreate(
123                        IBaseResource theBaseResource,
124                        RequestDetails theRequestDetails,
125                        ServletRequestDetails theServletRequestDetails) {
126                ourLog.debug(
127                                "Starting pre-storage resource created hook for {}, {}, {}",
128                                theBaseResource,
129                                theRequestDetails,
130                                theServletRequestDetails);
131                if (theBaseResource == null) {
132                        ourLog.warn("Attempting to block golden resource manipulation on a null resource");
133                        return;
134                }
135
136                // If running in single EID mode, forbid multiple eids.
137                if (myMdmSettings.isPreventMultipleEids()) {
138                        ourLog.debug("Forbidding multiple EIDs on {}", theBaseResource);
139                        forbidIfHasMultipleEids(theBaseResource);
140                }
141
142                // TODO GGG MDM find a better way to identify internal calls?
143                if (isInternalRequest(theRequestDetails)) {
144                        ourLog.debug("Internal request - completed processing");
145                        return;
146                }
147
148                forbidIfMdmManagedTagIsPresent(theBaseResource);
149        }
150
151        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED)
152        public void blockManualGoldenResourceManipulationOnUpdate(
153                        IBaseResource theOldResource,
154                        IBaseResource theUpdatedResource,
155                        RequestDetails theRequestDetails,
156                        ServletRequestDetails theServletRequestDetails) {
157                ourLog.debug(
158                                "Starting pre-storage resource updated hook for {}, {}, {}, {}",
159                                theOldResource,
160                                theUpdatedResource,
161                                theRequestDetails,
162                                theServletRequestDetails);
163
164                if (theUpdatedResource == null) {
165                        ourLog.warn("Attempting to block golden resource manipulation on a null resource");
166                        return;
167                }
168
169                // If running in single EID mode, forbid multiple eids.
170                if (myMdmSettings.isPreventMultipleEids()) {
171                        ourLog.debug("Forbidding multiple EIDs on {}", theUpdatedResource);
172                        forbidIfHasMultipleEids(theUpdatedResource);
173                }
174
175                if (MdmResourceUtil.isGoldenRecordRedirected(theUpdatedResource)) {
176                        ourLog.debug(
177                                        "Deleting MDM links to deactivated Golden resource {}",
178                                        theUpdatedResource.getIdElement().toUnqualifiedVersionless());
179                        int deleted = myMdmLinkDeleteSvc.deleteNonRedirectWithAnyReferenceTo(theUpdatedResource);
180                        if (deleted > 0) {
181                                ourLog.debug("Deleted {} MDM links", deleted);
182                        }
183                }
184
185                if (isInternalRequest(theRequestDetails)) {
186                        ourLog.debug("Internal request - completed processing");
187                        return;
188                }
189
190                if (theOldResource != null) {
191                        forbidIfMdmManagedTagIsPresent(theOldResource);
192                        forbidModifyingMdmTag(theUpdatedResource, theOldResource);
193                }
194
195                if (myMdmSettings.isPreventEidUpdates()) {
196                        forbidIfModifyingExternalEidOnTarget(theUpdatedResource, theOldResource);
197                }
198        }
199
200        @Hook(Pointcut.STORAGE_PRECOMMIT_RESOURCE_DELETED)
201        public void deletePostCommit(
202                        RequestDetails theRequest, IBaseResource theResource, TransactionDetails theTransactionDetails) {
203                if (myMdmSettings.isSupportedMdmType(myFhirContext.getResourceType(theResource))) {
204                        Map<IResourcePersistentId, Set<IResourcePersistentId>> goldenIdToSourceIdsMap =
205                                        theTransactionDetails.getUserData(GOLDEN_RESOURCES_TO_DELETE);
206                        if (goldenIdToSourceIdsMap != null) {
207                                IResourcePersistentId sourcePid =
208                                                myIdHelperSvc.getPidOrNull(RequestPartitionId.allPartitions(), theResource);
209                                if (sourcePid != null) {
210                                        for (IResourcePersistentId goldenPid : goldenIdToSourceIdsMap.keySet()) {
211                                                if (goldenIdToSourceIdsMap.get(goldenPid).contains(sourcePid)) {
212                                                        // we only delete the golden resource if it's matched to a source id;
213                                                        // there could be multiple of these, so we only delete the first
214                                                        if (!theTransactionDetails.getDeletedResourceIds().contains(goldenPid)) {
215                                                                IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(theResource);
216                                                                deleteGoldenResource(goldenPid, dao, theRequest);
217                                                                /*
218                                                                 * We will add the removed id to the deleted list so that
219                                                                 * the deletedResourceId list is accurate for what has been
220                                                                 * deleted.
221                                                                 *
222                                                                 * This benefits other interceptor writers who might want
223                                                                 * to do their own resource deletion on this same pre-commit
224                                                                 * hook (and wouldn't be aware if we did this deletion already).
225                                                                 */
226                                                                theTransactionDetails.addDeletedResourceId(goldenPid);
227                                                                // remove the golden resource id so it won't be 're-deleted'
228                                                                // if a second id related to this golden id (linked in some way)
229                                                                // is processed
230                                                                goldenIdToSourceIdsMap.remove(goldenPid);
231                                                        }
232                                                }
233                                        }
234                                }
235                        }
236                }
237        }
238
239        @SuppressWarnings("unchecked")
240        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_DELETED)
241        public void deleteMdmLinks(
242                        RequestDetails theRequest, IBaseResource theResource, TransactionDetails theTransactionDetails) {
243                if (ourLinksDeletedBeforehand.get()) {
244                        return;
245                }
246
247                if (myMdmSettings.isSupportedMdmType(myFhirContext.getResourceType(theResource))) {
248                        IIdType sourceId = theResource.getIdElement().toVersionless();
249                        IResourcePersistentId sourcePid =
250                                        myIdHelperSvc.getPidOrThrowException(RequestPartitionId.allPartitions(), sourceId);
251                        List<IMdmLink> allLinks =
252                                        myMdmLinkDao.findLinksAssociatedWithGoldenResourceOfSourceResourceExcludingNoMatch(sourcePid);
253
254                        Map<MdmMatchResultEnum, List<IMdmLink>> linksByMatchResult =
255                                        allLinks.stream().collect(Collectors.groupingBy(IMdmLink::getMatchResult));
256                        List<IMdmLink> matches =
257                                        linksByMatchResult.containsKey(MATCH) ? linksByMatchResult.get(MATCH) : new ArrayList<>();
258                        List<IMdmLink> possibleMatches = linksByMatchResult.containsKey(POSSIBLE_MATCH)
259                                        ? linksByMatchResult.get(POSSIBLE_MATCH)
260                                        : new ArrayList<>();
261
262                        if (isDeletingLastMatchedSourceResource(sourcePid, matches)) {
263                                /*
264                                 * We are attempting to delete the only source resource left linked to the golden resource.
265                                 * In this case, we'll clean up remaining links and mark the orphaned
266                                 * golden resource for deletion, which we'll do in STORAGE_PRECOMMIT_RESOURCE_DELETED
267                                 */
268                                IResourcePersistentId goldenPid = extractGoldenPid(theResource, matches.get(0));
269                                if (!theTransactionDetails.getDeletedResourceIds().contains(goldenPid)) {
270                                        IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(theResource);
271                                        cleanUpPossibleMatches(possibleMatches, dao, goldenPid, theRequest);
272                                        IAnyResource goldenResource = (IAnyResource) dao.readByPid(goldenPid);
273                                        myMdmLinkDeleteSvc.deleteWithAnyReferenceTo(goldenResource);
274                                        /*
275                                         * Mark the golden resource for deletion.
276                                         * We won't do it yet, because there might be additional deletes coming
277                                         * that include this exact golden resource
278                                         * (eg, if delete is done by a filter and multiple delete is enabled)
279                                         */
280                                        Map<IResourcePersistentId, Set<IResourcePersistentId>> goldenIdsToDelete2linkedSrcIds =
281                                                        theTransactionDetails.getUserData(GOLDEN_RESOURCES_TO_DELETE);
282                                        if (goldenIdsToDelete2linkedSrcIds == null) {
283                                                goldenIdsToDelete2linkedSrcIds = new ConcurrentHashMap<>();
284                                        }
285                                        if (!goldenIdsToDelete2linkedSrcIds.containsKey(goldenPid)) {
286                                                goldenIdsToDelete2linkedSrcIds.put(goldenPid, new HashSet<>());
287                                        }
288                                        goldenIdsToDelete2linkedSrcIds.get(goldenPid).add(sourcePid);
289                                        theTransactionDetails.putUserData(GOLDEN_RESOURCES_TO_DELETE, goldenIdsToDelete2linkedSrcIds);
290                                }
291                        }
292                        myMdmLinkDeleteSvc.deleteWithAnyReferenceTo(theResource);
293                }
294        }
295
296        @SuppressWarnings("rawtypes")
297        private void deleteGoldenResource(
298                        IResourcePersistentId goldenPid, IFhirResourceDao<?> theDao, RequestDetails theRequest) {
299                setLinksDeletedBeforehand();
300
301                if (myMdmSettings.isAutoExpungeGoldenResources()) {
302                        int numDeleted = deleteExpungeGoldenResource(goldenPid);
303                        if (numDeleted > 0) {
304                                ourLog.info("Removed {} golden resource(s).", numDeleted);
305                        }
306                } else {
307                        String url = theRequest == null ? "" : theRequest.getCompleteUrl();
308                        theDao.deletePidList(
309                                        url,
310                                        Collections.singleton(goldenPid),
311                                        new DeleteConflictList(),
312                                        theRequest,
313                                        new TransactionDetails());
314                }
315                resetLinksDeletedBeforehand();
316        }
317
318        /**
319         *  Clean up possible matches associated with a GR if they are the only link left
320         *  since they are no longer "real matches"
321         *  Possible match resources are resubmitted for matching
322         */
323        private void cleanUpPossibleMatches(
324                        List<IMdmLink> possibleMatches,
325                        IFhirResourceDao<?> theDao,
326                        IResourcePersistentId theGoldenPid,
327                        RequestDetails theRequestDetails) {
328                IAnyResource goldenResource = (IAnyResource) theDao.readByPid(theGoldenPid);
329                for (IMdmLink possibleMatch : possibleMatches) {
330                        if (possibleMatch.getGoldenResourcePersistenceId().equals(theGoldenPid)) {
331                                IBaseResource sourceResource = theDao.readByPid(possibleMatch.getSourcePersistenceId());
332                                MdmCreateOrUpdateParams params = new MdmCreateOrUpdateParams();
333                                params.setGoldenResource(goldenResource);
334                                params.setSourceResource((IAnyResource) sourceResource);
335                                params.setMatchResult(NO_MATCH);
336                                MdmTransactionContext mdmContext =
337                                                createMdmContext(MdmTransactionContext.OperationType.UPDATE_LINK, sourceResource.fhirType());
338                                params.setMdmContext(mdmContext);
339                                params.setRequestDetails(theRequestDetails);
340
341                                mdmLinkUpdaterSvc.updateLink(params);
342                        }
343                }
344        }
345
346        private IResourcePersistentId extractGoldenPid(IBaseResource theResource, IMdmLink theMdmLink) {
347                IResourcePersistentId goldenPid = theMdmLink.getGoldenResourcePersistenceId();
348                goldenPid = myIdHelperSvc.newPidFromStringIdAndResourceName(
349                                goldenPid.getPartitionId(), goldenPid.getId().toString(), theResource.fhirType());
350                return goldenPid;
351        }
352
353        private boolean isDeletingLastMatchedSourceResource(IResourcePersistentId theSourcePid, List<IMdmLink> theMatches) {
354                return theMatches.size() == 1
355                                && theMatches.get(0).getSourcePersistenceId().equals(theSourcePid);
356        }
357
358        private MdmTransactionContext createMdmContext(
359                        MdmTransactionContext.OperationType theOperation, String theResourceType) {
360                TransactionLogMessages transactionLogMessages = TransactionLogMessages.createNew();
361                MdmTransactionContext retVal = new MdmTransactionContext(transactionLogMessages, theOperation);
362                retVal.setResourceType(theResourceType);
363                return retVal;
364        }
365
366        @SuppressWarnings("unchecked")
367        private int deleteExpungeGoldenResource(IResourcePersistentId theGoldenPid) {
368                IDeleteExpungeSvc deleteExpungeSvc = myIMdmClearHelperSvc.getDeleteExpungeSvc();
369                return deleteExpungeSvc.deleteExpunge(new ArrayList<>(Collections.singleton(theGoldenPid)), false, null);
370        }
371
372        private void forbidIfModifyingExternalEidOnTarget(IBaseResource theNewResource, IBaseResource theOldResource) {
373                List<CanonicalEID> newExternalEids = Collections.emptyList();
374                List<CanonicalEID> oldExternalEids = Collections.emptyList();
375                if (theNewResource != null) {
376                        newExternalEids = myEIDHelper.getExternalEid(theNewResource);
377                }
378                if (theOldResource != null) {
379                        oldExternalEids = myEIDHelper.getExternalEid(theOldResource);
380                }
381
382                if (oldExternalEids.isEmpty()) {
383                        return;
384                }
385
386                if (!myEIDHelper.eidMatchExists(newExternalEids, oldExternalEids)) {
387                        throwBlockEidChange();
388                }
389        }
390
391        private void throwBlockEidChange() {
392                throw new ForbiddenOperationException(
393                                Msg.code(763) + "While running with EID updates disabled, EIDs may not be updated on source resources");
394        }
395
396        /*
397         * Will throw a forbidden error if a request attempts to add/remove the MDM tag on a Resource.
398         */
399        private void forbidModifyingMdmTag(IBaseResource theNewResource, IBaseResource theOldResource) {
400                if (MdmResourceUtil.isMdmManaged(theNewResource) != MdmResourceUtil.isMdmManaged(theOldResource)) {
401                        throwBlockMdmManagedTagChange();
402                }
403        }
404
405        private void forbidIfHasMultipleEids(IBaseResource theResource) {
406                String resourceType = extractResourceType(theResource);
407                if (myMdmSettings.isSupportedMdmType(resourceType)) {
408                        if (myEIDHelper.getExternalEid(theResource).size() > 1) {
409                                throwBlockMultipleEids();
410                        }
411                }
412        }
413
414        /*
415         * We assume that if we have RequestDetails, then this was an HTTP request and not an internal one.
416         */
417        private boolean isInternalRequest(RequestDetails theRequestDetails) {
418                return theRequestDetails == null || theRequestDetails instanceof SystemRequestDetails;
419        }
420
421        private void forbidIfMdmManagedTagIsPresent(IBaseResource theResource) {
422                if (theResource == null) {
423                        ourLog.warn("Attempting to forbid MDM on a null resource");
424                        return;
425                }
426
427                if (MdmResourceUtil.isMdmManaged(theResource)) {
428                        throwModificationBlockedByMdm();
429                }
430                if (MdmResourceUtil.hasGoldenRecordSystemTag(theResource)) {
431                        throwModificationBlockedByMdm();
432                }
433        }
434
435        private void throwBlockMdmManagedTagChange() {
436                throw new ForbiddenOperationException(Msg.code(764) + "The " + MdmConstants.CODE_HAPI_MDM_MANAGED
437                                + " tag on a resource may not be changed once created.");
438        }
439
440        private void throwModificationBlockedByMdm() {
441                throw new ForbiddenOperationException(Msg.code(765)
442                                + "Cannot create or modify Resources that are managed by MDM. This resource contains a tag with one of these systems: "
443                                + MdmConstants.SYSTEM_GOLDEN_RECORD_STATUS + " or " + MdmConstants.SYSTEM_MDM_MANAGED);
444        }
445
446        private void throwBlockMultipleEids() {
447                throw new ForbiddenOperationException(Msg.code(766)
448                                + "While running with multiple EIDs disabled, source resources may have at most one EID.");
449        }
450
451        private String extractResourceType(IBaseResource theResource) {
452                return myFhirContext.getResourceType(theResource);
453        }
454
455        @Hook(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_EVERYTHING)
456        public void expungeAllMdmLinks(AtomicInteger theCounter) {
457                ourLog.debug("Expunging all MdmLink records");
458                theCounter.addAndGet(myExpungeEverythingService.expungeEverythingMdmLinks());
459        }
460
461        @Hook(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_RESOURCE)
462        public void expungeAllMatchedMdmLinks(AtomicInteger theCounter, IBaseResource theResource) {
463                ourLog.debug("Expunging MdmLink records with reference to {}", theResource.getIdElement());
464                theCounter.addAndGet(myMdmLinkDeleteSvc.deleteWithAnyReferenceTo(theResource));
465        }
466
467        public static void setLinksDeletedBeforehand() {
468                ourLinksDeletedBeforehand.set(Boolean.TRUE);
469        }
470
471        public static void resetLinksDeletedBeforehand() {
472                ourLinksDeletedBeforehand.remove();
473        }
474}